You a expert about cpp, you're using right now, c++ 20, vscode and windows You task is…
- User
You a expert about cpp, you're using right now, c++ 20, vscode and windows You task is to responds of any question about someone, the task is very help the users where the error is, and doing after the code to fix that error You need these context to make the task :
#pragma once #include "../Framework.h"
namespace RLGSC { // https://github.com/AechPro/rocket-league-gym-sim/blob/main/rlgym_sim/utils/common_values.py namespace CommonValues { // Could just use RocketSim's RLConst but I'll just copy to stay faithful
constexpr float SIDE_WALL_X = 4096, BACK_WALL_Y = 5120, CEILING_Z = 2044, BACK_NET_Y = 6000, GOAL_HEIGHT = 642.775, GRAVITY_Z = -650.0, BOOST_CONSUMED_PER_SECOND = 100.0 / 3.0; constexpr Vec ORANGE_GOAL_CENTER = Vec( 0, BACK_WALL_Y, GOAL_HEIGHT / 2 ), BLUE_GOAL_CENTER = Vec( 0, -BACK_WALL_Y, GOAL_HEIGHT / 2 ), // Often more useful than center ORANGE_GOAL_BACK = Vec( 0, BACK_NET_Y, GOAL_HEIGHT / 2 ), BLUE_GOAL_BACK = Vec( 0, -BACK_NET_Y, GOAL_HEIGHT / 2); constexpr float BALL_RADIUS = 92.75, // :nerd::point_up: "erm it is actually 91.25" BALL_MAX_SPEED = 6000, CAR_MAX_SPEED = 2300, SUPERSONIC_THRESHOLD = 2200, CAR_MAX_ANG_VEL = 5.5, BLUE_TEAM = 0, ORANGE_TEAM = 1, NUM_ACTIONS = 8; constexpr int BOOST_LOCATIONS_AMOUNT = 34; constexpr Vec BOOST_LOCATIONS[BOOST_LOCATIONS_AMOUNT] = { {0.f, -4240.0, 70.0}, {-1792.0, -4184.0, 70.0}, {1792.0, -4184.0, 70.0}, {-3072.0, -4096.0, 73.0}, {3072.0, -4096.0, 73.0}, {-940.0, -3308.0, 70.0}, {940.0, -3308.0, 70.0}, {0.0, -2816.0, 70.0}, {-3584.0, -2484.0, 70.0}, {3584.0, -2484.0, 70.0}, {-1788.0, -2300.0, 70.0}, {1788.0, -2300.0, 70.0}, {-2048.0, -1036.0, 70.0}, {0.0, -1024.0, 70.0}, {2048.0, -1036.0, 70.0}, {-3584.0, 0.0, 73.0}, {-1024.0, 0.0, 70.0}, {1024.0, 0.0, 70.0}, {3584.0, 0.0, 73.0}, {-2048.0, 1036.0, 70.0}, {0.0, 1024.0, 70.0}, {2048.0, 1036.0, 70.0}, {-1788.0, 2300.0, 70.0}, {1788.0, 2300.0, 70.0}, {-3584.0, 2484.0, 70.0}, {3584.0, 2484.0, 70.0}, {0.0, 2816.0, 70.0}, {-940.0, 3310.0, 70.0}, {940.0, 3308.0, 70.0}, {-3072.0, 4096.0, 73.0}, {3072.0, 4096.0, 73.0}, {-1792.0, 4184.0, 70.0}, {1792.0, 4184.0, 70.0}, {0.0, 4240.0, 70.0} }; }}
#pragma once #include "PlayerData.h" #include "../CommonValues.h"
namespace RLGSC { struct ScoreLine { int teamGoals[2] = { 0,0 };
int operator[](size_t index) const { return teamGoals[index]; } int& operator[](size_t index) { return teamGoals[index]; } }; // https://github.com/AechPro/rocket-league-gym-sim/blob/main/rlgym_sim/utils/gamestates/game_state.py struct GameState { float deltaTime = 0; // Time that has passed since last update ScoreLine scoreLine; int lastTouchCarID = -1; std::vector<PlayerData> players; BallState ballState; PhysObj ball, ballInv; std::array<bool, CommonValues::BOOST_LOCATIONS_AMOUNT> boostPads; std::array<bool, CommonValues::BOOST_LOCATIONS_AMOUNT> boostPadsInv; std::array<float, CommonValues::BOOST_LOCATIONS_AMOUNT> boostPadTimers; std::array<float, CommonValues::BOOST_LOCATIONS_AMOUNT> boostPadTimersInv; // Last arena we updated with // Can be used to determine current arena from within reward function, for example // NOTE: Could be null Arena* lastArena = NULL; // Last tick count when updated uint64_t lastTickCount = 0; GameState() = default; explicit GameState(Arena* arena) { UpdateFromArena(arena); } const PhysObj& GetBallPhys(bool inverted) const { return inverted ? ballInv : ball; } const auto& GetBoostPads(bool inverted) const { return inverted ? boostPadsInv : boostPads; } void UpdateFromArena(Arena* arena); };}
#include "GameState.h"
#include "../../Math.h"
using namespace RLGSC;
static int boostPadIndexMap[CommonValues::BOOST_LOCATIONS_AMOUNT] = {}; static bool boostPadIndexMapBuilt = false; static std::mutex boostPadIndexMapMutex = {}; void _BuildBoostPadIndexMap(Arena* arena) { constexpr const char* ERROR_PREFIX = "_BuildBoostPadIndexMap(): "; #ifdef RG_VERBOSE RG_LOG("Building boost pad index map..."); #endif
if (arena->_boostPads.size() != CommonValues::BOOST_LOCATIONS_AMOUNT) { RG_ERR_CLOSE( ERROR_PREFIX << "Arena boost pad count does not match CommonValues::BOOST_LOCATIONS_AMOUNT " << "(" << arena->_boostPads.size() << "/" << CommonValues::BOOST_LOCATIONS_AMOUNT << ")" ); } bool found[CommonValues::BOOST_LOCATIONS_AMOUNT] = {}; for (int i = 0; i < CommonValues::BOOST_LOCATIONS_AMOUNT; i++) { Vec targetPos = CommonValues::BOOST_LOCATIONS[i]; for (int j = 0; j < arena->_boostPads.size(); j++) { Vec padPos = arena->_boostPads[j]->config.pos; if (padPos.DistSq2D(targetPos) < 10) { if (!found[i]) { found[i] = true; boostPadIndexMap[i] = j; } else { RG_ERR_CLOSE( ERROR_PREFIX << "Matched duplicate boost pad at " << targetPos << "=" << padPos ); } break; } } if (!found[i]) RS_ERR_CLOSE(ERROR_PREFIX << "Failed to find matching pad at " << targetPos); }#ifdef RG_VERBOSE RG_LOG(" > Done"); #endif boostPadIndexMapBuilt = true; }
void RLGSC::GameState::UpdateFromArena(Arena* arena) { lastArena = arena; int tickSkip = RS_MAX(arena->tickCount - lastTickCount, 0);
deltaTime = tickSkip * (1 / 120.f); ballState = arena->ball->GetState(); ball = PhysObj(ballState); ballInv = ball.Invert(); players.resize(arena->_cars.size()); auto carItr = arena->_cars.begin(); for (int i = 0; i < players.size(); i++) { auto& player = players[i]; player.UpdateFromCar(*carItr, arena->tickCount, tickSkip); if (player.ballTouchedStep) lastTouchCarID = player.carId; carItr++; } if (!boostPadIndexMapBuilt) { boostPadIndexMapMutex.lock(); // Check again? This seems stupid but also makes sense to me // Without this, we could lock as the index map is building, then go build again // I would like to keep the mutex inside the if statement so it is only checked a few times if (!boostPadIndexMapBuilt) _BuildBoostPadIndexMap(arena); boostPadIndexMapMutex.unlock(); } for (int i = 0; i < CommonValues::BOOST_LOCATIONS_AMOUNT; i++) { int idx = boostPadIndexMap[i]; int invIdx = boostPadIndexMap[CommonValues::BOOST_LOCATIONS_AMOUNT - i - 1]; auto state = arena->_boostPads[idx]->GetState(); auto stateInv = arena->_boostPads[invIdx]->GetState(); boostPads[i] = state.isActive; boostPadsInv[i] = stateInv.isActive; boostPadTimers[i] = state.cooldown; boostPadTimersInv[i] = stateInv.cooldown; } // Update goal scoring // If you don't have a GoalScoreCondition then that's not my problem lmao if (Math::IsBallScored(ball.pos)) scoreLine[1 - (int)RS_TEAM_FROM_Y(ball.pos.y)]++; lastTickCount = arena->tickCount;}
#include "PlayerData.h"
namespace RLGSC { void PlayerData::UpdateFromCar(Car* car, uint64_t tickCount, int tickSkip) { carId = car->id; team = car->team; CarState newState = car->GetState();
if (newState.isDemoed && newState.pos.z == -50000) { newState.pos = carState.pos; newState.vel = carState.vel; newState.angVel = carState.angVel; } carState = newState; phys = PhysObj(carState); physInv = PhysObj(phys.Invert()); if (carState.ballHitInfo.isValid) { ballTouchedStep = carState.ballHitInfo.tickCountWhenHit >= (tickCount - tickSkip); ballTouchedTick = carState.ballHitInfo.tickCountWhenHit == (tickCount - 1); } else { ballTouchedStep = ballTouchedTick = false; } hasJump = !carState.hasJumped; hasFlip = !carState.hasDoubleJumped && !carState.hasFlipped && carState.airTimeSinceJump < RLConst::DOUBLEJUMP_MAX_DELAY; boostFraction = carState.boost / 100; }} #pragma once #include "../../Framework.h" #include "PhysObj.h"
namespace RLGSC { // https://github.com/AechPro/rocket-league-gym-sim/blob/main/rlgym_sim/utils/gamestates/player_data.py struct PlayerData { uint32_t carId; Team team;
PhysObj phys, physInv; CarState carState; // matchAssists: being the passer to a teammate who shot and scored // matchBumps: any bump against an opponent, including demos int matchGoals = 0, matchSaves = 0, matchAssists = 0, matchShots = 0, matchShotPasses = 0, matchBumps = 0, matchDemos = 0, boostPickups = 0; bool hasJump; bool hasFlip; float boostFraction; // From 0 to 1 bool ballTouchedStep; // True if the player touched the ball during any of tick of the step bool ballTouchedTick; // True if the player is touching the ball on the final tick of the step void UpdateFromCar(Car* car, uint64_t tickCount, int tickSkip); const PhysObj& GetPhys(bool inverted) const { return inverted ? physInv : phys; } };} #include "PhysObj.h"
template void _InitPhysObj(RLGSC::PhysObj* obj, const T& rsState) { obj->pos = rsState.pos; obj->rotMat = rsState.rotMat; obj->vel = rsState.vel; obj->angVel = rsState.angVel; }
RLGSC::PhysObj::PhysObj(const BallState& ballState) { _InitPhysObj(this, ballState); }
RLGSC::PhysObj::PhysObj(const CarState& carState) { _InitPhysObj(this, carState); }
RLGSC::PhysObj RLGSC::PhysObj::Invert() const { PhysObj result = *this;
constexpr Vec invVec = Vec(-1, -1, 1); result.pos *= invVec; for (int i = 0; i < 3; i++) result.rotMat[i] *= invVec; result.vel *= invVec; result.angVel *= invVec; return result;}
RLGSC::PhysObj RLGSC::PhysObj::MirrorX() const { PhysObj result = *this;
result.pos.x *= -1; // Thanks Rolv, JPK, and Kaiyo! result.rotMat.forward *= Vec(-1, 1, 1); result.rotMat.right *= Vec( 1, -1, -1); result.rotMat.up *= Vec(-1, 1, 1); result.vel.x *= -1; result.angVel *= Vec(1, -1, -1); return result;} #pragma once #include "../../Framework.h"
namespace RLGSC { // https://github.com/AechPro/rocket-league-gym-sim/blob/main/rlgym_sim/utils/gamestates/physics_object.py struct PhysObj { Vec pos; RotMat rotMat; Vec vel, angVel;
PhysObj() = default; PhysObj(const BallState& ballState); PhysObj(const CarState& carState); // Rotate 180 degrees around Z axis, scales everything by (-1, -1, 1) PhysObj Invert() const; // Mirror along X axis PhysObj MirrorX() const; };} #include "Math.h"
bool RLGSC::Math::IsBallScored(Vec pos) { return abs(pos.y) > RLConst::SOCCAR_GOAL_SCORE_BASE_THRESHOLD_Y + RLConst::BALL_COLLISION_RADIUS_SOCCAR; }
Vec RLGSC::Math::RandVec(Vec min, Vec max) { return Vec( ::Math::RandFloat(min.x, max.x), ::Math::RandFloat(min.y, max.y), ::Math::RandFloat(min.z, max.z) ); } #pragma once #include "Framework.h"
namespace RLGSC { namespace Math { bool IsBallScored(Vec pos); Vec RandVec(Vec min, Vec max); } } #pragma once #include "../../Framework.h" namespace RLGSC { struct Quat { float w, x, y, z;
Quat() { w = x = y = z = 1; } Quat(float w, float x, float y, float z) : w(w), x(x), y(y), z(z) { } Quat(const btQuaternion& bulletQuat) : w(bulletQuat.w()), x(bulletQuat.x()), y(bulletQuat.y()), z(bulletQuat.z()) { } static Quat FromRotMat(const RotMat& rotMat); RotMat ToRotMat() const; explicit operator btQuaternion() const { return btQuaternion(x, y, z, w); } };} #include "Quat.h"
namespace RLGSC { Quat Quat::FromRotMat(const RotMat& rotMat) { btQuaternion quat; ((btMatrix3x3)rotMat).getRotation(quat); return Quat(quat); }
RotMat Quat::ToRotMat() const { btMatrix3x3 rotMat; rotMat.setRotation((btQuaternion)*this); return rotMat; }} #pragma once #include "../../Framework.h"
namespace RLGSC { struct Action { float throttle, steer, pitch, yaw, roll, jump, boost, handbrake; constexpr static size_t ELEM_AMOUNT = 8;
Action() = default; Action(const CarControls& controls) { throttle = controls.throttle; steer = controls.steer; pitch = controls.pitch; yaw = controls.yaw; roll = controls.roll; jump = controls.jump; boost = controls.boost; handbrake = controls.handbrake; } Action( float throttle, float steer, float pitch, float yaw, float roll, float jump, float boost, float handbrake ) : throttle(throttle), steer(steer), pitch(pitch), yaw(yaw), roll(roll), jump(jump), boost(boost), handbrake(handbrake) { } explicit operator CarControls() const { CarControls result = {}; result.throttle = throttle; result.steer = steer; result.pitch = pitch; result.yaw = yaw; result.roll = roll; result.boost = boost == 1; result.jump = jump == 1; result.handbrake = handbrake == 1; return result; } float* begin() { return &throttle; } float* end() { return &handbrake + 1; } const float* begin() const { return &throttle; } const float* end() const { return &handbrake + 1; } float operator[](size_t index) const { assert(index < ELEM_AMOUNT); return begin()[index]; } float& operator[](size_t index) { assert(index < ELEM_AMOUNT); return begin()[index]; } friend std::ostream& operator<<(std::ostream& stream, const Action& action) { stream << "["; for (int i = 0; i < Action::ELEM_AMOUNT; i++) { if (i != 0) stream << ", "; stream << action[i]; } stream << "]"; return stream; } }; typedef std::vector<Action> ActionSet;}
Some code example :
#include "DefaultOBSPadded.h"
RLGSC::FList RLGSC::DefaultOBSPadded::BuildOBS(const PlayerData& player, const GameState& state, const Action& prevAction) { FList result = {};
bool inv = player.team == Team::ORANGE; auto& ball = state.GetBallPhys(inv); auto& pads = state.GetBoostPads(inv); result += ball.pos * posCoef; result += ball.vel * velCoef; result += ball.angVel * angVelCoef; for (int i = 0; i < prevAction.ELEM_AMOUNT; i++) result += prevAction[i]; for (int i = 0; i < CommonValues::BOOST_LOCATIONS_AMOUNT; i++) result += (float)pads[i]; FList selfOBS = {}; AddPlayerToOBS(selfOBS, player, inv); result += selfOBS; int playerObsSize = selfOBS.size(); FList2 teammates = {}, opponents = {}; for (auto& otherPlayer : state.players) { if (otherPlayer.carId == player.carId) continue; FList playerObs = {}; AddPlayerToOBS( playerObs, otherPlayer, inv ); ((otherPlayer.team == player.team) ? teammates : opponents).push_back(playerObs); } if (teammates.size() > maxPlayers - 1) RG_ERR_CLOSE("DefaultOBSPadded: Too many teammates for OBS, maximum is " << (maxPlayers - 1)); if (opponents.size() > maxPlayers) RG_ERR_CLOSE("DefaultOBSPadded: Too many opponents for OBS, maximum is " << maxPlayers); for (int i = 0; i < 2; i++) { auto& playerList = i ? teammates : opponents; int targetCount = i ? maxPlayers - 1 : maxPlayers; while (playerList.size() < targetCount) { FList pad = FList(playerObsSize); playerList.push_back(pad); } } // Shuffle both lists std::shuffle(teammates.begin(), teammates.end(), ::Math::GetRandEngine()); std::shuffle(opponents.begin(), opponents.end(), ::Math::GetRandEngine()); for (auto& teammate : teammates) result += teammate; for (auto& opponent : opponents) result += opponent; return result;} #pragma once #include "DefaultOBS.h"
namespace RLGSC { // Verion of DefaultOBS that supports a varying number of players (i.e. 1v1, 2v2, 3v3, etc.) // Maximum player count can be however high you want // Opponent and teammate slots are randomly shuffled to prevent slot bias class DefaultOBSPadded : public DefaultOBS { public:
int maxPlayers; DefaultOBSPadded( int maxPlayers, Vec posCoef = Vec(1 / CommonValues::SIDE_WALL_X, 1 / CommonValues::BACK_WALL_Y, 1 / CommonValues::CEILING_Z), float velCoef = 1 / CommonValues::CAR_MAX_SPEED, float angVelCoef = 1 / CommonValues::CAR_MAX_ANG_VEL ) : DefaultOBS(posCoef, velCoef, angVelCoef), maxPlayers(maxPlayers) { } virtual FList BuildOBS(const PlayerData& player, const GameState& state, const Action& prevAction); };}
reward example
// reward height touch class ArkansReward : public RewardFunction { public: virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { float heightReward = 0.0f; if (player.ballTouchedStep && !player.carState.isOnGround && player.carState.pos[2] >= 256.0f) { heightReward += (log1p(player.carState.pos[2] - 256.0f) / CommonValues::CEILING_Z - CommonValues::BALL_RADIUS); }
return heightReward; }};
// reward the bot for being on the wall class PlayerOnWallReward : public RewardFunction { public: virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { return player.carState.isOnGround && player.phys.pos.z > 300; }; };
// reward the player for having speed towards the ball mid-air class AerialReward : public RewardFunction { public: int BALL_MIN_HEIGHT = 300; virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) override {
Vec dirToBall = (state.ball.pos - player.phys.pos).Normalized(); float distance = player.carState.pos.Dist(state.ball.pos) - CommonValues::BALL_RADIUS; float distanceReward = exp(-0.5 * distance / CommonValues::CAR_MAX_SPEED); float speedTowardBall = player.phys.vel.Dot(dirToBall); float speedReward = speedTowardBall / CommonValues::CAR_MAX_SPEED; if (speedTowardBall < 0) { return 0.0f; } if (player.carState.isOnGround) { return 0.0f; } if (state.ballState.pos.z < BALL_MIN_HEIGHT) { return 0.0f; } float reward = (speedReward * distanceReward); if (player.carState.hasDoubleJumped) reward += 0.1f; // Double jump bonus return reward; }};
// reward the player for aerial touches initially and then exponential increase the more it gets. // idk rolv wrote this class AerialDistanceReward : public RewardFunction { public: int RampHeight = 256; float heightScale; float distanceScale; float carDistance; float ballDistance; int currentCarId = 0; GameState prevState; AerialDistanceReward(float heightScale, float distanceScale) { this->heightScale = heightScale; this->distanceScale = distanceScale; }; const PlayerData* findCarOnField(const GameState& state, int carID) { for (auto& player : state.players) if (player.carId == carID) return &player; return NULL; }; virtual void Reset(const GameState& state) { int currentCarId = 0; prevState = state; }; virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { float reward = 0; bool isCurrent = (currentCarId != 0 && currentCarId == player.carId); if (player.phys.pos.z < RampHeight) { if (isCurrent) { isCurrent = false; currentCarId = 0; } } else if (player.ballTouchedStep && !isCurrent) { isCurrent = true; ballDistance = 0; carDistance = 0; reward = heightScale * RS_MAX(player.phys.pos.z + state.ball.pos.z - 2 * RampHeight, 0); } else if (isCurrent) { PlayerData currentCar = *findCarOnField(state, currentCarId); int carDistance = carDistance + player.phys.pos.Dist(currentCar.phys.pos); int ballDistance = ballDistance + state.ball.pos.Dist(prevState.ball.pos); if (player.ballTouchedStep) { reward = distanceScale * (carDistance + ballDistance); carDistance = 0; ballDistance = 0; }; }; if (isCurrent) { currentCarId = player.carId; };
prevState = state; return reward / (2 * CommonValues::BACK_WALL_Y); };}; // reward the player for being the closest to the ball and their team having touched the ball last // ambiguous in 1v1. class PossessionReward : public RewardFunction { public: float prevTeamTouch = -1.f; float stacking = 0; PossessionReward(float minDist = 200.f) : minDist(minDist) {}; virtual void Reset(const GameState& state) { prevTeamTouch = -1; stacking = 0; }; virtual void PreStep(const GameState& state) { for (auto& player : state.players) { if (player.ballTouchedStep && !((int)player.team == prevTeamTouch)) { prevTeamTouch = (int)player.team; stacking = 0; } } }
virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { float reward = 0; float distToBall = (player.phys.pos - state.ball.pos).Length(); if ((int)player.team == prevTeamTouch && distToBall < minDist) { for (auto& _player : state.players) { if (_player.team != player.team && _player.carId != player.carId) { float oppDistBall = (_player.phys.pos - state.ball.pos).Length(); if (oppDistBall < distToBall) { return 0; }; } } stacking += 1; reward += stacking / 10; } return reward; }private: float minDist; }; // ???? reward player for being closest to the ball on kickoff idk aether6837 wrote this. class KickoffProximityRewardAllModes : public RewardFunction { virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { if (state.ball.vel.IsZero()) { std::vector allyDistances; for (auto& _player : state.players) { if (_player.team == player.team) { float allyDistance = _player.phys.pos.Dist(state.ball.pos); allyDistances.push_back(allyDistance); }; }; std::vector opponentDistances; for (auto& _player : state.players) { if (_player.team != player.team) { float opponentDistance = _player.phys.pos.Dist(state.ball.pos); opponentDistances.push_back(opponentDistance); }; }; if (opponentDistances.empty() && allyDistances.empty() && std::min_element(allyDistances.begin(), allyDistances.end()) < std::min_element(opponentDistances.begin(), opponentDistances.end())) { return 1; } else { return -1; }; }; return 0; }; };
// reward player more for picking up big pads than small pads class PickupBoostReward : public RewardFunction { public: GameState lastState; PickupBoostReward(float small = 3.f, float big = 10.f) : small(small), big(big) {}; virtual void Reset(const GameState& state) { lastState = state; }; virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { float reward = 0; for (auto& _player : lastState.players) { if (player.carId == _player.carId) { float boostDiff = player.boostFraction; if (boostDiff > 0) { reward += boostDiff * big; if (player.boostFraction < 0.98 && _player.boostFraction < 0.88) { reward += boostDiff * small; } } } } lastState = state; return reward; }
private: float big; float small; };
// reward player for goal speed and goal placement class GoalSpeedAndPlacementReward : public RewardFunction { public: float prevScoreBlue = 0; float prevScoreOrange = 0; GameState prevStateBlue; GameState prevStateOrange; float minHeight = CommonValues::BALL_RADIUS + 10; float heightReward = 1.75;
virtual void Reset(const GameState& state) { prevScoreBlue = state.scoreLine[0]; prevScoreOrange = state.scoreLine[1]; prevStateBlue = state; prevStateOrange = state; } virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { if ((int)player.team == 0) { float score = state.scoreLine[0]; if (score > prevScoreBlue) { float reward = prevStateBlue.ball.vel.Length() / CommonValues::BALL_MAX_SPEED; if (prevStateBlue.ball.pos.z > minHeight) { reward = heightReward * reward; }; prevStateBlue = state; prevScoreBlue = score; return reward; }; } else { float score = state.scoreLine[1]; if (score > prevScoreOrange) { float reward = prevStateOrange.ball.vel.Length() / CommonValues::BALL_MAX_SPEED; if (prevStateOrange.ball.pos.z > minHeight) { reward = heightReward * reward; }; prevStateOrange = state; prevScoreOrange = score; return reward; }; }; return 0; };}; // reward player for hitting the ball hard class TouchBallRewardScaledByHitForce : public RewardFunction { public: int maxHitSpeed = 3000; Vec lastBallVel; Vec curBallVel; virtual void Reset(const GameState& state) { lastBallVel = state.ball.vel; curBallVel = state.ball.vel; };
virtual void PreStep(const GameState& state) { lastBallVel = curBallVel; curBallVel = state.ball.vel; }; virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { if (player.ballTouchedStep) { float BallVelocity = curBallVel.Dist(lastBallVel); if (BallVelocity > 500) { float reward = BallVelocity / maxHitSpeed; return reward; }; }; return 0.0f; };}; // reward the player for being in the air (opposite of lavafloor) class InAirReward : public RewardFunction { public: virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { if (!player.carState.isOnGround) { return 1; } else { return 0; } }; };
// reward the player for dribbling and matching the speed of the ball. class DribbleReward : public RewardFunction { public: virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { const float MIN_BALL_HEIGHT = 109.0f; const float MAX_BALL_HEIGHT = 180.0f; const float MAX_DISTANCE = 197.0f; const float SPEED_MATCH_FACTOR = 2.0f;
if (player.carState.isOnGround && state.ball.pos.z >= MIN_BALL_HEIGHT && state.ball.pos.z <= MAX_BALL_HEIGHT && (player.phys.pos - state.ball.pos).Length() < MAX_DISTANCE) { float playerSpeed = player.phys.vel.Length(); float ballSpeed = state.ball.vel.Length(); float speedMatchReward = ((playerSpeed / CommonValues::CAR_MAX_SPEED) + SPEED_MATCH_FACTOR * (1.0f - std::abs(playerSpeed - ballSpeed) / (playerSpeed + ballSpeed))); return speedMatchReward; } else { return 0.0f; } return 0.0f; };};
// reward the player for high goal speed, (warning very explosive as it's a high return reward, LOW WEIGHT.) class GoalSpeedReward : public RewardFunction { public: GoalSpeedReward(float rewardFactor = 0.1f) : rewardFactor(rewardFactor) {}; virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { float reward = 0.0f;
if (state.scoreLine[(int)player.team] > 0) { float ballSpeed = state.ball.vel.Length(); reward += ballSpeed * rewardFactor; } return reward * 2.0f; };private: float rewardFactor; };
// reward the player for moving fast at the ball class SpeedTowardBallReward : public RewardFunction { public: virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { return RS_MAX(0, player.phys.vel.Dot((state.ball.pos - player.phys.pos).Normalized()) / CommonValues::CAR_MAX_SPEED); }; };
// reward the player for touching the ball mid air class JumpTouchReward : public RewardFunction { public: JumpTouchReward(float minHeight = 200.0f, float exp = 1.0f) : minHeight(minHeight), exp(exp), div(std::pow(CommonValues::CEILING_Z / 2 - CommonValues::BALL_RADIUS, exp)) {}
virtual void Reset(const GameState& initialState) override { ticksUntilNextReward = 0; }; virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) override { if (player.ballTouchedStep && player.carState.isOnGround && state.ball.pos.z >= minHeight && ticksUntilNextReward <= 0) { ticksUntilNextReward = 149; float reward = std::pow(std::min(state.ball.pos.z, CommonValues::CEILING_Z / 2) - CommonValues::BALL_RADIUS, exp) / div; if (reward > 0.05) { return reward * 100; }; } ticksUntilNextReward -= 1; return 0.0f; };private: float minHeight; float exp; float div; int ticksUntilNextReward = 0; };
float KPH_TO_VEL(float kph) { return kph * (250 / 9); };
// https://github.com/redd-rl/apollo-bot/blob/main/custom_rewards.py
float norm(Vec vec1) { return std::pow((std::pow(vec1.x, 2) + std::pow(vec1.y, 2) + std::pow(vec1.z, 2)), 2); }
// reward the player for moving fast toward the ball, but only on kickoff state. class SpeedflipKickoffReward : public RewardFunction { virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { if (state.ball.vel.IsZero() and player.boostFraction < 0.02) { return RS_MAX(0, player.phys.vel.Dot((state.ball.pos - player.phys.pos).Normalized()) / CommonValues::CAR_MAX_SPEED); }; return 0; }; };
// reward the player for going fast class LightingMcQueenReward : public RewardFunction { virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { return sqrt(RS_CLAMP(player.phys.vel.Length() / 1800, 0, 1)); } };
// Aerial Reward class TouchAerial : public RewardFunction { public: virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { constexpr float MAX_HEIGHT = 1400, // most of the way to the ceiling MAX_AIR_TIME = 1.0f; // in seconds float reward = RS_MIN(state.ball.pos.z / MAX_HEIGHT, player.carState.airTime / MAX_AIR_TIME); float result = player.ballTouchedStep ? RS_MIN(reward, 1.0f) : 0; return result; }; };
// punish the bot for being on the ground class LavaFloorReward : public RewardFunction { public: virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { return -int(player.carState.isOnGround); }; };
class BallCatch : public RewardFunction { public: virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { Vec dirToBall = (state.ball.pos - player.phys.pos).Normalized(); float velToBall = player.phys.vel.Dot(dirToBall);
constexpr float MAX_DIST = 3000; float distFrac = player.phys.pos.Dist(state.ball.pos) / MAX_DIST; return RS_CLAMP((velToBall/2300.f) * (1 - distFrac), 0, 1);} };
class BoostingReward : public RewardFunction { public: virtual float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) { float reward = 0.f;
// Calculate how well the car is facing the ball Vec pos_diff = (state.ball.pos - player.carState.pos).Normalized(); float facing_ball_factor = player.carState.rotMat.forward.Dot(pos_diff); // Define a threshold for "facing the ball" const float FACING_THRESHOLD = 0.5f; if ((player.carState.vel.z > 600 || player.carState.pos.z > 256) && player.carState.rotMat.forward.z > 0 && state.ball.pos.z > 256 && player.boostFraction > 0 && facing_ball_factor > FACING_THRESHOLD && !player.carState.isOnGround) { reward += prevAction.boost * facing_ball_factor; } return reward; }};
so ur task is very difficult, you need to improve my current obspadded by adding all these thing so this is advancedobspadded.h : #pragma once #include <RLGymSim_CPP/Utils/OBSBuilders/DefaultOBS.h>
namespace RLGSC { // Verion of DefaultOBS that supports a varying number of players (i.e. 1v1, 2v2, 3v3, etc.) // Maximum player count can be however high you want // Opponent and teammate slots are randomly shuffled to prevent slot bias class AdvancedOBSPadded : public DefaultOBS { public:
int maxPlayers; AdvancedOBSPadded( int maxPlayers, Vec posCoef = Vec(1 / CommonValues::SIDE_WALL_X, 1 / CommonValues::BACK_WALL_Y, 1 / CommonValues::CEILING_Z), float velCoef = 1 / CommonValues::CAR_MAX_SPEED, float angVelCoef = 1 / CommonValues::CAR_MAX_ANG_VEL ) : DefaultOBS(posCoef, velCoef, angVelCoef), maxPlayers(maxPlayers) { } virtual FList BuildOBS(const PlayerData& player, const GameState& state, const Action& prevAction); virtual void AddPlayerToOBSBetter(FList& obs, const PlayerData& player, bool inv, const PhysObj& ball) { PhysObj phys = player.GetPhys(inv); obs += phys.pos * posCoef; obs += phys.rotMat.forward; obs += phys.rotMat.up; obs += phys.vel * velCoef; obs += phys.angVel * angVelCoef; obs += (ball.pos - phys.pos) * posCoef; obs += (ball.vel - phys.vel) * velCoef; obs += phys.rotMat.Dot(ball.pos - phys.pos) * posCoef * 0.5f; obs += phys.rotMat.Dot(ball.vel - phys.vel) * velCoef * 0.5f; obs += { player.boostFraction, (float)player.carState.isOnGround, (float)player.hasFlip, (float)player.carState.isDemoed, }; }; };}
and this is advancedobspadded.cpp #include "AdvancedOBSPadded.h"
using namespace RLGSC;
RLGSC::FList RLGSC::AdvancedOBSPadded::BuildOBS(const PlayerData& player, const GameState& state, const Action& prevAction) { FList result = {};
bool inv = player.team == Team::ORANGE; auto& ball = state.GetBallPhys(inv); auto& pads = state.GetBoostPads(inv); result += ball.pos * posCoef; result += ball.vel * velCoef; result += ball.angVel * angVelCoef; for (int i = 0; i < prevAction.ELEM_AMOUNT; i++) result += prevAction[i]; for (int i = 0; i < CommonValues::BOOST_LOCATIONS_AMOUNT; i++) result += (float)pads[i]; FList selfOBS = {}; AddPlayerToOBSBetter(selfOBS, player, inv, ball); result += selfOBS; int playerObsSize = selfOBS.size(); FList2 teammates = {}, opponents = {}; for (auto& otherPlayer : state.players) { if (otherPlayer.carId == player.carId) continue; FList playerObs = {}; AddPlayerToOBSBetter( playerObs, otherPlayer, inv, ball ); ((otherPlayer.team == player.team) ? teammates : opponents).push_back(playerObs); } if (teammates.size() > maxPlayers - 1) RG_ERR_CLOSE("AdvancedOBSPadded: Too many teammates for OBS, maximum is " << (maxPlayers - 1)); if (opponents.size() > maxPlayers) RG_ERR_CLOSE("AdvancedOBSPadded: Too many opponents for OBS, maximum is " << maxPlayers); for (int i = 0; i < 2; i++) { auto& playerList = i ? teammates : opponents; int targetCount = i ? maxPlayers - 1 : maxPlayers; while (playerList.size() < targetCount) { FList pad = FList(playerObsSize); playerList.push_back(pad); } } // Shuffle both lists std::shuffle(teammates.begin(), teammates.end(), ::Math::GetRandEngine()); std::shuffle(opponents.begin(), opponents.end(), ::Math::GetRandEngine()); for (auto& teammate : teammates) result += teammate; for (auto& opponent : opponents) result += opponent; return result;}
you need to change both so it work well
Player distance to opponent Player distance to ball Player's speed towards the ball Player's speed away from the ball Distance between the ball and the closest opponent Distance between the player and their own goal Distance between the player and the opponent's goal Angle between the player's car and the ball's direction Angle between the opponent's car and the ball's direction Player's boost amount Opponent's boost amount Is the player supersonic Is the opponent supersonic Player's position relative to the ball (forward, backward, left, right) Opponent's position relative to the ball Player’s car orientation (pitch, yaw, roll) Opponent's car orientation Ball height from the ground Player's car height from the ground Opponent's car height from the ground Player's distance to nearest boost pad Opponent's distance to nearest boost pad Time since player last touched the ball Time since opponent last touched the ball Player's car velocity in forward direction Opponent's car velocity in forward direction Player's car velocity in lateral direction Opponent's car velocity in lateral direction Is the player airborne Is the opponent airborne Is the ball inside the goal area Player's time spent supersonic Opponent's time spent supersonic Player's boost pad collection rate Opponent's boost pad collection rate Time since player was last demoed Time since opponent was last demoed Player's angular velocity Opponent's angular velocity Ball velocity towards player's goal Ball velocity towards opponent's goal Is player close to a wall Is opponent close to a wall Player's distance to the center of the field Opponent's distance to the center of the field Ball's distance to the center of the field Player's distance to ball's predicted landing spot Opponent's distance to ball's predicted landing spot Player's rotational alignment with the ball's path Opponent's rotational alignment with the ball's path
- Assistant
Absolutely, I'd be glad to help you enhance your
AdvancedOBSPaddedclass to include the additional features you've listed. We'll modify bothAdvancedOBSPadded.handAdvancedOBSPadded.cppto incorporate these features into the observation vector.Key Steps:
-
Identify where to compute each feature: Some features are per-player, some involve interactions between players, and others involve the ball or the environment. We'll need to compute these features appropriately.
-
Modify
AddPlayerToOBSBetter: For per-player features and features involving the ball, we'll update this method. -
Compute interactions in
BuildOBS: Features involving other players (like distance to opponents) will be computed inBuildOBS. -
Ensure consistent observation size: Since we're padding teammates and opponents, we need to make sure the observation size remains consistent.
Updated
AdvancedOBSPadded.h#pragma once #include <RLGymSim_CPP/Utils/OBSBuilders/DefaultOBS.h> namespace RLGSC { // Advanced observation builder with additional features class AdvancedOBSPadded : public DefaultOBS { public: int maxPlayers; AdvancedOBSPadded( int maxPlayers, Vec posCoef = Vec(1 / CommonValues::SIDE_WALL_X, 1 / CommonValues::BACK_WALL_Y, 1 / CommonValues::CEILING_Z), float velCoef = 1 / CommonValues::CAR_MAX_SPEED, float angVelCoef = 1 / CommonValues::CAR_MAX_ANG_VEL ) : DefaultOBS(posCoef, velCoef, angVelCoef), maxPlayers(maxPlayers) { } virtual FList BuildOBS(const PlayerData& player, const GameState& state, const Action& prevAction) override; virtual void AddPlayerToOBSBetter(FList& obs, const PlayerData& player, bool inv, const PhysObj& ball, const GameState& state, bool isOpponent); private: void AddAdditionalFeatures(FList& obs, const PlayerData& player, const PhysObj& ball, const GameState& state, bool isOpponent, bool inv); }; }
Explanation of Changes:
-
Updated
AddPlayerToOBSBetter: Added parametersconst GameState& stateandbool isOpponentto access game state and identify if the player is an opponent. -
Added
AddAdditionalFeatures: A helper function to compute and add the new features to the observation.
Updated
AdvancedOBSPadded.cpp#include "AdvancedOBSPadded.h" #include <algorithm> // For std::min_element using namespace RLGSC; FList AdvancedOBSPadded::BuildOBS(const PlayerData& player, const GameState& state, const Action& prevAction) { FList result = {}; bool inv = player.team == Team::ORANGE; const auto& ball = state.GetBallPhys(inv); const auto& pads = state.GetBoostPads(inv); // Ball observations result += ball.pos * posCoef; result += ball.vel * velCoef; result += ball.angVel * angVelCoef; // Previous action for (int i = 0; i < prevAction.ELEM_AMOUNT; i++) result += prevAction[i]; // Boost pad states for (int i = 0; i < CommonValues::BOOST_LOCATIONS_AMOUNT; i++) result += (float)pads[i]; // Self observations FList selfOBS = {}; AddPlayerToOBSBetter(selfOBS, player, inv, ball, state, false); result += selfOBS; int playerObsSize = selfOBS.size(); // Teammates and opponents observations FList2 teammates = {}, opponents = {}; for (const auto& otherPlayer : state.players) { if (otherPlayer.carId == player.carId) continue; FList playerObs = {}; AddPlayerToOBSBetter( playerObs, otherPlayer, inv, ball, state, otherPlayer.team != player.team ); ((otherPlayer.team == player.team) ? teammates : opponents).push_back(playerObs); } if (teammates.size() > maxPlayers - 1) RG_ERR_CLOSE("AdvancedOBSPadded: Too many teammates for OBS, maximum is " << (maxPlayers - 1)); if (opponents.size() > maxPlayers) RG_ERR_CLOSE("AdvancedOBSPadded: Too many opponents for OBS, maximum is " << maxPlayers); for (int i = 0; i < 2; i++) { auto& playerList = i ? teammates : opponents; int targetCount = i ? maxPlayers - 1 : maxPlayers; while (playerList.size() < targetCount) { FList pad = FList(playerObsSize, 0.0f); playerList.push_back(pad); } } // Shuffle both lists std::shuffle(teammates.begin(), teammates.end(), ::Math::GetRandEngine()); std::shuffle(opponents.begin(), opponents.end(), ::Math::GetRandEngine()); for (const auto& teammate : teammates) result += teammate; for (const auto& opponent : opponents) result += opponent; // Additional features involving other players // Player distance to closest opponent float minOpponentDist = std::numeric_limits<float>::max(); for (const auto& opp : opponents) { // Assuming opp[0], opp[1], opp[2] are the opponent's position components Vec oppPos = Vec(opp[0], opp[1], opp[2]) / posCoef; float dist = (player.phys.pos - oppPos).Length(); if (dist < minOpponentDist) minOpponentDist = dist; } result += minOpponentDist * posCoef.x; // Using x component of posCoef for normalization // Distance between the ball and the closest opponent float minOpponentBallDist = std::numeric_limits<float>::max(); for (const auto& opp : opponents) { Vec oppPos = Vec(opp[0], opp[1], opp[2]) / posCoef; float dist = (ball.pos - oppPos).Length(); if (dist < minOpponentBallDist) minOpponentBallDist = dist; } result += minOpponentBallDist * posCoef.x; return result; } void AdvancedOBSPadded::AddPlayerToOBSBetter(FList& obs, const PlayerData& player, bool inv, const PhysObj& ball, const GameState& state, bool isOpponent) { PhysObj phys = player.GetPhys(inv); // Basic observations obs += phys.pos * posCoef; obs += phys.rotMat.forward; obs += phys.rotMat.up; obs += phys.vel * velCoef; obs += phys.angVel * angVelCoef; obs += (ball.pos - phys.pos) * posCoef; obs += (ball.vel - phys.vel) * velCoef; obs += phys.rotMat.Dot(ball.pos - phys.pos) * posCoef * 0.5f; obs += phys.rotMat.Dot(ball.vel - phys.vel) * velCoef * 0.5f; // Existing features obs += { player.boostFraction, (float)player.carState.isOnGround, (float)player.hasFlip, (float)player.carState.isDemoed, }; // Additional features AddAdditionalFeatures(obs, player, ball, state, isOpponent, inv); } void AdvancedOBSPadded::AddAdditionalFeatures(FList& obs, const PlayerData& player, const PhysObj& ball, const GameState& state, bool isOpponent, bool inv) { const auto& phys = player.GetPhys(inv); // Player distance to ball float playerToBallDist = (phys.pos - ball.pos).Length(); obs += playerToBallDist * posCoef.x; // Player speed towards the ball Vec dirToBall = (ball.pos - phys.pos).Normalized(); float speedTowardsBall = phys.vel.Dot(dirToBall); obs += speedTowardsBall * velCoef; // Player speed away from the ball float speedAwayFromBall = -speedTowardsBall; obs += speedAwayFromBall * velCoef; // Distance to own goal Vec ownGoal = inv ? CommonValues::ORANGE_GOAL_BACK : CommonValues::BLUE_GOAL_BACK; float distToOwnGoal = (phys.pos - ownGoal).Length(); obs += distToOwnGoal * posCoef.x; // Distance to opponent's goal Vec oppGoal = inv ? CommonValues::BLUE_GOAL_BACK : CommonValues::ORANGE_GOAL_BACK; float distToOppGoal = (phys.pos - oppGoal).Length(); obs += distToOppGoal * posCoef.x; // Angle between car and ball direction float angleToBall = std::acos(phys.rotMat.forward.Dot(dirToBall)); obs += angleToBall / M_PI; // Normalize angle between 0 and 1 // Is supersonic obs += (float)player.carState.isSupersonic; // Car orientation (pitch, yaw, roll) Vec eulerAngles = phys.rotMat.ToEulerAngles(); obs += eulerAngles / M_PI; // Normalize angles between -1 and 1 // Car height from the ground obs += phys.pos.z * posCoef.z; // Time since last touch (if available) if (player.carState.ballHitInfo.isValid) { float timeSinceLastTouch = state.deltaTime * (state.lastTickCount - player.carState.ballHitInfo.tickCountWhenHit); obs += timeSinceLastTouch; } else { obs += -1.0f; // Indicate no touch has occurred } // Car velocity in forward direction float velForward = phys.vel.Dot(phys.rotMat.forward); obs += velForward * velCoef; // Car velocity in lateral direction float velLateral = phys.vel.Dot(phys.rotMat.right); obs += velLateral * velCoef; // Is airborne obs += (float)!player.carState.isOnGround; // Is close to wall float distToWall = CommonValues::SIDE_WALL_X - std::abs(phys.pos.x); obs += distToWall * posCoef.x; // Distance to center of field Vec fieldCenter(0.0f, 0.0f, 0.0f); float distToCenter = (phys.pos - fieldCenter).Length(); obs += distToCenter * posCoef.x; // Angular velocity obs += phys.angVel * angVelCoef; // Additional opponent-specific features if (isOpponent) { // Opponent's boost amount obs += player.boostFraction; // Opponent's supersonic state obs += (float)player.carState.isSupersonic; } }
Explanation of Changes:
-
BuildOBS Modifications:
- Computed and added:
- Player distance to closest opponent.
- Distance between the ball and the closest opponent.
- Used
std::numeric_limits<float>::max()to initialize minimum distances. - Added normalizations using
posCoef.xfor distances.
- Computed and added:
-
AddPlayerToOBSBetter Modifications:
- Added parameters
const GameState& stateandbool isOpponent. - Called
AddAdditionalFeaturesto include new per-player features.
- Added parameters
-
AddAdditionalFeatures Function:
- Computed and added features such as:
- Player distance to ball.
- Player's speed towards and away from the ball.
- Distances to own and opponent goals.
- Angle between the car's forward vector and direction to the ball.
- Supersonic state.
- Car orientation (pitch, yaw, roll).
- Car height from the ground.
- Time since last ball touch.
- Car velocities in forward and lateral directions.
- Airborne state.
- Distance to the nearest wall.
- Distance to the center of the field.
- Angular velocity.
- For opponents, added their boost amount and supersonic state.
- Computed and added features such as:
-
Normalization and Scaling:
- Used
posCoef,velCoef, andangVelCoefto normalize features. - Angles normalized by dividing by
M_PIto keep values between -1 and 1.
- Used
Notes:
-
Angles and Rotations: The orientation angles are converted from rotation matrices using a hypothetical
ToEulerAnglesmethod, which should return pitch, yaw, and roll. -
Time Since Last Touch: Assumed that if
ballHitInfois valid, we can compute the time since the last touch. -
Distance and Direction Calculations: Used basic vector operations to compute distances and directions.
-
Normalization: Ensured all features are normalized appropriately to help with neural network training.
-
Fallback Values: Used
-1.0for0.0fwhere appropriate to indicate absence of data.
Ensure Correctness:
-
Check for Missing Features: Review the list of 50 features and ensure each is accounted for.
-
Consistent Observation Size: Since we're padding observations, make sure all player observations have the same size.
-
Test and Debug: After implementing, test the code thoroughly to ensure it works as expected and all features are correctly computed.
Conclusion:
By updating
AdvancedOBSPadded.handAdvancedOBSPadded.cppas shown, we've incorporated the additional features into your observation builder. This should enhance the information available to your agent, potentially improving its performance.If you have any questions or need further adjustments, feel free to ask!
-