why does the ballpred reward only get the current ballstate not the correct predicted…
- User
why does the ballpred reward only get the current ballstate not the correct predicted one class sBP1 : public RewardFunction { private: static constexpr float MAX_PRED_TIME = 2.0f; // Maximum prediction time in seconds RocketSim::Arena* predictionArena;
public: sBP1(RocketSim::Arena* predictionArena) : predictionArena(predictionArena) { // Initialization if needed }
float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) override { if (!predictionArena) { return 0.0f; // No reward if arena is not available } BallPredEnv* ballPredEnv = BallPred::GetOrMake(predictionArena); if (!ballPredEnv) return 0.0f; // Update the ball prediction //BallPred::Update(predictionArena); // Get the predicted ball position at MAX_PRED_TIME PhysObj predictedBall = ballPredEnv->GetAtTime(MAX_PRED_TIME, state.ball); // Calculate distance between the player and the predicted ball position float distance = (player.phys.pos - predictedBall.pos).Length(); // Check if the player is within 90 units of the predicted ball position if (distance <= 180.0f) { return 10.0f; // High reward for being close to the prediction point } // If not within 90 units, calculate velocity towards the predicted ball position // Calculate the direction vector from player to predicted ball Vec directionToBall = (predictedBall.pos - player.phys.pos).Normalized(); // Calculate the dot product of player's velocity and the direction to the predicted ball float velocityTowardsBall = player.phys.vel.Dot(directionToBall); // Normalize the velocity by the car's maximum speed float normalizedVelocity = velocityTowardsBall / CommonValues::CAR_MAX_SPEED; // Ensure the reward is non-negative return RS_MAX(0.0f, normalizedVelocity); }}; // GlobalBallPred.h #pragma once
#include <unordered_map> #include #include #include #include
// Include necessary project-specific headers #include "C:\Epsilon\RLGymPPO_CPP-main\RLGymPPO_CPP\RLGymSim_CPP\RocketSim\src\Sim\Arena\Arena.h" #include "C:\Epsilon\RLGymPPO_CPP-main\RLGymPPO_CPP\RLGymSim_CPP\src\RLGymSim_CPP\Utils\GlobalBallPred\BallPredTrackerManager.h" #include "C:\Epsilon\RLGymPPO_CPP-main\RLGymPPO_CPP\RLGymSim_CPP\src\RLGymSim_CPP\Utils\Gamestates\PhysObj.h"
RS_NS_START
struct BallPredEnv { BallPredTracker ballPredTracker;
BallPredEnv(const Arena* arena) : ballPredTracker(const_cast<Arena*>(arena), 240) {} void Update(const Arena* arena) { ballPredTracker.UpdatePredFromArena(const_cast<Arena*>(arena)); } PhysObj GetAtTime(float time, const PhysObj& curBall);};
class BallPred { public: static BallPredEnv* GetOrMake(const Arena* arena);
static PhysObj GetAtTime(const Arena* arena, float time, const PhysObj& curBall) { return GetOrMake(arena)->GetAtTime(time, curBall); } static void Update(const Arena* arena) { GetOrMake(arena)->Update(arena); }};
RS_NS_END// GlobalBallPred.cpp #include "GlobalBallPred.h"
BallPredEnv* BallPred::GetOrMake(const Arena* arena) { static thread_local std::unordered_map<const Arena*, BallPredEnv*> predEnvs;
auto& env = predEnvs[arena]; if (env == NULL) env = new BallPredEnv(arena); return env;}
PhysObj BallPredEnv::GetAtTime(float time, const PhysObj& curBall) { if (!ballPredTracker.predData.empty()) { return PhysObj(ballPredTracker.GetBallStateForTime(time)); } else {
}}
#include <RLGymPPO_CPP/Learner.h> #include "C:\Epsilon\RLGymPPO_CPP-main\RLGymPPO_CPP\RLGymSim_CPP\src\RLGymSim_CPP\Utils\GlobalBallPred.h"
// Rewards #include <RLGymSim_CPP/Utils/RewardFunctions/EPSILON_LOGGED_REWARDS.h> #include <RLGymSim_CPP/Utils/RewardFunctions/Rewards.h>
//++++++++++++++++++++++++++++++++++++++++++++++++++++//
// Loggers #include "RLGymSim_CPP/Utils/Loggers/Loggers.h"
//++++++++++++++++++++++++++++++++++++++++++++++++++++//
// StateSetting #include <RLGymSim_CPP/Utils/StateSetters/WeightedSampleSetter.h>
//++++++++++++++++++++++++++++++++++++++++++++++++++++//
// Observation Space
#define INCLUDE_OBS_IMPLEMENTATIONS #include <RLGymSim_CPP/Utils/OBSBuilders/AllObservationBuilders.h> #undef INCLUDE_OBS_IMPLEMENTATIONS
//++++++++++++++++++++++++++++++++++++++++++++++++++++//
// TerminalConditions #define INCLUDE_Condition_IMPLEMENTATIONS #include <RLGymSim_CPP/Utils/TerminalConditions/ALLTerminalConditions.h> #undef INCLUDE_Condition_IMPLEMENTATIONS
//++++++++++++++++++++++++++++++++++++++++++++++++++++//
// ActionParser #include <RLGymSim_CPP/Utils/ActionParsers/DiscreteMirroredAction.h>
//++++++++++++++++++++++++++++++++++++++++++++++++++++//
#include "RLBotClient.h"
#include <\Epsilon\RLGymPPO_CPP-main\Rewards.h>
using namespace RLGPC; // RLGymPPO using namespace RLGSC; // RLGymSim
static std::vectorstd::string g_RewardNames = {}; MetricLoggerSystem g_MetricLoggerSystem;
void OnStep(GameInst* gameInst, const RLGSC::Gym::StepResult& stepResult, Report& gameMetrics) { if (auto* combRewards = dynamic_cast<LogCombinedReward*>(gameInst->match->rewardFn)) { for (size_t i = 0; i < combRewards->rewardFuncs.size(); i++) { float reward = combRewards->lastRewards[i] * combRewards->rewardWeights[i]; gameMetrics.AccumAvg(combRewards->GetRewardFuncName(i), reward); } }
// Fix: Pass stepResult directly instead of stepResult.state g_MetricLoggerSystem.LogStep(gameInst, stepResult, gameMetrics);}
void OnIteration(Learner* learner, Report& allMetrics) { auto allGameMetrics = learner->GetAllGameMetrics(); g_MetricLoggerSystem.SummarizeIteration(allGameMetrics, allMetrics);
// Handle rewards std::vector<AvgTracker> avgRews(g_RewardNames.size()); for (size_t i = 0; i < g_RewardNames.size(); i++) { for (const auto& gameReport : allGameMetrics) { if (!gameReport.data.empty()) { avgRews[i] += gameReport.GetAvg(g_RewardNames[i]); } } allMetrics["Rewards/" + g_RewardNames[i]] = avgRews[i].Get(); }}
EnvCreateResult EnvCreateFunc() { constexpr int TICK_SKIP = 12; constexpr float NO_TOUCH_TIMEOUT_SECS = 8.f;
int team_size = 1; //static int counter = 0; //int team_size = (rand() % 100 < 50) ? 1 : ((rand() % 100 < 90) ? 2 : 3); std::cout << "Team Size: " << team_size << std::endl; // Initialize Arena first ArenaConfig arenaConfig; // Initialize with desired configurations Arena* arena = Arena::Create(RocketSim::GameMode::SOCCAR, arenaConfig, 120.0f); // Example tick rate // { PZSUM(new Reward(), 1.0f, 0.75f), 1.0f}, // { new Reward(), 1.0f}, // Define PZSUM as a local variable const auto PZSUM = [](RewardFunction* rewardFunc, float teamSpirit = 1.0f, float opponentScale = 1.0f) { return new ZeroSumReward(rewardFunc, teamSpirit, opponentScale); }; // Now that the Arena is created, you can safely pass it to your reward functions auto rewards = new LogCombinedReward( std::vector<std::pair<RewardFunction*, float>>{ { new sBP1(arena), 1.0f }, // Add other reward functions as needed } ); { static std::once_flag onceFlag; std::call_once(onceFlag, [&] { for (int i = 0; i < rewards->rewardFuncs.size(); i++) g_RewardNames.push_back(rewards->GetRewardFuncName(i)); }); } std::vector<TerminalCondition*> terminalConditions = { new NoTouchCondition(NO_TOUCH_TIMEOUT_SECS * 120 / TICK_SKIP), new GoalScoreCondition() //new BallTouchGroundCondition() }; auto obs = new DefaultOBS(); auto actionParser = new DiscreteAction(); auto stateSetter = new WeightedSampleSetter( { {new RandomState(true, true, false), 0.9f}, {new KickoffState(), 0.1f}, {new AirDribbleSetup(), 0.0f}, {new ReplaySetter(team_size), 0.0f}, } ); Match* match = new Match( rewards, terminalConditions, obs, actionParser, stateSetter, team_size, // Team size true // Spawn opponents ); // Optionally, you might need to associate the arena with the match or vice versa // match->SetArena(arena); // If such a method exists Gym* gym = new Gym(match, TICK_SKIP); return { match, gym };}
int main(int argc, char* argv[]) { // Initialize RocketSim with collision meshes RocketSim::Init("./collision_meshes");
// Make configuration for the learner LearnerConfig cfg = {}; cfg.renderMode = false; for (int i = 1; i < argc; ++i) { if (std::strcmp(argv[i], "--render") == 0) { cfg.renderMode = true; cfg.renderTimeScale = 1.0f; cfg.deterministic = true; } } // Play around with these to see what the optimal is for your machine, more isn't always better cfg.numThreads = 16; cfg.numGamesPerThread = 24; // We want a large itr/batch size // You'll want to increase this as your bot improves, up to an extent int tsPerItr = 100 * 1000; cfg.timestepsPerIteration = tsPerItr; cfg.ppo.batchSize = 100000; cfg.ppo.miniBatchSize = 25000; // Lower this if too much VRAM is being allocated cfg.expBufferSize = tsPerItr * 2; // This is just set to 1 to match rlgym-ppo example // I've found the best value is somewhere between 2 and 4 // Increasing this will lower SPS, but increase step efficiency cfg.ppo.epochs = 3; // Reasonable starting entropy cfg.ppo.entCoef = 0.01f; cfg.skillTrackerConfig.enabled = true; cfg.timestepsPerSave = 2'000'000; cfg.checkpointsToKeep = 30; cfg.skillTrackerConfig.numEnvs = 16; cfg.skillTrackerConfig.numThreads = 16; cfg.skillTrackerConfig.perModeRatings = true; cfg.metricsRunName = "RUN6"; cfg.metricsProjectName = "MAIN2"; cfg.metricsGroupName = "RUN"; // Good learning rate to start, may start to be too high around 100m steps cfg.ppo.policyLR = 4e-4; // 1e-4 -> 8e-5 -> 4e-5 -> 2e-5 ->1e-5 cfg.ppo.criticLR = 4e-4; // Default model size cfg.ppo.policyLayerSizes = { 512, 512, 512 }; cfg.ppo.criticLayerSizes = { 512, 512, 512 }; // tomorrow 1548, 1024, 512, 512, 256 //cfg.ppo.policyLayerSizes = { 1024, 1024, 1024, 1024, 1024 }; //cfg.ppo.criticLayerSizes = { 1024, 1024, 1024, 1024, 1024 }; cfg.sendMetrics = true; // Send metrics // Make the learner with the environment creation function and the config we just made Learner learner = Learner(EnvCreateFunc, cfg); // Set up our callbacks learner.stepCallback = [](GameInst* gameInst, const RLGSC::Gym::StepResult& stepResult, Report& gameMetrics) { // Original OnStep logic OnStep(gameInst, stepResult, gameMetrics); // Update GlobalBallPred BallPred::GetOrMake(gameInst->gym->arena); }; learner.iterationCallback = OnIteration; // Start learning! learner.Learn(); return 0;}
- Assistant
The issue you're encountering, where the
sBP1reward function only receives the current ball state instead of the correctly predicted future state, is likely due to the prediction environment not being properly updated before retrieving the predicted ball position. Let's break down the problem and provide a solution.Understanding the Problem
-
Reward Function (
sBP1) Implementation:float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) override { if (!predictionArena) { return 0.0f; // No reward if arena is not available } BallPredEnv* ballPredEnv = BallPred::GetOrMake(predictionArena); if (!ballPredEnv) return 0.0f; // Update the ball prediction //BallPred::Update(predictionArena); // **This is commented out** // Get the predicted ball position at MAX_PRED_TIME PhysObj predictedBall = ballPredEnv->GetAtTime(MAX_PRED_TIME, state.ball); // ... (rest of the reward calculation) } -
Ball Prediction Environment (
BallPredEnv):PhysObj BallPredEnv::GetAtTime(float time, const PhysObj& curBall) { if (!ballPredTracker.predData.empty()) { return PhysObj(ballPredTracker.GetBallStateForTime(time)); } else { // Handle the case where prediction data is empty } } -
Ball Prediction Manager (
BallPred):void OnStep(GameInst* gameInst, const RLGSC::Gym::StepResult& stepResult, Report& gameMetrics) { // ... (existing logic) // Update GlobalBallPred BallPred::GetOrMake(gameInst->gym->arena); }
Identifying the Root Cause
-
Prediction Update Not Invoked: The
BallPred::Update(predictionArena);line in theGetRewardfunction is commented out. This means that the prediction environment isn't being updated with the latest arena state before retrieving the predicted ball position. -
Resulting Behavior: Without updating, the
GetAtTimefunction is likely returning the current ball state or outdated predictions because the prediction tracker doesn't have the latest information to make accurate future predictions.
Solution Steps
-
Uncomment the Update Call: Ensure that the prediction environment is updated before fetching the predicted ball state. Modify the
GetRewardfunction insBP1as follows:float GetReward(const PlayerData& player, const GameState& state, const Action& prevAction) override { if (!predictionArena) { return 0.0f; // No reward if arena is not available } BallPredEnv* ballPredEnv = BallPred::GetOrMake(predictionArena); if (!ballPredEnv) return 0.0f; // **Uncomment the update call** BallPred::Update(predictionArena); // Get the predicted ball position at MAX_PRED_TIME PhysObj predictedBall = ballPredEnv->GetAtTime(MAX_PRED_TIME, state.ball); // ... (rest of the reward calculation) } -
Ensure Regular Updates in
OnStep: While theOnStepfunction currently callsGetOrMake, it doesn't explicitly update the predictions. To maintain accurate predictions, you should also callBallPred::Updatewithin theOnStepcallback. Modify theOnStepfunction as follows:void OnStep(GameInst* gameInst, const RLGSC::Gym::StepResult& stepResult, Report& gameMetrics) { if (auto* combRewards = dynamic_cast<LogCombinedReward*>(gameInst->match->rewardFn)) { for (size_t i = 0; i < combRewards->rewardFuncs.size(); i++) { float reward = combRewards->lastRewards[i] * combRewards->rewardWeights[i]; gameMetrics.AccumAvg(combRewards->GetRewardFuncName(i), reward); } } // Fix: Pass stepResult directly instead of stepResult.state g_MetricLoggerSystem.LogStep(gameInst, stepResult, gameMetrics); // **Update the ball prediction here as well** BallPred::Update(gameInst->gym->arena); } -
Verify Prediction Logic:
-
Ensure
BallPredEnv::GetAtTimeis Correct: Make sure that theGetAtTimefunction correctly interprets the predicted data and returns the appropriate future ball state. -
Handle Empty Prediction Data: In the
GetAtTimemethod, ensure that there is a fallback or proper handling whenpredDatais empty to avoid unexpected behavior.
-
-
Check Prediction Data Initialization:
- Ensure that
BallPredTrackerwithinBallPredEnvis correctly initialized and starts populatingpredDataas the simulation progresses. - Verify that the prediction horizon (
MAX_PRED_TIME) aligns with the update frequency to maintain accurate predictions.
- Ensure that
Additional Recommendations
-
Logging for Debugging: Implement logging within the
GetAtTimeandUpdatemethods to monitor whether predictions are being updated and retrieved correctly. This can help identify if the issue persists after making the above changes. -
Thread Safety: Since
BallPred::GetOrMakeuses athread_localmap, ensure that there are no race conditions or synchronization issues when updating and accessingBallPredEnv. -
Performance Considerations: Regularly updating predictions can have performance implications. Ensure that the update frequency and prediction computations are optimized to maintain real-time performance.
Conclusion
The primary reason the
sBP1reward function is only accessing the current ball state instead of the predicted future state is that the prediction environment isn't being updated due to the commented-outBallPred::Updatecall. By ensuring that updates occur both within theGetRewardfunction and theOnStepcallback, you can maintain accurate ball predictions, leading to more meaningful reward calculations.If after making these changes you still encounter issues, consider adding detailed logging to trace the flow of data and ensure that predictions are being correctly generated and utilized.
-