Tune hyperparameters on the left, watch this algorithm learn on the right. Reward curve, seed-to-seed stability,
and successful episodes update live, so you can see whether a config is actually improving the policy.
);
};
// ---- Stability (mean ± σ band) ----
const StabilityCurve = ({ running, tick, fit }) => {
const W = 560, H = 150, pad = 24;
const pw = W - pad*2, ph = H - pad*2;
const N = 60;
// three seeds drift around a mean; bad fit = wide band
const data = useMemo(() => {
const arr = [];
const T = Math.max(5, Math.min(N, tick));
for (let i = 0; i < T; i++) {
const t = i / N;
const mean = fit * (1 - Math.exp(-t * 3)) * 0.9;
const noise = (1 - fit) * 0.55 + 0.08;
const sigma = noise * (0.6 + 0.4*(1-t));
arr.push({ mean, sigma });
}
return arr;
}, [tick, fit]);
const pts = (key, sign=0) => data.map((d, i) => {
const x = pad + (i / Math.max(1, N-1)) * pw;
const y = pad + ph - (Math.max(0, Math.min(1, d[key] + sign * d.sigma))) * ph;
return [x.toFixed(1), y.toFixed(1)];
});
const meanPts = pts('mean');
const hi = pts('mean', +1);
const lo = pts('mean', -1);
const band = [...hi, ...lo.slice().reverse()].map(p => p.join(',')).join(' ');
return (
);
};
// ---- Successful runs (episode strip) ----
const SuccessfulRuns = ({ episodes, envId }) => {
const envTerm = { warehouse: 'orders filled', inventory: 'demand met', portfolio: 'beat S&P', truck: 'on-time delivery' }[envId] || 'solved';
return (
{episodes.map((ok, i) => (
{ok ? '✓' : '·'}
))}
Each square is one recent evaluation episode, filled squares are successful ({envTerm}). A rising success rate confirms the policy is actually learning the task, not just accumulating reward from easy states.
);
};
const PgAlgorithms = ({ D, onPick }) => (
Algorithms // catalog of available policies
Every algorithm below is pre-wired to OptRL's orchestrator. Click an entry to
load it into the sandbox with sensible defaults for the chosen environment.
);
};
const Playground = ({ envId, setEnvId, algo, setAlgo, onPublish, pgView, onNav }) => {
const D = window.RLX_DATA;
const [running, setRunning] = useState(true);
const [tick, setTick] = useState(0);
const env = D.playgroundEnvs.find(e => e.id === envId) || D.playgroundEnvs[0];
const algoIdx = D.playgroundAlgos.indexOf(algo);
useEffect(() => {
if (!running) return;
if (pgView && pgView !== 'pg-envs') return;
const id = setInterval(() => setTick(t => t + 1), 120);
return () => clearInterval(id);
}, [running, pgView]);
// Per-(algo × env) suitability: algorithms are NOT equally good on every env.
// Scaled base reward per env, then a suitability multiplier keyed on the algo's class
// vs. the env's data regime. Also applies a per-algo quality prior.
const metrics = useMemo(() => {
// Baseline reward scale per environment
const envBase = { warehouse: 920, inventory: 1180, portfolio: 17.2, truck: 2080 }[env.id] || 800;
// Algorithm → class (mirrors playgroundAlgoGroups)
const algoClass = {
// on-policy
'PPO': 'policy', 'TRPO': 'policy', 'GRPO': 'policy',
// off-policy
'SAC': 'value', 'TD-MPC2': 'value', 'DreamerV3': 'value', 'Rainbow-DQN': 'value',
// inverse RL
'GAIL': 'irl', 'AIRL': 'irl',
// offline
'CQL': 'offline', 'IQL': 'offline', 'Decision Transformer': 'offline',
// proprietary (hybrid model-based + MPC)
'OptRL-Warehouse-v3': 'hybrid-warehouse', 'OptRL-Truck-v1': 'hybrid-truck',
}[algo] || 'value';
// Env profile (what the env rewards)
// reg: data regime it lives in (online-sim / logged / imitation-friendly / continuous / discrete)
// longHorizon: gains from model-based planning
const envProfile = {
warehouse: { kind: 'discrete-combinatorial', longHorizon: 0.8, imitable: 1.0 },
inventory: { kind: 'discrete-sequential', longHorizon: 0.3, imitable: 0.6 },
portfolio: { kind: 'offline-continuous', longHorizon: 0.7, imitable: 0.2 },
truck: { kind: 'continuous-planning', longHorizon: 1.0, imitable: 0.8 },
}[env.id];
// Suitability: how well this class matches this env (0.55 = bad fit, 1.0 = great fit)
let fit = 0.85;
if (algoClass === 'policy') {
// on-policy: fine in online sim, weaker in long-horizon and bad in offline
fit = env.id === 'portfolio' ? 0.60 : (envProfile.longHorizon > 0.8 ? 0.78 : 0.92);
} else if (algoClass === 'value') {
// off-policy: strong in continuous/long-horizon, variable on discrete
if (algo === 'Rainbow-DQN') fit = envProfile.kind.includes('discrete') ? 1.00 : 0.58;
else if (algo === 'DreamerV3' || algo === 'TD-MPC2') fit = 0.85 + 0.15 * envProfile.longHorizon;
else /* SAC */ fit = envProfile.kind.includes('continuous') ? 0.98 : 0.80;
if (env.id === 'portfolio') fit -= 0.15; // portfolio really wants offline
} else if (algoClass === 'irl') {
// IRL: only good if env has expert demos available
fit = 0.55 + 0.40 * envProfile.imitable;
} else if (algoClass === 'offline') {
// offline RL: huge on portfolio (logged data), mediocre online
fit = env.id === 'portfolio' ? 1.02 : 0.70;
if (algo === 'Decision Transformer' && env.id === 'portfolio') fit = 1.05;
} else if (algoClass === 'hybrid-warehouse') {
fit = env.id === 'warehouse' ? 1.08 : 0.82;
} else if (algoClass === 'hybrid-truck') {
fit = env.id === 'truck' ? 1.10 : 0.80;
}
// Per-algo quality prior (same order as playgroundAlgos list; falls back to 0.9)
const qualityPrior = [1.0, 0.98, 0.95, 0.92, 0.88, 0.86, 0.84, 0.80, 0.76, 0.74, 0.70, 0.68][algoIdx];
const algoMod = fit * (qualityPrior || 0.9);
// Learning curve: rises with tick, asymptotes at base * algoMod
const curScore = Math.round(envBase * algoMod * (1 - Math.exp(-tick / 30)) * 100) / 100;
// Success-% ceiling also depends on fit (bad fit plateaus lower)
const successCeiling = Math.min(97, 45 + fit * 52);
const success = Math.min(successCeiling, 25 + tick * 0.9 * fit).toFixed(1);
const episode = Math.floor(tick / 8) + 1;
const step = (tick * 1247) % env.episodeLen;
return {
curScore, episode, step, success,
fps: 842 + (tick % 10),
fit,
algoClass,
};
}, [env.id, algoIdx, tick, algo]);
if (pgView === 'pg-algos') {
return { setAlgo(name); onNav && onNav('pg-envs'); }} />;
}
if (pgView === 'pg-sessions') {
return onNav && onNav('pg-envs')} />;
}
if (pgView === 'pg-train') {
return ;
}
const VizComponent = {
warehouse: WarehouseViz, inventory: InventoryViz, portfolio: PortfolioViz, truck: TruckViz
}[env.id];
return (
OptRL Playground
// interactive product · runs feed back to RLX
A separate interactive product from the RLX leaderboard. Pick an environment,
pick an algorithm, we spin up a sandboxed rollout on our compute and stream
metrics back. Submit completed runs to publish on RLX.
{metrics.fit >= 1.0 ? 'Near-optimal match, this algo class is purpose-built for this env.'
: metrics.fit >= 0.9 ? 'Strong fit, expect convergence well above baseline.'
: metrics.fit >= 0.75 ? 'Workable fit, will learn, but won\'t top the leaderboard here.'
: 'Poor fit, algorithm class doesn\'t match this env\'s data regime. Try a different family.'}