# ppo_walker.py - PPO (stable-baselines3), same settings as v1, trained on a fixed score
import sys, time, json, numpy as np, torch
from stable_baselines3 import PPO
from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3.common.vec_env import VecNormalize
from stable_baselines3.common.callbacks import BaseCallback
from walker_env import WalkerEnv, STEPS, final_score

torch.set_num_threads(1)

def evaluate(model, venv, rule):
    """Deterministic policy, one 5-second trial, scored with the same rule it trains on."""
    env = WalkerEnv(rule); obs, _ = env.reset()
    for _ in range(STEPS // env.FRAME_SKIP):
        a, _ = model.predict(venv.normalize_obs(obs), deterministic=True)
        obs, r, term, trunc, _ = env.step(a)
        if term or trunc:
            break
    return final_score(float(env.data.qpos[0]), term, rule)

class Log(BaseCallback):
    def __init__(self, every, t0, rule):
        super().__init__(); self.every, self.t0, self.rows, self.next, self.rule = every, t0, [], every, rule
    def _on_step(self):
        if self.num_timesteps >= self.next:
            self.next += self.every
            dist = evaluate(self.model, self.training_env, self.rule)
            steps = self.num_timesteps * WalkerEnv.FRAME_SKIP
            self.rows.append(dict(physics_steps=steps, trials=steps // STEPS,
                                  seconds=round(time.process_time() - self.t0, 1), distance=dist))
            print(f"physics steps {steps:8d}  trials {steps//STEPS:5d}  policy {dist:5.2f} m  cpu {time.process_time()-self.t0:6.1f} s", flush=True)
        return True

if __name__ == "__main__":
    seed = int(sys.argv[1]) if len(sys.argv) > 1 else 1
    agent_steps = int(sys.argv[2]) if len(sys.argv) > 2 else 600_000   # = 3.0 M physics steps
    rule = sys.argv[3] if len(sys.argv) > 3 else "forfeit"
    venv = VecNormalize(make_vec_env(WalkerEnv, n_envs=4, seed=seed, env_kwargs={"rule": rule}), norm_obs=True, norm_reward=True)
    model = PPO("MlpPolicy", venv, n_steps=512, batch_size=64, seed=seed, verbose=0)
    t0 = time.process_time(); cb = Log(24_000, t0, rule)
    model.learn(agent_steps, callback=cb)
    model.save(f"ppo_{rule}_seed{seed}"); venv.save(f"ppo_{rule}_seed{seed}_norm.pkl")
    json.dump(cb.rows, open(f"ppo_{rule}_seed{seed}.json", "w"))
