# ppo_walker.py - PPO (stable-baselines3) learns a closed-loop neural-network policy
import sys, time, json, numpy as np, torch
from stable_baselines3 import PPO
from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3.common.vec_env import VecNormalize
from stable_baselines3.common.callbacks import BaseCallback
from walker_env import WalkerEnv, STEPS, score

torch.set_num_threads(1)

def evaluate(model, venv):
    """Run the deterministic policy for one 5-second trial and return score()."""
    env = WalkerEnv(); obs, _ = env.reset()
    for _ in range(STEPS // env.FRAME_SKIP):
        a, _ = model.predict(venv.normalize_obs(obs), deterministic=True)
        obs, *_ = env.step(a)
    return score(env.data)

class Log(BaseCallback):
    def __init__(self, every, t0):
        super().__init__(); self.every, self.t0, self.rows, self.next = every, t0, [], every
    def _on_step(self):
        if self.num_timesteps >= self.next:
            self.next += self.every
            dist = evaluate(self.model, self.training_env)
            steps = self.num_timesteps * WalkerEnv.FRAME_SKIP
            self.rows.append(dict(physics_steps=steps, trials=steps // STEPS,
                                  seconds=round(time.process_time() - self.t0, 1), distance=dist))
            print(f"physics steps {steps:8d}  trials {steps//STEPS:5d}  policy {dist:5.2f} m  cpu {time.process_time()-self.t0:6.1f} s", flush=True)
        return True

if __name__ == "__main__":
    seed = int(sys.argv[1]) if len(sys.argv) > 1 else 1
    agent_steps = int(sys.argv[2]) if len(sys.argv) > 2 else 600_000   # = 3.0 M physics steps
    venv = VecNormalize(make_vec_env(WalkerEnv, n_envs=4, seed=seed), norm_obs=True, norm_reward=True)
    model = PPO("MlpPolicy", venv, n_steps=512, batch_size=64, seed=seed, verbose=0)
    t0 = time.process_time(); cb = Log(24_000, t0)       # log every 96 trials
    model.learn(agent_steps, callback=cb)
    model.save(f"ppo_seed{seed}"); venv.save(f"ppo_seed{seed}_norm.pkl")
    json.dump(cb.rows, open(f"ppo_seed{seed}.json", "w"))
