poetry run python sac_rlhf.py \
  --actor_and_q_net_hidden_dim=512 \
  --actor_and_q_net_hidden_layers=5 \
  --alpha=0.2 \
  --autotune \
  --batch_size=256 \
  --buffer_size=1000000 \
  --capture_video \
  --cuda \
  --no-early_stopping \
  --early_stop_patience=5 \
  --early_stopping_mean=1000 \
  --early_stopping_step=500000 \
  --no-enable_greater_or_smaller_check \
  --env_id="Hopper-v5" \
  --evaluation_episodes=10 \
  --evaluation_frequency=10000 \
  --no-exploration_load \
  --explore_batch_size=256 \
  --explore_learning_starts=512 \
  --gamma=0.9939604765683844 \
  --lambda_ssl=0.1 \
  --learning_starts=5000 \
  --log_file \
  --log_level="INFO" \
  --max_augmentation_offset=10 \
  --min_augmentation_offset=5 \
  --path_to_model="" \
  --path_to_replay_buffer="" \
  --policy_frequency=2 \
  --policy_lr=0.0002236151342771317 \
  --pref_buffer_size_sessions=10 \
  --preference_sampling="disagree" \
  --q_lr=0.0006143918780906629 \
  --reward_net_dropout=0.1 \
  --reward_net_hidden_dim=256 \
  --reward_net_hidden_layers=5 \
  --reward_net_val_split=0.2 \
  --no-rune \
  --rune_beta=0.08936083059656459 \
  --rune_beta_decay=0.0001 \
  --seed=1 \
  --no-surf \
  --surf_sampling_strategy="disagree" \
  --surf_tau=0.999 \
  --target_network_frequency=1 \
  --tau=0.005 \
  --teacher_batch_strategy="minibatch" \
  --teacher_feedback_batch_size=16 \
  --teacher_feedback_exponential_lambda=0.1 \
  --teacher_feedback_num_queries_per_session=40 \
  --teacher_feedback_schedule="linear" \
  --teacher_feedback_total_queries=6000 \
  --teacher_learning_rate=0.0008147316395014214 \
  --teacher_minibatch_size=20 \
  --teacher_sim_beta=-1 \
  --teacher_sim_delta_equal=0 \
  --teacher_sim_delta_skip=-10000000 \
  --teacher_sim_epsilon=0 \
  --teacher_sim_gamma=1 \
  --teacher_update_epochs=17 \
  --torch_deterministic \
  --total_explore_steps=10000 \
  --total_timesteps=4000000 \
  --track \
  --trajectory_length=128 \
  --unlabeled_batch_ratio=2 \
  --unsupervised_exploration \
  --wandb_entity="cleanRLHF" \
  --wandb_project_name="Hopper_human_feedback" \
  --feedback_server_url="http://localhost:5001" \
  --feedback_server_autostart \
  --teacher_feedback_mode="human"