feat: add Sharpa sim2real and HORA training support

benchmark/benchmark_sharpa_init_dr_construct.py

@@ -21,10 +21,10 @@
 - `/home/admin1/ws/unilabsim/mujoco_uni/python/mujoco/batch_env.cc`
 
 Usage:
-    uv run python benchmark/benchmark_sharpa_init_dr_construct.py
-    uv run python benchmark/benchmark_sharpa_init_dr_construct.py --env-nums 256,512,1024
-    uv run python benchmark/benchmark_sharpa_init_dr_construct.py --variant-counts 1,2,4,8
-    uv run python benchmark/benchmark_sharpa_init_dr_construct.py --measure construct_plus_pool
+    uv run benchmark/benchmark_sharpa_init_dr_construct.py
+    uv run benchmark/benchmark_sharpa_init_dr_construct.py --env-nums 256,512,1024
+    uv run benchmark/benchmark_sharpa_init_dr_construct.py --variant-counts 1,2,4,8
+    uv run benchmark/benchmark_sharpa_init_dr_construct.py --measure construct_plus_pool
 """
 
 from __future__ import annotations
@@ -79,7 +79,7 @@ class ConstructRecord:
     mode: str
     variant_count: int
     num_envs: int
-    scale_range: list[float]
+    scale_list: list[float]
     repeats: int
     samples_sec: list[float]
     mean_sec: float
@@ -122,9 +122,11 @@ def _parse_variant_counts(value: str | None) -> list[int]:
 
 def _compose_cfg(task: str, *, lower: float, upper: float, variant_count: int):
     config_dir = str(ROOT_DIR / "conf" / "ppo")
+    scale_list = np.linspace(lower, upper, variant_count, dtype=np.float64)
+    scale_override = ",".join(f"{float(scale):g}" for scale in scale_list)
     overrides = [
         f"task={task}",
-        f"env.scale_range=[{lower:g},{upper:g},{variant_count}]",
+        f"env.domain_rand.scale_list=[{scale_override}]",
         "hydra.run.dir=.",
         "hydra.output_subdir=null",
         "hydra/job_logging=disabled",
@@ -254,7 +256,7 @@ def _summarize_record(
         mode=mode,
         variant_count=variant_count,
         num_envs=num_envs,
-        scale_range=[lower, upper, float(variant_count)],
+        scale_list=np.linspace(lower, upper, variant_count, dtype=np.float64).tolist(),
         repeats=len(samples),
         samples_sec=[float(sample) for sample in samples],
         mean_sec=float(mean(samples)),

conf/appo/config.yaml

@@ -61,6 +61,7 @@ training:
   wandb_notes: null
   wandb_mode: null
   sim_backend: mujoco
+  log_root: null
   log_dir: null
   play_only: false
   no_play: false

conf/appo/task/allegro_inhand/motrix.yaml

@@ -0,0 +1,63 @@
+# @package _global_
+training:
+  task_name: AllegroInhandRotation
+  sim_backend: motrix
+  play_steps: 200
+  render_spacing: 0.5
+  cam_distance: 1.5
+  cam_lookat: [0.75, 0.75, 0]
+  cam_elevation: -20.0
+algo:
+  num_envs: 16384
+  steps_per_env: 8
+  max_iterations: 201
+  save_interval: 100
+  algorithm:
+    value_loss_coef: 4.0
+    entropy_coef: 0.01
+    learning_rate: 0.001
+    desired_kl: 0.02
+    num_learning_epochs: 5
+    num_mini_batches: 4
+    clip_param: 0.2
+    gamma: 0.99
+    lam: 0.95
+    max_grad_norm: 1.0
+    use_clipped_value_loss: true
+    schedule: adaptive
+  actor:
+    hidden_dims: [512, 256, 128]
+    activation: elu
+    obs_normalization: true
+    distribution_cfg:
+      class_name: rsl_rl.modules.distribution.GaussianDistribution
+      init_std: 1.0
+      std_type: scalar
+  critic:
+    hidden_dims: [512, 256, 128]
+    activation: elu
+    obs_normalization: true
+reward:
+  scales:
+    rotate: 1.25
+    obj_linvel: -0.3
+    pose_diff: -0.3
+    torque: -0.1
+    work: -2.0
+    drop: 0.0
+  angvel_clip_min: -0.5
+  angvel_clip_max: 0.5
+  reset_z_threshold: 0.125
+env:
+  gen_grasp: false
+  max_episode_seconds: 20.0
+  grasp_cache_path: cache/allegro_grasp_50k.npy
+  # Keep only grasp/pose reset variation. All online DR terms stay disabled.
+  domain_rand:
+    randomize_base_mass: false
+    random_com: false
+    randomize_gravity: false
+    push_robots: false
+    joint_noise: 0.0
+    ball_vel_noise: 0.0
+    ball_z_offset: 0.0

conf/appo/task/allegro_inhand/mujoco.yaml

@@ -2,16 +2,40 @@
 training:
   task_name: AllegroInhandRotation
   sim_backend: mujoco
+  play_steps: 200
+  render_spacing: 0.5
+  cam_distance: 1.5
+  cam_lookat: [0.75, 0.75, 0]
+  cam_elevation: -20.0
 algo:
   num_envs: 16384
   steps_per_env: 8
-  max_iterations: 501
+  max_iterations: 201
+  save_interval: 100
   algorithm:
     value_loss_coef: 4.0
+    entropy_coef: 0.01
+    learning_rate: 0.001
     desired_kl: 0.02
+    num_learning_epochs: 5
+    num_mini_batches: 4
+    clip_param: 0.2
+    gamma: 0.99
+    lam: 0.95
+    max_grad_norm: 1.0
+    use_clipped_value_loss: true
+    schedule: adaptive
   actor:
+    hidden_dims: [512, 256, 128]
+    activation: elu
     obs_normalization: true
+    distribution_cfg:
+      class_name: rsl_rl.modules.distribution.GaussianDistribution
+      init_std: 1.0
+      std_type: scalar
   critic:
+    hidden_dims: [512, 256, 128]
+    activation: elu
     obs_normalization: true
 reward:
   scales:
@@ -24,3 +48,16 @@ reward:
   angvel_clip_min: -0.5
   angvel_clip_max: 0.5
   reset_z_threshold: 0.125
+env:
+  gen_grasp: false
+  max_episode_seconds: 20.0
+  grasp_cache_path: cache/allegro_grasp_50k.npy
+  # Keep only grasp/pose reset variation. All online DR terms stay disabled.
+  domain_rand:
+    randomize_base_mass: false
+    random_com: false
+    randomize_gravity: false
+    push_robots: false
+    joint_noise: 0.0
+    ball_vel_noise: 0.0
+    ball_z_offset: 0.0

conf/appo/task/sharpa_inhand/mujoco.yaml

@@ -0,0 +1,116 @@
+# @package _global_
+# Base Sharpa APPO MuJoCo owner config. The HORA variant inherits this file
+# so backend support and shared hyperparameters stay visible at the backend
+# owner layer rather than being duplicated in a HORA-only variant.
+
+training:
+  task_name: SharpaInhandRotation
+  sim_backend: mujoco
+  play_steps: 200
+  render_spacing: 0.5
+  cam_distance: 1.5
+  cam_lookat: [0.75, 0.75, 0.4]
+  cam_elevation: -20.0
+
+algo:
+  num_envs: 16384
+  steps_per_env: 8
+  max_iterations: 301
+  save_interval: 50
+  actor:
+    hidden_dims: [512, 256, 128]
+    activation: elu
+    obs_normalization: true
+    distribution_cfg:
+      class_name: rsl_rl.modules.distribution.GaussianDistribution
+      init_std: 1.0
+      std_type: scalar
+  critic:
+    hidden_dims: [512, 256, 128]
+    activation: elu
+    obs_normalization: true
+  algorithm:
+    value_loss_coef: 4.0
+    entropy_coef: 0.01
+    learning_rate: 0.001
+    desired_kl: 0.02
+    num_learning_epochs: 5
+    num_mini_batches: 4
+    clip_param: 0.2
+    gamma: 0.99
+    lam: 0.95
+
+reward:
+  scales:
+    rotate: 2.5
+    obj_linvel: -0.3
+    pose_diff: -0.4
+    torque: -0.1
+    work: -0.5
+    object_pos: 0.003
+  angvel_clip_min: -0.5
+  angvel_clip_max: 0.5
+
+env:
+  zero_action_test_mode: false
+  clip_obs: 5.0
+  clip_actions: 1.0
+  reset_height_lower: 0.59906
+  reset_height_upper: 0.63906
+  reset_angle_diff: 0.7853981633974483
+  rot_axis: [0.0, 0.0, 1.0]
+  grasp_cache_path: cache/sharpa_grasp_linspace
+  sensor:
+    tactile_force_sensor_names:
+      - contact_right_thumb_elastomer_force
+      - contact_right_index_elastomer_force
+      - contact_right_middle_elastomer_force
+      - contact_right_ring_elastomer_force
+      - contact_right_pinky_elastomer_force
+  disable_tactile_ids: []
+  use_default_object_pose_for_object_pos_anchor: false
+  obs:
+    observation_mode: flattened
+    enable_tactile: true
+    binary_contact: false
+    enable_contact_pos: false
+    contact_smooth: 0.5
+    contact_threshold: 0.05
+    tactile_force_clip_max: 4.0
+  priv_info:
+    include_friction_scale: true
+    include_gravity_direction: false
+  control_config:
+    action_scale: 0.041666666666666664
+    p_gain: 1.0 # use the XML value instead of this value by default
+    d_gain: 0.1
+    torque_control: false # can only be false
+    dof_limits_scale: 0.9
+  domain_rand:
+    scale_list: [0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6]
+    randomize_gravity_direction: true
+    gravity_direction_magnitude: 9.81
+    randomize_pd_gains: true
+    randomize_p_gain_scale_lower: 0.5
+    randomize_p_gain_scale_upper: 2.0
+    randomize_d_gain_scale_lower: 0.5
+    randomize_d_gain_scale_upper: 2.0
+    randomize_friction: true
+    randomize_friction_scale_lower: 0.75
+    randomize_friction_scale_upper: 1.25
+    elastomer_base_friction: 2.0
+    metal_base_friction: 1.0
+    object_base_friction: 2.0
+    randomize_com: true
+    randomize_com_lower: -0.01
+    randomize_com_upper: 0.01
+    randomize_mass: true
+    randomize_mass_lower: 0.01
+    randomize_mass_upper: 0.25
+    force_scale: 2.0
+    random_force_prob_scalar: 0.25
+    force_decay: 0.9
+    force_decay_interval: 0.08
+    joint_noise_scale: 0.02
+    contact_latency: 0.005
+    contact_sensor_noise: 0.01

conf/appo/task/sharpa_inhand/mujoco_hora.yaml

@@ -0,0 +1,32 @@
+# @package _global_
+# HORA Sharpa APPO variant. Inherit the shared MuJoCo owner so backend support
+# and shared hyperparameters stay aligned with the non-HORA Sharpa APPO config.
+defaults:
+  - /task/sharpa_inhand/mujoco
+  - _self_
+
+algo:
+  algo_log_name: hora_appo
+  runtime_impl: hora_appo
+  runtime_resolver: unilab.algos.torch.hora.appo:resolve_hora_appo_runtime
+  obs_groups:
+    # Keep grouped keys explicit in the owner YAML; runtime support for these
+    # grouped observations lands in the next implementation step.
+    actor:
+      actor: 0
+      priv_info: 0
+    critic:
+      actor: 0
+      priv_info: 0
+  actor:
+    class_name: unilab.algos.torch.hora:HoraActorModel
+    priv_info_embed_dim: 9
+    priv_mlp_hidden_dims: [256, 128, 9]
+  critic:
+    class_name: unilab.algos.torch.hora:HoraCriticModel
+    priv_info_embed_dim: 9
+    priv_mlp_hidden_dims: [256, 128, 9]
+
+env:
+  obs:
+    observation_mode: separated

conf/hora_distill/config.yaml

@@ -0,0 +1,44 @@
+defaults:
+  - _self_
+  - task: sharpa_inhand/mujoco
+
+algo:
+  algo_log_name: hora_distill
+  seed: 1
+  num_envs: 4096
+  max_agent_steps: 1000000000
+  save_interval_steps: 100000000
+  log_interval_steps: 32768
+  learning_rate: 3.0e-4
+  load_run: "-1"
+  checkpoint: -1
+  model: {}
+
+training:
+  task_name: SharpaInhandRotation
+  device: null
+  logger: tensorboard
+  sim_backend: mujoco
+  play_only: false
+  play_env_num: 16
+  play_steps: 200
+  render_spacing: 1.0
+  cam_distance: 6.0
+  cam_elevation: -20.0
+  cam_azimuth: 90.0
+  cam_lookat: null
+  cam_tracking: false
+  cam_tracking_env_idx: 0
+  cam_tracking_extra_envs: 2
+  log_root: null
+  log_dir: null
+
+hydra:
+  run:
+    dir: .
+  output_subdir: null
+  job:
+    chdir: false
+  job_logging:
+    root:
+      handlers: [console]

conf/hora_distill/task/sharpa_inhand/mujoco.yaml

@@ -0,0 +1,19 @@
+# @package _global_
+teacher:
+  algo_family: ppo
+  task: sharpa_inhand/mujoco_hora
+
+training:
+  task_name: SharpaInhandRotation
+  sim_backend: mujoco
+  render_spacing: 0.5
+  cam_distance: 1.5
+  cam_lookat: [0.75, 0.75, 0.4]
+  cam_elevation: -20.0
+  cam_azimuth: 90.0
+
+algo:
+  algo_log_name: hora_distill
+  num_envs: 16384
+  max_agent_steps: 100000000
+  save_interval_steps: 10000000

conf/offpolicy/config.yaml

@@ -25,6 +25,7 @@ training:
   cam_distance: 6.0
   cam_elevation: -20.0
   cam_azimuth: 90.0
+  log_root: null
   log_dir: null
   no_sync_collection: false
   env_steps_per_sync: 1