ArXiv.orgPub Date : 2025-04-17DOI: 10.48550/arxiv.2504.12609
Tyler Ga Wei Lum, Olivia Y. Lee, Chengyuan Liu, Jeannette Bohg
{"title":"Crossing the Human-Robot Embodiment Gap with Sim-to-Real RL using One Human Demonstration","authors":"Tyler Ga Wei Lum, Olivia Y. Lee, Chengyuan Liu, Jeannette Bohg","doi":"10.48550/arxiv.2504.12609","DOIUrl":"https://doi.org/10.48550/arxiv.2504.12609","url":null,"abstract":"Teaching robots dexterous manipulation skills often requires collecting hundreds of demonstrations using wearables or teleoperation, a process that is challenging to scale. Videos of human-object interactions are easier to collect and scale, but leveraging them directly for robot learning is difficult due to the lack of explicit action labels and human-robot embodiment differences. We propose Human2Sim2Robot, a novel real-to-sim-to-real framework for training dexterous manipulation policies using only one RGB-D video of a human demonstrating a task. Our method utilizes reinforcement learning (RL) in simulation to cross the embodiment gap without relying on wearables, teleoperation, or large-scale data collection. From the video, we extract: (1) the object pose trajectory to define an object-centric, embodiment-agnostic reward, and (2) the pre-manipulation hand pose to initialize and guide exploration during RL training. These components enable effective policy learning without any task-specific reward tuning. In the single human demo regime, Human2Sim2Robot outperforms object-aware replay by over 55% and imitation learning by over 68% on grasping, non-prehensile manipulation, and multi-step tasks. Website: https://human2sim2robot.github.io","PeriodicalId":522157,"journal":{"name":"ArXiv.org","volume":"1 1","pages":""},"PeriodicalIF":0.0,"publicationDate":"2025-04-17","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"https://arxiv.org/pdf/2504.12609","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"147892595","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":0,"RegionCategory":"","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"OA","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}