原路径：/SSD2/yangzijing/egoview-v9-foundationhand-s9l10-20260901/v9_benchmark/ego0824_rgb_demo.py
SHA256：bc6b4e4e838b6ff40ab6acbdcbe790c95f334659b2b6d514fd011add415eb8ff
1 """Isolated, explicitly uncalibrated RGB demo branch for the 08479125 archive.""" 2 import argparse, os, time, json 3 from pathlib import Path 4 from dataclasses import replace, asdict 5 os.environ.setdefault('PYOPENGL_PLATFORM','egl') 6 import numpy as np 7 from v9_benchmark.ego0824_p3r2 import OUT, PILOT, read 8 from v9_benchmark.hf_0907_s9_worker import load_npz as load 9 from v9_fusion.io import write_json as write, sha256_file as sha 10 ROOT=Path(__file__).resolve().parents[1] 11 DEST=Path(os.environ.get('EGO0824_DEMO_ROOT',str(OUT/'rgb_demo'))) 12 13 def rows():return read(OUT/'input_inventory.json')['sequences'] 14 def rgb_fit_receipt(report): 15 report=dict(report) 16 report['estimated_canonical_K']=report.pop('native_sensor_K',None) 17 report['measured_sensor_intrinsics_used']=False 18 report['ground_truth_used']=False 19 report['wrist_depth_policy']='RGB and metric MANO size prior jointly fitted with assumed canonical K; estimated scale, not calibrated GT' 20 report['palm_initialization']=dict(report['palm_initialization'],method='S9_palm_SQPNP_seed_with_assumed_canonical_K') 21 report['receipt_label_correction']='Shared fit engine contains historical calibrated/native labels; this batch supplies explicitly estimated K. Numerical outputs unchanged.' 22 return report 23 def link(src,dst): 24 if not dst.exists():dst.symlink_to(Path(src).resolve(),target_is_directory=Path(src).is_dir()) 25 26 def prepare(row): 27 sid=row['sequence_id'];src=OUT/'native_rgb_analysis/work'/sid;w=DEST/'work'/sid;w.mkdir(parents=True,exist_ok=True) 28 for name in ['rgb','sam_dense']:link(src/name,w/name) 29 link(read(src/'raw_s9_receipt.json')['prediction_path'],w/'raw_s9.npz') 30 p=read(src/'prepared.json');p.update(profile='V9P3R2_RGB_demo',camera_geometry='canonical_not_calibrated',production_geometry_ready=True,ground_truth_used=False,intrinsics_source='historical rgb_raster_half_width assumption',depth_payload=False) 31 write(w/'prepared.json',p) 32 z=load(src/'rgb_candidates.npz');m=load(OUT/'native_tracking_ablations_v3'/sid/'repair_candidate.npz');mapping=m['mapping'].copy();tracks=m['tracks'].copy();n=row['frame_count'] 33 audit=OUT/'native_inputs'/sid/'strict_frame_audit.json' 34 bad=read(audit)['strict_missing_frame_ids'] if audit.exists() else [];mapping[bad]=-1;tracks[bad]=-1 35 e=load(src/'semantic_observations.npz') 36 for k in ['valid','detected','detector_observed','v5r_mmpose_valid']:e[k][:]=False 37 for k in ['confidence','sam_score','v5r_mmpose_keypoint_scores']:e[k][:]=0 38 for k in ['boxes_xyxy','v5r_mmpose_keypoints_2d']:e[k][:]=np.nan 39 e['sam_index'][:]=-1;e['observation_track_ids']=tracks 40 for f,s in np.argwhere(mapping>=0): 41 i=z['offsets'][f]+mapping[f,s] 42 for k in ['valid','detected','v5r_mmpose_valid']:e[k][f,s]=True 43 for target,source in [('boxes_xyxy','box'),('v5r_mmpose_keypoints_2d','keypoints'),('v5r_mmpose_keypoint_scores','scores'),('confidence','yolo_score'),('sam_score','sam_score'),('sam_index','sam_index')]:e[target][f,s]=z[source][i] 44 e['detector_observed'][f,s]=z['yolo_score'][i]>=.3 45 # Without measured depth, a SAM mask and fewer than the fitter's four 46 # reliable joints cannot constrain a partial hand's pose and root depth. 47 # Retain the observation in the source cache, but do not fabricate 3D. 48 unsupported=e['valid']&((e['v5r_mmpose_keypoint_scores']>=.35).sum(-1)<4) 49 for key in ['valid','detected','v5r_mmpose_valid']:e[key][unsupported]=False 50 e['observation_track_ids'][unsupported]=-1 51 fragment=np.zeros_like(unsupported) 52 for s in range(2): 53 ids=np.flatnonzero(e['valid'][:,s]);runs=np.split(ids,np.flatnonzero((np.diff(ids)!=1)|(e['observation_track_ids'][ids[1:],s]!=e['observation_track_ids'][ids[:-1],s]))+1) 54 for run in runs: 55 if not len(run) or len(run)>2:continue 56 score=e['v5r_mmpose_keypoint_scores'][run,s] 57 if ((score>=.35).sum(-1)<12).all() and not (score[:,[0,5,9,13,17]]>=.6).any():fragment[run,s]=True 58 for key in ['valid','detected','v5r_mmpose_valid']:e[key][fragment]=False 59 e['observation_track_ids'][fragment]=-1 60 write(w/'rgb_observability.json',dict(policy='minimum 4 reliable same-frame RGB joints; reject isolated 1-2 frame runs with <12 reliable joints and no confident palm joint, lacking depth/pose constraints',insufficient_joints_slots=np.argwhere(unsupported).tolist(),weak_fragment_slots=np.argwhere(fragment).tolist(),rejected_slots=np.argwhere(unsupported|fragment).tolist(),rejected_count=int((unsupported|fragment).sum()),smoothing_used=False,presence_invention=False,scope='isolated uncalibrated RGB batch; P3R2 production defaults unchanged')) 61 np.savez_compressed(w/'semantic_observations.npz',**e) 62 k=np.broadcast_to(np.array([[960.,0,960],[0,960,540],[0,0,1]],np.float32),(n,3,3)).copy() 63 np.savez_compressed(w/'camera_intrinsics.npz',K=k,timestamps_s=row['timestamps_s'],source_pts_s=row['source_pts_s'],source_frame_ids=np.arange(n),source_frame_valid=~np.isin(np.arange(n),bad)) 64 write(w/'input_binding.json',dict(source_sha256=row['source_sha256'],source_archive_sha256=row['source_archive_sha256'],source_video=row['source_video'],raw_s9_sha256=sha(w/'raw_s9.npz'),observations_sha256=sha(w/'semantic_observations.npz'),damaged_frames=bad,tracking_variant='native_tracking_ablations_v3/repair_candidate',tracking_acceptance='isolated RGB assignment regression checked; final 3D visual acceptance pending',camera_geometry='uncalibrated canonical pinhole',K=k[0].tolist(),depth_for_hand_fit=False,depth_trajectory_authorization='User explicit batch-only historical RGB override; no production defaults changed')) 65 return w 66 67 def fit(row,mano): 68 from v9_benchmark.p3r_fit_worker import constraints_for,repaired_config 69 from v9_benchmark.p3r1_anomaly_experiment import hand_arrays 70 from v9_fusion.s9_sequence_fit import fit_sequence 71 from v9_fusion.hand_surface import SurfaceConfig 72 import torch 73 w=DEST/'work'/row['sequence_id'] 74 if (w/'fit_complete.json').exists():return 75 w=prepare(row);e=load(w/'semantic_observations.npz');raw=load(w/'raw_s9.npz');k=load(w/'camera_intrinsics.npz')['K'];surface=constraints_for(w,e) 76 # R2 requires explicit absence of depth, and disables containment on partial 77 # masks without reliable depth. RGB predicted depth is not a sensor factor. 78 surface['mask_depth_seed_m']=np.zeros(e['valid'].shape,np.float32) 79 partial=(e['v5r_mmpose_keypoint_scores']>=.35).sum(-1)<12 80 surface['containment_weight']=np.where(partial,0,surface['mask_weight']).astype(np.float32) 81 np.savez_compressed(w/'surface_constraints.npz',**surface) 82 conf=replace(repaired_config(False,quality_refinement=True),respect_observation_track_boundaries=True,partial_mask_initialization=True,palm_confidence_guard=True) 83 sc=SurfaceConfig(containment_weight=.8,coverage_weight=.25,optimize_subject_shape=True) 84 torch.manual_seed(20260914);np.random.seed(20260914);start=time.monotonic() 85 def progress(r): 86 if r['step']%120==1:print(row['sequence_id'],r['step'],r['loss'],flush=True) 87 result,report=fit_sequence(raw,e,k,mano,conf,surface_constraints=surface,surface_config=sc,progress=progress) 88 np.savez_compressed(w/'fitted.npz',**result);write(w/'fit_engine_receipt.json',report);write(w/'fit.json',rgb_fit_receipt(report)) 89 h=hand_arrays(result,e);np.savez_compressed(w/'final_hand_camera.npz',**h) 90 write(w/'fit_complete.json',dict(frames=row['frame_count'],valid_slots=h['hand_valid'].sum(0).tolist(),config=asdict(conf),surface_config=asdict(sc),seconds=time.monotonic()-start,model='FoundationHand-S9L10 (1e-6)',hand_geometry_source='S9L10 + P3R2 RGB observation/MANO fit only',hawor_geometry_used=False,code_sha256={str(p.relative_to(ROOT)):sha(p) for p in [Path(__file__),ROOT/'v9_fusion/s9_sequence_fit.py',ROOT/'v9_fusion/s9_pose_initialization.py']})) 91 print('FIT_COMPLETE',row['sequence_id'],flush=True) 92 93 def build_state(row): 94 import cv2 95 from egoview.v5_pipeline.state import NormalizedInput,HandPrediction,CameraTrajectory,SceneExclusionPrediction,SceneDepthPrediction,PredictionState,DA3_SCENE_DEPTH_SCHEMA_VERSION,thaw_json 96 from egoview.v5_pipeline.fusion import build_da3_scene_map,fuse_v5_hands_to_world 97 from egoview.v5r_pipeline.runner import V7_SCENE_FUSION_CONFIG 98 from egoview.v5r_scene_scale import align_da3_scene_scale,_MANO_SCALE_POLICY 99 from v9_benchmark.hf_0907_contract import mano_topology 100 sid=row['sequence_id'];w=DEST/'work'/sid;n=row['frame_count'];rec,seg=sid.split('__');cache=ROOT/'artifacts/v9_eval/work'/rec/seg/'v7_prediction_state_cache' 101 receipt=read(cache/'cache_receipt.json');m=read(cache/'prediction_state_metadata.json');binding=receipt['input_binding'] 102 assert binding['source_video_sha256']==row['source_sha256']==sha(row['source_video']) 103 assert binding['hand_rgb_transform']=='sbs_left_eye' and binding['frame_count']==n and binding['fps']==row['fps'] 104 # Verify recorded cache bytes, independent of current mutable old pipeline code. 105 for name,record in receipt['artifacts'].items():assert sha(cache/name)==record['sha256'],name 106 raw=load(cache/'prediction_state.npz');k=load(w/'camera_intrinsics.npz')['K'];e=load(w/'semantic_observations.npz');h=load(w/'final_hand_camera.npz');bad=read(w/'input_binding.json')['damaged_frames'] 107 assert np.array_equal(raw['normalized/frame_ids'],np.arange(n)) 108 ids=np.arange(n);keys=raw['normalized/frame_keys'];nm=m['normalized']['metadata'];nm.update(source=row['source_video'],source_video_sha256=row['source_sha256'],foundationhand_used=True,ground_truth_used=False) 109 norm=NormalizedInput(rgb_dir=w/'rgb',rgb_paths=tuple(sorted((w/'rgb').glob('*.png'))),frame_ids=ids,frame_keys=keys,timestamp_ns=np.rint(np.array(row['timestamps_s'])*1e9).astype(np.int64),intrinsics=k[0],image_size_hw=np.array([1080,1920]),fps=30.,metadata=nm) 110 a=dict(e,frame_keys=keys,takes=np.full(n,sid),intrinsics=k[0],joints_camera_m=h['joints_camera_m'],vertices_camera_m=h['vertices_camera_m'],translation_camera_m=h['joints_camera_m'][:,:,0],root_depth_camera_z_m=h['joints_camera_m'][:,:,0,2],valid=h['hand_valid'],mano_rotation_matrices=h['mano_rotation_matrices'],mano_shape=h['mano_shape']) 111 from scipy.spatial.transform import Rotation 112 aa=np.full((n,2,16,3),np.nan,np.float32);hv=h['hand_valid'] 113 if hv.any():aa[hv]=Rotation.from_matrix(h['mano_rotation_matrices'][hv].reshape(-1,3,3)).as_rotvec().reshape(-1,16,3) 114 a.update(global_orient_camera_aa=aa[:,:,0],hand_pose_aa=aa[:,:,1:],betas=h['mano_shape'],mano_hand_size_m=np.full(hv.shape,np.nan,np.float32),root_depth_delta_ray_m=np.zeros(hv.shape,np.float32),root_depth_translation_delta_camera_m=np.zeros((n,2,3),np.float32),root_depth_correction_accepted=np.zeros(hv.shape,bool),root_depth_extent_source=np.full(hv.shape,'P3R2_RGB_joint_fit'),root_depth_solver_reason=np.full(hv.shape,'S9_MANO_RGB_estimated_Z')) 115 hand=HandPrediction.from_payload(a,dict(profile='V9P3R2_RGB_demo',provider='FoundationHand-S9L10 (1e-6)',mano_scale_policy=_MANO_SCALE_POLICY,mano_scale_modified=False,metric_depth_target_used=False,ground_truth_used=False,sensor_depth_used=False,camera_geometry='canonical_not_calibrated',scale_uncertainty='learned MANO anthropometric prior and assumed focal length; not calibrated metric GT',hawor_geometry_used=False)) 116 # Remove the old hand-driven scene scale before computing this batch's scale. 117 oldscale=float(m['scene_depth']['metadata']['scene_scale_alignment']['scale_factor']) 118 dm=m['scene_depth']['metadata'];rawmeta={key:dm[key] for key in ['provider','runtime','provider_metadata','coordinate_convention','depth_quantity','input_modality','backend_invocations','da3_model_invocations']} 119 rawmeta.update(schema_version=DA3_SCENE_DEPTH_SCHEMA_VERSION,role='shared_rgb_metric_depth_and_scene_map',shared_for_vipe=True,hand_prediction_consumed=False,hand_root_depth_target_used=False,ground_truth_used=False,sensor_depth_used=False,estimated_not_ground_truth=True,old_scene_scale_removed=oldscale) 120 depthvalues=raw['scene_depth/depth_camera_z_m']/np.float32(oldscale);validdepth=raw['scene_depth/valid_mask'].copy();validdepth[bad]=False 121 depth=SceneDepthPrediction(frame_ids=ids,frame_keys=keys,depth_camera_z_m=depthvalues,valid_mask=validdepth,intrinsics=raw['scene_depth/intrinsics'],metadata=rawmeta) 122 faces,model=mano_topology();scale=align_da3_scene_scale(hand,depth,faces[1]);depth=scale.scene_depth 123 poses=raw['trajectory/camera_to_world'].copy();poses[:,:3,3]*=scale.scale_factor/oldscale;tv=raw['trajectory/valid'].copy();tv[bad]=False 124 tm=m['trajectory']['metadata'];tm.update(estimated_not_ground_truth=True,old_scene_scale_removed=oldscale,current_sequence_scale=scale.scale_factor,shared_da3_prediction_digest=depth.digest,scale_retargeting='single sequence multiplicative factor applied to all camera translations and all scene depths; no per-frame rescaling') 125 trajectory=CameraTrajectory(frame_ids=ids,frame_keys=keys,camera_to_world=poses,valid=tv,confidence=raw['trajectory/confidence'],metadata=tm) 126 exclusion=SceneExclusionPrediction(frame_ids=ids,frame_keys=keys,excluded_mask=raw['scene_exclusion/excluded_mask'],metadata=m['scene_exclusion']['metadata']) 127 print('SCENE_REBUILD',sid,'scale',scale.scale_factor,scale.reason,flush=True) 128 scene=build_da3_scene_map(norm,hand,depth,trajectory,exclusion,config=V7_SCENE_FUSION_CONFIG) 129 world=fuse_v5_hands_to_world(hand,trajectory) 130 state=PredictionState(normalized=norm,hand=hand,trajectory=trajectory,scene_exclusion=exclusion,scene_depth=depth,scene_map=scene,world_arrays=world,metadata=dict(schema_version='egoview_v5_prediction_state_v3',v5_prediction_digest=hand.digest,shared_da3_prediction_digest=depth.digest,profile='V9P3R2_RGB_demo',estimated_not_ground_truth=True)) 131 np.savez_compressed(w/'world_reconstruction.npz',**world,camera_to_world=poses,camera_valid=tv,K=k,timestamps_s=row['timestamps_s'],faces=faces,scene_points=scene.points_world_m,scene_colors=scene.colors_rgb,scene_support=scene.frame_support) 132 write(w/'scene_provenance.json',dict(cache=str(cache),cache_artifacts=receipt['artifacts'],input_binding=binding,current_source_sha256=row['source_sha256'],current_decoded_rgb_sha256=row['native_left_decoded_rgb_sha256'],old_hand_geometry_reused=False,old_scene_map_reused=False,old_scene_scale_removed=oldscale,current_scale=thaw_json(depth.metadata['scene_scale_alignment']),trajectory=tm,depth_runtime=rawmeta,scene=thaw_json(scene.metadata),mano=model,all_sources_estimated=True,not_calibrated_ground_truth=True,points=len(scene.points_world_m))) 133 return state,faces,read(cache/'v7_run_manifest.json')['outputs']['demo'] 134 135 def render(row): 136 import cv2,subprocess 137 from egoview.v5_pipeline import render as renderer 138 from v9_fusion.display import _patched_v7_renderer 139 from v9_benchmark.hand_overlay import HandOverlay 140 from v9_fusion.constants import V9_MANO_RIGHT 141 sid=row['sequence_id'];w=DEST/'work'/sid;final=DEST/'final';final.mkdir(parents=True,exist_ok=True);target=final/(sid+'__V9P3R2_RGB_two_column.mp4') 142 if (w/'render_complete.json').exists():return 143 state,faces,old=build_state(row);cv2.setNumThreads(2) 144 config=renderer.DemoRenderConfig(panel_height=720,fps=30.,max_scene_points=300000,point_size=old['point_size'],mano_faces_path=V9_MANO_RIGHT,trajectory_trail_duration_seconds=2.,**old['display_view']) 145 oldwriter=renderer._write_video;oldoverlay=renderer._CameraMeshOverlayRenderer.render;oldlines=renderer._line_annotations 146 visibility=np.asarray(state.hand.arrays['valid'])&state.trajectory.valid[:,None] 147 # Same historical camera/scene view. Upgrade encoding to avoid intermediate 148 # MPEG4 loss; use native per-side MANO winding in the camera overlay. 149 index=[0] 150 def overlay(obj,im,vertices,colors,intrinsics,*,opacity): 151 f=index[0];index[0]+=1 152 return HandOverlay(im.shape[1],im.shape[0],faces,obj.renderer).render(im,vertices,colors,intrinsics,opacity=opacity,sides=np.flatnonzero(visibility[f])) 153 def writer(path,frames,fps): 154 enc=None;count=0 155 try: 156 for count,frame in enumerate(frames,1): 157 frame=frame.copy();height,width=frame.shape[:2] 158 cv2.rectangle(frame,(0,0),(width,31),(25,25,25),-1) 159 text=f'V9P3R2 | RGB estimated geometry - NOT calibrated GT | {sid} | f{count-1} {(count-1)/fps:.3f}s' 160 cv2.putText(frame,text,(10,22),cv2.FONT_HERSHEY_SIMPLEX,.53,(245,245,245),1,cv2.LINE_AA) 161 if count-1 in read(w/'input_binding.json')['damaged_frames']:cv2.putText(frame,'SOURCE DECODE DAMAGE - reconstruction invalid',(12,height-20),cv2.FONT_HERSHEY_SIMPLEX,.8,(0,180,255),2) 162 if enc is None:enc=subprocess.Popen(['ffmpeg','-hide_banner','-loglevel','error','-y','-f','rawvideo','-pix_fmt','bgr24','-s',f'{width}x{height}','-r',str(fps),'-i','pipe:0','-an','-c:v','libx264','-preset','fast','-crf','18','-threads','2','-pix_fmt','yuv420p','-movflags','+faststart',str(path)],stdin=subprocess.PIPE) 163 enc.stdin.write(frame.tobytes()) 164 if count%150==0:print('RENDER',sid,count,flush=True) 165 finally: 166 if enc is not None:enc.stdin.close() 167 assert enc is not None and enc.wait()==0 168 return count 169 renderer._write_video=writer 170 def lines(frame_index,*args,**kwargs): 171 if not state.trajectory.valid[frame_index]:return np.empty((0,2,3),np.float32),np.empty((0,3),np.uint8),np.empty((0,3),np.float32),np.empty((0,3),np.uint8) 172 return oldlines(frame_index,*args,**kwargs) 173 renderer._line_annotations=lines 174 try: 175 with _patched_v7_renderer(state,config,None,occlusion_radius_m=.018,occlusion_depth_margin_m=.04,overlay_annotations=True,hand_wireframe_rgb=(28,104,65),enable_hand_alignment=False) as patch: 176 renderer._CameraMeshOverlayRenderer.render=overlay 177 report=renderer.render_demo(state,target,config=config,demo_hand_visibility=visibility) 178 report['display_patch']={key:(value() if callable(value) else value) for key,value in patch.items()} 179 report['display_patch']['hand_alignment']['reason']='same canonical K and shared P3R2 geometry used by both panels; no display-only hand shift' 180 finally:renderer._write_video=oldwriter;renderer._CameraMeshOverlayRenderer.render=oldoverlay;renderer._line_annotations=oldlines 181 report.update(camera_hand_source='FoundationHand-S9L10 P3R2 same-frame fitted MANO',world_hand_source='same camera mesh rigidly transformed through estimated ViPE T_wc',estimated_not_ground_truth=True,display_hand_alignment_applied=False,native_left_right_topology=True) 182 write(w/'render_complete.json',report);print('RENDER_COMPLETE',sid,flush=True) 183 184 def main(): 185 p=argparse.ArgumentParser();p.add_argument('action',choices=['fit','render']);p.add_argument('--sequence',default=PILOT);a=p.parse_args();selected=rows() if a.sequence=='all' else [r for r in rows() if r['sequence_id'] in a.sequence.split(',')] 186 if a.action=='fit': 187 from v9_fusion import s9l10_worker as locked 188 locked._activate_official_runtime() 189 import torch,yaml 190 from foundation_hand.models.mano import ManoCameraLayer 191 from v9_fusion.s9l10 import S9L10_CONFIG,S9L10_PYTHON 192 assert Path(__import__('sys').executable).samefile(S9L10_PYTHON) 193 torch.set_num_threads(3);cfg=yaml.safe_load(S9L10_CONFIG.read_text());mano=ManoCameraLayer(cfg['assets']['mano_root'],fix_left_shapedirs=True).cuda().eval();mano.requires_grad_(False) 194 for row in selected:fit(row,mano) 195 else: 196 for row in selected:render(row) 197 198 if __name__=='__main__':main()