{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "0_wHveSGjXyDY", "split": "train", "caption": "An older man repeatedly strikes a metal assembly. He straightens as a red-clad woman watches. Two other men stand nearby.", "mentions": [{"text": "An older man", "span": [0, 12], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a metal assembly", "span": [32, 48], "occurrence": 1, "mark_ids": [2], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "He", "span": [50, 52], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "a red-clad woman", "span": [68, 84], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "Two other men", "span": [94, 107], "occurrence": 1, "mark_ids": [3, 6], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 49], [50, 93], [94, 121]], "groups": [{"group_id": 0, "mark_id": 4, "category": "person", "isthing": 1, "mentions": [{"text": "An older man", "span": [0, 12], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "He", "span": [50, 52], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 2, "category": "other machine", "isthing": 0, "mentions": [{"text": "a metal assembly", "span": [32, 48], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown"], "chain_agreement": "unknown"}, {"group_id": 2, "mark_id": 5, "category": "person", "isthing": 1, "mentions": [{"text": "a red-clad woman", "span": [68, 84], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 3, "category": "person", "isthing": 1, "mentions": [{"text": "Two other men", "span": [94, 107], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 6, "category": "person", "isthing": 1, "mentions": [{"text": "Two other men", "span": [94, 107], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "strikes", "span": [24, 31], "spans": [[24, 31]], "particle": null, "mentions": [{"text": "strikes", "span": [24, 31], "spans": [[24, 31]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "An older man", "span": [0, 12], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a metal assembly", "span": [32, 48], "groups": [1], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "strikes", "status": "visible", "segments_frames": [[7, 13]], "segments": [[3.4, 6.9]], "segment_details": [{"start_frame": 7, "end_frame": 13, "start_frame_1based": 8, "end_frame_1based": 14, "start_s": 3.4, "end_s": 6.9, "last_observed_s": 6.4, "start_source_frame": 217, "end_source_frame": 262, "start_filename": "00000217.jpg", "end_filename": "00000262.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The older man repeatedly raises and lowers his striking arm toward the assembly.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [166, 172, 181, 187, 196, 202, 211, 217, 226, 232, 241, 247, 256, 262, 271, 277, 286, 292, 301]}}, {"text": "straightens", "span": [53, 64], "spans": [[53, 64]], "particle": null, "mentions": [{"text": "straightens", "span": [53, 64], "spans": [[53, 64]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "He", "span": [50, 52], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 1, "text": "straightens", "status": "visible", "segments_frames": [[14, 17]], "segments": [[7.0, 8.9]], "segment_details": [{"start_frame": 14, "end_frame": 17, "start_frame_1based": 15, "end_frame_1based": 18, "start_s": 7.0, "end_s": 8.9, "last_observed_s": 8.4, "start_source_frame": 271, "end_source_frame": 292, "start_filename": "00000271.jpg", "end_filename": "00000292.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He raises his torso from the deeply bent working position.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [166, 172, 181, 187, 196, 202, 211, 217, 226, 232, 241, 247, 256, 262, 271, 277, 286, 292, 301]}}, {"text": "watches", "span": [85, 92], "spans": [[85, 92]], "particle": null, "mentions": [{"text": "watches", "span": [85, 92], "spans": [[85, 92]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "a red-clad woman", "span": [68, 84], "groups": [2], "group_mentions": [0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 2, "text": "watches", "status": "visible", "segments_frames": [[0, 18]], "segments": [[0.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 18, "start_frame_1based": 1, "end_frame_1based": 19, "start_s": 0.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 166, "end_source_frame": 301, "start_filename": "00000166.jpg", "end_filename": "00000301.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The woman remains behind the assembly, looking toward the man's work.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [166, 172, 181, 187, 196, 202, 211, 217, 226, 232, 241, 247, 256, 262, 271, 277, 286, 292, 301]}}, {"text": "stand", "span": [108, 113], "spans": [[108, 113]], "particle": null, "mentions": [{"text": "stand", "span": [108, 113], "spans": [[108, 113]], "particle": null, "sentence_index": 2}], "type": "posture", "sentence_index": 2, "subject": {"text": "Two other men", "span": [94, 107], "groups": [3, 4], "group_mentions": [0, 0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 3, "text": "stand", "status": "visible", "segments_frames": [[9, 13], [16, 18]], "segments": [[4.4, 6.9], [8.0, 9.2]], "segment_details": [{"start_frame": 9, "end_frame": 13, "start_frame_1based": 10, "end_frame_1based": 14, "start_s": 4.4, "end_s": 6.9, "last_observed_s": 6.4, "start_source_frame": 232, "end_source_frame": 262, "start_filename": "00000232.jpg", "end_filename": "00000262.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 16, "end_frame": 18, "start_frame_1based": 17, "end_frame_1based": 19, "start_s": 8.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 286, "end_source_frame": 301, "start_filename": "00000286.jpg", "end_filename": "00000301.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Both men hold standing positions beside the assembly during these intervals; the left man steps between them.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [166, 172, 181, 187, 196, 202, 211, 217, 226, 232, 241, 247, 256, 262, 271, 277, 286, 292, 301]}}], "coverage": {"marks_shown": 7, "marks_grounded": 5, "things_shown": 5, "things_grounded": 4, "stuff_shown": 2, "stuff_grounded": 1, "n_mentions": 6, "category_disagree": 0, "category_checked": 4}, "marks": [{"mark_id": 3, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 0, "max_rel_area": 0.2215, "mean_rel_area": 0.06232, "centroid": [0.923, 0.469], "motion": 0.162}, {"mark_id": 4, "category": "person", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 10, "max_rel_area": 0.12553, "mean_rel_area": 0.10222, "centroid": [0.736, 0.485], "motion": 0.053}, {"mark_id": 5, "category": "person", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 0, "max_rel_area": 0.05673, "mean_rel_area": 0.04343, "centroid": [0.647, 0.327], "motion": 0.05}, {"mark_id": 6, "category": "person", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 31, "first_frame": 15, "last_frame": 45, "best_frame": 31, "max_rel_area": 0.10878, "mean_rel_area": 0.08473, "centroid": [0.155, 0.401], "motion": 0.198}, {"mark_id": 7, "category": "person", "isthing": 1, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 21, "first_frame": 24, "last_frame": 44, "best_frame": 29, "max_rel_area": 0.00357, "mean_rel_area": 0.00221, "centroid": [0.255, 0.052], "motion": 0.222}, {"mark_id": 2, "category": "other machine", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 10, "max_rel_area": 0.49916, "mean_rel_area": 0.47164, "centroid": [0.508, 0.5], "motion": 0.052}, {"mark_id": 1, "category": "ground", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 12, "max_rel_area": 0.2841, "mean_rel_area": 0.24341, "centroid": [0.484, 0.504], "motion": 0.058}], "num_frames": 46, "duration_s": 9.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0, 4.2, 4.4, 4.6, 4.8, 5.0, 5.2, 5.4, 5.6, 5.8, 6.0, 6.2, 6.4, 6.6, 6.8, 7.0, 7.2, 7.4, 7.6, 7.8, 8.0, 8.2, 8.4, 8.6, 8.8, 9.0], "source_frame_indices": [166, 169, 172, 175, 178, 181, 184, 187, 190, 193, 196, 199, 202, 205, 208, 211, 214, 217, 220, 223, 226, 229, 232, 235, 238, 241, 244, 247, 250, 253, 256, 259, 262, 265, 268, 271, 274, 277, 280, 283, 286, 289, 292, 295, 298, 301], "timing_sha": "d84660d56c21", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 5, "unlinked": 0, "snapped": 0, "collective": 1, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 21911, "output_tokens": 1249, "thinking_tokens": 453, "total_tokens": 23160}, "seconds": 26.02, "response_id": "resp_0514c8468985cc23006ac32f6b7c4c87d09d7499b9daa3eedb", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 166, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 172, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 181, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 187, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 196, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 202, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 211, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 217, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 226, "time_s": 4.0}, {"role": "clean", "path": "clean/f022.jpg", "frame": 9, "source_frame": 232, "time_s": 4.4}, {"role": "clean", "path": "clean/f025.jpg", "frame": 10, "source_frame": 241, "time_s": 5.0}, {"role": "clean", "path": "clean/f027.jpg", "frame": 11, "source_frame": 247, "time_s": 5.4}, {"role": "clean", "path": "clean/f030.jpg", "frame": 12, "source_frame": 256, "time_s": 6.0}, {"role": "clean", "path": "clean/f032.jpg", "frame": 13, "source_frame": 262, "time_s": 6.4}, {"role": "clean", "path": "clean/f035.jpg", "frame": 14, "source_frame": 271, "time_s": 7.0}, {"role": "clean", "path": "clean/f037.jpg", "frame": 15, "source_frame": 277, "time_s": 7.4}, {"role": "clean", "path": "clean/f040.jpg", "frame": 16, "source_frame": 286, "time_s": 8.0}, {"role": "clean", "path": "clean/f042.jpg", "frame": 17, "source_frame": 292, "time_s": 8.4}, {"role": "clean", "path": "clean/f045.jpg", "frame": 18, "source_frame": 301, "time_s": 9.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 166, "time_s": 0.0, "mark_ids": [3, 4, 5, 2, 1]}, {"role": "marks", "path": "marks/f015_p0.jpg", "frame": 15, "panel": 0, "source_frame": 211, "time_s": 3.0, "mark_ids": [3, 4, 5, 6, 2, 1]}, {"role": "marks", "path": "marks/f030_p0.jpg", "frame": 30, "panel": 0, "source_frame": 256, "time_s": 6.0, "mark_ids": [3, 4, 5, 6, 7, 2, 1]}, {"role": "marks", "path": "marks/f045_p0.jpg", "frame": 45, "panel": 0, "source_frame": 301, "time_s": 9.0, "mark_ids": [3, 4, 5, 6, 2, 1]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [3, 4, 5, 6, 7, 2, 1]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "d84660d56c21", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [166, 172, 181, 187, 196, 202, 211, 217, 226, 232, 241, 247, 256, 262, 271, 277, 286, 292, 301], "actions": [{"action_index": 0, "text": "strikes", "status": "visible", "segments_frames": [[7, 13]], "segments": [[3.4, 6.9]], "segment_details": [{"start_frame": 7, "end_frame": 13, "start_frame_1based": 8, "end_frame_1based": 14, "start_s": 3.4, "end_s": 6.9, "last_observed_s": 6.4, "start_source_frame": 217, "end_source_frame": 262, "start_filename": "00000217.jpg", "end_filename": "00000262.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The older man repeatedly raises and lowers his striking arm toward the assembly.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [166, 172, 181, 187, 196, 202, 211, 217, 226, 232, 241, 247, 256, 262, 271, 277, 286, 292, 301]}, {"action_index": 1, "text": "straightens", "status": "visible", "segments_frames": [[14, 17]], "segments": [[7.0, 8.9]], "segment_details": [{"start_frame": 14, "end_frame": 17, "start_frame_1based": 15, "end_frame_1based": 18, "start_s": 7.0, "end_s": 8.9, "last_observed_s": 8.4, "start_source_frame": 271, "end_source_frame": 292, "start_filename": "00000271.jpg", "end_filename": "00000292.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He raises his torso from the deeply bent working position.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [166, 172, 181, 187, 196, 202, 211, 217, 226, 232, 241, 247, 256, 262, 271, 277, 286, 292, 301]}, {"action_index": 2, "text": "watches", "status": "visible", "segments_frames": [[0, 18]], "segments": [[0.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 18, "start_frame_1based": 1, "end_frame_1based": 19, "start_s": 0.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 166, "end_source_frame": 301, "start_filename": "00000166.jpg", "end_filename": "00000301.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The woman remains behind the assembly, looking toward the man's work.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [166, 172, 181, 187, 196, 202, 211, 217, 226, 232, 241, 247, 256, 262, 271, 277, 286, 292, 301]}, {"action_index": 3, "text": "stand", "status": "visible", "segments_frames": [[9, 13], [16, 18]], "segments": [[4.4, 6.9], [8.0, 9.2]], "segment_details": [{"start_frame": 9, "end_frame": 13, "start_frame_1based": 10, "end_frame_1based": 14, "start_s": 4.4, "end_s": 6.9, "last_observed_s": 6.4, "start_source_frame": 232, "end_source_frame": 262, "start_filename": "00000232.jpg", "end_filename": "00000262.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 16, "end_frame": 18, "start_frame_1based": 17, "end_frame_1based": 19, "start_s": 8.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 286, "end_source_frame": 301, "start_filename": "00000286.jpg", "end_filename": "00000301.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Both men hold standing positions beside the assembly during these intervals; the left man steps between them.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [166, 172, 181, 187, 196, 202, 211, 217, 226, 232, 241, 247, 256, 262, 271, 277, 286, 292, 301]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1093_TZUYCiN-7T0", "split": "train", "caption": "Two dogs run side by side across the snow. They kick up snow behind them.", "mentions": [{"text": "Two dogs", "span": [0, 8], "occurrence": 1, "mark_ids": [4, 5], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "the snow", "span": [33, 41], "occurrence": 1, "mark_ids": [2], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "They", "span": [43, 47], "occurrence": 1, "mark_ids": [4, 5], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "snow", "span": [56, 60], "occurrence": 2, "mark_ids": [2], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "them", "span": [68, 72], "occurrence": 1, "mark_ids": [4, 5], "confidence": "high", "sentence_index": 1, "is_pronoun": true}], "sentences": [[0, 42], [43, 73]], "groups": [{"group_id": 0, "mark_id": 4, "category": "dog", "isthing": 1, "mentions": [{"text": "Two dogs", "span": [0, 8], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "They", "span": [43, 47], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [68, 72], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 5, "category": "dog", "isthing": 1, "mentions": [{"text": "Two dogs", "span": [0, 8], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "They", "span": [43, 47], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [68, 72], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "pronoun"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 2, "category": "snowfield", "isthing": 0, "mentions": [{"text": "the snow", "span": [33, 41], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "snow", "span": [56, 60], "occurrence": 2, "confidence": "high", "collective": false}], "agreement": ["agree", "agree"], "chain_agreement": "agree"}], "actions": [{"text": "run", "span": [9, 12], "spans": [[9, 12]], "particle": null, "mentions": [{"text": "run", "span": [9, 12], "spans": [[9, 12]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "Two dogs", "span": [0, 8], "groups": [0, 1], "group_mentions": [0, 0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 0, "text": "run", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 828, "end_source_frame": 870, "start_filename": "00000828.jpg", "end_filename": "00000870.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "Both dogs show changing running strides as they advance together throughout the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [828, 834, 843, 849, 858, 864, 870]}}, {"text": "kick up", "span": [48, 55], "spans": [[48, 55]], "particle": null, "mentions": [{"text": "kick up", "span": [48, 55], "spans": [[48, 55]], "particle": null, "sentence_index": 1}], "type": "phrasal", "sentence_index": 1, "subject": {"text": "They", "span": [43, 47], "groups": [0, 1], "group_mentions": [1, 1], "group_match": "exact"}, "objects": [{"text": "snow", "span": [56, 60], "groups": [2], "group_mentions": [1], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 1, "text": "kick up", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 828, "end_source_frame": 870, "start_filename": "00000828.jpg", "end_filename": "00000870.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Disturbed snow sprays around and behind the dogs' feet as they run.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [828, 834, 843, 849, 858, 864, 870]}}], "coverage": {"marks_shown": 5, "marks_grounded": 3, "things_shown": 2, "things_grounded": 2, "stuff_shown": 3, "stuff_grounded": 1, "n_mentions": 8, "category_disagree": 0, "category_checked": 3}, "marks": [{"mark_id": 4, "category": "dog", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.00969, "mean_rel_area": 0.00794, "centroid": [0.631, 0.712], "motion": 0.12}, {"mark_id": 5, "category": "dog", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.00769, "mean_rel_area": 0.00601, "centroid": [0.593, 0.735], "motion": 0.106}, {"mark_id": 3, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 5, "max_rel_area": 0.61372, "mean_rel_area": 0.56964, "centroid": [0.5, 0.465], "motion": 0.045}, {"mark_id": 2, "category": "snowfield", "isthing": 0, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 7, "max_rel_area": 0.37415, "mean_rel_area": 0.33355, "centroid": [0.49, 0.743], "motion": 0.066}, {"mark_id": 1, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 1, "max_rel_area": 0.12232, "mean_rel_area": 0.08161, "centroid": [0.417, 0.305], "motion": 0.191}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [828, 831, 834, 837, 840, 843, 846, 849, 852, 855, 858, 861, 864, 867, 870], "timing_sha": "92c2fb36e570", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 3, "unlinked": 0, "snapped": 0, "collective": 2, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 16718, "output_tokens": 795, "thinking_tokens": 309, "total_tokens": 17513}, "seconds": 16.7, "response_id": "resp_06270fec4e9adba3006ac32f6b780487d0959711d433a7938b", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 828, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 834, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 843, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 849, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 858, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 864, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 870, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 828, "time_s": 0.0, "mark_ids": [4, 5, 3, 2, 1]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 843, "time_s": 1.0, "mark_ids": [4, 5, 3, 2, 1]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 858, "time_s": 2.0, "mark_ids": [4, 5, 3, 2, 1]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 870, "time_s": 2.8, "mark_ids": [4, 5, 3, 2, 1]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [4, 5, 3, 2, 1]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "92c2fb36e570", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [828, 834, 843, 849, 858, 864, 870], "actions": [{"action_index": 0, "text": "run", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 828, "end_source_frame": 870, "start_filename": "00000828.jpg", "end_filename": "00000870.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "Both dogs show changing running strides as they advance together throughout the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [828, 834, 843, 849, 858, 864, 870]}, {"action_index": 1, "text": "kick up", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 828, "end_source_frame": 870, "start_filename": "00000828.jpg", "end_filename": "00000870.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Disturbed snow sprays around and behind the dogs' feet as they run.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [828, 834, 843, 849, 858, 864, 870]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1182_tNbLX2wX4Bs", "split": "train", "caption": "Many children stand in a circle indoors beside two women. A boy steps in place among them. One woman gestures toward the children.", "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "mark_ids": [21, 17, 23, 18, 14, 11, 9, 8, 24, 7, 27, 13, 16, 20, 22, 25, 15, 38, 12, 26, 39], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "two women", "span": [47, 56], "occurrence": 1, "mark_ids": [10, 19], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "A boy", "span": [58, 63], "occurrence": 1, "mark_ids": [17], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "them", "span": [85, 89], "occurrence": 1, "mark_ids": [21, 17, 23, 18, 14, 11, 9, 8, 24, 7, 27, 13, 16, 20, 22, 25, 15, 38, 12, 26, 39], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "One woman", "span": [91, 100], "occurrence": 1, "mark_ids": [10], "confidence": "high", "sentence_index": 2, "is_pronoun": false}, {"text": "the children", "span": [117, 129], "occurrence": 1, "mark_ids": [21, 17, 23, 18, 14, 11, 9, 8, 24, 7, 27, 13, 16, 20, 22, 25, 15, 38, 12, 26, 39], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 57], [58, 90], [91, 130]], "groups": [{"group_id": 0, "mark_id": 21, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 17, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "A boy", "span": [58, 63], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 23, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 18, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 14, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 5, "mark_id": 11, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 6, "mark_id": 9, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 7, "mark_id": 8, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 8, "mark_id": 24, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 9, "mark_id": 7, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 10, "mark_id": 27, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 11, "mark_id": 13, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 12, "mark_id": 16, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 13, "mark_id": 20, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 14, "mark_id": 22, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 15, "mark_id": 25, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 16, "mark_id": 15, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 17, "mark_id": 38, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 18, "mark_id": 12, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 19, "mark_id": 26, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 20, "mark_id": 39, "category": "person", "isthing": 1, "mentions": [{"text": "Many children", "span": [0, 13], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [85, 89], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "the children", "span": [117, 129], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 21, "mark_id": 10, "category": "person", "isthing": 1, "mentions": [{"text": "two women", "span": [47, 56], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "One woman", "span": [91, 100], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "agree"], "chain_agreement": "agree"}, {"group_id": 22, "mark_id": 19, "category": "person", "isthing": 1, "mentions": [{"text": "two women", "span": [47, 56], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "stand", "span": [14, 19], "spans": [[14, 19]], "particle": null, "mentions": [{"text": "stand", "span": [14, 19], "spans": [[14, 19]], "particle": null, "sentence_index": 0}], "type": "posture", "sentence_index": 0, "subject": {"text": "Many children", "span": [0, 13], "groups": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], "group_mentions": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "group_match": "exact"}, "objects": [{"text": "two women", "span": [47, 56], "groups": [21, 22], "group_mentions": [0, 0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "stand", "status": "visible", "segments_frames": [[0, 18]], "segments": [[0.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 18, "start_frame_1based": 1, "end_frame_1based": 19, "start_s": 0.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 752, "end_source_frame": 887, "start_filename": "00000752.jpg", "end_filename": "00000887.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The children remain upright in a circle throughout, although several are partly obscured.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [752, 758, 767, 773, 782, 788, 797, 803, 812, 818, 827, 833, 842, 848, 857, 863, 872, 878, 887]}}, {"text": "steps", "span": [64, 69], "spans": [[64, 69]], "particle": null, "mentions": [{"text": "steps", "span": [64, 69], "spans": [[64, 69]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "A boy", "span": [58, 63], "groups": [1], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "them", "span": [85, 89], "groups": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], "group_mentions": [1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 1, "text": "steps", "status": "visible", "segments_frames": [[0, 9]], "segments": [[0.0, 4.9]], "segment_details": [{"start_frame": 0, "end_frame": 9, "start_frame_1based": 1, "end_frame_1based": 10, "start_s": 0.0, "end_s": 4.9, "last_observed_s": 4.4, "start_source_frame": 752, "end_source_frame": 818, "start_filename": "00000752.jpg", "end_filename": "00000818.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The central boy repeatedly lifts and lowers his feet in place before settling into a stationary stance.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [752, 758, 767, 773, 782, 788, 797, 803, 812, 818, 827, 833, 842, 848, 857, 863, 872, 878, 887]}}, {"text": "gestures", "span": [101, 109], "spans": [[101, 109]], "particle": null, "mentions": [{"text": "gestures", "span": [101, 109], "spans": [[101, 109]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "One woman", "span": [91, 100], "groups": [21], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "the children", "span": [117, 129], "groups": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], "group_mentions": [2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 2, "text": "gestures", "status": "visible", "segments_frames": [[0, 1], [15, 17]], "segments": [[0.0, 0.9], [7.4, 8.9]], "segment_details": [{"start_frame": 0, "end_frame": 1, "start_frame_1based": 1, "end_frame_1based": 2, "start_s": 0.0, "end_s": 0.9, "last_observed_s": 0.4, "start_source_frame": 752, "end_source_frame": 758, "start_filename": "00000752.jpg", "end_filename": "00000758.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 15, "end_frame": 17, "start_frame_1based": 16, "end_frame_1based": 18, "start_s": 7.4, "end_s": 8.9, "last_observed_s": 8.4, "start_source_frame": 863, "end_source_frame": 878, "start_filename": "00000863.jpg", "end_filename": "00000878.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The woman on the left extends a hand toward the circle initially and raises it again near the end.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [752, 758, 767, 773, 782, 788, 797, 803, 812, 818, 827, 833, 842, 848, 857, 863, 872, 878, 887]}}], "coverage": {"marks_shown": 39, "marks_grounded": 23, "things_shown": 36, "things_grounded": 23, "stuff_shown": 3, "stuff_grounded": 0, "n_mentions": 67, "category_disagree": 0, "category_checked": 23}, "marks": [{"mark_id": 6, "category": "window", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 0, "max_rel_area": 0.05568, "mean_rel_area": 0.04996, "centroid": [0.126, 0.245], "motion": 0.054}, {"mark_id": 36, "category": "table or desk", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 12, "max_rel_area": 0.04439, "mean_rel_area": 0.0414, "centroid": [0.418, 0.424], "motion": 0.037}, {"mark_id": 21, "category": "person", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 8, "max_rel_area": 0.04287, "mean_rel_area": 0.04062, "centroid": [0.801, 0.562], "motion": 0.013}, {"mark_id": 4, "category": "door", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 15, "max_rel_area": 0.03997, "mean_rel_area": 0.03839, "centroid": [0.811, 0.311], "motion": 0.038}, {"mark_id": 17, "category": "person", "isthing": 1, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 20, "max_rel_area": 0.03991, "mean_rel_area": 0.03472, "centroid": [0.602, 0.549], "motion": 0.026}, {"mark_id": 23, "category": "person", "isthing": 1, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 0, "max_rel_area": 0.03507, "mean_rel_area": 0.03289, "centroid": [0.908, 0.549], "motion": 0.01}, {"mark_id": 18, "category": "person", "isthing": 1, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 8, "max_rel_area": 0.03389, "mean_rel_area": 0.03192, "centroid": [0.701, 0.571], "motion": 0.025}, {"mark_id": 14, "category": "person", "isthing": 1, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 40, "max_rel_area": 0.03385, "mean_rel_area": 0.03248, "centroid": [0.443, 0.57], "motion": 0.049}, {"mark_id": 11, "category": "person", "isthing": 1, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 30, "max_rel_area": 0.03303, "mean_rel_area": 0.03151, "centroid": [0.238, 0.554], "motion": 0.015}, {"mark_id": 9, "category": "person", "isthing": 1, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 4, "max_rel_area": 0.02692, "mean_rel_area": 0.025, "centroid": [0.162, 0.542], "motion": 0.018}, {"mark_id": 8, "category": "person", "isthing": 1, "iscrowd": 0, "order": 10, "color": [139, 195, 74], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 35, "max_rel_area": 0.02085, "mean_rel_area": 0.01979, "centroid": [0.079, 0.51], "motion": 0.017}, {"mark_id": 24, "category": "person", "isthing": 1, "iscrowd": 0, "order": 11, "color": [255, 87, 34], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 35, "max_rel_area": 0.02006, "mean_rel_area": 0.01804, "centroid": [0.97, 0.527], "motion": 0.013}, {"mark_id": 10, "category": "person", "isthing": 1, "iscrowd": 0, "order": 12, "color": [156, 39, 176], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 16, "max_rel_area": 0.01957, "mean_rel_area": 0.01388, "centroid": [0.141, 0.424], "motion": 0.045}, {"mark_id": 7, "category": "person", "isthing": 1, "iscrowd": 0, "order": 13, "color": [0, 188, 212], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 40, "max_rel_area": 0.0149, "mean_rel_area": 0.01272, "centroid": [0.038, 0.493], "motion": 0.019}, {"mark_id": 27, "category": "person", "isthing": 1, "iscrowd": 0, "order": 14, "color": [121, 85, 72], "panel": 1, "n_present": 45, "first_frame": 0, "last_frame": 45, "best_frame": 33, "max_rel_area": 0.01284, "mean_rel_area": 0.00419, "centroid": [0.167, 0.512], "motion": 0.237}, {"mark_id": 5, "category": "door", "isthing": 1, "iscrowd": 0, "order": 15, "color": [233, 30, 99], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 9, "max_rel_area": 0.01177, "mean_rel_area": 0.01043, "centroid": [0.61, 0.331], "motion": 0.033}, {"mark_id": 13, "category": "person", "isthing": 1, "iscrowd": 0, "order": 16, "color": [63, 81, 181], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 0, "max_rel_area": 0.01108, "mean_rel_area": 0.00939, "centroid": [0.387, 0.455], "motion": 0.026}, {"mark_id": 16, "category": "person", "isthing": 1, "iscrowd": 0, "order": 17, "color": [205, 220, 57], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.01094, "mean_rel_area": 0.00997, "centroid": [0.539, 0.454], "motion": 0.008}, {"mark_id": 20, "category": "person", "isthing": 1, "iscrowd": 0, "order": 18, "color": [255, 152, 0], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 25, "max_rel_area": 0.01043, "mean_rel_area": 0.00659, "centroid": [0.747, 0.457], "motion": 0.039}, {"mark_id": 22, "category": "person", "isthing": 1, "iscrowd": 0, "order": 19, "color": [96, 125, 139], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 14, "max_rel_area": 0.00854, "mean_rel_area": 0.00653, "centroid": [0.864, 0.462], "motion": 0.02}, {"mark_id": 19, "category": "person", "isthing": 1, "iscrowd": 0, "order": 20, "color": [54, 67, 244], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.00843, "mean_rel_area": 0.0059, "centroid": [0.696, 0.292], "motion": 0.041}, {"mark_id": 25, "category": "person", "isthing": 1, "iscrowd": 0, "order": 21, "color": [34, 189, 255], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 26, "max_rel_area": 0.00835, "mean_rel_area": 0.00644, "centroid": [0.946, 0.517], "motion": 0.015}, {"mark_id": 15, "category": "person", "isthing": 1, "iscrowd": 0, "order": 22, "color": [60, 200, 60], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 1, "max_rel_area": 0.00616, "mean_rel_area": 0.00295, "centroid": [0.478, 0.393], "motion": 0.042}, {"mark_id": 37, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 23, "color": [240, 120, 40], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 5, "max_rel_area": 0.0052, "mean_rel_area": 0.00465, "centroid": [0.938, 0.226], "motion": 0.013}, {"mark_id": 38, "category": "person", "isthing": 1, "iscrowd": 0, "order": 24, "color": [180, 60, 220], "panel": 2, "n_present": 44, "first_frame": 1, "last_frame": 44, "best_frame": 18, "max_rel_area": 0.00363, "mean_rel_area": 0.0015, "centroid": [0.678, 0.363], "motion": 0.044}, {"mark_id": 12, "category": "person", "isthing": 1, "iscrowd": 0, "order": 25, "color": [0, 215, 255], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 15, "max_rel_area": 0.00307, "mean_rel_area": 0.00228, "centroid": [0.261, 0.416], "motion": 0.041}, {"mark_id": 26, "category": "person", "isthing": 1, "iscrowd": 0, "order": 26, "color": [200, 90, 130], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 30, "max_rel_area": 0.00171, "mean_rel_area": 0.00131, "centroid": [0.954, 0.288], "motion": 0.011}, {"mark_id": 35, "category": "box", "isthing": 1, "iscrowd": 0, "order": 27, "color": [80, 175, 76], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 35, "max_rel_area": 0.00142, "mean_rel_area": 0.0011, "centroid": [0.396, 0.271], "motion": 0.018}, {"mark_id": 34, "category": "box", "isthing": 1, "iscrowd": 0, "order": 28, "color": [255, 100, 180], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 38, "max_rel_area": 0.0014, "mean_rel_area": 0.00128, "centroid": [0.366, 0.272], "motion": 0.015}, {"mark_id": 29, "category": "box", "isthing": 1, "iscrowd": 0, "order": 29, "color": [0, 140, 255], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 38, "max_rel_area": 0.00134, "mean_rel_area": 0.00089, "centroid": [0.525, 0.275], "motion": 0.013}, {"mark_id": 30, "category": "box", "isthing": 1, "iscrowd": 0, "order": 30, "color": [139, 195, 74], "panel": 3, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 23, "max_rel_area": 0.00126, "mean_rel_area": 0.00114, "centroid": [0.499, 0.275], "motion": 0.014}, {"mark_id": 33, "category": "box", "isthing": 1, "iscrowd": 0, "order": 31, "color": [255, 87, 34], "panel": 3, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 35, "max_rel_area": 0.00118, "mean_rel_area": 0.00094, "centroid": [0.427, 0.271], "motion": 0.015}, {"mark_id": 32, "category": "box", "isthing": 1, "iscrowd": 0, "order": 32, "color": [156, 39, 176], "panel": 3, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 44, "max_rel_area": 0.00116, "mean_rel_area": 0.00098, "centroid": [0.453, 0.273], "motion": 0.015}, {"mark_id": 28, "category": "box", "isthing": 1, "iscrowd": 0, "order": 33, "color": [0, 188, 212], "panel": 3, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 9, "max_rel_area": 0.00108, "mean_rel_area": 0.00071, "centroid": [0.544, 0.274], "motion": 0.011}, {"mark_id": 31, "category": "box", "isthing": 1, "iscrowd": 0, "order": 34, "color": [121, 85, 72], "panel": 3, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 39, "max_rel_area": 0.00088, "mean_rel_area": 0.00069, "centroid": [0.475, 0.272], "motion": 0.015}, {"mark_id": 39, "category": "person", "isthing": 1, "iscrowd": 0, "order": 35, "color": [233, 30, 99], "panel": 3, "n_present": 20, "first_frame": 18, "last_frame": 44, "best_frame": 39, "max_rel_area": 0.00173, "mean_rel_area": 0.00107, "centroid": [0.137, 0.356], "motion": 0.043}, {"mark_id": 2, "category": "floor", "isthing": 0, "iscrowd": 0, "order": 36, "color": [63, 81, 181], "panel": 3, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 1, "max_rel_area": 0.3133, "mean_rel_area": 0.30594, "centroid": [0.5, 0.753], "motion": 0.025}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 37, "color": [205, 220, 57], "panel": 3, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.17015, "mean_rel_area": 0.16119, "centroid": [0.5, 0.301], "motion": 0.005}, {"mark_id": 3, "category": "curtain", "isthing": 0, "iscrowd": 0, "order": 38, "color": [255, 152, 0], "panel": 3, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 35, "max_rel_area": 0.02774, "mean_rel_area": 0.02169, "centroid": [0.07, 0.263], "motion": 0.033}], "num_frames": 46, "duration_s": 9.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0, 4.2, 4.4, 4.6, 4.8, 5.0, 5.2, 5.4, 5.6, 5.8, 6.0, 6.2, 6.4, 6.6, 6.8, 7.0, 7.2, 7.4, 7.6, 7.8, 8.0, 8.2, 8.4, 8.6, 8.8, 9.0], "source_frame_indices": [752, 755, 758, 761, 764, 767, 770, 773, 776, 779, 782, 785, 788, 791, 794, 797, 800, 803, 806, 809, 812, 815, 818, 821, 824, 827, 830, 833, 836, 839, 842, 845, 848, 851, 854, 857, 860, 863, 866, 869, 872, 875, 878, 881, 884, 887], "timing_sha": "840b7d076e85", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 6, "unlinked": 0, "snapped": 0, "collective": 4, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 30910, "output_tokens": 1599, "thinking_tokens": 648, "total_tokens": 32509}, "seconds": 32.55, "response_id": "resp_0033717b0568e49e006ac32f7bd9e887d0a8463c2ea00735f8", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 752, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 758, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 767, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 773, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 782, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 788, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 797, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 803, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 812, "time_s": 4.0}, {"role": "clean", "path": "clean/f022.jpg", "frame": 9, "source_frame": 818, "time_s": 4.4}, {"role": "clean", "path": "clean/f025.jpg", "frame": 10, "source_frame": 827, "time_s": 5.0}, {"role": "clean", "path": "clean/f027.jpg", "frame": 11, "source_frame": 833, "time_s": 5.4}, {"role": "clean", "path": "clean/f030.jpg", "frame": 12, "source_frame": 842, "time_s": 6.0}, {"role": "clean", "path": "clean/f032.jpg", "frame": 13, "source_frame": 848, "time_s": 6.4}, {"role": "clean", "path": "clean/f035.jpg", "frame": 14, "source_frame": 857, "time_s": 7.0}, {"role": "clean", "path": "clean/f037.jpg", "frame": 15, "source_frame": 863, "time_s": 7.4}, {"role": "clean", "path": "clean/f040.jpg", "frame": 16, "source_frame": 872, "time_s": 8.0}, {"role": "clean", "path": "clean/f042.jpg", "frame": 17, "source_frame": 878, "time_s": 8.4}, {"role": "clean", "path": "clean/f045.jpg", "frame": 18, "source_frame": 887, "time_s": 9.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 752, "time_s": 0.0, "mark_ids": [6, 36, 21, 4, 17, 23, 18, 14, 11, 9]}, {"role": "marks", "path": "marks/f000_p1.jpg", "frame": 0, "panel": 1, "source_frame": 752, "time_s": 0.0, "mark_ids": [8, 24, 10, 7, 27, 5, 13, 16, 20, 22]}, {"role": "marks", "path": "marks/f000_p2.jpg", "frame": 0, "panel": 2, "source_frame": 752, "time_s": 0.0, "mark_ids": [19, 25, 15, 37, 12, 26, 35, 34, 29]}, {"role": "marks", "path": "marks/f000_p3.jpg", "frame": 0, "panel": 3, "source_frame": 752, "time_s": 0.0, "mark_ids": [30, 33, 32, 28, 31, 2, 1, 3]}, {"role": "marks", "path": "marks/f015_p0.jpg", "frame": 15, "panel": 0, "source_frame": 797, "time_s": 3.0, "mark_ids": [6, 36, 21, 4, 17, 23, 18, 14, 11, 9]}, {"role": "marks", "path": "marks/f015_p1.jpg", "frame": 15, "panel": 1, "source_frame": 797, "time_s": 3.0, "mark_ids": [8, 24, 10, 7, 27, 5, 13, 16, 20, 22]}, {"role": "marks", "path": "marks/f015_p2.jpg", "frame": 15, "panel": 2, "source_frame": 797, "time_s": 3.0, "mark_ids": [19, 25, 15, 37, 38, 12, 26, 35, 34, 29]}, {"role": "marks", "path": "marks/f015_p3.jpg", "frame": 15, "panel": 3, "source_frame": 797, "time_s": 3.0, "mark_ids": [30, 33, 32, 28, 31, 2, 1, 3]}, {"role": "marks", "path": "marks/f030_p0.jpg", "frame": 30, "panel": 0, "source_frame": 842, "time_s": 6.0, "mark_ids": [6, 36, 21, 4, 17, 23, 18, 14, 11, 9]}, {"role": "marks", "path": "marks/f030_p1.jpg", "frame": 30, "panel": 1, "source_frame": 842, "time_s": 6.0, "mark_ids": [8, 24, 10, 7, 27, 5, 13, 16, 20, 22]}, {"role": "marks", "path": "marks/f030_p2.jpg", "frame": 30, "panel": 2, "source_frame": 842, "time_s": 6.0, "mark_ids": [19, 25, 15, 37, 38, 12, 26, 35, 34, 29]}, {"role": "marks", "path": "marks/f030_p3.jpg", "frame": 30, "panel": 3, "source_frame": 842, "time_s": 6.0, "mark_ids": [30, 33, 32, 28, 31, 2, 1, 3]}, {"role": "marks", "path": "marks/f045_p0.jpg", "frame": 45, "panel": 0, "source_frame": 887, "time_s": 9.0, "mark_ids": [6, 36, 21, 4, 17, 23, 18, 14, 11, 9]}, {"role": "marks", "path": "marks/f045_p1.jpg", "frame": 45, "panel": 1, "source_frame": 887, "time_s": 9.0, "mark_ids": [8, 24, 10, 7, 27, 5, 13, 16, 20, 22]}, {"role": "marks", "path": "marks/f045_p2.jpg", "frame": 45, "panel": 2, "source_frame": 887, "time_s": 9.0, "mark_ids": [19, 25, 15, 37, 12, 26, 35, 34, 29]}, {"role": "marks", "path": "marks/f045_p3.jpg", "frame": 45, "panel": 3, "source_frame": 887, "time_s": 9.0, "mark_ids": [30, 33, 32, 28, 31, 2, 1, 3]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [6, 36, 21, 4, 17, 23, 18, 14, 11, 9, 8, 24, 10, 7, 27, 5]}, {"role": "ids", "path": "ids/sheet1.jpg", "mark_ids": [13, 16, 20, 22, 19, 25, 15, 37, 38, 12, 26, 35, 34, 29, 30, 33]}, {"role": "ids", "path": "ids/sheet2.jpg", "mark_ids": [32, 28, 31, 39, 2, 1, 3]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "840b7d076e85", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [752, 758, 767, 773, 782, 788, 797, 803, 812, 818, 827, 833, 842, 848, 857, 863, 872, 878, 887], "actions": [{"action_index": 0, "text": "stand", "status": "visible", "segments_frames": [[0, 18]], "segments": [[0.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 18, "start_frame_1based": 1, "end_frame_1based": 19, "start_s": 0.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 752, "end_source_frame": 887, "start_filename": "00000752.jpg", "end_filename": "00000887.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The children remain upright in a circle throughout, although several are partly obscured.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [752, 758, 767, 773, 782, 788, 797, 803, 812, 818, 827, 833, 842, 848, 857, 863, 872, 878, 887]}, {"action_index": 1, "text": "steps", "status": "visible", "segments_frames": [[0, 9]], "segments": [[0.0, 4.9]], "segment_details": [{"start_frame": 0, "end_frame": 9, "start_frame_1based": 1, "end_frame_1based": 10, "start_s": 0.0, "end_s": 4.9, "last_observed_s": 4.4, "start_source_frame": 752, "end_source_frame": 818, "start_filename": "00000752.jpg", "end_filename": "00000818.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The central boy repeatedly lifts and lowers his feet in place before settling into a stationary stance.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [752, 758, 767, 773, 782, 788, 797, 803, 812, 818, 827, 833, 842, 848, 857, 863, 872, 878, 887]}, {"action_index": 2, "text": "gestures", "status": "visible", "segments_frames": [[0, 1], [15, 17]], "segments": [[0.0, 0.9], [7.4, 8.9]], "segment_details": [{"start_frame": 0, "end_frame": 1, "start_frame_1based": 1, "end_frame_1based": 2, "start_s": 0.0, "end_s": 0.9, "last_observed_s": 0.4, "start_source_frame": 752, "end_source_frame": 758, "start_filename": "00000752.jpg", "end_filename": "00000758.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 15, "end_frame": 17, "start_frame_1based": 16, "end_frame_1based": 18, "start_s": 7.4, "end_s": 8.9, "last_observed_s": 8.4, "start_source_frame": 863, "end_source_frame": 878, "start_filename": "00000863.jpg", "end_filename": "00000878.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The woman on the left extends a hand toward the circle initially and raises it again near the end.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [752, 758, 767, 773, 782, 788, 797, 803, 812, 818, 827, 833, 842, 848, 857, 863, 872, 878, 887]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1270_9oDHw4wwTQI", "split": "train", "caption": "A boy and a girl dance on the turquoise floor. They turn together. Many spectators watch nearby, and some hold video cameras.", "mentions": [{"text": "A boy", "span": [0, 5], "occurrence": 1, "mark_ids": [14], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a girl", "span": [10, 16], "occurrence": 1, "mark_ids": [15], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "the turquoise floor", "span": [26, 45], "occurrence": 1, "mark_ids": [2], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "They", "span": [47, 51], "occurrence": 1, "mark_ids": [14, 15], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "Many spectators", "span": [67, 82], "occurrence": 1, "mark_ids": [13, 12, 25, 26, 29, 19, 30, 18, 32, 33, 20, 37, 31, 38, 21, 22], "confidence": "medium", "sentence_index": 2, "is_pronoun": false}, {"text": "some", "span": [101, 105], "occurrence": 1, "mark_ids": [13, 12, 19, 29], "confidence": "high", "sentence_index": 2, "is_pronoun": true}, {"text": "video cameras", "span": [111, 124], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 46], [47, 66], [67, 125]], "groups": [{"group_id": 0, "mark_id": 14, "category": "person", "isthing": 1, "mentions": [{"text": "A boy", "span": [0, 5], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "They", "span": [47, 51], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 15, "category": "person", "isthing": 1, "mentions": [{"text": "a girl", "span": [10, 16], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "They", "span": [47, 51], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 2, "category": "floor", "isthing": 0, "mentions": [{"text": "the turquoise floor", "span": [26, 45], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 13, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}, {"text": "some", "span": [101, 105], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 12, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}, {"text": "some", "span": [101, 105], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 5, "mark_id": 25, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 6, "mark_id": 26, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 7, "mark_id": 29, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}, {"text": "some", "span": [101, 105], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 8, "mark_id": 19, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}, {"text": "some", "span": [101, 105], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 9, "mark_id": 30, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 10, "mark_id": 18, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 11, "mark_id": 32, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 12, "mark_id": 33, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 13, "mark_id": 20, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 14, "mark_id": 37, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 15, "mark_id": 31, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 16, "mark_id": 38, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 17, "mark_id": 21, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 18, "mark_id": 22, "category": "person", "isthing": 1, "mentions": [{"text": "Many spectators", "span": [67, 82], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 19, "mark_id": 3, "category": "other electronic product", "isthing": 0, "mentions": [{"text": "video cameras", "span": [111, 124], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "dance", "span": [17, 22], "spans": [[17, 22]], "particle": null, "mentions": [{"text": "dance", "span": [17, 22], "spans": [[17, 22]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "A boy", "span": [0, 5], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a girl", "span": [10, 16], "groups": [1], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "dance", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 715, "end_source_frame": 757, "start_filename": "00000715.jpg", "end_filename": "00000757.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The boy and girl move together in a partnered dance hold throughout the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [715, 721, 730, 736, 745, 751, 757]}}, {"text": "turn", "span": [52, 56], "spans": [[52, 56]], "particle": null, "mentions": [{"text": "turn", "span": [52, 56], "spans": [[52, 56]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "They", "span": [47, 51], "groups": [0, 1], "group_mentions": [1, 1], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 1, "text": "turn", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 715, "end_source_frame": 757, "start_filename": "00000715.jpg", "end_filename": "00000757.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The pair progressively changes orientation while maintaining their dance hold.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [715, 721, 730, 736, 745, 751, 757]}}, {"text": "watch", "span": [83, 88], "spans": [[83, 88]], "particle": null, "mentions": [{"text": "watch", "span": [83, 88], "spans": [[83, 88]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "Many spectators", "span": [67, 82], "groups": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], "group_mentions": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "group_match": "exact"}, "objects": [], "confidence": "medium", "temporal": {"action_index": 2, "text": "watch", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 715, "end_source_frame": 757, "start_filename": "00000715.jpg", "end_filename": "00000757.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Spectators face the dancing area throughout, although different members are visible at different times.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [715, 721, 730, 736, 745, 751, 757]}}, {"text": "hold", "span": [106, 110], "spans": [[106, 110]], "particle": null, "mentions": [{"text": "hold", "span": [106, 110], "spans": [[106, 110]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "some", "span": [101, 105], "groups": [3, 4, 7, 8], "group_mentions": [1, 1, 1, 1], "group_match": "exact"}, "objects": [{"text": "video cameras", "span": [111, 124], "groups": [19], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 3, "text": "hold", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 715, "end_source_frame": 757, "start_filename": "00000715.jpg", "end_filename": "00000757.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "Members of the referenced subset visibly hold cameras toward the dancers across the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [715, 721, 730, 736, 745, 751, 757]}}], "coverage": {"marks_shown": 40, "marks_grounded": 20, "things_shown": 37, "things_grounded": 18, "stuff_shown": 3, "stuff_grounded": 2, "n_mentions": 26, "category_disagree": 0, "category_checked": 20}, "marks": [{"mark_id": 14, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 3, "max_rel_area": 0.22898, "mean_rel_area": 0.2014, "centroid": [0.483, 0.553], "motion": 0.126}, {"mark_id": 11, "category": "person", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.16641, "mean_rel_area": 0.09552, "centroid": [0.149, 0.504], "motion": 0.164}, {"mark_id": 15, "category": "person", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.15466, "mean_rel_area": 0.06767, "centroid": [0.529, 0.405], "motion": 0.161}, {"mark_id": 13, "category": "person", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 14, "first_frame": 0, "last_frame": 13, "best_frame": 11, "max_rel_area": 0.08054, "mean_rel_area": 0.04121, "centroid": [0.703, 0.383], "motion": 0.59}, {"mark_id": 12, "category": "person", "isthing": 1, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 13, "max_rel_area": 0.06148, "mean_rel_area": 0.03575, "centroid": [0.6, 0.336], "motion": 0.594}, {"mark_id": 25, "category": "person", "isthing": 1, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 14, "first_frame": 1, "last_frame": 14, "best_frame": 2, "max_rel_area": 0.05943, "mean_rel_area": 0.0297, "centroid": [0.451, 0.317], "motion": 0.545}, {"mark_id": 26, "category": "person", "isthing": 1, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 13, "first_frame": 2, "last_frame": 14, "best_frame": 4, "max_rel_area": 0.04652, "mean_rel_area": 0.02158, "centroid": [0.396, 0.27], "motion": 0.527}, {"mark_id": 29, "category": "person", "isthing": 1, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 12, "first_frame": 3, "last_frame": 14, "best_frame": 5, "max_rel_area": 0.04257, "mean_rel_area": 0.02046, "centroid": [0.337, 0.284], "motion": 0.512}, {"mark_id": 19, "category": "person", "isthing": 1, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 9, "first_frame": 0, "last_frame": 8, "best_frame": 6, "max_rel_area": 0.07317, "mean_rel_area": 0.04302, "centroid": [0.746, 0.328], "motion": 0.412}, {"mark_id": 30, "category": "person", "isthing": 1, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 11, "first_frame": 4, "last_frame": 14, "best_frame": 7, "max_rel_area": 0.04185, "mean_rel_area": 0.01487, "centroid": [0.294, 0.228], "motion": 0.491}, {"mark_id": 17, "category": "person", "isthing": 1, "iscrowd": 0, "order": 10, "color": [139, 195, 74], "panel": 1, "n_present": 10, "first_frame": 0, "last_frame": 11, "best_frame": 4, "max_rel_area": 0.04677, "mean_rel_area": 0.01885, "centroid": [0.966, 0.589], "motion": 0.213}, {"mark_id": 16, "category": "person", "isthing": 1, "iscrowd": 0, "order": 11, "color": [255, 87, 34], "panel": 1, "n_present": 13, "first_frame": 0, "last_frame": 12, "best_frame": 0, "max_rel_area": 0.02678, "mean_rel_area": 0.01509, "centroid": [0.064, 0.559], "motion": 0.148}, {"mark_id": 18, "category": "person", "isthing": 1, "iscrowd": 0, "order": 12, "color": [156, 39, 176], "panel": 1, "n_present": 10, "first_frame": 0, "last_frame": 14, "best_frame": 3, "max_rel_area": 0.0327, "mean_rel_area": 0.01568, "centroid": [0.581, 0.396], "motion": 0.537}, {"mark_id": 4, "category": "window", "isthing": 1, "iscrowd": 0, "order": 13, "color": [0, 188, 212], "panel": 1, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.00995, "mean_rel_area": 0.00517, "centroid": [0.505, 0.151], "motion": 0.622}, {"mark_id": 24, "category": "window", "isthing": 1, "iscrowd": 0, "order": 14, "color": [121, 85, 72], "panel": 1, "n_present": 14, "first_frame": 1, "last_frame": 14, "best_frame": 8, "max_rel_area": 0.01043, "mean_rel_area": 0.00617, "centroid": [0.526, 0.058], "motion": 0.653}, {"mark_id": 5, "category": "window", "isthing": 1, "iscrowd": 0, "order": 15, "color": [233, 30, 99], "panel": 1, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 8, "max_rel_area": 0.0071, "mean_rel_area": 0.00504, "centroid": [0.528, 0.057], "motion": 0.686}, {"mark_id": 32, "category": "person", "isthing": 1, "iscrowd": 0, "order": 16, "color": [63, 81, 181], "panel": 1, "n_present": 7, "first_frame": 8, "last_frame": 14, "best_frame": 9, "max_rel_area": 0.03145, "mean_rel_area": 0.015, "centroid": [0.286, 0.233], "motion": 0.261}, {"mark_id": 23, "category": "window", "isthing": 1, "iscrowd": 0, "order": 17, "color": [205, 220, 57], "panel": 1, "n_present": 14, "first_frame": 1, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.00694, "mean_rel_area": 0.00332, "centroid": [0.469, 0.051], "motion": 0.632}, {"mark_id": 27, "category": "window", "isthing": 1, "iscrowd": 0, "order": 18, "color": [255, 152, 0], "panel": 1, "n_present": 12, "first_frame": 3, "last_frame": 14, "best_frame": 8, "max_rel_area": 0.00791, "mean_rel_area": 0.00466, "centroid": [0.397, 0.059], "motion": 0.561}, {"mark_id": 39, "category": "person", "isthing": 1, "iscrowd": 0, "order": 19, "color": [96, 125, 139], "panel": 1, "n_present": 4, "first_frame": 11, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.04569, "mean_rel_area": 0.02998, "centroid": [0.192, 0.332], "motion": 0.211}, {"mark_id": 28, "category": "window", "isthing": 1, "iscrowd": 0, "order": 20, "color": [54, 67, 244], "panel": 2, "n_present": 12, "first_frame": 3, "last_frame": 14, "best_frame": 10, "max_rel_area": 0.00487, "mean_rel_area": 0.00283, "centroid": [0.301, 0.137], "motion": 0.402}, {"mark_id": 33, "category": "person", "isthing": 1, "iscrowd": 0, "order": 21, "color": [34, 189, 255], "panel": 2, "n_present": 6, "first_frame": 9, "last_frame": 14, "best_frame": 10, "max_rel_area": 0.01916, "mean_rel_area": 0.01123, "centroid": [0.28, 0.216], "motion": 0.261}, {"mark_id": 20, "category": "person", "isthing": 1, "iscrowd": 0, "order": 22, "color": [60, 200, 60], "panel": 2, "n_present": 4, "first_frame": 0, "last_frame": 3, "best_frame": 2, "max_rel_area": 0.03637, "mean_rel_area": 0.02635, "centroid": [0.903, 0.36], "motion": 0.158}, {"mark_id": 37, "category": "person", "isthing": 1, "iscrowd": 0, "order": 23, "color": [240, 120, 40], "panel": 2, "n_present": 5, "first_frame": 10, "last_frame": 14, "best_frame": 10, "max_rel_area": 0.01691, "mean_rel_area": 0.00895, "centroid": [0.267, 0.201], "motion": 0.236}, {"mark_id": 31, "category": "person", "isthing": 1, "iscrowd": 0, "order": 24, "color": [180, 60, 220], "panel": 2, "n_present": 4, "first_frame": 7, "last_frame": 10, "best_frame": 8, "max_rel_area": 0.02212, "mean_rel_area": 0.01403, "centroid": [0.218, 0.34], "motion": 0.16}, {"mark_id": 40, "category": "person", "isthing": 1, "iscrowd": 0, "order": 25, "color": [0, 215, 255], "panel": 2, "n_present": 3, "first_frame": 12, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.02114, "mean_rel_area": 0.01416, "centroid": [0.147, 0.196], "motion": 0.137}, {"mark_id": 38, "category": "person", "isthing": 1, "iscrowd": 0, "order": 26, "color": [200, 90, 130], "panel": 2, "n_present": 4, "first_frame": 10, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.01021, "mean_rel_area": 0.00713, "centroid": [0.21, 0.278], "motion": 0.234}, {"mark_id": 21, "category": "person", "isthing": 1, "iscrowd": 0, "order": 27, "color": [80, 175, 76], "panel": 2, "n_present": 2, "first_frame": 0, "last_frame": 1, "best_frame": 0, "max_rel_area": 0.03218, "mean_rel_area": 0.02886, "centroid": [0.936, 0.332], "motion": 0.056}, {"mark_id": 36, "category": "window", "isthing": 1, "iscrowd": 0, "order": 28, "color": [255, 100, 180], "panel": 2, "n_present": 5, "first_frame": 10, "last_frame": 14, "best_frame": 10, "max_rel_area": 0.00277, "mean_rel_area": 0.00212, "centroid": [0.245, 0.068], "motion": 0.22}, {"mark_id": 34, "category": "window", "isthing": 1, "iscrowd": 0, "order": 29, "color": [0, 140, 255], "panel": 2, "n_present": 5, "first_frame": 10, "last_frame": 14, "best_frame": 11, "max_rel_area": 0.00229, "mean_rel_area": 0.00159, "centroid": [0.221, 0.047], "motion": 0.224}, {"mark_id": 10, "category": "window", "isthing": 1, "iscrowd": 0, "order": 30, "color": [139, 195, 74], "panel": 3, "n_present": 3, "first_frame": 0, "last_frame": 2, "best_frame": 1, "max_rel_area": 0.0037, "mean_rel_area": 0.00264, "centroid": [0.897, 0.069], "motion": 0.137}, {"mark_id": 35, "category": "window", "isthing": 1, "iscrowd": 0, "order": 31, "color": [255, 87, 34], "panel": 3, "n_present": 5, "first_frame": 10, "last_frame": 14, "best_frame": 11, "max_rel_area": 0.00127, "mean_rel_area": 0.00089, "centroid": [0.273, 0.039], "motion": 0.213}, {"mark_id": 7, "category": "window", "isthing": 1, "iscrowd": 0, "order": 32, "color": [156, 39, 176], "panel": 3, "n_present": 2, "first_frame": 0, "last_frame": 1, "best_frame": 1, "max_rel_area": 0.00506, "mean_rel_area": 0.00417, "centroid": [0.905, 0.066], "motion": 0.061}, {"mark_id": 22, "category": "person", "isthing": 1, "iscrowd": 0, "order": 33, "color": [0, 188, 212], "panel": 3, "n_present": 1, "first_frame": 0, "last_frame": 0, "best_frame": 0, "max_rel_area": 0.01793, "mean_rel_area": 0.01793, "centroid": [0.968, 0.308], "motion": 0.0}, {"mark_id": 8, "category": "window", "isthing": 1, "iscrowd": 0, "order": 34, "color": [121, 85, 72], "panel": 3, "n_present": 2, "first_frame": 0, "last_frame": 1, "best_frame": 0, "max_rel_area": 0.00234, "mean_rel_area": 0.00205, "centroid": [0.897, 0.168], "motion": 0.06}, {"mark_id": 6, "category": "window", "isthing": 1, "iscrowd": 0, "order": 35, "color": [233, 30, 99], "panel": 3, "n_present": 2, "first_frame": 0, "last_frame": 1, "best_frame": 0, "max_rel_area": 0.00172, "mean_rel_area": 0.00168, "centroid": [0.949, 0.06], "motion": 0.058}, {"mark_id": 9, "category": "window", "isthing": 1, "iscrowd": 0, "order": 36, "color": [63, 81, 181], "panel": 3, "n_present": 2, "first_frame": 0, "last_frame": 1, "best_frame": 1, "max_rel_area": 0.00102, "mean_rel_area": 0.00058, "centroid": [0.872, 0.151], "motion": 0.069}, {"mark_id": 2, "category": "floor", "isthing": 0, "iscrowd": 0, "order": 37, "color": [205, 220, 57], "panel": 3, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.22598, "mean_rel_area": 0.17503, "centroid": [0.552, 0.763], "motion": 0.095}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 38, "color": [255, 152, 0], "panel": 3, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 5, "max_rel_area": 0.18843, "mean_rel_area": 0.14643, "centroid": [0.566, 0.36], "motion": 0.12}, {"mark_id": 3, "category": "other electronic product", "isthing": 0, "iscrowd": 0, "order": 39, "color": [96, 125, 139], "panel": 3, "n_present": 14, "first_frame": 0, "last_frame": 14, "best_frame": 11, "max_rel_area": 0.00532, "mean_rel_area": 0.00182, "centroid": [0.646, 0.12], "motion": 0.631}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [715, 718, 721, 724, 727, 730, 733, 736, 739, 742, 745, 748, 751, 754, 757], "timing_sha": "4f169848ce4f", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 6, "unlinked": 0, "snapped": 0, "collective": 3, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 25761, "output_tokens": 1228, "thinking_tokens": 360, "total_tokens": 26989}, "seconds": 21.8, "response_id": "resp_07721f4f38265cd5006ac32f84bd3487d0b18df5e2b857220d", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 715, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 721, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 730, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 736, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 745, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 751, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 757, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 715, "time_s": 0.0, "mark_ids": [14, 11, 15, 13, 12, 19]}, {"role": "marks", "path": "marks/f000_p1.jpg", "frame": 0, "panel": 1, "source_frame": 715, "time_s": 0.0, "mark_ids": [17, 16, 18, 4, 5]}, {"role": "marks", "path": "marks/f000_p2.jpg", "frame": 0, "panel": 2, "source_frame": 715, "time_s": 0.0, "mark_ids": [20, 21]}, {"role": "marks", "path": "marks/f000_p3.jpg", "frame": 0, "panel": 3, "source_frame": 715, "time_s": 0.0, "mark_ids": [10, 7, 22, 8, 6, 9, 2, 1, 3]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 730, "time_s": 1.0, "mark_ids": [14, 11, 15, 13, 12, 25, 26, 29, 19, 30]}, {"role": "marks", "path": "marks/f005_p1.jpg", "frame": 5, "panel": 1, "source_frame": 730, "time_s": 1.0, "mark_ids": [17, 16, 18, 4, 24, 5, 23, 27]}, {"role": "marks", "path": "marks/f005_p2.jpg", "frame": 5, "panel": 2, "source_frame": 730, "time_s": 1.0, "mark_ids": [28]}, {"role": "marks", "path": "marks/f005_p3.jpg", "frame": 5, "panel": 3, "source_frame": 730, "time_s": 1.0, "mark_ids": [2, 1, 3]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 745, "time_s": 2.0, "mark_ids": [14, 11, 15, 13, 12, 25, 26, 29, 30]}, {"role": "marks", "path": "marks/f010_p1.jpg", "frame": 10, "panel": 1, "source_frame": 745, "time_s": 2.0, "mark_ids": [17, 16, 4, 24, 5, 32, 23, 27]}, {"role": "marks", "path": "marks/f010_p2.jpg", "frame": 10, "panel": 2, "source_frame": 745, "time_s": 2.0, "mark_ids": [28, 33, 37, 31, 38, 36, 34]}, {"role": "marks", "path": "marks/f010_p3.jpg", "frame": 10, "panel": 3, "source_frame": 745, "time_s": 2.0, "mark_ids": [35, 2, 1, 3]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 757, "time_s": 2.8, "mark_ids": [14, 11, 15, 12, 25, 26, 29, 30]}, {"role": "marks", "path": "marks/f014_p1.jpg", "frame": 14, "panel": 1, "source_frame": 757, "time_s": 2.8, "mark_ids": [18, 4, 24, 5, 32, 23, 27, 39]}, {"role": "marks", "path": "marks/f014_p2.jpg", "frame": 14, "panel": 2, "source_frame": 757, "time_s": 2.8, "mark_ids": [28, 33, 37, 40, 38, 36, 34]}, {"role": "marks", "path": "marks/f014_p3.jpg", "frame": 14, "panel": 3, "source_frame": 757, "time_s": 2.8, "mark_ids": [35, 2, 1, 3]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [14, 11, 15, 13, 12, 25, 26, 29, 19, 30, 17, 16, 18, 4, 24, 5]}, {"role": "ids", "path": "ids/sheet1.jpg", "mark_ids": [32, 23, 27, 39, 28, 33, 20, 37, 31, 40, 38, 21, 36, 34, 10, 35]}, {"role": "ids", "path": "ids/sheet2.jpg", "mark_ids": [7, 22, 8, 6, 9, 2, 1, 3]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "4f169848ce4f", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [715, 721, 730, 736, 745, 751, 757], "actions": [{"action_index": 0, "text": "dance", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 715, "end_source_frame": 757, "start_filename": "00000715.jpg", "end_filename": "00000757.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The boy and girl move together in a partnered dance hold throughout the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [715, 721, 730, 736, 745, 751, 757]}, {"action_index": 1, "text": "turn", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 715, "end_source_frame": 757, "start_filename": "00000715.jpg", "end_filename": "00000757.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The pair progressively changes orientation while maintaining their dance hold.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [715, 721, 730, 736, 745, 751, 757]}, {"action_index": 2, "text": "watch", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 715, "end_source_frame": 757, "start_filename": "00000715.jpg", "end_filename": "00000757.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Spectators face the dancing area throughout, although different members are visible at different times.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [715, 721, 730, 736, 745, 751, 757]}, {"action_index": 3, "text": "hold", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 715, "end_source_frame": 757, "start_filename": "00000715.jpg", "end_filename": "00000757.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "Members of the referenced subset visibly hold cameras toward the dancers across the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [715, 721, 730, 736, 745, 751, 757]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1363_QwwaDqYgOYo", "split": "train", "caption": "A black-and-white dog wriggles against a leopard-print blanket on a bed. It then lies stretched out beside the blanket.", "mentions": [{"text": "A black-and-white dog", "span": [0, 21], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a leopard-print blanket", "span": [39, 62], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a bed", "span": [66, 71], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "It", "span": [73, 75], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "the blanket", "span": [107, 118], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 1, "is_pronoun": false}], "sentences": [[0, 72], [73, 119]], "groups": [{"group_id": 0, "mark_id": 5, "category": "dog", "isthing": 1, "mentions": [{"text": "A black-and-white dog", "span": [0, 21], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "It", "span": [73, 75], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 3, "category": "textiles", "isthing": 0, "mentions": [{"text": "a leopard-print blanket", "span": [39, 62], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "the blanket", "span": [107, 118], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 6, "category": "bed", "isthing": 1, "mentions": [{"text": "a bed", "span": [66, 71], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "wriggles", "span": [22, 30], "spans": [[22, 30]], "particle": null, "mentions": [{"text": "wriggles", "span": [22, 30], "spans": [[22, 30]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "A black-and-white dog", "span": [0, 21], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a leopard-print blanket", "span": [39, 62], "groups": [1], "group_mentions": [0], "group_match": "exact"}, {"text": "a bed", "span": [66, 71], "groups": [2], "group_mentions": [0], "group_match": "exact"}], "confidence": "medium", "temporal": {"action_index": 0, "text": "wriggles", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.9]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 2448, "end_source_frame": 2469, "start_filename": "00002448.jpg", "end_filename": "00002469.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The dog shifts its head and body against the blanket during the early frames.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2448, 2454, 2463, 2469, 2478, 2484, 2490]}}, {"text": "lies", "span": [81, 85], "spans": [[81, 85]], "particle": null, "mentions": [{"text": "lies", "span": [81, 85], "spans": [[81, 85]], "particle": null, "sentence_index": 1}], "type": "posture", "sentence_index": 1, "subject": {"text": "It", "span": [73, 75], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "the blanket", "span": [107, 118], "groups": [1], "group_mentions": [1], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 1, "text": "lies", "status": "visible", "segments_frames": [[3, 6]], "segments": [[1.4, 3.0]], "segment_details": [{"start_frame": 3, "end_frame": 6, "start_frame_1based": 4, "end_frame_1based": 7, "start_s": 1.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 2469, "end_source_frame": 2490, "start_filename": "00002469.jpg", "end_filename": "00002490.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The dog remains stretched out with its belly down and hind legs extended in the later frames.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2448, 2454, 2463, 2469, 2478, 2484, 2490]}}], "coverage": {"marks_shown": 8, "marks_grounded": 3, "things_shown": 5, "things_grounded": 2, "stuff_shown": 3, "stuff_grounded": 1, "n_mentions": 5, "category_disagree": 0, "category_checked": 3}, "marks": [{"mark_id": 6, "category": "bed", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.19316, "mean_rel_area": 0.17586, "centroid": [0.624, 0.578], "motion": 0.08}, {"mark_id": 4, "category": "window", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.13627, "mean_rel_area": 0.10758, "centroid": [0.832, 0.2], "motion": 0.063}, {"mark_id": 5, "category": "dog", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 4, "max_rel_area": 0.0374, "mean_rel_area": 0.03504, "centroid": [0.598, 0.61], "motion": 0.099}, {"mark_id": 7, "category": "pillow", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.02549, "mean_rel_area": 0.02339, "centroid": [0.369, 0.375], "motion": 0.114}, {"mark_id": 8, "category": "pillow", "isthing": 1, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.01585, "mean_rel_area": 0.01466, "centroid": [0.535, 0.282], "motion": 0.11}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 5, "max_rel_area": 0.34942, "mean_rel_area": 0.29436, "centroid": [0.5, 0.384], "motion": 0.035}, {"mark_id": 3, "category": "textiles", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.23161, "mean_rel_area": 0.1958, "centroid": [0.677, 0.753], "motion": 0.081}, {"mark_id": 2, "category": "floor", "isthing": 0, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.20028, "mean_rel_area": 0.15197, "centroid": [0.5, 0.793], "motion": 0.039}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [2448, 2451, 2454, 2457, 2460, 2463, 2466, 2469, 2472, 2475, 2478, 2481, 2484, 2487, 2490], "timing_sha": "2d1ce32242b9", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 5, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 16894, "output_tokens": 915, "thinking_tokens": 380, "total_tokens": 17809}, "seconds": 17.14, "response_id": "resp_0db1245ac2fff64e006ac32f9a95c087d09e942c66ca193ec1", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 2448, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 2454, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 2463, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 2469, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 2478, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 2484, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 2490, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 2448, "time_s": 0.0, "mark_ids": [6, 4, 5, 7, 8, 1, 3, 2]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 2463, "time_s": 1.0, "mark_ids": [6, 4, 5, 7, 8, 1, 3, 2]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 2478, "time_s": 2.0, "mark_ids": [6, 4, 5, 7, 8, 1, 3, 2]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 2490, "time_s": 2.8, "mark_ids": [6, 4, 5, 7, 8, 1, 3, 2]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [6, 4, 5, 7, 8, 1, 3, 2]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "2d1ce32242b9", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2448, 2454, 2463, 2469, 2478, 2484, 2490], "actions": [{"action_index": 0, "text": "wriggles", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.9]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 2448, "end_source_frame": 2469, "start_filename": "00002448.jpg", "end_filename": "00002469.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The dog shifts its head and body against the blanket during the early frames.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2448, 2454, 2463, 2469, 2478, 2484, 2490]}, {"action_index": 1, "text": "lies", "status": "visible", "segments_frames": [[3, 6]], "segments": [[1.4, 3.0]], "segment_details": [{"start_frame": 3, "end_frame": 6, "start_frame_1based": 4, "end_frame_1based": 7, "start_s": 1.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 2469, "end_source_frame": 2490, "start_filename": "00002469.jpg", "end_filename": "00002490.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The dog remains stretched out with its belly down and hind legs extended in the later frames.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2448, 2454, 2463, 2469, 2478, 2484, 2490]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1529_N6hl1No1Syc", "split": "train", "caption": "A dark wooden chair and a lighter chair are on bare ground among trees. They are beside scattered logs and a pale stone.", "mentions": [{"text": "A dark wooden chair", "span": [0, 19], "occurrence": 1, "mark_ids": [7], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a lighter chair", "span": [24, 39], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "bare ground", "span": [47, 58], "occurrence": 1, "mark_ids": [1], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "trees", "span": [65, 70], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "They", "span": [72, 76], "occurrence": 1, "mark_ids": [7, 8], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "scattered logs", "span": [88, 102], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "a pale stone", "span": [107, 119], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 1, "is_pronoun": false}], "sentences": [[0, 71], [72, 120]], "groups": [{"group_id": 0, "mark_id": 7, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "A dark wooden chair", "span": [0, 19], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "They", "span": [72, 76], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 8, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "a lighter chair", "span": [24, 39], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "They", "span": [72, 76], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 1, "category": "ground", "isthing": 0, "mentions": [{"text": "bare ground", "span": [47, 58], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 6, "category": "tree", "isthing": 0, "mentions": [{"text": "trees", "span": [65, 70], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 5, "category": "wood", "isthing": 0, "mentions": [{"text": "scattered logs", "span": [88, 102], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 5, "mark_id": 4, "category": "stone", "isthing": 0, "mentions": [{"text": "a pale stone", "span": [107, 119], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [], "coverage": {"marks_shown": 8, "marks_grounded": 6, "things_shown": 2, "things_grounded": 2, "stuff_shown": 6, "stuff_grounded": 4, "n_mentions": 8, "category_disagree": 0, "category_checked": 6}, "marks": [{"mark_id": 7, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.01095, "mean_rel_area": 0.01041, "centroid": [0.339, 0.515], "motion": 0.021}, {"mark_id": 8, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 13, "max_rel_area": 0.00556, "mean_rel_area": 0.00445, "centroid": [0.399, 0.476], "motion": 0.031}, {"mark_id": 6, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 10, "max_rel_area": 0.51754, "mean_rel_area": 0.5158, "centroid": [0.5, 0.466], "motion": 0.031}, {"mark_id": 1, "category": "ground", "isthing": 0, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 3, "max_rel_area": 0.27534, "mean_rel_area": 0.26632, "centroid": [0.516, 0.759], "motion": 0.104}, {"mark_id": 2, "category": "grass", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.15831, "mean_rel_area": 0.14815, "centroid": [0.5, 0.702], "motion": 0.015}, {"mark_id": 5, "category": "wood", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 6, "max_rel_area": 0.03006, "mean_rel_area": 0.02878, "centroid": [0.456, 0.529], "motion": 0.025}, {"mark_id": 3, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.02084, "mean_rel_area": 0.01761, "centroid": [0.368, 0.063], "motion": 0.071}, {"mark_id": 4, "category": "stone", "isthing": 0, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.00784, "mean_rel_area": 0.0063, "centroid": [0.275, 0.728], "motion": 0.091}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [888, 891, 894, 897, 900, 903, 906, 909, 912, 915, 918, 921, 924, 927, 930], "timing_sha": "653eae423fc3", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 0, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 16894, "output_tokens": 454, "thinking_tokens": 144, "total_tokens": 17348}, "seconds": 10.22, "response_id": "resp_035fd182f19ac3e7006ac32f9c2a9487d08ea38def156955eb", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 888, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 894, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 903, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 909, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 918, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 924, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 930, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 888, "time_s": 0.0, "mark_ids": [7, 8, 6, 1, 2, 5, 3, 4]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 903, "time_s": 1.0, "mark_ids": [7, 8, 6, 1, 2, 5, 3, 4]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 918, "time_s": 2.0, "mark_ids": [7, 8, 6, 1, 2, 5, 3, 4]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 930, "time_s": 2.8, "mark_ids": [7, 8, 6, 1, 2, 5, 3, 4]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [7, 8, 6, 1, 2, 5, 3, 4]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "653eae423fc3", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [888, 894, 903, 909, 918, 924, 930], "actions": [], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1665_7wOrT_feKvA", "split": "train", "caption": "A blue-and-white boat travels along the river beneath a raised bridge. It moves steadily to the right.", "mentions": [{"text": "A blue-and-white boat", "span": [0, 21], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "the river", "span": [36, 45], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a raised bridge", "span": [54, 69], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "It", "span": [71, 73], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 1, "is_pronoun": true}], "sentences": [[0, 70], [71, 102]], "groups": [{"group_id": 0, "mark_id": 8, "category": "ship or boat", "isthing": 1, "mentions": [{"text": "A blue-and-white boat", "span": [0, 21], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "It", "span": [71, 73], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 6, "category": "river", "isthing": 0, "mentions": [{"text": "the river", "span": [36, 45], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 4, "category": "bridge", "isthing": 0, "mentions": [{"text": "a raised bridge", "span": [54, 69], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "travels", "span": [22, 29], "spans": [[22, 29], [74, 79]], "particle": null, "mentions": [{"text": "travels", "span": [22, 29], "spans": [[22, 29]], "particle": null, "sentence_index": 0}, {"text": "moves", "span": [74, 79], "spans": [[74, 79]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 0, "subject": {"text": "A blue-and-white boat", "span": [0, 21], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a raised bridge", "span": [54, 69], "groups": [2], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "travels", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 2628, "end_source_frame": 2670, "start_filename": "00002628.jpg", "end_filename": "00002670.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The boat advances steadily rightward beneath the stationary bridge throughout the clean-frame sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2628, 2634, 2643, 2649, 2658, 2664, 2670]}}], "coverage": {"marks_shown": 8, "marks_grounded": 3, "things_shown": 1, "things_grounded": 1, "stuff_shown": 7, "stuff_grounded": 2, "n_mentions": 4, "category_disagree": 0, "category_checked": 3}, "marks": [{"mark_id": 8, "category": "ship or boat", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.01658, "mean_rel_area": 0.01511, "centroid": [0.452, 0.871], "motion": 0.031}, {"mark_id": 5, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 10, "max_rel_area": 0.45785, "mean_rel_area": 0.45452, "centroid": [0.5, 0.379], "motion": 0.002}, {"mark_id": 4, "category": "bridge", "isthing": 0, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.34849, "mean_rel_area": 0.3479, "centroid": [0.5, 0.513], "motion": 0.002}, {"mark_id": 7, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.07646, "mean_rel_area": 0.07351, "centroid": [0.609, 0.756], "motion": 0.007}, {"mark_id": 6, "category": "river", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 5, "max_rel_area": 0.07454, "mean_rel_area": 0.07308, "centroid": [0.5, 0.955], "motion": 0.001}, {"mark_id": 3, "category": "house", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.0239, "mean_rel_area": 0.02325, "centroid": [0.609, 0.774], "motion": 0.007}, {"mark_id": 1, "category": "grass", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 5, "max_rel_area": 0.00838, "mean_rel_area": 0.00781, "centroid": [0.631, 0.91], "motion": 0.025}, {"mark_id": 2, "category": "building", "isthing": 0, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 6, "max_rel_area": 0.00472, "mean_rel_area": 0.00463, "centroid": [0.534, 0.689], "motion": 0.001}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [2628, 2631, 2634, 2637, 2640, 2643, 2646, 2649, 2652, 2655, 2658, 2661, 2664, 2667, 2670], "timing_sha": "ff2998d705c0", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 2, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 16891, "output_tokens": 586, "thinking_tokens": 156, "total_tokens": 17477}, "seconds": 11.68, "response_id": "resp_0eabc9591c8846a4006ac32fa634d887d0860d701212ff8580", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 2628, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 2634, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 2643, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 2649, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 2658, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 2664, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 2670, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 2628, "time_s": 0.0, "mark_ids": [8, 5, 4, 7, 6, 3, 1, 2]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 2643, "time_s": 1.0, "mark_ids": [8, 5, 4, 7, 6, 3, 1, 2]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 2658, "time_s": 2.0, "mark_ids": [8, 5, 4, 7, 6, 3, 1, 2]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 2670, "time_s": 2.8, "mark_ids": [8, 5, 4, 7, 6, 3, 1, 2]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [8, 5, 4, 7, 6, 3, 1, 2]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "ff2998d705c0", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2628, 2634, 2643, 2649, 2658, 2664, 2670], "actions": [{"action_index": 0, "text": "travels", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 2628, "end_source_frame": 2670, "start_filename": "00002628.jpg", "end_filename": "00002670.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The boat advances steadily rightward beneath the stationary bridge throughout the clean-frame sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2628, 2634, 2643, 2649, 2658, 2664, 2670]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1804_H89b4qUm51w", "split": "train", "caption": "A woman walks along a stone path toward a wooden planter beside a white house. She briefly reaches toward the leafy plants.", "mentions": [{"text": "A woman", "span": [0, 7], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a stone path", "span": [20, 32], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a wooden planter", "span": [40, 56], "occurrence": 1, "mark_ids": [9], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a white house", "span": [64, 77], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "She", "span": [79, 82], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "the leafy plants", "span": [106, 122], "occurrence": 1, "mark_ids": [7], "confidence": "high", "sentence_index": 1, "is_pronoun": false}], "sentences": [[0, 78], [79, 123]], "groups": [{"group_id": 0, "mark_id": 8, "category": "person", "isthing": 1, "mentions": [{"text": "A woman", "span": [0, 7], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "She", "span": [79, 82], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 3, "category": "path", "isthing": 0, "mentions": [{"text": "a stone path", "span": [20, 32], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 9, "category": "box", "isthing": 1, "mentions": [{"text": "a wooden planter", "span": [40, 56], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown"], "chain_agreement": "unknown"}, {"group_id": 3, "mark_id": 4, "category": "house", "isthing": 0, "mentions": [{"text": "a white house", "span": [64, 77], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 7, "category": "other plant", "isthing": 0, "mentions": [{"text": "the leafy plants", "span": [106, 122], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "walks", "span": [8, 13], "spans": [[8, 13]], "particle": null, "mentions": [{"text": "walks", "span": [8, 13], "spans": [[8, 13]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "A woman", "span": [0, 7], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a wooden planter", "span": [40, 56], "groups": [2], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "walks", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 3498, "end_source_frame": 3540, "start_filename": "00003498.jpg", "end_filename": "00003540.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "She advances along the stone path toward the planter throughout the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3498, 3504, 3513, 3519, 3528, 3534, 3540]}}, {"text": "reaches", "span": [91, 98], "spans": [[91, 98]], "particle": null, "mentions": [{"text": "reaches", "span": [91, 98], "spans": [[91, 98]], "particle": null, "sentence_index": 1}], "type": "prepositional", "sentence_index": 1, "subject": {"text": "She", "span": [79, 82], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "the leafy plants", "span": [106, 122], "groups": [4], "group_mentions": [0], "group_match": "exact"}], "confidence": "medium", "temporal": {"action_index": 1, "text": "reaches", "status": "visible", "segments_frames": [[2, 3]], "segments": [[1.0, 1.9]], "segment_details": [{"start_frame": 2, "end_frame": 3, "start_frame_1based": 3, "end_frame_1based": 4, "start_s": 1.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 3513, "end_source_frame": 3519, "start_filename": "00003513.jpg", "end_filename": "00003519.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "She extends her right arm toward the foliage, most clearly in frame 3, then lowers it.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3498, 3504, 3513, 3519, 3528, 3534, 3540]}}], "coverage": {"marks_shown": 9, "marks_grounded": 5, "things_shown": 2, "things_grounded": 2, "stuff_shown": 7, "stuff_grounded": 3, "n_mentions": 6, "category_disagree": 0, "category_checked": 4}, "marks": [{"mark_id": 8, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 9, "max_rel_area": 0.15527, "mean_rel_area": 0.1363, "centroid": [0.438, 0.598], "motion": 0.164}, {"mark_id": 9, "category": "box", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 3, "max_rel_area": 0.01796, "mean_rel_area": 0.00753, "centroid": [0.555, 0.548], "motion": 0.15}, {"mark_id": 7, "category": "other plant", "isthing": 0, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 11, "max_rel_area": 0.54471, "mean_rel_area": 0.41529, "centroid": [0.496, 0.515], "motion": 0.037}, {"mark_id": 4, "category": "house", "isthing": 0, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 8, "max_rel_area": 0.40115, "mean_rel_area": 0.29401, "centroid": [0.747, 0.428], "motion": 0.061}, {"mark_id": 5, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 4, "max_rel_area": 0.09916, "mean_rel_area": 0.05399, "centroid": [0.373, 0.096], "motion": 0.13}, {"mark_id": 1, "category": "handrail or fence", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.0578, "mean_rel_area": 0.04232, "centroid": [0.12, 0.533], "motion": 0.056}, {"mark_id": 6, "category": "mountain", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 3, "max_rel_area": 0.04286, "mean_rel_area": 0.02786, "centroid": [0.148, 0.131], "motion": 0.107}, {"mark_id": 3, "category": "path", "isthing": 0, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.03073, "mean_rel_area": 0.01893, "centroid": [0.466, 0.806], "motion": 0.125}, {"mark_id": 2, "category": "pole", "isthing": 0, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 13, "first_frame": 0, "last_frame": 14, "best_frame": 3, "max_rel_area": 0.00906, "mean_rel_area": 0.00432, "centroid": [0.516, 0.655], "motion": 0.136}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [3498, 3501, 3504, 3507, 3510, 3513, 3516, 3519, 3522, 3525, 3528, 3531, 3534, 3537, 3540], "timing_sha": "3f997e876600", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 4, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 17184, "output_tokens": 911, "thinking_tokens": 372, "total_tokens": 18095}, "seconds": 19.61, "response_id": "resp_0a343dafd2f44fc6006ac32fabc1a087d09bcfeba1063c4e67", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 3498, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 3504, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 3513, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 3519, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 3528, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 3534, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 3540, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 3498, "time_s": 0.0, "mark_ids": [8, 9, 7, 4, 5, 1, 6, 3, 2]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 3513, "time_s": 1.0, "mark_ids": [8, 9, 7, 4, 5, 1, 6, 3, 2]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 3528, "time_s": 2.0, "mark_ids": [8, 9, 7, 4, 5, 1, 6, 3]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 3540, "time_s": 2.8, "mark_ids": [8, 9, 7, 4, 5, 1, 6, 3, 2]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [8, 9, 7, 4, 5, 1, 6, 3, 2]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "3f997e876600", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3498, 3504, 3513, 3519, 3528, 3534, 3540], "actions": [{"action_index": 0, "text": "walks", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 3498, "end_source_frame": 3540, "start_filename": "00003498.jpg", "end_filename": "00003540.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "She advances along the stone path toward the planter throughout the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3498, 3504, 3513, 3519, 3528, 3534, 3540]}, {"action_index": 1, "text": "reaches", "status": "visible", "segments_frames": [[2, 3]], "segments": [[1.0, 1.9]], "segment_details": [{"start_frame": 2, "end_frame": 3, "start_frame_1based": 3, "end_frame_1based": 4, "start_s": 1.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 3513, "end_source_frame": 3519, "start_filename": "00003513.jpg", "end_filename": "00003519.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "She extends her right arm toward the foliage, most clearly in frame 3, then lowers it.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3498, 3504, 3513, 3519, 3528, 3534, 3540]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1959_Ejgrd03o2F8", "split": "train", "caption": "A woman leans on a railing beside two white cabinets. She looks over the water and colorful waterslides. A man in red stands beside the blue slide entrance.", "mentions": [{"text": "A woman", "span": [0, 7], "occurrence": 1, "mark_ids": [10], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a railing", "span": [17, 26], "occurrence": 1, "mark_ids": [1], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "two white cabinets", "span": [34, 52], "occurrence": 1, "mark_ids": [12, 13], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "She", "span": [54, 57], "occurrence": 1, "mark_ids": [10], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "the water", "span": [69, 78], "occurrence": 1, "mark_ids": [7], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "colorful waterslides", "span": [83, 103], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "A man", "span": [105, 110], "occurrence": 1, "mark_ids": [11], "confidence": "high", "sentence_index": 2, "is_pronoun": false}, {"text": "the blue slide entrance", "span": [132, 155], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 53], [54, 104], [105, 156]], "groups": [{"group_id": 0, "mark_id": 10, "category": "person", "isthing": 1, "mentions": [{"text": "A woman", "span": [0, 7], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "She", "span": [54, 57], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 1, "category": "handrail or fence", "isthing": 0, "mentions": [{"text": "a railing", "span": [17, 26], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 12, "category": "box", "isthing": 1, "mentions": [{"text": "two white cabinets", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["disagree"], "chain_agreement": "disagree"}, {"group_id": 3, "mark_id": 13, "category": "box", "isthing": 1, "mentions": [{"text": "two white cabinets", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["disagree"], "chain_agreement": "disagree"}, {"group_id": 4, "mark_id": 7, "category": "water", "isthing": 0, "mentions": [{"text": "the water", "span": [69, 78], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 5, "mark_id": 8, "category": "pipeline", "isthing": 0, "mentions": [{"text": "colorful waterslides", "span": [83, 103], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown"], "chain_agreement": "unknown"}, {"group_id": 6, "mark_id": 11, "category": "person", "isthing": 1, "mentions": [{"text": "A man", "span": [105, 110], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 7, "mark_id": 4, "category": "other construction", "isthing": 0, "mentions": [{"text": "the blue slide entrance", "span": [132, 155], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["disagree"], "chain_agreement": "disagree"}], "actions": [{"text": "leans", "span": [8, 13], "spans": [[8, 13]], "particle": null, "mentions": [{"text": "leans", "span": [8, 13], "spans": [[8, 13]], "particle": null, "sentence_index": 0}], "type": "posture", "sentence_index": 0, "subject": {"text": "A woman", "span": [0, 7], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a railing", "span": [17, 26], "groups": [1], "group_mentions": [0], "group_match": "exact"}, {"text": "two white cabinets", "span": [34, 52], "groups": [2, 3], "group_mentions": [0, 0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "leans", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.9]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 1668, "end_source_frame": 1689, "start_filename": "00001668.jpg", "end_filename": "00001689.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The woman maintains a forward lean against the railing until she leaves the visible scene.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1668, 1674, 1683, 1689, 1698, 1704, 1710]}}, {"text": "looks", "span": [58, 63], "spans": [[58, 63]], "particle": null, "mentions": [{"text": "looks", "span": [58, 63], "spans": [[58, 63]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "She", "span": [54, 57], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "the water", "span": [69, 78], "groups": [4], "group_mentions": [0], "group_match": "exact"}, {"text": "colorful waterslides", "span": [83, 103], "groups": [5], "group_mentions": [0], "group_match": "exact"}], "confidence": "medium", "temporal": {"action_index": 1, "text": "looks", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.9]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 1668, "end_source_frame": 1689, "start_filename": "00001668.jpg", "end_filename": "00001689.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Her head faces outward toward the water and slide area, though her precise gaze target is unclear.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1668, 1674, 1683, 1689, 1698, 1704, 1710]}}, {"text": "stands", "span": [118, 124], "spans": [[118, 124]], "particle": null, "mentions": [{"text": "stands", "span": [118, 124], "spans": [[118, 124]], "particle": null, "sentence_index": 2}], "type": "posture", "sentence_index": 2, "subject": {"text": "A man", "span": [105, 110], "groups": [6], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "the blue slide entrance", "span": [132, 155], "groups": [7], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 2, "text": "stands", "status": "visible", "segments_frames": [[0, 0]], "segments": [[0.0, 0.4]], "segment_details": [{"start_frame": 0, "end_frame": 0, "start_frame_1based": 1, "end_frame_1based": 1, "start_s": 0.0, "end_s": 0.4, "last_observed_s": 0.0, "start_source_frame": 1668, "end_source_frame": 1668, "start_filename": "00001668.jpg", "end_filename": "00001668.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The partially cropped man is upright beside the slide entrance in the first clean frame.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1668, 1674, 1683, 1689, 1698, 1704, 1710]}}], "coverage": {"marks_shown": 13, "marks_grounded": 8, "things_shown": 4, "things_grounded": 4, "stuff_shown": 9, "stuff_grounded": 4, "n_mentions": 9, "category_disagree": 3, "category_checked": 7}, "marks": [{"mark_id": 13, "category": "box", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 9, "first_frame": 0, "last_frame": 8, "best_frame": 5, "max_rel_area": 0.11316, "mean_rel_area": 0.06695, "centroid": [0.449, 0.861], "motion": 0.768}, {"mark_id": 12, "category": "box", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 8, "first_frame": 0, "last_frame": 7, "best_frame": 0, "max_rel_area": 0.04993, "mean_rel_area": 0.0347, "centroid": [0.359, 0.625], "motion": 0.615}, {"mark_id": 10, "category": "person", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 5, "first_frame": 0, "last_frame": 4, "best_frame": 3, "max_rel_area": 0.02285, "mean_rel_area": 0.02196, "centroid": [0.42, 0.502], "motion": 0.337}, {"mark_id": 11, "category": "person", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 2, "first_frame": 0, "last_frame": 1, "best_frame": 0, "max_rel_area": 0.03117, "mean_rel_area": 0.01603, "centroid": [0.038, 0.831], "motion": 0.155}, {"mark_id": 4, "category": "other construction", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.41298, "mean_rel_area": 0.10665, "centroid": [0.363, 0.534], "motion": 0.113}, {"mark_id": 9, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.40923, "mean_rel_area": 0.30627, "centroid": [0.516, 0.581], "motion": 0.128}, {"mark_id": 8, "category": "pipeline", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.20741, "mean_rel_area": 0.11387, "centroid": [0.723, 0.72], "motion": 0.306}, {"mark_id": 5, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 3, "max_rel_area": 0.20379, "mean_rel_area": 0.1479, "centroid": [0.507, 0.115], "motion": 0.072}, {"mark_id": 2, "category": "ground", "isthing": 0, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.10813, "mean_rel_area": 0.07337, "centroid": [0.547, 0.585], "motion": 0.121}, {"mark_id": 7, "category": "water", "isthing": 0, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 9, "max_rel_area": 0.08732, "mean_rel_area": 0.06251, "centroid": [0.456, 0.614], "motion": 0.563}, {"mark_id": 6, "category": "mountain", "isthing": 0, "iscrowd": 0, "order": 10, "color": [139, 195, 74], "panel": 1, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 8, "max_rel_area": 0.07326, "mean_rel_area": 0.06132, "centroid": [0.508, 0.211], "motion": 0.102}, {"mark_id": 1, "category": "handrail or fence", "isthing": 0, "iscrowd": 0, "order": 11, "color": [255, 87, 34], "panel": 1, "n_present": 12, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.09614, "mean_rel_area": 0.03409, "centroid": [0.3, 0.689], "motion": 0.517}, {"mark_id": 3, "category": "grass", "isthing": 0, "iscrowd": 0, "order": 12, "color": [156, 39, 176], "panel": 1, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.04655, "mean_rel_area": 0.03168, "centroid": [0.542, 0.749], "motion": 0.467}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [1668, 1671, 1674, 1677, 1680, 1683, 1686, 1689, 1692, 1695, 1698, 1701, 1704, 1707, 1710], "timing_sha": "cd8f0a3d2b8c", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 8, "unlinked": 0, "snapped": 0, "collective": 1, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 19320, "output_tokens": 1021, "thinking_tokens": 214, "total_tokens": 20341}, "seconds": 19.35, "response_id": "resp_0acb70b5649d678f006ac32fb296f087d08c80195d50c18a8f", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 1668, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 1674, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 1683, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 1689, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 1698, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 1704, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 1710, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 1668, "time_s": 0.0, "mark_ids": [13, 12, 10, 11, 4, 9, 8, 5, 2, 7]}, {"role": "marks", "path": "marks/f000_p1.jpg", "frame": 0, "panel": 1, "source_frame": 1668, "time_s": 0.0, "mark_ids": [6, 1, 3]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 1683, "time_s": 1.0, "mark_ids": [13, 12, 4, 9, 8, 5, 2, 7]}, {"role": "marks", "path": "marks/f005_p1.jpg", "frame": 5, "panel": 1, "source_frame": 1683, "time_s": 1.0, "mark_ids": [6, 1, 3]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 1698, "time_s": 2.0, "mark_ids": [4, 9, 8, 5, 2, 7]}, {"role": "marks", "path": "marks/f010_p1.jpg", "frame": 10, "panel": 1, "source_frame": 1698, "time_s": 2.0, "mark_ids": [6, 3]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 1710, "time_s": 2.8, "mark_ids": [4, 9, 8, 5, 2, 7]}, {"role": "marks", "path": "marks/f014_p1.jpg", "frame": 14, "panel": 1, "source_frame": 1710, "time_s": 2.8, "mark_ids": [6, 1, 3]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [13, 12, 10, 11, 4, 9, 8, 5, 2, 7, 6, 1, 3]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "cd8f0a3d2b8c", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1668, 1674, 1683, 1689, 1698, 1704, 1710], "actions": [{"action_index": 0, "text": "leans", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.9]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 1668, "end_source_frame": 1689, "start_filename": "00001668.jpg", "end_filename": "00001689.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The woman maintains a forward lean against the railing until she leaves the visible scene.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1668, 1674, 1683, 1689, 1698, 1704, 1710]}, {"action_index": 1, "text": "looks", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.9]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 1668, "end_source_frame": 1689, "start_filename": "00001668.jpg", "end_filename": "00001689.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Her head faces outward toward the water and slide area, though her precise gaze target is unclear.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1668, 1674, 1683, 1689, 1698, 1704, 1710]}, {"action_index": 2, "text": "stands", "status": "visible", "segments_frames": [[0, 0]], "segments": [[0.0, 0.4]], "segment_details": [{"start_frame": 0, "end_frame": 0, "start_frame_1based": 1, "end_frame_1based": 1, "start_s": 0.0, "end_s": 0.4, "last_observed_s": 0.0, "start_source_frame": 1668, "end_source_frame": 1668, "start_filename": "00001668.jpg", "end_filename": "00001668.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The partially cropped man is upright beside the slide entrance in the first clean frame.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1668, 1674, 1683, 1689, 1698, 1704, 1710]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "2106_m0-AalaDlqU", "split": "train", "caption": "A boy is seated cross-legged on the patterned floor beside a laptop. He operates it and watches it.", "mentions": [{"text": "A boy", "span": [0, 5], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "the patterned floor", "span": [32, 51], "occurrence": 1, "mark_ids": [2], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a laptop", "span": [59, 67], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "He", "span": [69, 71], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "it", "span": [81, 83], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "it", "span": [96, 98], "occurrence": 2, "mark_ids": [8], "confidence": "high", "sentence_index": 1, "is_pronoun": true}], "sentences": [[0, 68], [69, 99]], "groups": [{"group_id": 0, "mark_id": 5, "category": "person", "isthing": 1, "mentions": [{"text": "A boy", "span": [0, 5], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "He", "span": [69, 71], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 2, "category": "floor", "isthing": 0, "mentions": [{"text": "the patterned floor", "span": [32, 51], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 8, "category": "computer", "isthing": 1, "mentions": [{"text": "a laptop", "span": [59, 67], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "it", "span": [81, 83], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "it", "span": [96, 98], "occurrence": 2, "confidence": "high", "collective": false}], "agreement": ["disagree", "pronoun", "pronoun"], "chain_agreement": "disagree"}], "actions": [{"text": "is seated", "span": [6, 15], "spans": [[6, 15]], "particle": null, "mentions": [{"text": "is seated", "span": [6, 15], "spans": [[6, 15]], "particle": null, "sentence_index": 0}], "type": "posture", "sentence_index": 0, "subject": {"text": "A boy", "span": [0, 5], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a laptop", "span": [59, 67], "groups": [2], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "is seated", "status": "visible", "segments_frames": [[0, 4]], "segments": [[0.0, 2.2]], "segment_details": [{"start_frame": 0, "end_frame": 4, "start_frame_1based": 1, "end_frame_1based": 5, "start_s": 0.0, "end_s": 2.2, "last_observed_s": 2.0, "start_source_frame": 3258, "end_source_frame": 3288, "start_filename": "00003258.jpg", "end_filename": "00003288.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The boy maintains a cross-legged seated posture beside the laptop throughout the sequence.", "fps": 2.0, "input_frame_count": 5, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0], "source_frame_indices": [3258, 3264, 3273, 3279, 3288]}}, {"text": "operates", "span": [72, 80], "spans": [[72, 80]], "particle": null, "mentions": [{"text": "operates", "span": [72, 80], "spans": [[72, 80]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "He", "span": [69, 71], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "it", "span": [81, 83], "groups": [2], "group_mentions": [1], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 1, "text": "operates", "status": "visible", "segments_frames": [[0, 4]], "segments": [[0.0, 2.2]], "segment_details": [{"start_frame": 0, "end_frame": 4, "start_frame_1based": 1, "end_frame_1based": 5, "start_s": 0.0, "end_s": 2.2, "last_observed_s": 2.0, "start_source_frame": 3258, "end_source_frame": 3288, "start_filename": "00003258.jpg", "end_filename": "00003288.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "His hand remains over the laptop keyboard and changes position slightly as he uses it.", "fps": 2.0, "input_frame_count": 5, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0], "source_frame_indices": [3258, 3264, 3273, 3279, 3288]}}, {"text": "watches", "span": [88, 95], "spans": [[88, 95]], "particle": null, "mentions": [{"text": "watches", "span": [88, 95], "spans": [[88, 95]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "He", "span": [69, 71], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "it", "span": [96, 98], "groups": [2], "group_mentions": [2], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 2, "text": "watches", "status": "visible", "segments_frames": [[0, 4]], "segments": [[0.0, 2.2]], "segment_details": [{"start_frame": 0, "end_frame": 4, "start_frame_1based": 1, "end_frame_1based": 5, "start_s": 0.0, "end_s": 2.2, "last_observed_s": 2.0, "start_source_frame": 3258, "end_source_frame": 3288, "start_filename": "00003258.jpg", "end_filename": "00003288.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "His gaze stays directed toward the laptop across the clean frames.", "fps": 2.0, "input_frame_count": 5, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0], "source_frame_indices": [3258, 3264, 3273, 3279, 3288]}}], "coverage": {"marks_shown": 8, "marks_grounded": 3, "things_shown": 4, "things_grounded": 2, "stuff_shown": 4, "stuff_grounded": 1, "n_mentions": 6, "category_disagree": 1, "category_checked": 3}, "marks": [{"mark_id": 8, "category": "computer", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 11, "first_frame": 0, "last_frame": 10, "best_frame": 4, "max_rel_area": 0.28302, "mean_rel_area": 0.24585, "centroid": [0.3, 0.584], "motion": 0.027}, {"mark_id": 5, "category": "person", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 11, "first_frame": 0, "last_frame": 10, "best_frame": 5, "max_rel_area": 0.26326, "mean_rel_area": 0.24228, "centroid": [0.604, 0.484], "motion": 0.024}, {"mark_id": 6, "category": "bag or package", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 11, "first_frame": 0, "last_frame": 10, "best_frame": 0, "max_rel_area": 0.04046, "mean_rel_area": 0.03891, "centroid": [0.128, 0.159], "motion": 0.038}, {"mark_id": 7, "category": "bag or package", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 11, "first_frame": 0, "last_frame": 10, "best_frame": 3, "max_rel_area": 0.03225, "mean_rel_area": 0.02938, "centroid": [0.897, 0.38], "motion": 0.057}, {"mark_id": 2, "category": "floor", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 11, "first_frame": 0, "last_frame": 10, "best_frame": 9, "max_rel_area": 0.24772, "mean_rel_area": 0.21139, "centroid": [0.5, 0.584], "motion": 0.071}, {"mark_id": 3, "category": "cupboard or showcase or storage rack", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 11, "first_frame": 0, "last_frame": 10, "best_frame": 3, "max_rel_area": 0.18412, "mean_rel_area": 0.16025, "centroid": [0.607, 0.289], "motion": 0.026}, {"mark_id": 4, "category": "clothes", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 11, "first_frame": 0, "last_frame": 10, "best_frame": 0, "max_rel_area": 0.07997, "mean_rel_area": 0.05626, "centroid": [0.894, 0.154], "motion": 0.039}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 11, "first_frame": 0, "last_frame": 10, "best_frame": 2, "max_rel_area": 0.00773, "mean_rel_area": 0.00378, "centroid": [0.171, 0.055], "motion": 0.113}], "num_frames": 11, "duration_s": 2.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0], "source_frame_indices": [3258, 3261, 3264, 3267, 3270, 3273, 3276, 3279, 3282, 3285, 3288], "timing_sha": "1c7fb8713775", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 6, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 16052, "output_tokens": 1106, "thinking_tokens": 435, "total_tokens": 17158}, "seconds": 20.48, "response_id": "resp_0abbff2acadc602d006ac32fbf46ec87d0aa421f752ecf85d3", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 3258, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 3264, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 3273, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 3279, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 3288, "time_s": 2.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 3258, "time_s": 0.0, "mark_ids": [8, 5, 6, 7, 2, 3, 4, 1]}, {"role": "marks", "path": "marks/f002_p0.jpg", "frame": 2, "panel": 0, "source_frame": 3264, "time_s": 0.4, "mark_ids": [8, 5, 6, 7, 2, 3, 4, 1]}, {"role": "marks", "path": "marks/f007_p0.jpg", "frame": 7, "panel": 0, "source_frame": 3279, "time_s": 1.4, "mark_ids": [8, 5, 6, 7, 2, 3, 4, 1]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 3288, "time_s": 2.0, "mark_ids": [8, 5, 6, 7, 2, 3, 4, 1]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [8, 5, 6, 7, 2, 3, 4, 1]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "1c7fb8713775", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 5, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0], "source_frame_indices": [3258, 3264, 3273, 3279, 3288], "actions": [{"action_index": 0, "text": "is seated", "status": "visible", "segments_frames": [[0, 4]], "segments": [[0.0, 2.2]], "segment_details": [{"start_frame": 0, "end_frame": 4, "start_frame_1based": 1, "end_frame_1based": 5, "start_s": 0.0, "end_s": 2.2, "last_observed_s": 2.0, "start_source_frame": 3258, "end_source_frame": 3288, "start_filename": "00003258.jpg", "end_filename": "00003288.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The boy maintains a cross-legged seated posture beside the laptop throughout the sequence.", "fps": 2.0, "input_frame_count": 5, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0], "source_frame_indices": [3258, 3264, 3273, 3279, 3288]}, {"action_index": 1, "text": "operates", "status": "visible", "segments_frames": [[0, 4]], "segments": [[0.0, 2.2]], "segment_details": [{"start_frame": 0, "end_frame": 4, "start_frame_1based": 1, "end_frame_1based": 5, "start_s": 0.0, "end_s": 2.2, "last_observed_s": 2.0, "start_source_frame": 3258, "end_source_frame": 3288, "start_filename": "00003258.jpg", "end_filename": "00003288.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "His hand remains over the laptop keyboard and changes position slightly as he uses it.", "fps": 2.0, "input_frame_count": 5, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0], "source_frame_indices": [3258, 3264, 3273, 3279, 3288]}, {"action_index": 2, "text": "watches", "status": "visible", "segments_frames": [[0, 4]], "segments": [[0.0, 2.2]], "segment_details": [{"start_frame": 0, "end_frame": 4, "start_frame_1based": 1, "end_frame_1based": 5, "start_s": 0.0, "end_s": 2.2, "last_observed_s": 2.0, "start_source_frame": 3258, "end_source_frame": 3288, "start_filename": "00003258.jpg", "end_filename": "00003288.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "His gaze stays directed toward the laptop across the clean frames.", "fps": 2.0, "input_frame_count": 5, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0], "source_frame_indices": [3258, 3264, 3273, 3279, 3288]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "2331_WP9xxG7oYik", "split": "train", "caption": "A woman stands in front of a wall of framed posters. She then gestures inward.", "mentions": [{"text": "A woman", "span": [0, 7], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a wall", "span": [27, 33], "occurrence": 1, "mark_ids": [1], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "framed posters", "span": [37, 51], "occurrence": 1, "mark_ids": [7, 8, 9, 10, 11], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "She", "span": [53, 56], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 1, "is_pronoun": true}], "sentences": [[0, 52], [53, 78]], "groups": [{"group_id": 0, "mark_id": 6, "category": "person", "isthing": 1, "mentions": [{"text": "A woman", "span": [0, 7], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "She", "span": [53, 56], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 1, "category": "wall", "isthing": 0, "mentions": [{"text": "a wall", "span": [27, 33], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 7, "category": "painting or poster", "isthing": 1, "mentions": [{"text": "framed posters", "span": [37, 51], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 8, "category": "painting or poster", "isthing": 1, "mentions": [{"text": "framed posters", "span": [37, 51], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 9, "category": "painting or poster", "isthing": 1, "mentions": [{"text": "framed posters", "span": [37, 51], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 5, "mark_id": 10, "category": "painting or poster", "isthing": 1, "mentions": [{"text": "framed posters", "span": [37, 51], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 6, "mark_id": 11, "category": "painting or poster", "isthing": 1, "mentions": [{"text": "framed posters", "span": [37, 51], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "stands", "span": [8, 14], "spans": [[8, 14]], "particle": null, "mentions": [{"text": "stands", "span": [8, 14], "spans": [[8, 14]], "particle": null, "sentence_index": 0}], "type": "posture", "sentence_index": 0, "subject": {"text": "A woman", "span": [0, 7], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 0, "text": "stands", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 227, "end_source_frame": 269, "start_filename": "00000227.jpg", "end_filename": "00000269.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The woman remains upright in the same place throughout the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [227, 233, 242, 248, 257, 263, 269]}}, {"text": "gestures", "span": [62, 70], "spans": [[62, 70]], "particle": null, "mentions": [{"text": "gestures", "span": [62, 70], "spans": [[62, 70]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "She", "span": [53, 56], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [], "confidence": "medium", "temporal": {"action_index": 1, "text": "gestures", "status": "visible", "segments_frames": [[3, 5]], "segments": [[1.4, 2.8]], "segment_details": [{"start_frame": 3, "end_frame": 5, "start_frame_1based": 4, "end_frame_1based": 6, "start_s": 1.4, "end_s": 2.8, "last_observed_s": 2.4, "start_source_frame": 248, "end_source_frame": 263, "start_filename": "00000248.jpg", "end_filename": "00000263.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Her extended arms move inward and slightly downward before holding a closer position.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [227, 233, 242, 248, 257, 263, 269]}}], "coverage": {"marks_shown": 11, "marks_grounded": 7, "things_shown": 7, "things_grounded": 6, "stuff_shown": 4, "stuff_grounded": 1, "n_mentions": 8, "category_disagree": 0, "category_checked": 7}, "marks": [{"mark_id": 6, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.08601, "mean_rel_area": 0.08277, "centroid": [0.743, 0.553], "motion": 0.012}, {"mark_id": 7, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.04111, "mean_rel_area": 0.04032, "centroid": [0.103, 0.184], "motion": 0.002}, {"mark_id": 5, "category": "door", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.03479, "mean_rel_area": 0.03459, "centroid": [0.959, 0.456], "motion": 0.063}, {"mark_id": 8, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 4, "max_rel_area": 0.02841, "mean_rel_area": 0.02795, "centroid": [0.343, 0.21], "motion": 0.003}, {"mark_id": 9, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 4, "max_rel_area": 0.02126, "mean_rel_area": 0.02071, "centroid": [0.538, 0.233], "motion": 0.003}, {"mark_id": 10, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.01419, "mean_rel_area": 0.01321, "centroid": [0.689, 0.249], "motion": 0.006}, {"mark_id": 11, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 10, "max_rel_area": 0.00128, "mean_rel_area": 0.00086, "centroid": [0.835, 0.241], "motion": 0.005}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 10, "max_rel_area": 0.33589, "mean_rel_area": 0.33178, "centroid": [0.5, 0.375], "motion": 0.05}, {"mark_id": 3, "category": "handrail or fence", "isthing": 0, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 5, "max_rel_area": 0.26118, "mean_rel_area": 0.26005, "centroid": [0.5, 0.79], "motion": 0.027}, {"mark_id": 4, "category": "floor", "isthing": 0, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 10, "max_rel_area": 0.05183, "mean_rel_area": 0.05032, "centroid": [0.739, 0.893], "motion": 0.003}, {"mark_id": 2, "category": "ceiling", "isthing": 0, "iscrowd": 0, "order": 10, "color": [139, 195, 74], "panel": 1, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 5, "max_rel_area": 0.01109, "mean_rel_area": 0.01069, "centroid": [0.873, 0.086], "motion": 0.125}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [227, 230, 233, 236, 239, 242, 245, 248, 251, 254, 257, 260, 263, 266, 269], "timing_sha": "0c97d8246ba6", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 2, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 19014, "output_tokens": 802, "thinking_tokens": 276, "total_tokens": 19816}, "seconds": 17.59, "response_id": "resp_0b5f3af25df1d911006ac32fc5331487d0b7bf8cf5f11a5b2d", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 227, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 233, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 242, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 248, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 257, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 263, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 269, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 227, "time_s": 0.0, "mark_ids": [6, 7, 5, 8, 9, 10, 11, 1, 3, 4]}, {"role": "marks", "path": "marks/f000_p1.jpg", "frame": 0, "panel": 1, "source_frame": 227, "time_s": 0.0, "mark_ids": [2]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 242, "time_s": 1.0, "mark_ids": [6, 7, 5, 8, 9, 10, 11, 1, 3, 4]}, {"role": "marks", "path": "marks/f005_p1.jpg", "frame": 5, "panel": 1, "source_frame": 242, "time_s": 1.0, "mark_ids": [2]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 257, "time_s": 2.0, "mark_ids": [6, 7, 5, 8, 9, 10, 11, 1, 3, 4]}, {"role": "marks", "path": "marks/f010_p1.jpg", "frame": 10, "panel": 1, "source_frame": 257, "time_s": 2.0, "mark_ids": [2]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 269, "time_s": 2.8, "mark_ids": [6, 7, 5, 8, 9, 10, 11, 1, 3, 4]}, {"role": "marks", "path": "marks/f014_p1.jpg", "frame": 14, "panel": 1, "source_frame": 269, "time_s": 2.8, "mark_ids": [2]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [6, 7, 5, 8, 9, 10, 11, 1, 3, 4, 2]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "0c97d8246ba6", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [227, 233, 242, 248, 257, 263, 269], "actions": [{"action_index": 0, "text": "stands", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 227, "end_source_frame": 269, "start_filename": "00000227.jpg", "end_filename": "00000269.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The woman remains upright in the same place throughout the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [227, 233, 242, 248, 257, 263, 269]}, {"action_index": 1, "text": "gestures", "status": "visible", "segments_frames": [[3, 5]], "segments": [[1.4, 2.8]], "segment_details": [{"start_frame": 3, "end_frame": 5, "start_frame_1based": 4, "end_frame_1based": 6, "start_s": 1.4, "end_s": 2.8, "last_observed_s": 2.4, "start_source_frame": 248, "end_source_frame": 263, "start_filename": "00000248.jpg", "end_filename": "00000263.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Her extended arms move inward and slightly downward before holding a closer position.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [227, 233, 242, 248, 257, 263, 269]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "298_ma7hngzCd-I", "split": "train", "caption": "A girl watches many long-snouted fish swim past. She gestures toward them. A large fish glides close by.", "mentions": [{"text": "A girl", "span": [0, 6], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "mark_ids": [29, 18, 10, 7, 12, 6, 8, 11, 5, 4, 16, 19, 15, 22, 14, 21, 17, 13, 25, 24, 23, 31, 9, 26, 30, 27, 28, 32, 20], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "She", "span": [49, 52], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "mark_ids": [29, 18, 10, 7, 12, 6, 8, 11, 5, 4, 16, 19, 15, 22, 14, 21, 17, 13, 25, 24, 23, 31, 9, 26, 30, 27, 28, 32, 20], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "A large fish", "span": [75, 87], "occurrence": 1, "mark_ids": [29], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 48], [49, 74], [75, 104]], "groups": [{"group_id": 0, "mark_id": 3, "category": "person", "isthing": 1, "mentions": [{"text": "A girl", "span": [0, 6], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "She", "span": [49, 52], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 29, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "A large fish", "span": [75, 87], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 18, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 10, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 7, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 5, "mark_id": 12, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 6, "mark_id": 6, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 7, "mark_id": 8, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 8, "mark_id": 11, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 9, "mark_id": 5, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 10, "mark_id": 4, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 11, "mark_id": 16, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 12, "mark_id": 19, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 13, "mark_id": 15, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 14, "mark_id": 22, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 15, "mark_id": 14, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 16, "mark_id": 21, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 17, "mark_id": 17, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 18, "mark_id": 13, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 19, "mark_id": 25, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 20, "mark_id": 24, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 21, "mark_id": 23, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 22, "mark_id": 31, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 23, "mark_id": 9, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 24, "mark_id": 26, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 25, "mark_id": 30, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 26, "mark_id": 27, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 27, "mark_id": 28, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 28, "mark_id": 32, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 29, "mark_id": 20, "category": "other animal", "isthing": 1, "mentions": [{"text": "many long-snouted fish", "span": [15, 37], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [69, 73], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}], "actions": [{"text": "watches", "span": [7, 14], "spans": [[7, 14]], "particle": null, "mentions": [{"text": "watches", "span": [7, 14], "spans": [[7, 14]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "A girl", "span": [0, 6], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "many long-snouted fish", "span": [15, 37], "groups": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29], "group_mentions": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "watches", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.4]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.4, "last_observed_s": 3.0, "start_source_frame": 1337, "end_source_frame": 1382, "start_filename": "00001337.jpg", "end_filename": "00001382.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The girl faces the fish and follows their movement with her gaze.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1337, 1343, 1352, 1358, 1367, 1373, 1382, 1388, 1397, 1403, 1412, 1418, 1427, 1433, 1442, 1448, 1457, 1463, 1472]}}, {"text": "swim", "span": [38, 42], "spans": [[38, 42]], "particle": null, "mentions": [{"text": "swim", "span": [38, 42], "spans": [[38, 42]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "many long-snouted fish", "span": [15, 37], "groups": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29], "group_mentions": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 1, "text": "swim", "status": "visible", "segments_frames": [[0, 18]], "segments": [[0.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 18, "start_frame_1based": 1, "end_frame_1based": 19, "start_s": 0.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 1337, "end_source_frame": 1472, "start_filename": "00001337.jpg", "end_filename": "00001472.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Fish continually swim through the scene, although individual members appear for different portions of the clip.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1337, 1343, 1352, 1358, 1367, 1373, 1382, 1388, 1397, 1403, 1412, 1418, 1427, 1433, 1442, 1448, 1457, 1463, 1472]}}, {"text": "gestures", "span": [53, 61], "spans": [[53, 61]], "particle": null, "mentions": [{"text": "gestures", "span": [53, 61], "spans": [[53, 61]], "particle": null, "sentence_index": 1}], "type": "prepositional", "sentence_index": 1, "subject": {"text": "She", "span": [49, 52], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "them", "span": [69, 73], "groups": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29], "group_mentions": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 2, "text": "gestures", "status": "visible", "segments_frames": [[5, 5], [7, 7], [14, 14]], "segments": [[2.4, 2.9], [3.4, 3.9], [7.0, 7.4]], "segment_details": [{"start_frame": 5, "end_frame": 5, "start_frame_1based": 6, "end_frame_1based": 6, "start_s": 2.4, "end_s": 2.9, "last_observed_s": 2.4, "start_source_frame": 1373, "end_source_frame": 1373, "start_filename": "00001373.jpg", "end_filename": "00001373.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 7, "end_frame": 7, "start_frame_1based": 8, "end_frame_1based": 8, "start_s": 3.4, "end_s": 3.9, "last_observed_s": 3.4, "start_source_frame": 1388, "end_source_frame": 1388, "start_filename": "00001388.jpg", "end_filename": "00001388.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 14, "end_frame": 14, "start_frame_1based": 15, "end_frame_1based": 15, "start_s": 7.0, "end_s": 7.4, "last_observed_s": 7.0, "start_source_frame": 1442, "end_source_frame": 1442, "start_filename": "00001442.jpg", "end_filename": "00001442.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "She extends a hand toward the fish in three observed gestures, returning it toward her face between the first two.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1337, 1343, 1352, 1358, 1367, 1373, 1382, 1388, 1397, 1403, 1412, 1418, 1427, 1433, 1442, 1448, 1457, 1463, 1472]}}, {"text": "glides", "span": [88, 94], "spans": [[88, 94]], "particle": null, "mentions": [{"text": "glides", "span": [88, 94], "spans": [[88, 94]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "A large fish", "span": [75, 87], "groups": [1], "group_mentions": [2], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 3, "text": "glides", "status": "visible", "segments_frames": [[12, 18]], "segments": [[6.0, 9.2]], "segment_details": [{"start_frame": 12, "end_frame": 18, "start_frame_1based": 13, "end_frame_1based": 19, "start_s": 6.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 1427, "end_source_frame": 1472, "start_filename": "00001427.jpg", "end_filename": "00001472.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The large foreground fish passes along the upper scene and becomes prominently visible close by.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1337, 1343, 1352, 1358, 1367, 1373, 1382, 1388, 1397, 1403, 1412, 1418, 1427, 1433, 1442, 1448, 1457, 1463, 1472]}}], "coverage": {"marks_shown": 32, "marks_grounded": 30, "things_shown": 30, "things_grounded": 30, "stuff_shown": 2, "stuff_grounded": 0, "n_mentions": 61, "category_disagree": 0, "category_checked": 30}, "marks": [{"mark_id": 3, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 21, "first_frame": 0, "last_frame": 35, "best_frame": 11, "max_rel_area": 0.46561, "mean_rel_area": 0.31227, "centroid": [0.741, 0.477], "motion": 0.283}, {"mark_id": 29, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 29, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.16217, "mean_rel_area": 0.08604, "centroid": [0.337, 0.268], "motion": 0.268}, {"mark_id": 18, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 28, "first_frame": 18, "last_frame": 45, "best_frame": 30, "max_rel_area": 0.03708, "mean_rel_area": 0.02337, "centroid": [0.19, 0.595], "motion": 0.312}, {"mark_id": 10, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 32, "first_frame": 0, "last_frame": 45, "best_frame": 22, "max_rel_area": 0.02567, "mean_rel_area": 0.00806, "centroid": [0.452, 0.575], "motion": 0.591}, {"mark_id": 7, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 37, "first_frame": 0, "last_frame": 41, "best_frame": 39, "max_rel_area": 0.01706, "mean_rel_area": 0.00518, "centroid": [0.512, 0.533], "motion": 0.623}, {"mark_id": 12, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 24, "first_frame": 0, "last_frame": 36, "best_frame": 25, "max_rel_area": 0.03795, "mean_rel_area": 0.01829, "centroid": [0.195, 0.277], "motion": 0.337}, {"mark_id": 6, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 36, "first_frame": 0, "last_frame": 37, "best_frame": 34, "max_rel_area": 0.01586, "mean_rel_area": 0.00442, "centroid": [0.471, 0.7], "motion": 0.723}, {"mark_id": 8, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 38, "first_frame": 0, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.01057, "mean_rel_area": 0.00524, "centroid": [0.528, 0.668], "motion": 0.508}, {"mark_id": 11, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 33, "first_frame": 0, "last_frame": 45, "best_frame": 15, "max_rel_area": 0.01276, "mean_rel_area": 0.00647, "centroid": [0.546, 0.547], "motion": 0.47}, {"mark_id": 5, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 26, "first_frame": 0, "last_frame": 45, "best_frame": 35, "max_rel_area": 0.01749, "mean_rel_area": 0.00573, "centroid": [0.293, 0.455], "motion": 0.507}, {"mark_id": 4, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 10, "color": [139, 195, 74], "panel": 1, "n_present": 32, "first_frame": 0, "last_frame": 45, "best_frame": 20, "max_rel_area": 0.0105, "mean_rel_area": 0.00519, "centroid": [0.553, 0.545], "motion": 0.497}, {"mark_id": 16, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 11, "color": [255, 87, 34], "panel": 1, "n_present": 23, "first_frame": 17, "last_frame": 39, "best_frame": 37, "max_rel_area": 0.01574, "mean_rel_area": 0.01036, "centroid": [0.511, 0.636], "motion": 0.623}, {"mark_id": 19, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 12, "color": [156, 39, 176], "panel": 1, "n_present": 28, "first_frame": 18, "last_frame": 45, "best_frame": 19, "max_rel_area": 0.01031, "mean_rel_area": 0.00669, "centroid": [0.415, 0.697], "motion": 0.561}, {"mark_id": 15, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 13, "color": [0, 188, 212], "panel": 1, "n_present": 33, "first_frame": 11, "last_frame": 45, "best_frame": 25, "max_rel_area": 0.0074, "mean_rel_area": 0.00389, "centroid": [0.708, 0.335], "motion": 0.348}, {"mark_id": 22, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 14, "color": [121, 85, 72], "panel": 1, "n_present": 26, "first_frame": 20, "last_frame": 45, "best_frame": 22, "max_rel_area": 0.00937, "mean_rel_area": 0.00643, "centroid": [0.168, 0.725], "motion": 0.339}, {"mark_id": 14, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 15, "color": [233, 30, 99], "panel": 1, "n_present": 29, "first_frame": 5, "last_frame": 40, "best_frame": 25, "max_rel_area": 0.00735, "mean_rel_area": 0.00258, "centroid": [0.327, 0.509], "motion": 0.818}, {"mark_id": 21, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 16, "color": [63, 81, 181], "panel": 1, "n_present": 27, "first_frame": 19, "last_frame": 45, "best_frame": 20, "max_rel_area": 0.00746, "mean_rel_area": 0.00393, "centroid": [0.392, 0.644], "motion": 0.499}, {"mark_id": 17, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 17, "color": [205, 220, 57], "panel": 1, "n_present": 29, "first_frame": 17, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.00536, "mean_rel_area": 0.00328, "centroid": [0.54, 0.377], "motion": 0.654}, {"mark_id": 13, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 18, "color": [255, 152, 0], "panel": 1, "n_present": 16, "first_frame": 0, "last_frame": 17, "best_frame": 17, "max_rel_area": 0.01124, "mean_rel_area": 0.00341, "centroid": [0.654, 0.432], "motion": 0.184}, {"mark_id": 25, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 19, "color": [96, 125, 139], "panel": 1, "n_present": 17, "first_frame": 26, "last_frame": 45, "best_frame": 36, "max_rel_area": 0.00957, "mean_rel_area": 0.00438, "centroid": [0.193, 0.483], "motion": 0.269}, {"mark_id": 24, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 20, "color": [54, 67, 244], "panel": 2, "n_present": 21, "first_frame": 25, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.00419, "mean_rel_area": 0.00244, "centroid": [0.501, 0.499], "motion": 0.268}, {"mark_id": 23, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 21, "color": [34, 189, 255], "panel": 2, "n_present": 21, "first_frame": 25, "last_frame": 45, "best_frame": 35, "max_rel_area": 0.0036, "mean_rel_area": 0.00221, "centroid": [0.514, 0.627], "motion": 0.311}, {"mark_id": 31, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 22, "color": [60, 200, 60], "panel": 2, "n_present": 11, "first_frame": 35, "last_frame": 45, "best_frame": 40, "max_rel_area": 0.00936, "mean_rel_area": 0.00606, "centroid": [0.15, 0.606], "motion": 0.233}, {"mark_id": 9, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 23, "color": [240, 120, 40], "panel": 2, "n_present": 17, "first_frame": 0, "last_frame": 45, "best_frame": 19, "max_rel_area": 0.00364, "mean_rel_area": 0.00085, "centroid": [0.628, 0.567], "motion": 0.38}, {"mark_id": 26, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 24, "color": [180, 60, 220], "panel": 2, "n_present": 7, "first_frame": 28, "last_frame": 34, "best_frame": 28, "max_rel_area": 0.01134, "mean_rel_area": 0.00958, "centroid": [0.224, 0.551], "motion": 0.157}, {"mark_id": 30, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 25, "color": [0, 215, 255], "panel": 2, "n_present": 5, "first_frame": 32, "last_frame": 45, "best_frame": 34, "max_rel_area": 0.00898, "mean_rel_area": 0.00628, "centroid": [0.713, 0.685], "motion": 0.399}, {"mark_id": 27, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 26, "color": [200, 90, 130], "panel": 2, "n_present": 4, "first_frame": 28, "last_frame": 31, "best_frame": 30, "max_rel_area": 0.00406, "mean_rel_area": 0.00336, "centroid": [0.042, 0.654], "motion": 0.043}, {"mark_id": 28, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 27, "color": [80, 175, 76], "panel": 2, "n_present": 3, "first_frame": 29, "last_frame": 31, "best_frame": 30, "max_rel_area": 0.00455, "mean_rel_area": 0.00218, "centroid": [0.024, 0.59], "motion": 0.025}, {"mark_id": 32, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 28, "color": [255, 100, 180], "panel": 2, "n_present": 6, "first_frame": 40, "last_frame": 45, "best_frame": 44, "max_rel_area": 0.00022, "mean_rel_area": 0.00017, "centroid": [0.341, 0.915], "motion": 0.036}, {"mark_id": 20, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 29, "color": [0, 140, 255], "panel": 2, "n_present": 2, "first_frame": 18, "last_frame": 19, "best_frame": 18, "max_rel_area": 0.00011, "mean_rel_area": 0.0001, "centroid": [0.831, 0.467], "motion": 0.154}, {"mark_id": 2, "category": "floor", "isthing": 0, "iscrowd": 0, "order": 30, "color": [139, 195, 74], "panel": 3, "n_present": 22, "first_frame": 0, "last_frame": 27, "best_frame": 17, "max_rel_area": 0.03825, "mean_rel_area": 0.01279, "centroid": [0.635, 0.922], "motion": 0.345}, {"mark_id": 1, "category": "ceiling", "isthing": 0, "iscrowd": 0, "order": 31, "color": [255, 87, 34], "panel": 3, "n_present": 19, "first_frame": 0, "last_frame": 22, "best_frame": 3, "max_rel_area": 0.0312, "mean_rel_area": 0.01788, "centroid": [0.559, 0.086], "motion": 0.343}], "num_frames": 46, "duration_s": 9.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0, 4.2, 4.4, 4.6, 4.8, 5.0, 5.2, 5.4, 5.6, 5.8, 6.0, 6.2, 6.4, 6.6, 6.8, 7.0, 7.2, 7.4, 7.6, 7.8, 8.0, 8.2, 8.4, 8.6, 8.8, 9.0], "source_frame_indices": [1337, 1340, 1343, 1346, 1349, 1352, 1355, 1358, 1361, 1364, 1367, 1370, 1373, 1376, 1379, 1382, 1385, 1388, 1391, 1394, 1397, 1400, 1403, 1406, 1409, 1412, 1415, 1418, 1421, 1424, 1427, 1430, 1433, 1436, 1439, 1442, 1445, 1448, 1451, 1454, 1457, 1460, 1463, 1466, 1469, 1472], "timing_sha": "6ece758397a7", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 6, "unlinked": 0, "snapped": 0, "collective": 3, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 29869, "output_tokens": 1297, "thinking_tokens": 315, "total_tokens": 31166}, "seconds": 23.13, "response_id": "resp_07fb3c88bf47feea006ac32fd3df2c87d0991f6b7fa25b6f95", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 1337, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 1343, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 1352, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 1358, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 1367, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 1373, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 1382, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 1388, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 1397, "time_s": 4.0}, {"role": "clean", "path": "clean/f022.jpg", "frame": 9, "source_frame": 1403, "time_s": 4.4}, {"role": "clean", "path": "clean/f025.jpg", "frame": 10, "source_frame": 1412, "time_s": 5.0}, {"role": "clean", "path": "clean/f027.jpg", "frame": 11, "source_frame": 1418, "time_s": 5.4}, {"role": "clean", "path": "clean/f030.jpg", "frame": 12, "source_frame": 1427, "time_s": 6.0}, {"role": "clean", "path": "clean/f032.jpg", "frame": 13, "source_frame": 1433, "time_s": 6.4}, {"role": "clean", "path": "clean/f035.jpg", "frame": 14, "source_frame": 1442, "time_s": 7.0}, {"role": "clean", "path": "clean/f037.jpg", "frame": 15, "source_frame": 1448, "time_s": 7.4}, {"role": "clean", "path": "clean/f040.jpg", "frame": 16, "source_frame": 1457, "time_s": 8.0}, {"role": "clean", "path": "clean/f042.jpg", "frame": 17, "source_frame": 1463, "time_s": 8.4}, {"role": "clean", "path": "clean/f045.jpg", "frame": 18, "source_frame": 1472, "time_s": 9.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 1337, "time_s": 0.0, "mark_ids": [3, 10, 7, 12, 6, 8, 11, 5]}, {"role": "marks", "path": "marks/f000_p1.jpg", "frame": 0, "panel": 1, "source_frame": 1337, "time_s": 0.0, "mark_ids": [4, 13]}, {"role": "marks", "path": "marks/f000_p2.jpg", "frame": 0, "panel": 2, "source_frame": 1337, "time_s": 0.0, "mark_ids": [9]}, {"role": "marks", "path": "marks/f000_p3.jpg", "frame": 0, "panel": 3, "source_frame": 1337, "time_s": 0.0, "mark_ids": [2, 1]}, {"role": "marks", "path": "marks/f015_p0.jpg", "frame": 15, "panel": 0, "source_frame": 1382, "time_s": 3.0, "mark_ids": [3, 7, 6, 8, 11]}, {"role": "marks", "path": "marks/f015_p1.jpg", "frame": 15, "panel": 1, "source_frame": 1382, "time_s": 3.0, "mark_ids": [4, 15, 14, 13]}, {"role": "marks", "path": "marks/f015_p2.jpg", "frame": 15, "panel": 2, "source_frame": 1382, "time_s": 3.0, "mark_ids": []}, {"role": "marks", "path": "marks/f015_p3.jpg", "frame": 15, "panel": 3, "source_frame": 1382, "time_s": 3.0, "mark_ids": [2, 1]}, {"role": "marks", "path": "marks/f030_p0.jpg", "frame": 30, "panel": 0, "source_frame": 1427, "time_s": 6.0, "mark_ids": [29, 18, 10, 12]}, {"role": "marks", "path": "marks/f030_p1.jpg", "frame": 30, "panel": 1, "source_frame": 1427, "time_s": 6.0, "mark_ids": [16, 19, 15, 22, 21, 17]}, {"role": "marks", "path": "marks/f030_p2.jpg", "frame": 30, "panel": 2, "source_frame": 1427, "time_s": 6.0, "mark_ids": [24, 23, 26, 27, 28]}, {"role": "marks", "path": "marks/f030_p3.jpg", "frame": 30, "panel": 3, "source_frame": 1427, "time_s": 6.0, "mark_ids": []}, {"role": "marks", "path": "marks/f045_p0.jpg", "frame": 45, "panel": 0, "source_frame": 1472, "time_s": 9.0, "mark_ids": [29, 18, 10, 8, 11, 5]}, {"role": "marks", "path": "marks/f045_p1.jpg", "frame": 45, "panel": 1, "source_frame": 1472, "time_s": 9.0, "mark_ids": [4, 19, 15, 22, 21, 17, 25]}, {"role": "marks", "path": "marks/f045_p2.jpg", "frame": 45, "panel": 2, "source_frame": 1472, "time_s": 9.0, "mark_ids": [24, 23, 31, 9, 30, 32]}, {"role": "marks", "path": "marks/f045_p3.jpg", "frame": 45, "panel": 3, "source_frame": 1472, "time_s": 9.0, "mark_ids": []}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [3, 29, 18, 10, 7, 12, 6, 8, 11, 5, 4, 16, 19, 15, 22, 14]}, {"role": "ids", "path": "ids/sheet1.jpg", "mark_ids": [21, 17, 13, 25, 24, 23, 31, 9, 26, 30, 27, 28, 32, 20, 2, 1]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "6ece758397a7", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1337, 1343, 1352, 1358, 1367, 1373, 1382, 1388, 1397, 1403, 1412, 1418, 1427, 1433, 1442, 1448, 1457, 1463, 1472], "actions": [{"action_index": 0, "text": "watches", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.4]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.4, "last_observed_s": 3.0, "start_source_frame": 1337, "end_source_frame": 1382, "start_filename": "00001337.jpg", "end_filename": "00001382.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The girl faces the fish and follows their movement with her gaze.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1337, 1343, 1352, 1358, 1367, 1373, 1382, 1388, 1397, 1403, 1412, 1418, 1427, 1433, 1442, 1448, 1457, 1463, 1472]}, {"action_index": 1, "text": "swim", "status": "visible", "segments_frames": [[0, 18]], "segments": [[0.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 18, "start_frame_1based": 1, "end_frame_1based": 19, "start_s": 0.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 1337, "end_source_frame": 1472, "start_filename": "00001337.jpg", "end_filename": "00001472.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Fish continually swim through the scene, although individual members appear for different portions of the clip.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1337, 1343, 1352, 1358, 1367, 1373, 1382, 1388, 1397, 1403, 1412, 1418, 1427, 1433, 1442, 1448, 1457, 1463, 1472]}, {"action_index": 2, "text": "gestures", "status": "visible", "segments_frames": [[5, 5], [7, 7], [14, 14]], "segments": [[2.4, 2.9], [3.4, 3.9], [7.0, 7.4]], "segment_details": [{"start_frame": 5, "end_frame": 5, "start_frame_1based": 6, "end_frame_1based": 6, "start_s": 2.4, "end_s": 2.9, "last_observed_s": 2.4, "start_source_frame": 1373, "end_source_frame": 1373, "start_filename": "00001373.jpg", "end_filename": "00001373.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 7, "end_frame": 7, "start_frame_1based": 8, "end_frame_1based": 8, "start_s": 3.4, "end_s": 3.9, "last_observed_s": 3.4, "start_source_frame": 1388, "end_source_frame": 1388, "start_filename": "00001388.jpg", "end_filename": "00001388.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 14, "end_frame": 14, "start_frame_1based": 15, "end_frame_1based": 15, "start_s": 7.0, "end_s": 7.4, "last_observed_s": 7.0, "start_source_frame": 1442, "end_source_frame": 1442, "start_filename": "00001442.jpg", "end_filename": "00001442.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "She extends a hand toward the fish in three observed gestures, returning it toward her face between the first two.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1337, 1343, 1352, 1358, 1367, 1373, 1382, 1388, 1397, 1403, 1412, 1418, 1427, 1433, 1442, 1448, 1457, 1463, 1472]}, {"action_index": 3, "text": "glides", "status": "visible", "segments_frames": [[12, 18]], "segments": [[6.0, 9.2]], "segment_details": [{"start_frame": 12, "end_frame": 18, "start_frame_1based": 13, "end_frame_1based": 19, "start_s": 6.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 1427, "end_source_frame": 1472, "start_filename": "00001427.jpg", "end_filename": "00001472.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The large foreground fish passes along the upper scene and becomes prominently visible close by.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1337, 1343, 1352, 1358, 1367, 1373, 1382, 1388, 1397, 1403, 1412, 1418, 1427, 1433, 1442, 1448, 1457, 1463, 1472]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "390_TDIMeOVXQr0", "split": "train", "caption": "Many people are beside black lockers. Some walk past them, and others sit. One seated boy stands up and turns toward the lockers.", "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "mark_ids": [11, 10, 18, 14, 16, 21, 8, 15, 9, 5, 7, 6, 23, 12, 22, 17, 24, 27, 13, 26, 29, 28, 30], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "black lockers", "span": [23, 36], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "Some", "span": [38, 42], "occurrence": 1, "mark_ids": [5, 7, 9, 12, 8, 15, 18, 23], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "them", "span": [53, 57], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "others", "span": [63, 69], "occurrence": 1, "mark_ids": [10, 11, 16], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "One seated boy", "span": [75, 89], "occurrence": 1, "mark_ids": [10], "confidence": "high", "sentence_index": 2, "is_pronoun": false}, {"text": "the lockers", "span": [117, 128], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 37], [38, 74], [75, 129]], "groups": [{"group_id": 0, "mark_id": 11, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "others", "span": [63, 69], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 10, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "others", "span": [63, 69], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "One seated boy", "span": [75, 89], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 18, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "Some", "span": [38, 42], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 14, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 16, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "others", "span": [63, 69], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 5, "mark_id": 21, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 6, "mark_id": 8, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "Some", "span": [38, 42], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 7, "mark_id": 15, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "Some", "span": [38, 42], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 8, "mark_id": 9, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "Some", "span": [38, 42], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 9, "mark_id": 5, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "Some", "span": [38, 42], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 10, "mark_id": 7, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "Some", "span": [38, 42], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 11, "mark_id": 6, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 12, "mark_id": 23, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "Some", "span": [38, 42], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 13, "mark_id": 12, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "Some", "span": [38, 42], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 14, "mark_id": 22, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 15, "mark_id": 17, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 16, "mark_id": 24, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 17, "mark_id": 27, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 18, "mark_id": 13, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 19, "mark_id": 26, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 20, "mark_id": 29, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 21, "mark_id": 28, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 22, "mark_id": 30, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 23, "mark_id": 3, "category": "cupboard or showcase or storage rack", "isthing": 0, "mentions": [{"text": "black lockers", "span": [23, 36], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "them", "span": [53, 57], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "the lockers", "span": [117, 128], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}], "actions": [{"text": "walk", "span": [43, 47], "spans": [[43, 47]], "particle": null, "mentions": [{"text": "walk", "span": [43, 47], "spans": [[43, 47]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "Some", "span": [38, 42], "groups": [2, 6, 7, 8, 9, 10, 12, 13], "group_mentions": [1, 1, 1, 1, 1, 1, 1, 1], "group_match": "exact"}, "objects": [{"text": "them", "span": [53, 57], "groups": [23], "group_mentions": [1], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "walk", "status": "visible", "segments_frames": [[0, 5], [6, 10], [14, 18]], "segments": [[0.0, 2.9], [3.0, 5.4], [7.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 5, "start_frame_1based": 1, "end_frame_1based": 6, "start_s": 0.0, "end_s": 2.9, "last_observed_s": 2.4, "start_source_frame": 602, "end_source_frame": 638, "start_filename": "00000602.jpg", "end_filename": "00000638.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 6, "end_frame": 10, "start_frame_1based": 7, "end_frame_1based": 11, "start_s": 3.0, "end_s": 5.4, "last_observed_s": 5.0, "start_source_frame": 647, "end_source_frame": 677, "start_filename": "00000647.jpg", "end_filename": "00000677.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 14, "end_frame": 18, "start_frame_1based": 15, "end_frame_1based": 19, "start_s": 7.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 707, "end_source_frame": 737, "start_filename": "00000707.jpg", "end_filename": "00000737.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Different members walk past the lockers early, across the foreground midway, and through the room near the end.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}}, {"text": "sit", "span": [70, 73], "spans": [[70, 73]], "particle": null, "mentions": [{"text": "sit", "span": [70, 73], "spans": [[70, 73]], "particle": null, "sentence_index": 1}], "type": "posture", "sentence_index": 1, "subject": {"text": "others", "span": [63, 69], "groups": [0, 1, 4], "group_mentions": [1, 1, 1], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 1, "text": "sit", "status": "visible", "segments_frames": [[0, 13]], "segments": [[0.0, 6.9]], "segment_details": [{"start_frame": 0, "end_frame": 13, "start_frame_1based": 1, "end_frame_1based": 14, "start_s": 0.0, "end_s": 6.9, "last_observed_s": 6.4, "start_source_frame": 602, "end_source_frame": 698, "start_filename": "00000602.jpg", "end_filename": "00000698.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "People remain seated along the lockers; the number-nine boy leaves his seat before the other seated people disappear from view.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}}, {"text": "seated", "span": [79, 85], "spans": [[79, 85]], "particle": null, "mentions": [{"text": "seated", "span": [79, 85], "spans": [[79, 85]], "particle": null, "sentence_index": 2}], "type": "posture", "sentence_index": 2, "subject": {"text": "One seated boy", "span": [75, 89], "groups": [1], "group_mentions": [2], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 2, "text": "seated", "status": "visible", "segments_frames": [[0, 10]], "segments": [[0.0, 5.4]], "segment_details": [{"start_frame": 0, "end_frame": 10, "start_frame_1based": 1, "end_frame_1based": 11, "start_s": 0.0, "end_s": 5.4, "last_observed_s": 5.0, "start_source_frame": 602, "end_source_frame": 677, "start_filename": "00000602.jpg", "end_filename": "00000677.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The number-nine boy sits at the lockers before beginning to rise.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}}, {"text": "stands up", "span": [90, 99], "spans": [[90, 99]], "particle": null, "mentions": [{"text": "stands up", "span": [90, 99], "spans": [[90, 99]], "particle": null, "sentence_index": 2}], "type": "phrasal", "sentence_index": 2, "subject": {"text": "One seated boy", "span": [75, 89], "groups": [1], "group_mentions": [2], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 3, "text": "stands up", "status": "visible", "segments_frames": [[10, 11]], "segments": [[5.0, 5.9]], "segment_details": [{"start_frame": 10, "end_frame": 11, "start_frame_1based": 11, "end_frame_1based": 12, "start_s": 5.0, "end_s": 5.9, "last_observed_s": 5.4, "start_source_frame": 677, "end_source_frame": 683, "start_filename": "00000677.jpg", "end_filename": "00000683.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The boy lifts himself from his seat and straightens his legs.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}}, {"text": "turns", "span": [104, 109], "spans": [[104, 109]], "particle": null, "mentions": [{"text": "turns", "span": [104, 109], "spans": [[104, 109]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "One seated boy", "span": [75, 89], "groups": [1], "group_mentions": [2], "group_match": "exact"}, "objects": [{"text": "the lockers", "span": [117, 128], "groups": [23], "group_mentions": [2], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 4, "text": "turns", "status": "visible", "segments_frames": [[11, 12]], "segments": [[5.4, 6.4]], "segment_details": [{"start_frame": 11, "end_frame": 12, "start_frame_1based": 12, "end_frame_1based": 13, "start_s": 5.4, "end_s": 6.4, "last_observed_s": 6.0, "start_source_frame": 683, "end_source_frame": 692, "start_filename": "00000683.jpg", "end_filename": "00000692.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "After rising, the boy rotates until his back faces the viewer and he faces the lockers.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}}], "coverage": {"marks_shown": 30, "marks_grounded": 24, "things_shown": 25, "things_grounded": 23, "stuff_shown": 5, "stuff_grounded": 1, "n_mentions": 38, "category_disagree": 0, "category_checked": 24}, "marks": [{"mark_id": 11, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 38, "first_frame": 0, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.08701, "mean_rel_area": 0.02867, "centroid": [0.589, 0.634], "motion": 0.769}, {"mark_id": 10, "category": "person", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 32, "first_frame": 0, "last_frame": 35, "best_frame": 31, "max_rel_area": 0.10836, "mean_rel_area": 0.03624, "centroid": [0.569, 0.679], "motion": 0.83}, {"mark_id": 18, "category": "person", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 19, "first_frame": 19, "last_frame": 45, "best_frame": 20, "max_rel_area": 0.26736, "mean_rel_area": 0.08568, "centroid": [0.639, 0.56], "motion": 0.825}, {"mark_id": 19, "category": "door", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 27, "last_frame": 41, "best_frame": 38, "max_rel_area": 0.27679, "mean_rel_area": 0.15412, "centroid": [0.559, 0.469], "motion": 0.835}, {"mark_id": 14, "category": "person", "isthing": 1, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 23, "first_frame": 11, "last_frame": 36, "best_frame": 33, "max_rel_area": 0.09182, "mean_rel_area": 0.05136, "centroid": [0.535, 0.594], "motion": 0.852}, {"mark_id": 16, "category": "person", "isthing": 1, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 20, "first_frame": 15, "last_frame": 34, "best_frame": 34, "max_rel_area": 0.08934, "mean_rel_area": 0.05485, "centroid": [0.604, 0.668], "motion": 0.751}, {"mark_id": 21, "category": "person", "isthing": 1, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 11, "first_frame": 35, "last_frame": 45, "best_frame": 41, "max_rel_area": 0.28943, "mean_rel_area": 0.19754, "centroid": [0.575, 0.568], "motion": 0.641}, {"mark_id": 8, "category": "person", "isthing": 1, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 13, "first_frame": 0, "last_frame": 45, "best_frame": 3, "max_rel_area": 0.12231, "mean_rel_area": 0.07913, "centroid": [0.843, 0.638], "motion": 0.184}, {"mark_id": 15, "category": "person", "isthing": 1, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 12, "first_frame": 13, "last_frame": 24, "best_frame": 22, "max_rel_area": 0.13703, "mean_rel_area": 0.08108, "centroid": [0.468, 0.574], "motion": 0.824}, {"mark_id": 9, "category": "person", "isthing": 1, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.06064, "mean_rel_area": 0.03894, "centroid": [0.416, 0.625], "motion": 0.546}, {"mark_id": 5, "category": "person", "isthing": 1, "iscrowd": 0, "order": 10, "color": [139, 195, 74], "panel": 1, "n_present": 18, "first_frame": 0, "last_frame": 44, "best_frame": 2, "max_rel_area": 0.04019, "mean_rel_area": 0.02208, "centroid": [0.329, 0.486], "motion": 0.736}, {"mark_id": 7, "category": "person", "isthing": 1, "iscrowd": 0, "order": 11, "color": [255, 87, 34], "panel": 1, "n_present": 14, "first_frame": 0, "last_frame": 13, "best_frame": 11, "max_rel_area": 0.05601, "mean_rel_area": 0.0408, "centroid": [0.285, 0.62], "motion": 0.405}, {"mark_id": 6, "category": "person", "isthing": 1, "iscrowd": 0, "order": 12, "color": [156, 39, 176], "panel": 1, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 2, "max_rel_area": 0.03802, "mean_rel_area": 0.02443, "centroid": [0.248, 0.594], "motion": 0.232}, {"mark_id": 23, "category": "person", "isthing": 1, "iscrowd": 0, "order": 13, "color": [0, 188, 212], "panel": 1, "n_present": 6, "first_frame": 40, "last_frame": 45, "best_frame": 42, "max_rel_area": 0.18528, "mean_rel_area": 0.12077, "centroid": [0.724, 0.631], "motion": 0.418}, {"mark_id": 12, "category": "person", "isthing": 1, "iscrowd": 0, "order": 14, "color": [121, 85, 72], "panel": 1, "n_present": 11, "first_frame": 0, "last_frame": 10, "best_frame": 0, "max_rel_area": 0.02954, "mean_rel_area": 0.01009, "centroid": [0.034, 0.533], "motion": 0.067}, {"mark_id": 22, "category": "person", "isthing": 1, "iscrowd": 0, "order": 15, "color": [233, 30, 99], "panel": 1, "n_present": 8, "first_frame": 37, "last_frame": 45, "best_frame": 39, "max_rel_area": 0.03918, "mean_rel_area": 0.02264, "centroid": [0.773, 0.478], "motion": 0.401}, {"mark_id": 4, "category": "door", "isthing": 1, "iscrowd": 0, "order": 16, "color": [63, 81, 181], "panel": 1, "n_present": 11, "first_frame": 0, "last_frame": 10, "best_frame": 8, "max_rel_area": 0.02002, "mean_rel_area": 0.01348, "centroid": [0.038, 0.4], "motion": 0.043}, {"mark_id": 17, "category": "person", "isthing": 1, "iscrowd": 0, "order": 17, "color": [205, 220, 57], "panel": 1, "n_present": 7, "first_frame": 16, "last_frame": 22, "best_frame": 18, "max_rel_area": 0.03672, "mean_rel_area": 0.02452, "centroid": [0.968, 0.746], "motion": 0.122}, {"mark_id": 24, "category": "person", "isthing": 1, "iscrowd": 0, "order": 18, "color": [255, 152, 0], "panel": 1, "n_present": 4, "first_frame": 42, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.00754, "mean_rel_area": 0.00566, "centroid": [0.788, 0.472], "motion": 0.208}, {"mark_id": 27, "category": "person", "isthing": 1, "iscrowd": 0, "order": 19, "color": [96, 125, 139], "panel": 1, "n_present": 2, "first_frame": 44, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.00692, "mean_rel_area": 0.00489, "centroid": [0.766, 0.856], "motion": 0.031}, {"mark_id": 13, "category": "person", "isthing": 1, "iscrowd": 0, "order": 20, "color": [54, 67, 244], "panel": 2, "n_present": 1, "first_frame": 0, "last_frame": 0, "best_frame": 0, "max_rel_area": 0.02735, "mean_rel_area": 0.02735, "centroid": [0.036, 0.722], "motion": 0.0}, {"mark_id": 26, "category": "person", "isthing": 1, "iscrowd": 0, "order": 21, "color": [34, 189, 255], "panel": 2, "n_present": 2, "first_frame": 44, "last_frame": 45, "best_frame": 44, "max_rel_area": 0.00291, "mean_rel_area": 0.00277, "centroid": [0.689, 0.491], "motion": 0.033}, {"mark_id": 29, "category": "person", "isthing": 1, "iscrowd": 0, "order": 22, "color": [60, 200, 60], "panel": 2, "n_present": 1, "first_frame": 45, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.00803, "mean_rel_area": 0.00803, "centroid": [0.98, 0.829], "motion": 0.0}, {"mark_id": 28, "category": "person", "isthing": 1, "iscrowd": 0, "order": 23, "color": [240, 120, 40], "panel": 2, "n_present": 1, "first_frame": 45, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.00535, "mean_rel_area": 0.00535, "centroid": [0.591, 0.428], "motion": 0.0}, {"mark_id": 30, "category": "person", "isthing": 1, "iscrowd": 0, "order": 24, "color": [180, 60, 220], "panel": 2, "n_present": 1, "first_frame": 45, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.00013, "mean_rel_area": 0.00013, "centroid": [0.611, 0.701], "motion": 0.0}, {"mark_id": 3, "category": "cupboard or showcase or storage rack", "isthing": 0, "iscrowd": 0, "order": 25, "color": [0, 215, 255], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 25, "max_rel_area": 0.53844, "mean_rel_area": 0.33791, "centroid": [0.52, 0.553], "motion": 0.293}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 26, "color": [200, 90, 130], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.34634, "mean_rel_area": 0.21088, "centroid": [0.54, 0.392], "motion": 0.337}, {"mark_id": 2, "category": "floor", "isthing": 0, "iscrowd": 0, "order": 27, "color": [80, 175, 76], "panel": 2, "n_present": 45, "first_frame": 0, "last_frame": 44, "best_frame": 6, "max_rel_area": 0.1533, "mean_rel_area": 0.0825, "centroid": [0.473, 0.87], "motion": 0.431}, {"mark_id": 20, "category": "trash can", "isthing": 0, "iscrowd": 0, "order": 28, "color": [255, 100, 180], "panel": 2, "n_present": 11, "first_frame": 33, "last_frame": 43, "best_frame": 34, "max_rel_area": 0.04691, "mean_rel_area": 0.03598, "centroid": [0.422, 0.809], "motion": 0.813}, {"mark_id": 25, "category": "ceiling", "isthing": 0, "iscrowd": 0, "order": 29, "color": [0, 140, 255], "panel": 2, "n_present": 3, "first_frame": 43, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.04174, "mean_rel_area": 0.0273, "centroid": [0.685, 0.046], "motion": 0.092}], "num_frames": 46, "duration_s": 9.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0, 4.2, 4.4, 4.6, 4.8, 5.0, 5.2, 5.4, 5.6, 5.8, 6.0, 6.2, 6.4, 6.6, 6.8, 7.0, 7.2, 7.4, 7.6, 7.8, 8.0, 8.2, 8.4, 8.6, 8.8, 9.0], "source_frame_indices": [602, 605, 608, 611, 614, 617, 620, 623, 626, 629, 632, 635, 638, 641, 644, 647, 650, 653, 656, 659, 662, 665, 668, 671, 674, 677, 680, 683, 686, 689, 692, 695, 698, 701, 704, 707, 710, 713, 716, 719, 722, 725, 728, 731, 734, 737], "timing_sha": "06779b767870", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 7, "unlinked": 0, "snapped": 0, "collective": 2, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 27990, "output_tokens": 1710, "thinking_tokens": 516, "total_tokens": 29700}, "seconds": 29.38, "response_id": "resp_0ebe6edb16088dc8006ac32fd705f887d09bbcc7d643263470", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 602, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 608, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 617, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 623, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 632, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 638, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 647, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 653, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 662, "time_s": 4.0}, {"role": "clean", "path": "clean/f022.jpg", "frame": 9, "source_frame": 668, "time_s": 4.4}, {"role": "clean", "path": "clean/f025.jpg", "frame": 10, "source_frame": 677, "time_s": 5.0}, {"role": "clean", "path": "clean/f027.jpg", "frame": 11, "source_frame": 683, "time_s": 5.4}, {"role": "clean", "path": "clean/f030.jpg", "frame": 12, "source_frame": 692, "time_s": 6.0}, {"role": "clean", "path": "clean/f032.jpg", "frame": 13, "source_frame": 698, "time_s": 6.4}, {"role": "clean", "path": "clean/f035.jpg", "frame": 14, "source_frame": 707, "time_s": 7.0}, {"role": "clean", "path": "clean/f037.jpg", "frame": 15, "source_frame": 713, "time_s": 7.4}, {"role": "clean", "path": "clean/f040.jpg", "frame": 16, "source_frame": 722, "time_s": 8.0}, {"role": "clean", "path": "clean/f042.jpg", "frame": 17, "source_frame": 728, "time_s": 8.4}, {"role": "clean", "path": "clean/f045.jpg", "frame": 18, "source_frame": 737, "time_s": 9.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 602, "time_s": 0.0, "mark_ids": [11, 10, 8, 9]}, {"role": "marks", "path": "marks/f000_p1.jpg", "frame": 0, "panel": 1, "source_frame": 602, "time_s": 0.0, "mark_ids": [5, 7, 6, 12, 4]}, {"role": "marks", "path": "marks/f000_p2.jpg", "frame": 0, "panel": 2, "source_frame": 602, "time_s": 0.0, "mark_ids": [13, 3, 1, 2]}, {"role": "marks", "path": "marks/f015_p0.jpg", "frame": 15, "panel": 0, "source_frame": 647, "time_s": 3.0, "mark_ids": [11, 10, 14, 16, 15]}, {"role": "marks", "path": "marks/f015_p1.jpg", "frame": 15, "panel": 1, "source_frame": 647, "time_s": 3.0, "mark_ids": []}, {"role": "marks", "path": "marks/f015_p2.jpg", "frame": 15, "panel": 2, "source_frame": 647, "time_s": 3.0, "mark_ids": [3, 1, 2]}, {"role": "marks", "path": "marks/f030_p0.jpg", "frame": 30, "panel": 0, "source_frame": 692, "time_s": 6.0, "mark_ids": [11, 10, 19, 14, 16]}, {"role": "marks", "path": "marks/f030_p1.jpg", "frame": 30, "panel": 1, "source_frame": 692, "time_s": 6.0, "mark_ids": []}, {"role": "marks", "path": "marks/f030_p2.jpg", "frame": 30, "panel": 2, "source_frame": 692, "time_s": 6.0, "mark_ids": [3, 1, 2]}, {"role": "marks", "path": "marks/f045_p0.jpg", "frame": 45, "panel": 0, "source_frame": 737, "time_s": 9.0, "mark_ids": [11, 18, 21, 8]}, {"role": "marks", "path": "marks/f045_p1.jpg", "frame": 45, "panel": 1, "source_frame": 737, "time_s": 9.0, "mark_ids": [23, 22, 24, 27]}, {"role": "marks", "path": "marks/f045_p2.jpg", "frame": 45, "panel": 2, "source_frame": 737, "time_s": 9.0, "mark_ids": [26, 29, 28, 30, 3, 1, 25]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [11, 10, 18, 19, 14, 16, 21, 8, 15, 9, 5, 7, 6, 23, 12, 22]}, {"role": "ids", "path": "ids/sheet1.jpg", "mark_ids": [4, 17, 24, 27, 13, 26, 29, 28, 30, 3, 1, 2, 20, 25]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "06779b767870", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737], "actions": [{"action_index": 0, "text": "walk", "status": "visible", "segments_frames": [[0, 5], [6, 10], [14, 18]], "segments": [[0.0, 2.9], [3.0, 5.4], [7.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 5, "start_frame_1based": 1, "end_frame_1based": 6, "start_s": 0.0, "end_s": 2.9, "last_observed_s": 2.4, "start_source_frame": 602, "end_source_frame": 638, "start_filename": "00000602.jpg", "end_filename": "00000638.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 6, "end_frame": 10, "start_frame_1based": 7, "end_frame_1based": 11, "start_s": 3.0, "end_s": 5.4, "last_observed_s": 5.0, "start_source_frame": 647, "end_source_frame": 677, "start_filename": "00000647.jpg", "end_filename": "00000677.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 14, "end_frame": 18, "start_frame_1based": 15, "end_frame_1based": 19, "start_s": 7.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 707, "end_source_frame": 737, "start_filename": "00000707.jpg", "end_filename": "00000737.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Different members walk past the lockers early, across the foreground midway, and through the room near the end.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}, {"action_index": 1, "text": "sit", "status": "visible", "segments_frames": [[0, 13]], "segments": [[0.0, 6.9]], "segment_details": [{"start_frame": 0, "end_frame": 13, "start_frame_1based": 1, "end_frame_1based": 14, "start_s": 0.0, "end_s": 6.9, "last_observed_s": 6.4, "start_source_frame": 602, "end_source_frame": 698, "start_filename": "00000602.jpg", "end_filename": "00000698.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "People remain seated along the lockers; the number-nine boy leaves his seat before the other seated people disappear from view.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}, {"action_index": 2, "text": "seated", "status": "visible", "segments_frames": [[0, 10]], "segments": [[0.0, 5.4]], "segment_details": [{"start_frame": 0, "end_frame": 10, "start_frame_1based": 1, "end_frame_1based": 11, "start_s": 0.0, "end_s": 5.4, "last_observed_s": 5.0, "start_source_frame": 602, "end_source_frame": 677, "start_filename": "00000602.jpg", "end_filename": "00000677.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The number-nine boy sits at the lockers before beginning to rise.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}, {"action_index": 3, "text": "stands up", "status": "visible", "segments_frames": [[10, 11]], "segments": [[5.0, 5.9]], "segment_details": [{"start_frame": 10, "end_frame": 11, "start_frame_1based": 11, "end_frame_1based": 12, "start_s": 5.0, "end_s": 5.9, "last_observed_s": 5.4, "start_source_frame": 677, "end_source_frame": 683, "start_filename": "00000677.jpg", "end_filename": "00000683.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The boy lifts himself from his seat and straightens his legs.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}, {"action_index": 4, "text": "turns", "status": "visible", "segments_frames": [[11, 12]], "segments": [[5.4, 6.4]], "segment_details": [{"start_frame": 11, "end_frame": 12, "start_frame_1based": 12, "end_frame_1based": 13, "start_s": 5.4, "end_s": 6.4, "last_observed_s": 6.0, "start_source_frame": 683, "end_source_frame": 692, "start_filename": "00000683.jpg", "end_filename": "00000692.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "After rising, the boy rotates until his back faces the viewer and he faces the lockers.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "473_PWeWs8Uf9TA", "split": "train", "caption": "A person stands inside a transparent dome beside a wooden deck. Two ladders, two tables, plant pots and bags occupy the deck. A backhoe and a cement mixer are near a smaller dome.", "mentions": [{"text": "A person", "span": [0, 8], "occurrence": 1, "mark_ids": [10], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a transparent dome", "span": [23, 41], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a wooden deck", "span": [49, 62], "occurrence": 1, "mark_ids": [1], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "Two ladders", "span": [64, 75], "occurrence": 1, "mark_ids": [8, 9], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "two tables", "span": [77, 87], "occurrence": 1, "mark_ids": [12, 13], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "plant pots", "span": [89, 99], "occurrence": 1, "mark_ids": [14, 15, 19], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "bags", "span": [104, 108], "occurrence": 1, "mark_ids": [11, 17, 18], "confidence": "medium", "sentence_index": 1, "is_pronoun": false}, {"text": "the deck", "span": [116, 124], "occurrence": 1, "mark_ids": [1], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "A backhoe", "span": [126, 135], "occurrence": 1, "mark_ids": [16], "confidence": "high", "sentence_index": 2, "is_pronoun": false}, {"text": "a cement mixer", "span": [140, 154], "occurrence": 1, "mark_ids": [16], "confidence": "high", "sentence_index": 2, "is_pronoun": false}, {"text": "a smaller dome", "span": [164, 178], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 63], [64, 125], [126, 179]], "groups": [{"group_id": 0, "mark_id": 10, "category": "person", "isthing": 1, "mentions": [{"text": "A person", "span": [0, 8], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 3, "category": "other construction", "isthing": 0, "mentions": [{"text": "a transparent dome", "span": [23, 41], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "a smaller dome", "span": [164, 178], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown", "unknown"], "chain_agreement": "unknown"}, {"group_id": 2, "mark_id": 1, "category": "ground", "isthing": 0, "mentions": [{"text": "a wooden deck", "span": [49, 62], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "the deck", "span": [116, 124], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown", "unknown"], "chain_agreement": "unknown"}, {"group_id": 3, "mark_id": 8, "category": "ladder", "isthing": 1, "mentions": [{"text": "Two ladders", "span": [64, 75], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 9, "category": "ladder", "isthing": 1, "mentions": [{"text": "Two ladders", "span": [64, 75], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 5, "mark_id": 12, "category": "table or desk", "isthing": 1, "mentions": [{"text": "two tables", "span": [77, 87], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 6, "mark_id": 13, "category": "table or desk", "isthing": 1, "mentions": [{"text": "two tables", "span": [77, 87], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 7, "mark_id": 14, "category": "flower pot or vase", "isthing": 1, "mentions": [{"text": "plant pots", "span": [89, 99], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 8, "mark_id": 15, "category": "flower pot or vase", "isthing": 1, "mentions": [{"text": "plant pots", "span": [89, 99], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 9, "mark_id": 19, "category": "flower pot or vase", "isthing": 1, "mentions": [{"text": "plant pots", "span": [89, 99], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 10, "mark_id": 11, "category": "bag or package", "isthing": 1, "mentions": [{"text": "bags", "span": [104, 108], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 11, "mark_id": 17, "category": "bag or package", "isthing": 1, "mentions": [{"text": "bags", "span": [104, 108], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 12, "mark_id": 18, "category": "bag or package", "isthing": 1, "mentions": [{"text": "bags", "span": [104, 108], "occurrence": 1, "confidence": "medium", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 13, "mark_id": 16, "category": "wheeled machine", "isthing": 0, "mentions": [{"text": "A backhoe", "span": [126, 135], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "a cement mixer", "span": [140, 154], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown", "disagree"], "chain_agreement": "disagree"}], "actions": [{"text": "stands", "span": [9, 15], "spans": [[9, 15]], "particle": null, "mentions": [{"text": "stands", "span": [9, 15], "spans": [[9, 15]], "particle": null, "sentence_index": 0}], "type": "posture", "sentence_index": 0, "subject": {"text": "A person", "span": [0, 8], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a transparent dome", "span": [23, 41], "groups": [1], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "stands", "status": "visible", "segments_frames": [[0, 2]], "segments": [[0.0, 1.4]], "segment_details": [{"start_frame": 0, "end_frame": 2, "start_frame_1based": 1, "end_frame_1based": 3, "start_s": 0.0, "end_s": 1.4, "last_observed_s": 1.0, "start_source_frame": 152, "end_source_frame": 167, "start_filename": "00000152.jpg", "end_filename": "00000167.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The person is upright inside the doorway in the first three clean frames before becoming obscured.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272, 278, 287]}}], "coverage": {"marks_shown": 20, "marks_grounded": 14, "things_shown": 11, "things_grounded": 11, "stuff_shown": 9, "stuff_grounded": 3, "n_mentions": 17, "category_disagree": 1, "category_checked": 12}, "marks": [{"mark_id": 8, "category": "ladder", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 23, "first_frame": 0, "last_frame": 23, "best_frame": 19, "max_rel_area": 0.03246, "mean_rel_area": 0.02036, "centroid": [0.329, 0.489], "motion": 0.427}, {"mark_id": 9, "category": "ladder", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.07247, "mean_rel_area": 0.04553, "centroid": [0.203, 0.613], "motion": 0.359}, {"mark_id": 12, "category": "table or desk", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 30, "first_frame": 0, "last_frame": 29, "best_frame": 25, "max_rel_area": 0.00918, "mean_rel_area": 0.00599, "centroid": [0.562, 0.634], "motion": 0.675}, {"mark_id": 13, "category": "table or desk", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 25, "first_frame": 0, "last_frame": 24, "best_frame": 22, "max_rel_area": 0.00882, "mean_rel_area": 0.00628, "centroid": [0.465, 0.648], "motion": 0.533}, {"mark_id": 14, "category": "flower pot or vase", "isthing": 1, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 40, "first_frame": 0, "last_frame": 39, "best_frame": 35, "max_rel_area": 0.00238, "mean_rel_area": 0.00089, "centroid": [0.524, 0.638], "motion": 0.784}, {"mark_id": 17, "category": "bag or package", "isthing": 1, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 24, "first_frame": 18, "last_frame": 43, "best_frame": 35, "max_rel_area": 0.00345, "mean_rel_area": 0.00221, "centroid": [0.487, 0.925], "motion": 0.605}, {"mark_id": 11, "category": "bag or package", "isthing": 1, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 23, "first_frame": 0, "last_frame": 22, "best_frame": 21, "max_rel_area": 0.00228, "mean_rel_area": 0.00125, "centroid": [0.43, 0.764], "motion": 0.526}, {"mark_id": 19, "category": "flower pot or vase", "isthing": 1, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 26, "first_frame": 20, "last_frame": 45, "best_frame": 23, "max_rel_area": 0.00165, "mean_rel_area": 0.0013, "centroid": [0.553, 0.65], "motion": 0.472}, {"mark_id": 15, "category": "flower pot or vase", "isthing": 1, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 18, "first_frame": 0, "last_frame": 17, "best_frame": 14, "max_rel_area": 0.00325, "mean_rel_area": 0.00248, "centroid": [0.252, 0.831], "motion": 0.399}, {"mark_id": 18, "category": "bag or package", "isthing": 1, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 17, "first_frame": 18, "last_frame": 34, "best_frame": 29, "max_rel_area": 0.00282, "mean_rel_area": 0.00184, "centroid": [0.557, 0.93], "motion": 0.596}, {"mark_id": 10, "category": "person", "isthing": 1, "iscrowd": 0, "order": 10, "color": [139, 195, 74], "panel": 1, "n_present": 8, "first_frame": 0, "last_frame": 7, "best_frame": 2, "max_rel_area": 0.00838, "mean_rel_area": 0.00491, "centroid": [0.229, 0.574], "motion": 0.08}, {"mark_id": 3, "category": "other construction", "isthing": 0, "iscrowd": 0, "order": 11, "color": [255, 87, 34], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 5, "max_rel_area": 0.46941, "mean_rel_area": 0.21579, "centroid": [0.413, 0.476], "motion": 0.15}, {"mark_id": 1, "category": "ground", "isthing": 0, "iscrowd": 0, "order": 12, "color": [156, 39, 176], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 40, "max_rel_area": 0.40931, "mean_rel_area": 0.32522, "centroid": [0.505, 0.766], "motion": 0.035}, {"mark_id": 4, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 13, "color": [0, 188, 212], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 31, "max_rel_area": 0.37746, "mean_rel_area": 0.27307, "centroid": [0.547, 0.201], "motion": 0.156}, {"mark_id": 5, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 14, "color": [121, 85, 72], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.12679, "mean_rel_area": 0.07856, "centroid": [0.581, 0.374], "motion": 0.181}, {"mark_id": 2, "category": "grass", "isthing": 0, "iscrowd": 0, "order": 15, "color": [233, 30, 99], "panel": 1, "n_present": 40, "first_frame": 0, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.06099, "mean_rel_area": 0.02821, "centroid": [0.712, 0.545], "motion": 0.402}, {"mark_id": 6, "category": "other plant", "isthing": 0, "iscrowd": 0, "order": 16, "color": [63, 81, 181], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 14, "max_rel_area": 0.04365, "mean_rel_area": 0.01625, "centroid": [0.48, 0.627], "motion": 0.36}, {"mark_id": 7, "category": "shelf", "isthing": 0, "iscrowd": 0, "order": 17, "color": [205, 220, 57], "panel": 1, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 39, "max_rel_area": 0.02441, "mean_rel_area": 0.01394, "centroid": [0.503, 0.603], "motion": 0.355}, {"mark_id": 16, "category": "wheeled machine", "isthing": 0, "iscrowd": 0, "order": 18, "color": [255, 152, 0], "panel": 1, "n_present": 45, "first_frame": 1, "last_frame": 45, "best_frame": 44, "max_rel_area": 0.01545, "mean_rel_area": 0.0094, "centroid": [0.665, 0.518], "motion": 0.463}, {"mark_id": 20, "category": "water", "isthing": 0, "iscrowd": 0, "order": 19, "color": [96, 125, 139], "panel": 1, "n_present": 18, "first_frame": 28, "last_frame": 45, "best_frame": 44, "max_rel_area": 0.00424, "mean_rel_area": 0.00298, "centroid": [0.929, 0.654], "motion": 0.079}], "num_frames": 46, "duration_s": 9.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0, 4.2, 4.4, 4.6, 4.8, 5.0, 5.2, 5.4, 5.6, 5.8, 6.0, 6.2, 6.4, 6.6, 6.8, 7.0, 7.2, 7.4, 7.6, 7.8, 8.0, 8.2, 8.4, 8.6, 8.8, 9.0], "source_frame_indices": [152, 155, 158, 161, 164, 167, 170, 173, 176, 179, 182, 185, 188, 191, 194, 197, 200, 203, 206, 209, 212, 215, 218, 221, 224, 227, 230, 233, 236, 239, 242, 245, 248, 251, 254, 257, 260, 263, 266, 269, 272, 275, 278, 281, 284, 287], "timing_sha": "2ea96d3c64f6", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 2, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 25031, "output_tokens": 867, "thinking_tokens": 305, "total_tokens": 25898}, "seconds": 20.16, "response_id": "resp_08bb40f3e732d1ad006ac32feb37cc87d0b433e2d0fc67a9c0", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 152, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 158, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 167, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 173, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 182, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 188, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 197, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 203, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 212, "time_s": 4.0}, {"role": "clean", "path": "clean/f022.jpg", "frame": 9, "source_frame": 218, "time_s": 4.4}, {"role": "clean", "path": "clean/f025.jpg", "frame": 10, "source_frame": 227, "time_s": 5.0}, {"role": "clean", "path": "clean/f027.jpg", "frame": 11, "source_frame": 233, "time_s": 5.4}, {"role": "clean", "path": "clean/f030.jpg", "frame": 12, "source_frame": 242, "time_s": 6.0}, {"role": "clean", "path": "clean/f032.jpg", "frame": 13, "source_frame": 248, "time_s": 6.4}, {"role": "clean", "path": "clean/f035.jpg", "frame": 14, "source_frame": 257, "time_s": 7.0}, {"role": "clean", "path": "clean/f037.jpg", "frame": 15, "source_frame": 263, "time_s": 7.4}, {"role": "clean", "path": "clean/f040.jpg", "frame": 16, "source_frame": 272, "time_s": 8.0}, {"role": "clean", "path": "clean/f042.jpg", "frame": 17, "source_frame": 278, "time_s": 8.4}, {"role": "clean", "path": "clean/f045.jpg", "frame": 18, "source_frame": 287, "time_s": 9.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 152, "time_s": 0.0, "mark_ids": [8, 9, 12, 13, 14, 11, 15]}, {"role": "marks", "path": "marks/f000_p1.jpg", "frame": 0, "panel": 1, "source_frame": 152, "time_s": 0.0, "mark_ids": [10, 3, 1, 4, 5, 2, 6, 7]}, {"role": "marks", "path": "marks/f015_p0.jpg", "frame": 15, "panel": 0, "source_frame": 197, "time_s": 3.0, "mark_ids": [8, 12, 13, 14, 11, 15]}, {"role": "marks", "path": "marks/f015_p1.jpg", "frame": 15, "panel": 1, "source_frame": 197, "time_s": 3.0, "mark_ids": [3, 1, 4, 5, 2, 6, 7, 16]}, {"role": "marks", "path": "marks/f030_p0.jpg", "frame": 30, "panel": 0, "source_frame": 242, "time_s": 6.0, "mark_ids": [14, 17, 19, 18]}, {"role": "marks", "path": "marks/f030_p1.jpg", "frame": 30, "panel": 1, "source_frame": 242, "time_s": 6.0, "mark_ids": [3, 1, 4, 5, 2, 6, 7, 16, 20]}, {"role": "marks", "path": "marks/f045_p0.jpg", "frame": 45, "panel": 0, "source_frame": 287, "time_s": 9.0, "mark_ids": [19]}, {"role": "marks", "path": "marks/f045_p1.jpg", "frame": 45, "panel": 1, "source_frame": 287, "time_s": 9.0, "mark_ids": [3, 1, 4, 5, 2, 6, 7, 16, 20]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [8, 9, 12, 13, 14, 17, 11, 19, 15, 18, 10, 3, 1, 4, 5, 2]}, {"role": "ids", "path": "ids/sheet1.jpg", "mark_ids": [6, 7, 16, 20]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "2ea96d3c64f6", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272, 278, 287], "actions": [{"action_index": 0, "text": "stands", "status": "visible", "segments_frames": [[0, 2]], "segments": [[0.0, 1.4]], "segment_details": [{"start_frame": 0, "end_frame": 2, "start_frame_1based": 1, "end_frame_1based": 3, "start_s": 0.0, "end_s": 1.4, "last_observed_s": 1.0, "start_source_frame": 152, "end_source_frame": 167, "start_filename": "00000152.jpg", "end_filename": "00000167.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The person is upright inside the doorway in the first three clean frames before becoming obscured.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272, 278, 287]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "562_OA61thiz9wU", "split": "train", "caption": "A gray naval ship moves slowly right along the river. It is visible beyond a metal fence near a large leafy tree.", "mentions": [{"text": "A gray naval ship", "span": [0, 17], "occurrence": 1, "mark_ids": [7], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "the river", "span": [43, 52], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "It", "span": [54, 56], "occurrence": 1, "mark_ids": [7], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "a metal fence", "span": [75, 88], "occurrence": 1, "mark_ids": [1], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "a large leafy tree", "span": [94, 112], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 1, "is_pronoun": false}], "sentences": [[0, 53], [54, 113]], "groups": [{"group_id": 0, "mark_id": 7, "category": "ship or boat", "isthing": 1, "mentions": [{"text": "A gray naval ship", "span": [0, 17], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "It", "span": [54, 56], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 5, "category": "river", "isthing": 0, "mentions": [{"text": "the river", "span": [43, 52], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 1, "category": "handrail or fence", "isthing": 0, "mentions": [{"text": "a metal fence", "span": [75, 88], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 6, "category": "tree", "isthing": 0, "mentions": [{"text": "a large leafy tree", "span": [94, 112], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "moves", "span": [18, 23], "spans": [[18, 23]], "particle": null, "mentions": [{"text": "moves", "span": [18, 23], "spans": [[18, 23]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "A gray naval ship", "span": [0, 17], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 0, "text": "moves", "status": "visible", "segments_frames": [[0, 16]], "segments": [[0.0, 8.2]], "segment_details": [{"start_frame": 0, "end_frame": 16, "start_frame_1based": 1, "end_frame_1based": 17, "start_s": 0.0, "end_s": 8.2, "last_observed_s": 8.0, "start_source_frame": 152, "end_source_frame": 272, "start_filename": "00000152.jpg", "end_filename": "00000272.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The ship steadily advances right relative to the fence and tree throughout the sequence.", "fps": 2.0, "input_frame_count": 17, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272]}}], "coverage": {"marks_shown": 7, "marks_grounded": 4, "things_shown": 1, "things_grounded": 1, "stuff_shown": 6, "stuff_grounded": 3, "n_mentions": 5, "category_disagree": 0, "category_checked": 4}, "marks": [{"mark_id": 7, "category": "ship or boat", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 41, "first_frame": 0, "last_frame": 40, "best_frame": 40, "max_rel_area": 0.11418, "mean_rel_area": 0.06694, "centroid": [0.159, 0.545], "motion": 0.119}, {"mark_id": 6, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 41, "first_frame": 0, "last_frame": 40, "best_frame": 1, "max_rel_area": 0.46394, "mean_rel_area": 0.4493, "centroid": [0.517, 0.399], "motion": 0.063}, {"mark_id": 4, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 41, "first_frame": 0, "last_frame": 40, "best_frame": 0, "max_rel_area": 0.26112, "mean_rel_area": 0.24397, "centroid": [0.5, 0.294], "motion": 0.008}, {"mark_id": 1, "category": "handrail or fence", "isthing": 0, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 41, "first_frame": 0, "last_frame": 40, "best_frame": 5, "max_rel_area": 0.15269, "mean_rel_area": 0.14948, "centroid": [0.5, 0.875], "motion": 0.002}, {"mark_id": 3, "category": "path", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 41, "first_frame": 0, "last_frame": 40, "best_frame": 15, "max_rel_area": 0.03801, "mean_rel_area": 0.03511, "centroid": [0.359, 0.941], "motion": 0.012}, {"mark_id": 2, "category": "grass", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 41, "first_frame": 0, "last_frame": 40, "best_frame": 30, "max_rel_area": 0.03626, "mean_rel_area": 0.03311, "centroid": [0.5, 0.963], "motion": 0.001}, {"mark_id": 5, "category": "river", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 41, "first_frame": 0, "last_frame": 40, "best_frame": 1, "max_rel_area": 0.02065, "mean_rel_area": 0.01827, "centroid": [0.591, 0.773], "motion": 0.056}], "num_frames": 41, "duration_s": 8.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0, 4.2, 4.4, 4.6, 4.8, 5.0, 5.2, 5.4, 5.6, 5.8, 6.0, 6.2, 6.4, 6.6, 6.8, 7.0, 7.2, 7.4, 7.6, 7.8, 8.0], "source_frame_indices": [152, 155, 158, 161, 164, 167, 170, 173, 176, 179, 182, 185, 188, 191, 194, 197, 200, 203, 206, 209, 212, 215, 218, 221, 224, 227, 230, 233, 236, 239, 242, 245, 248, 251, 254, 257, 260, 263, 266, 269, 272], "timing_sha": "7ed3703bc0b5", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 1, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 21075, "output_tokens": 604, "thinking_tokens": 261, "total_tokens": 21679}, "seconds": 13.02, "response_id": "resp_03e6a3e28ff17491006ac32ff450f087d09fcd0e951bb1e3a8", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 152, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 158, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 167, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 173, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 182, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 188, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 197, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 203, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 212, "time_s": 4.0}, {"role": "clean", "path": "clean/f022.jpg", "frame": 9, "source_frame": 218, "time_s": 4.4}, {"role": "clean", "path": "clean/f025.jpg", "frame": 10, "source_frame": 227, "time_s": 5.0}, {"role": "clean", "path": "clean/f027.jpg", "frame": 11, "source_frame": 233, "time_s": 5.4}, {"role": "clean", "path": "clean/f030.jpg", "frame": 12, "source_frame": 242, "time_s": 6.0}, {"role": "clean", "path": "clean/f032.jpg", "frame": 13, "source_frame": 248, "time_s": 6.4}, {"role": "clean", "path": "clean/f035.jpg", "frame": 14, "source_frame": 257, "time_s": 7.0}, {"role": "clean", "path": "clean/f037.jpg", "frame": 15, "source_frame": 263, "time_s": 7.4}, {"role": "clean", "path": "clean/f040.jpg", "frame": 16, "source_frame": 272, "time_s": 8.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 152, "time_s": 0.0, "mark_ids": [7, 6, 4, 1, 3, 2, 5]}, {"role": "marks", "path": "marks/f012_p0.jpg", "frame": 12, "panel": 0, "source_frame": 188, "time_s": 2.4, "mark_ids": [7, 6, 4, 1, 3, 2, 5]}, {"role": "marks", "path": "marks/f027_p0.jpg", "frame": 27, "panel": 0, "source_frame": 233, "time_s": 5.4, "mark_ids": [7, 6, 4, 1, 3, 2, 5]}, {"role": "marks", "path": "marks/f040_p0.jpg", "frame": 40, "panel": 0, "source_frame": 272, "time_s": 8.0, "mark_ids": [7, 6, 4, 1, 3, 2, 5]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [7, 6, 4, 1, 3, 2, 5]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "7ed3703bc0b5", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 17, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272], "actions": [{"action_index": 0, "text": "moves", "status": "visible", "segments_frames": [[0, 16]], "segments": [[0.0, 8.2]], "segment_details": [{"start_frame": 0, "end_frame": 16, "start_frame_1based": 1, "end_frame_1based": 17, "start_s": 0.0, "end_s": 8.2, "last_observed_s": 8.0, "start_source_frame": 152, "end_source_frame": 272, "start_filename": "00000152.jpg", "end_filename": "00000272.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The ship steadily advances right relative to the fence and tree throughout the sequence.", "fps": 2.0, "input_frame_count": 17, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "64_8_IdHgf2G6g", "split": "train", "caption": "A yellow-shirted player skates across the ice. A white-shirted player makes tight turns holding a hockey stick. This player then skates toward a goalie standing before the net.", "mentions": [{"text": "A yellow-shirted player", "span": [0, 23], "occurrence": 1, "mark_ids": [10], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "the ice", "span": [38, 45], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "A white-shirted player", "span": [47, 69], "occurrence": 1, "mark_ids": [14], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "a hockey stick", "span": [96, 110], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "This player", "span": [112, 123], "occurrence": 1, "mark_ids": [14], "confidence": "high", "sentence_index": 2, "is_pronoun": false}, {"text": "a goalie", "span": [143, 151], "occurrence": 1, "mark_ids": [12], "confidence": "high", "sentence_index": 2, "is_pronoun": false}, {"text": "the net", "span": [168, 175], "occurrence": 1, "mark_ids": [7], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 46], [47, 111], [112, 176]], "groups": [{"group_id": 0, "mark_id": 10, "category": "person", "isthing": 1, "mentions": [{"text": "A yellow-shirted player", "span": [0, 23], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 3, "category": "ice", "isthing": 0, "mentions": [{"text": "the ice", "span": [38, 45], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 14, "category": "person", "isthing": 1, "mentions": [{"text": "A white-shirted player", "span": [47, 69], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "This player", "span": [112, 123], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 5, "category": "bat", "isthing": 0, "mentions": [{"text": "a hockey stick", "span": [96, 110], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["disagree"], "chain_agreement": "disagree"}, {"group_id": 4, "mark_id": 12, "category": "person", "isthing": 1, "mentions": [{"text": "a goalie", "span": [143, 151], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown"], "chain_agreement": "unknown"}, {"group_id": 5, "mark_id": 7, "category": "goal", "isthing": 1, "mentions": [{"text": "the net", "span": [168, 175], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["disagree"], "chain_agreement": "disagree"}], "actions": [{"text": "skates", "span": [24, 30], "spans": [[24, 30]], "particle": null, "mentions": [{"text": "skates", "span": [24, 30], "spans": [[24, 30]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "A yellow-shirted player", "span": [0, 23], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 0, "text": "skates", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.9]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 692, "end_source_frame": 713, "start_filename": "00000692.jpg", "end_filename": "00000713.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The yellow-shirted player glides rightward with alternating skating strides.", "fps": 2.0, "input_frame_count": 15, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0], "source_frame_indices": [692, 698, 707, 713, 722, 728, 737, 743, 752, 758, 767, 773, 782, 788, 797]}}, {"text": "makes tight turns", "span": [70, 87], "spans": [[70, 87]], "particle": null, "mentions": [{"text": "makes tight turns", "span": [70, 87], "spans": [[70, 87]], "particle": null, "sentence_index": 1}], "type": "light_verb", "sentence_index": 1, "subject": {"text": "A white-shirted player", "span": [47, 69], "groups": [2], "group_mentions": [0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 1, "text": "makes tight turns", "status": "visible", "segments_frames": [[7, 12]], "segments": [[3.4, 6.4]], "segment_details": [{"start_frame": 7, "end_frame": 12, "start_frame_1based": 8, "end_frame_1based": 13, "start_s": 3.4, "end_s": 6.4, "last_observed_s": 6.0, "start_source_frame": 743, "end_source_frame": 782, "start_filename": "00000743.jpg", "end_filename": "00000782.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The white-shirted player leans into successive curves and changes skating direction.", "fps": 2.0, "input_frame_count": 15, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0], "source_frame_indices": [692, 698, 707, 713, 722, 728, 737, 743, 752, 758, 767, 773, 782, 788, 797]}}, {"text": "holding", "span": [88, 95], "spans": [[88, 95]], "particle": null, "mentions": [{"text": "holding", "span": [88, 95], "spans": [[88, 95]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "A white-shirted player", "span": [47, 69], "groups": [2], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a hockey stick", "span": [96, 110], "groups": [3], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 2, "text": "holding", "status": "visible", "segments_frames": [[4, 14]], "segments": [[2.0, 7.2]], "segment_details": [{"start_frame": 4, "end_frame": 14, "start_frame_1based": 5, "end_frame_1based": 15, "start_s": 2.0, "end_s": 7.2, "last_observed_s": 7.0, "start_source_frame": 722, "end_source_frame": 797, "start_filename": "00000722.jpg", "end_filename": "00000797.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The player visibly grips the hockey stick throughout the skating sequence.", "fps": 2.0, "input_frame_count": 15, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0], "source_frame_indices": [692, 698, 707, 713, 722, 728, 737, 743, 752, 758, 767, 773, 782, 788, 797]}}, {"text": "skates", "span": [129, 135], "spans": [[129, 135]], "particle": null, "mentions": [{"text": "skates", "span": [129, 135], "spans": [[129, 135]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "This player", "span": [112, 123], "groups": [2], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "a goalie", "span": [143, 151], "groups": [4], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 3, "text": "skates", "status": "visible", "segments_frames": [[12, 14]], "segments": [[6.0, 7.2]], "segment_details": [{"start_frame": 12, "end_frame": 14, "start_frame_1based": 13, "end_frame_1based": 15, "start_s": 6.0, "end_s": 7.2, "last_observed_s": 7.0, "start_source_frame": 782, "end_source_frame": 797, "start_filename": "00000782.jpg", "end_filename": "00000797.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "After the turns, the player advances toward the goalie and net.", "fps": 2.0, "input_frame_count": 15, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0], "source_frame_indices": [692, 698, 707, 713, 722, 728, 737, 743, 752, 758, 767, 773, 782, 788, 797]}}, {"text": "standing", "span": [152, 160], "spans": [[152, 160]], "particle": null, "mentions": [{"text": "standing", "span": [152, 160], "spans": [[152, 160]], "particle": null, "sentence_index": 2}], "type": "posture", "sentence_index": 2, "subject": {"text": "a goalie", "span": [143, 151], "groups": [4], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "the net", "span": [168, 175], "groups": [5], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 4, "text": "standing", "status": "visible", "segments_frames": [[10, 14]], "segments": [[5.0, 7.2]], "segment_details": [{"start_frame": 10, "end_frame": 14, "start_frame_1based": 11, "end_frame_1based": 15, "start_s": 5.0, "end_s": 7.2, "last_observed_s": 7.0, "start_source_frame": 767, "end_source_frame": 797, "start_filename": "00000767.jpg", "end_filename": "00000797.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The goalie remains upright on skates in front of the net.", "fps": 2.0, "input_frame_count": 15, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0], "source_frame_indices": [692, 698, 707, 713, 722, 728, 737, 743, 752, 758, 767, 773, 782, 788, 797]}}], "coverage": {"marks_shown": 28, "marks_grounded": 6, "things_shown": 23, "things_grounded": 4, "stuff_shown": 5, "stuff_grounded": 2, "n_mentions": 7, "category_disagree": 2, "category_checked": 5}, "marks": [{"mark_id": 14, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 35, "first_frame": 1, "last_frame": 35, "best_frame": 26, "max_rel_area": 0.07885, "mean_rel_area": 0.01995, "centroid": [0.513, 0.437], "motion": 0.615}, {"mark_id": 26, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 17, "first_frame": 9, "last_frame": 29, "best_frame": 13, "max_rel_area": 0.0683, "mean_rel_area": 0.05441, "centroid": [0.328, 0.287], "motion": 0.286}, {"mark_id": 10, "category": "person", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 29, "first_frame": 0, "last_frame": 31, "best_frame": 1, "max_rel_area": 0.02076, "mean_rel_area": 0.00889, "centroid": [0.567, 0.283], "motion": 0.559}, {"mark_id": 7, "category": "goal", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 22, "first_frame": 0, "last_frame": 35, "best_frame": 8, "max_rel_area": 0.03199, "mean_rel_area": 0.0208, "centroid": [0.629, 0.3], "motion": 0.416}, {"mark_id": 6, "category": "window", "isthing": 1, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 13, "first_frame": 0, "last_frame": 35, "best_frame": 35, "max_rel_area": 0.06778, "mean_rel_area": 0.04123, "centroid": [0.477, 0.039], "motion": 0.426}, {"mark_id": 8, "category": "person", "isthing": 1, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 32, "first_frame": 0, "last_frame": 33, "best_frame": 6, "max_rel_area": 0.01027, "mean_rel_area": 0.00648, "centroid": [0.544, 0.251], "motion": 0.722}, {"mark_id": 12, "category": "person", "isthing": 1, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 22, "first_frame": 0, "last_frame": 35, "best_frame": 28, "max_rel_area": 0.01503, "mean_rel_area": 0.00995, "centroid": [0.635, 0.285], "motion": 0.412}, {"mark_id": 18, "category": "person", "isthing": 1, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 28, "first_frame": 4, "last_frame": 32, "best_frame": 5, "max_rel_area": 0.00752, "mean_rel_area": 0.00531, "centroid": [0.61, 0.255], "motion": 0.841}, {"mark_id": 9, "category": "person", "isthing": 1, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 9, "first_frame": 0, "last_frame": 20, "best_frame": 1, "max_rel_area": 0.02336, "mean_rel_area": 0.00752, "centroid": [0.413, 0.372], "motion": 0.666}, {"mark_id": 11, "category": "person", "isthing": 1, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 9, "first_frame": 0, "last_frame": 8, "best_frame": 7, "max_rel_area": 0.01528, "mean_rel_area": 0.01035, "centroid": [0.244, 0.413], "motion": 0.416}, {"mark_id": 13, "category": "person", "isthing": 1, "iscrowd": 0, "order": 10, "color": [139, 195, 74], "panel": 1, "n_present": 7, "first_frame": 0, "last_frame": 6, "best_frame": 4, "max_rel_area": 0.02038, "mean_rel_area": 0.01322, "centroid": [0.165, 0.482], "motion": 0.348}, {"mark_id": 15, "category": "person", "isthing": 1, "iscrowd": 0, "order": 11, "color": [255, 87, 34], "panel": 1, "n_present": 8, "first_frame": 1, "last_frame": 8, "best_frame": 3, "max_rel_area": 0.00446, "mean_rel_area": 0.00413, "centroid": [0.706, 0.364], "motion": 0.357}, {"mark_id": 17, "category": "person", "isthing": 1, "iscrowd": 0, "order": 12, "color": [156, 39, 176], "panel": 1, "n_present": 6, "first_frame": 3, "last_frame": 8, "best_frame": 5, "max_rel_area": 0.00675, "mean_rel_area": 0.00621, "centroid": [0.876, 0.424], "motion": 0.161}, {"mark_id": 25, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 13, "color": [0, 188, 212], "panel": 1, "n_present": 10, "first_frame": 9, "last_frame": 18, "best_frame": 9, "max_rel_area": 0.00156, "mean_rel_area": 0.00088, "centroid": [0.74, 0.018], "motion": 0.158}, {"mark_id": 16, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 14, "color": [121, 85, 72], "panel": 1, "n_present": 7, "first_frame": 2, "last_frame": 8, "best_frame": 2, "max_rel_area": 0.00294, "mean_rel_area": 0.00255, "centroid": [0.784, 0.111], "motion": 0.28}, {"mark_id": 21, "category": "person", "isthing": 1, "iscrowd": 0, "order": 15, "color": [233, 30, 99], "panel": 1, "n_present": 4, "first_frame": 5, "last_frame": 8, "best_frame": 6, "max_rel_area": 0.00856, "mean_rel_area": 0.0078, "centroid": [0.852, 0.435], "motion": 0.123}, {"mark_id": 23, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 16, "color": [63, 81, 181], "panel": 1, "n_present": 10, "first_frame": 9, "last_frame": 18, "best_frame": 9, "max_rel_area": 0.00117, "mean_rel_area": 0.00065, "centroid": [0.696, 0.014], "motion": 0.223}, {"mark_id": 28, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 17, "color": [205, 220, 57], "panel": 1, "n_present": 9, "first_frame": 9, "last_frame": 17, "best_frame": 10, "max_rel_area": 0.00144, "mean_rel_area": 0.00098, "centroid": [0.752, 0.021], "motion": 0.135}, {"mark_id": 24, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 18, "color": [255, 152, 0], "panel": 1, "n_present": 9, "first_frame": 9, "last_frame": 17, "best_frame": 9, "max_rel_area": 0.00142, "mean_rel_area": 0.00091, "centroid": [0.7, 0.018], "motion": 0.13}, {"mark_id": 22, "category": "person", "isthing": 1, "iscrowd": 0, "order": 19, "color": [96, 125, 139], "panel": 1, "n_present": 4, "first_frame": 5, "last_frame": 8, "best_frame": 5, "max_rel_area": 0.00531, "mean_rel_area": 0.00515, "centroid": [0.877, 0.372], "motion": 0.138}, {"mark_id": 20, "category": "window", "isthing": 1, "iscrowd": 0, "order": 20, "color": [54, 67, 244], "panel": 2, "n_present": 4, "first_frame": 5, "last_frame": 8, "best_frame": 8, "max_rel_area": 0.00523, "mean_rel_area": 0.00347, "centroid": [0.702, 0.029], "motion": 0.124}, {"mark_id": 19, "category": "window", "isthing": 1, "iscrowd": 0, "order": 21, "color": [34, 189, 255], "panel": 2, "n_present": 4, "first_frame": 5, "last_frame": 8, "best_frame": 8, "max_rel_area": 0.00293, "mean_rel_area": 0.00187, "centroid": [0.765, 0.027], "motion": 0.124}, {"mark_id": 27, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 22, "color": [60, 200, 60], "panel": 2, "n_present": 6, "first_frame": 9, "last_frame": 14, "best_frame": 9, "max_rel_area": 0.00109, "mean_rel_area": 0.0007, "centroid": [0.511, 0.009], "motion": 0.03}, {"mark_id": 3, "category": "ice", "isthing": 0, "iscrowd": 0, "order": 23, "color": [240, 120, 40], "panel": 2, "n_present": 36, "first_frame": 0, "last_frame": 35, "best_frame": 27, "max_rel_area": 0.72029, "mean_rel_area": 0.60561, "centroid": [0.5, 0.659], "motion": 0.056}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 24, "color": [180, 60, 220], "panel": 2, "n_present": 36, "first_frame": 0, "last_frame": 35, "best_frame": 9, "max_rel_area": 0.17685, "mean_rel_area": 0.13176, "centroid": [0.496, 0.213], "motion": 0.076}, {"mark_id": 2, "category": "handrail or fence", "isthing": 0, "iscrowd": 0, "order": 25, "color": [0, 215, 255], "panel": 2, "n_present": 36, "first_frame": 0, "last_frame": 35, "best_frame": 6, "max_rel_area": 0.16153, "mean_rel_area": 0.13684, "centroid": [0.5, 0.153], "motion": 0.087}, {"mark_id": 5, "category": "bat", "isthing": 0, "iscrowd": 0, "order": 26, "color": [200, 90, 130], "panel": 2, "n_present": 35, "first_frame": 0, "last_frame": 35, "best_frame": 25, "max_rel_area": 0.00762, "mean_rel_area": 0.002, "centroid": [0.465, 0.5], "motion": 0.354}, {"mark_id": 4, "category": "lamp", "isthing": 0, "iscrowd": 0, "order": 27, "color": [80, 175, 76], "panel": 2, "n_present": 2, "first_frame": 0, "last_frame": 1, "best_frame": 1, "max_rel_area": 0.00248, "mean_rel_area": 0.00209, "centroid": [0.086, 0.069], "motion": 0.078}], "num_frames": 36, "duration_s": 7.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0, 4.2, 4.4, 4.6, 4.8, 5.0, 5.2, 5.4, 5.6, 5.8, 6.0, 6.2, 6.4, 6.6, 6.8, 7.0], "source_frame_indices": [692, 695, 698, 701, 704, 707, 710, 713, 716, 719, 722, 725, 728, 731, 734, 737, 740, 743, 746, 749, 752, 755, 758, 761, 764, 767, 770, 773, 776, 779, 782, 785, 788, 791, 794, 797], "timing_sha": "49e5e9446a44", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 8, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 25950, "output_tokens": 1440, "thinking_tokens": 451, "total_tokens": 27390}, "seconds": 26.21, "response_id": "resp_0497c8003c81a45b006ac32fff24f087d0a1873509b954699f", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 692, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 698, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 707, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 713, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 722, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 728, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 737, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 743, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 752, "time_s": 4.0}, {"role": "clean", "path": "clean/f022.jpg", "frame": 9, "source_frame": 758, "time_s": 4.4}, {"role": "clean", "path": "clean/f025.jpg", "frame": 10, "source_frame": 767, "time_s": 5.0}, {"role": "clean", "path": "clean/f027.jpg", "frame": 11, "source_frame": 773, "time_s": 5.4}, {"role": "clean", "path": "clean/f030.jpg", "frame": 12, "source_frame": 782, "time_s": 6.0}, {"role": "clean", "path": "clean/f032.jpg", "frame": 13, "source_frame": 788, "time_s": 6.4}, {"role": "clean", "path": "clean/f035.jpg", "frame": 14, "source_frame": 797, "time_s": 7.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 692, "time_s": 0.0, "mark_ids": [10, 7, 6, 8, 12, 9, 11]}, {"role": "marks", "path": "marks/f000_p1.jpg", "frame": 0, "panel": 1, "source_frame": 692, "time_s": 0.0, "mark_ids": [13]}, {"role": "marks", "path": "marks/f000_p2.jpg", "frame": 0, "panel": 2, "source_frame": 692, "time_s": 0.0, "mark_ids": [3, 1, 2, 5, 4]}, {"role": "marks", "path": "marks/f012_p0.jpg", "frame": 12, "panel": 0, "source_frame": 728, "time_s": 2.4, "mark_ids": [14, 26, 10, 8, 18]}, {"role": "marks", "path": "marks/f012_p1.jpg", "frame": 12, "panel": 1, "source_frame": 728, "time_s": 2.4, "mark_ids": [25, 23, 28, 24]}, {"role": "marks", "path": "marks/f012_p2.jpg", "frame": 12, "panel": 2, "source_frame": 728, "time_s": 2.4, "mark_ids": [27, 3, 1, 2, 5]}, {"role": "marks", "path": "marks/f022_p0.jpg", "frame": 22, "panel": 0, "source_frame": 758, "time_s": 4.4, "mark_ids": [14, 26, 10, 8, 18]}, {"role": "marks", "path": "marks/f022_p1.jpg", "frame": 22, "panel": 1, "source_frame": 758, "time_s": 4.4, "mark_ids": []}, {"role": "marks", "path": "marks/f022_p2.jpg", "frame": 22, "panel": 2, "source_frame": 758, "time_s": 4.4, "mark_ids": [3, 1, 2, 5]}, {"role": "marks", "path": "marks/f035_p0.jpg", "frame": 35, "panel": 0, "source_frame": 797, "time_s": 7.0, "mark_ids": [14, 7, 6, 12]}, {"role": "marks", "path": "marks/f035_p1.jpg", "frame": 35, "panel": 1, "source_frame": 797, "time_s": 7.0, "mark_ids": []}, {"role": "marks", "path": "marks/f035_p2.jpg", "frame": 35, "panel": 2, "source_frame": 797, "time_s": 7.0, "mark_ids": [3, 1, 2, 5]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [14, 26, 10, 7, 6, 8, 12, 18, 9, 11, 13, 15, 17, 25, 16, 21]}, {"role": "ids", "path": "ids/sheet1.jpg", "mark_ids": [23, 28, 24, 22, 20, 19, 27, 3, 1, 2, 5, 4]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "49e5e9446a44", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 15, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0], "source_frame_indices": [692, 698, 707, 713, 722, 728, 737, 743, 752, 758, 767, 773, 782, 788, 797], "actions": [{"action_index": 0, "text": "skates", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.9]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 692, "end_source_frame": 713, "start_filename": "00000692.jpg", "end_filename": "00000713.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The yellow-shirted player glides rightward with alternating skating strides.", "fps": 2.0, "input_frame_count": 15, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0], "source_frame_indices": [692, 698, 707, 713, 722, 728, 737, 743, 752, 758, 767, 773, 782, 788, 797]}, {"action_index": 1, "text": "makes tight turns", "status": "visible", "segments_frames": [[7, 12]], "segments": [[3.4, 6.4]], "segment_details": [{"start_frame": 7, "end_frame": 12, "start_frame_1based": 8, "end_frame_1based": 13, "start_s": 3.4, "end_s": 6.4, "last_observed_s": 6.0, "start_source_frame": 743, "end_source_frame": 782, "start_filename": "00000743.jpg", "end_filename": "00000782.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The white-shirted player leans into successive curves and changes skating direction.", "fps": 2.0, "input_frame_count": 15, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0], "source_frame_indices": [692, 698, 707, 713, 722, 728, 737, 743, 752, 758, 767, 773, 782, 788, 797]}, {"action_index": 2, "text": "holding", "status": "visible", "segments_frames": [[4, 14]], "segments": [[2.0, 7.2]], "segment_details": [{"start_frame": 4, "end_frame": 14, "start_frame_1based": 5, "end_frame_1based": 15, "start_s": 2.0, "end_s": 7.2, "last_observed_s": 7.0, "start_source_frame": 722, "end_source_frame": 797, "start_filename": "00000722.jpg", "end_filename": "00000797.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The player visibly grips the hockey stick throughout the skating sequence.", "fps": 2.0, "input_frame_count": 15, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0], "source_frame_indices": [692, 698, 707, 713, 722, 728, 737, 743, 752, 758, 767, 773, 782, 788, 797]}, {"action_index": 3, "text": "skates", "status": "visible", "segments_frames": [[12, 14]], "segments": [[6.0, 7.2]], "segment_details": [{"start_frame": 12, "end_frame": 14, "start_frame_1based": 13, "end_frame_1based": 15, "start_s": 6.0, "end_s": 7.2, "last_observed_s": 7.0, "start_source_frame": 782, "end_source_frame": 797, "start_filename": "00000782.jpg", "end_filename": "00000797.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "After the turns, the player advances toward the goalie and net.", "fps": 2.0, "input_frame_count": 15, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0], "source_frame_indices": [692, 698, 707, 713, 722, 728, 737, 743, 752, 758, 767, 773, 782, 788, 797]}, {"action_index": 4, "text": "standing", "status": "visible", "segments_frames": [[10, 14]], "segments": [[5.0, 7.2]], "segment_details": [{"start_frame": 10, "end_frame": 14, "start_frame_1based": 11, "end_frame_1based": 15, "start_s": 5.0, "end_s": 7.2, "last_observed_s": 7.0, "start_source_frame": 767, "end_source_frame": 797, "start_filename": "00000767.jpg", "end_filename": "00000797.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The goalie remains upright on skates in front of the net.", "fps": 2.0, "input_frame_count": 15, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0], "source_frame_indices": [692, 698, 707, 713, 722, 728, 737, 743, 752, 758, 767, 773, 782, 788, 797]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "737_GU3ZbsUq3MA", "split": "train", "caption": "A masked man holds a brush beside rows of blue seats. He reaches toward the right-hand bags, then gestures. Behind him, an orange-shirted man moves a long tool along the left window.", "mentions": [{"text": "A masked man", "span": [0, 12], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a brush", "span": [19, 26], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "mark_ids": [10, 14, 11, 21, 18, 15, 22, 12, 13, 24, 16, 25], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "He", "span": [54, 56], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "the right-hand bags", "span": [72, 91], "occurrence": 1, "mark_ids": [9, 23], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "him", "span": [115, 118], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 2, "is_pronoun": true}, {"text": "an orange-shirted man", "span": [120, 141], "occurrence": 1, "mark_ids": [17], "confidence": "high", "sentence_index": 2, "is_pronoun": false}, {"text": "a long tool", "span": [148, 159], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 2, "is_pronoun": false}, {"text": "the left window", "span": [166, 181], "occurrence": 1, "mark_ids": [19], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 53], [54, 107], [108, 182]], "groups": [{"group_id": 0, "mark_id": 8, "category": "person", "isthing": 1, "mentions": [{"text": "A masked man", "span": [0, 12], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "He", "span": [54, 56], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "him", "span": [115, 118], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 4, "category": "tool", "isthing": 0, "mentions": [{"text": "a brush", "span": [19, 26], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "a long tool", "span": [148, 159], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 10, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 14, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 11, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 5, "mark_id": 21, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 6, "mark_id": 18, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 7, "mark_id": 15, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 8, "mark_id": 22, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 9, "mark_id": 12, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 10, "mark_id": 13, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 11, "mark_id": 24, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 12, "mark_id": 16, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 13, "mark_id": 25, "category": "chair or seat", "isthing": 1, "mentions": [{"text": "rows of blue seats", "span": [34, 52], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 14, "mark_id": 9, "category": "bag or package", "isthing": 1, "mentions": [{"text": "the right-hand bags", "span": [72, 91], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 15, "mark_id": 23, "category": "bag or package", "isthing": 1, "mentions": [{"text": "the right-hand bags", "span": [72, 91], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 16, "mark_id": 17, "category": "person", "isthing": 1, "mentions": [{"text": "an orange-shirted man", "span": [120, 141], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 17, "mark_id": 19, "category": "window", "isthing": 1, "mentions": [{"text": "the left window", "span": [166, 181], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "holds", "span": [13, 18], "spans": [[13, 18]], "particle": null, "mentions": [{"text": "holds", "span": [13, 18], "spans": [[13, 18]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "A masked man", "span": [0, 12], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a brush", "span": [19, 26], "groups": [1], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "holds", "status": "visible", "segments_frames": [[0, 18]], "segments": [[0.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 18, "start_frame_1based": 1, "end_frame_1based": 19, "start_s": 0.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 602, "end_source_frame": 737, "start_filename": "00000602.jpg", "end_filename": "00000737.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The masked man holds the upright brush throughout the observed sequence.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}}, {"text": "reaches", "span": [57, 64], "spans": [[57, 64]], "particle": null, "mentions": [{"text": "reaches", "span": [57, 64], "spans": [[57, 64]], "particle": null, "sentence_index": 1}], "type": "prepositional", "sentence_index": 1, "subject": {"text": "He", "span": [54, 56], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "the right-hand bags", "span": [72, 91], "groups": [14, 15], "group_mentions": [0, 0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 1, "text": "reaches", "status": "visible", "segments_frames": [[4, 5], [7, 9]], "segments": [[2.0, 2.9], [3.4, 4.9]], "segment_details": [{"start_frame": 4, "end_frame": 5, "start_frame_1based": 5, "end_frame_1based": 6, "start_s": 2.0, "end_s": 2.9, "last_observed_s": 2.4, "start_source_frame": 632, "end_source_frame": 638, "start_filename": "00000632.jpg", "end_filename": "00000638.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 7, "end_frame": 9, "start_frame_1based": 8, "end_frame_1based": 10, "start_s": 3.4, "end_s": 4.9, "last_observed_s": 4.4, "start_source_frame": 653, "end_source_frame": 668, "start_filename": "00000653.jpg", "end_filename": "00000668.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He extends toward the bags on his right, retracts, and then reaches down toward them again.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}}, {"text": "gestures", "span": [98, 106], "spans": [[98, 106]], "particle": null, "mentions": [{"text": "gestures", "span": [98, 106], "spans": [[98, 106]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "He", "span": [54, 56], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 2, "text": "gestures", "status": "visible", "segments_frames": [[10, 18]], "segments": [[5.0, 9.2]], "segment_details": [{"start_frame": 10, "end_frame": 18, "start_frame_1based": 11, "end_frame_1based": 19, "start_s": 5.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 677, "end_source_frame": 737, "start_filename": "00000677.jpg", "end_filename": "00000737.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He repeatedly raises and changes the position of his free gloved hand.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}}, {"text": "moves", "span": [142, 147], "spans": [[142, 147]], "particle": null, "mentions": [{"text": "moves", "span": [142, 147], "spans": [[142, 147]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "an orange-shirted man", "span": [120, 141], "groups": [16], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a long tool", "span": [148, 159], "groups": [1], "group_mentions": [1], "group_match": "exact"}, {"text": "the left window", "span": [166, 181], "groups": [17], "group_mentions": [0], "group_match": "exact"}], "confidence": "medium", "temporal": {"action_index": 3, "text": "moves", "status": "visible", "segments_frames": [[4, 15]], "segments": [[2.0, 7.9]], "segment_details": [{"start_frame": 4, "end_frame": 15, "start_frame_1based": 5, "end_frame_1based": 16, "start_s": 2.0, "end_s": 7.9, "last_observed_s": 7.4, "start_source_frame": 632, "end_source_frame": 713, "start_filename": "00000632.jpg", "end_filename": "00000713.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The orange-shirted man shifts the long tool along the left window area before lifting it away.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}}], "coverage": {"marks_shown": 25, "marks_grounded": 18, "things_shown": 21, "things_grounded": 17, "stuff_shown": 4, "stuff_grounded": 1, "n_mentions": 21, "category_disagree": 0, "category_checked": 18}, "marks": [{"mark_id": 8, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 19, "max_rel_area": 0.30749, "mean_rel_area": 0.22435, "centroid": [0.588, 0.545], "motion": 0.172}, {"mark_id": 17, "category": "person", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 45, "first_frame": 1, "last_frame": 45, "best_frame": 14, "max_rel_area": 0.12322, "mean_rel_area": 0.06952, "centroid": [0.27, 0.591], "motion": 0.191}, {"mark_id": 5, "category": "window", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 4, "max_rel_area": 0.10351, "mean_rel_area": 0.0792, "centroid": [0.913, 0.491], "motion": 0.066}, {"mark_id": 19, "category": "window", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 37, "first_frame": 9, "last_frame": 45, "best_frame": 40, "max_rel_area": 0.09158, "mean_rel_area": 0.04774, "centroid": [0.066, 0.446], "motion": 0.088}, {"mark_id": 10, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 9, "max_rel_area": 0.04664, "mean_rel_area": 0.02434, "centroid": [0.188, 0.772], "motion": 0.262}, {"mark_id": 14, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 41, "first_frame": 0, "last_frame": 45, "best_frame": 7, "max_rel_area": 0.04609, "mean_rel_area": 0.02403, "centroid": [0.781, 0.747], "motion": 0.125}, {"mark_id": 11, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 44, "first_frame": 0, "last_frame": 45, "best_frame": 0, "max_rel_area": 0.03693, "mean_rel_area": 0.01313, "centroid": [0.198, 0.683], "motion": 0.196}, {"mark_id": 9, "category": "bag or package", "isthing": 1, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 40, "max_rel_area": 0.03152, "mean_rel_area": 0.02121, "centroid": [0.854, 0.9], "motion": 0.168}, {"mark_id": 7, "category": "window", "isthing": 1, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 45, "first_frame": 0, "last_frame": 45, "best_frame": 6, "max_rel_area": 0.03233, "mean_rel_area": 0.01119, "centroid": [0.165, 0.429], "motion": 0.208}, {"mark_id": 20, "category": "bag or package", "isthing": 1, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 36, "first_frame": 10, "last_frame": 45, "best_frame": 22, "max_rel_area": 0.04279, "mean_rel_area": 0.01934, "centroid": [0.148, 0.901], "motion": 0.191}, {"mark_id": 21, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 10, "color": [139, 195, 74], "panel": 1, "n_present": 36, "first_frame": 10, "last_frame": 45, "best_frame": 22, "max_rel_area": 0.04197, "mean_rel_area": 0.02782, "centroid": [0.294, 0.9], "motion": 0.141}, {"mark_id": 6, "category": "window", "isthing": 1, "iscrowd": 0, "order": 11, "color": [255, 87, 34], "panel": 1, "n_present": 44, "first_frame": 0, "last_frame": 45, "best_frame": 0, "max_rel_area": 0.02544, "mean_rel_area": 0.0121, "centroid": [0.743, 0.421], "motion": 0.12}, {"mark_id": 18, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 12, "color": [156, 39, 176], "panel": 1, "n_present": 37, "first_frame": 4, "last_frame": 45, "best_frame": 28, "max_rel_area": 0.02492, "mean_rel_area": 0.00783, "centroid": [0.726, 0.898], "motion": 0.1}, {"mark_id": 15, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 13, "color": [0, 188, 212], "panel": 1, "n_present": 33, "first_frame": 0, "last_frame": 39, "best_frame": 0, "max_rel_area": 0.02412, "mean_rel_area": 0.00949, "centroid": [0.742, 0.686], "motion": 0.116}, {"mark_id": 22, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 14, "color": [121, 85, 72], "panel": 1, "n_present": 29, "first_frame": 17, "last_frame": 45, "best_frame": 19, "max_rel_area": 0.02524, "mean_rel_area": 0.01128, "centroid": [0.798, 0.926], "motion": 0.094}, {"mark_id": 12, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 15, "color": [233, 30, 99], "panel": 1, "n_present": 29, "first_frame": 0, "last_frame": 45, "best_frame": 2, "max_rel_area": 0.02069, "mean_rel_area": 0.01056, "centroid": [0.228, 0.68], "motion": 0.172}, {"mark_id": 23, "category": "bag or package", "isthing": 1, "iscrowd": 0, "order": 16, "color": [63, 81, 181], "panel": 1, "n_present": 28, "first_frame": 18, "last_frame": 45, "best_frame": 21, "max_rel_area": 0.01389, "mean_rel_area": 0.00784, "centroid": [0.945, 0.942], "motion": 0.062}, {"mark_id": 13, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 17, "color": [205, 220, 57], "panel": 1, "n_present": 25, "first_frame": 0, "last_frame": 45, "best_frame": 5, "max_rel_area": 0.01427, "mean_rel_area": 0.00761, "centroid": [0.26, 0.65], "motion": 0.149}, {"mark_id": 24, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 18, "color": [255, 152, 0], "panel": 1, "n_present": 28, "first_frame": 18, "last_frame": 45, "best_frame": 19, "max_rel_area": 0.00735, "mean_rel_area": 0.0034, "centroid": [0.968, 0.916], "motion": 0.05}, {"mark_id": 16, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 19, "color": [96, 125, 139], "panel": 1, "n_present": 18, "first_frame": 0, "last_frame": 32, "best_frame": 7, "max_rel_area": 0.00929, "mean_rel_area": 0.00531, "centroid": [0.69, 0.653], "motion": 0.085}, {"mark_id": 25, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 20, "color": [54, 67, 244], "panel": 2, "n_present": 7, "first_frame": 25, "last_frame": 31, "best_frame": 28, "max_rel_area": 0.00482, "mean_rel_area": 0.00279, "centroid": [0.675, 0.56], "motion": 0.031}, {"mark_id": 2, "category": "ceiling", "isthing": 0, "iscrowd": 0, "order": 21, "color": [34, 189, 255], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 13, "max_rel_area": 0.18146, "mean_rel_area": 0.14728, "centroid": [0.497, 0.185], "motion": 0.043}, {"mark_id": 4, "category": "tool", "isthing": 0, "iscrowd": 0, "order": 22, "color": [60, 200, 60], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 5, "max_rel_area": 0.12016, "mean_rel_area": 0.09411, "centroid": [0.498, 0.491], "motion": 0.056}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 23, "color": [240, 120, 40], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 0, "max_rel_area": 0.09356, "mean_rel_area": 0.06321, "centroid": [0.507, 0.543], "motion": 0.053}, {"mark_id": 3, "category": "lamp", "isthing": 0, "iscrowd": 0, "order": 24, "color": [180, 60, 220], "panel": 2, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 7, "max_rel_area": 0.04261, "mean_rel_area": 0.03657, "centroid": [0.504, 0.173], "motion": 0.074}], "num_frames": 46, "duration_s": 9.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0, 4.2, 4.4, 4.6, 4.8, 5.0, 5.2, 5.4, 5.6, 5.8, 6.0, 6.2, 6.4, 6.6, 6.8, 7.0, 7.2, 7.4, 7.6, 7.8, 8.0, 8.2, 8.4, 8.6, 8.8, 9.0], "source_frame_indices": [602, 605, 608, 611, 614, 617, 620, 623, 626, 629, 632, 635, 638, 641, 644, 647, 650, 653, 656, 659, 662, 665, 668, 671, 674, 677, 680, 683, 686, 689, 692, 695, 698, 701, 704, 707, 710, 713, 716, 719, 722, 725, 728, 731, 734, 737], "timing_sha": "06779b767870", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 8, "unlinked": 0, "snapped": 0, "collective": 1, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 27625, "output_tokens": 1084, "thinking_tokens": 270, "total_tokens": 28709}, "seconds": 21.0, "response_id": "resp_09d574a34ad985a7006ac33001693487d095bcd6b6a1737383", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 602, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 608, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 617, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 623, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 632, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 638, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 647, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 653, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 662, "time_s": 4.0}, {"role": "clean", "path": "clean/f022.jpg", "frame": 9, "source_frame": 668, "time_s": 4.4}, {"role": "clean", "path": "clean/f025.jpg", "frame": 10, "source_frame": 677, "time_s": 5.0}, {"role": "clean", "path": "clean/f027.jpg", "frame": 11, "source_frame": 683, "time_s": 5.4}, {"role": "clean", "path": "clean/f030.jpg", "frame": 12, "source_frame": 692, "time_s": 6.0}, {"role": "clean", "path": "clean/f032.jpg", "frame": 13, "source_frame": 698, "time_s": 6.4}, {"role": "clean", "path": "clean/f035.jpg", "frame": 14, "source_frame": 707, "time_s": 7.0}, {"role": "clean", "path": "clean/f037.jpg", "frame": 15, "source_frame": 713, "time_s": 7.4}, {"role": "clean", "path": "clean/f040.jpg", "frame": 16, "source_frame": 722, "time_s": 8.0}, {"role": "clean", "path": "clean/f042.jpg", "frame": 17, "source_frame": 728, "time_s": 8.4}, {"role": "clean", "path": "clean/f045.jpg", "frame": 18, "source_frame": 737, "time_s": 9.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 602, "time_s": 0.0, "mark_ids": [8, 5, 10, 14, 11, 9, 7]}, {"role": "marks", "path": "marks/f000_p1.jpg", "frame": 0, "panel": 1, "source_frame": 602, "time_s": 0.0, "mark_ids": [6, 15, 12, 13, 16]}, {"role": "marks", "path": "marks/f000_p2.jpg", "frame": 0, "panel": 2, "source_frame": 602, "time_s": 0.0, "mark_ids": [2, 4, 1, 3]}, {"role": "marks", "path": "marks/f015_p0.jpg", "frame": 15, "panel": 0, "source_frame": 647, "time_s": 3.0, "mark_ids": [8, 17, 5, 19, 10, 14, 11, 9, 7, 20]}, {"role": "marks", "path": "marks/f015_p1.jpg", "frame": 15, "panel": 1, "source_frame": 647, "time_s": 3.0, "mark_ids": [21, 6, 18, 15, 12]}, {"role": "marks", "path": "marks/f015_p2.jpg", "frame": 15, "panel": 2, "source_frame": 647, "time_s": 3.0, "mark_ids": [2, 4, 1, 3]}, {"role": "marks", "path": "marks/f030_p0.jpg", "frame": 30, "panel": 0, "source_frame": 692, "time_s": 6.0, "mark_ids": [8, 17, 5, 19, 10, 14, 11, 9, 7, 20]}, {"role": "marks", "path": "marks/f030_p1.jpg", "frame": 30, "panel": 1, "source_frame": 692, "time_s": 6.0, "mark_ids": [21, 6, 18, 15, 22, 23, 24, 16]}, {"role": "marks", "path": "marks/f030_p2.jpg", "frame": 30, "panel": 2, "source_frame": 692, "time_s": 6.0, "mark_ids": [25, 2, 4, 1, 3]}, {"role": "marks", "path": "marks/f045_p0.jpg", "frame": 45, "panel": 0, "source_frame": 737, "time_s": 9.0, "mark_ids": [8, 17, 5, 19, 10, 14, 11, 9, 7, 20]}, {"role": "marks", "path": "marks/f045_p1.jpg", "frame": 45, "panel": 1, "source_frame": 737, "time_s": 9.0, "mark_ids": [21, 6, 18, 22, 12, 23, 13, 24]}, {"role": "marks", "path": "marks/f045_p2.jpg", "frame": 45, "panel": 2, "source_frame": 737, "time_s": 9.0, "mark_ids": [2, 4, 1, 3]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [8, 17, 5, 19, 10, 14, 11, 9, 7, 20, 21, 6, 18, 15, 22, 12]}, {"role": "ids", "path": "ids/sheet1.jpg", "mark_ids": [23, 13, 24, 16, 25, 2, 4, 1, 3]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "06779b767870", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737], "actions": [{"action_index": 0, "text": "holds", "status": "visible", "segments_frames": [[0, 18]], "segments": [[0.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 18, "start_frame_1based": 1, "end_frame_1based": 19, "start_s": 0.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 602, "end_source_frame": 737, "start_filename": "00000602.jpg", "end_filename": "00000737.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The masked man holds the upright brush throughout the observed sequence.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}, {"action_index": 1, "text": "reaches", "status": "visible", "segments_frames": [[4, 5], [7, 9]], "segments": [[2.0, 2.9], [3.4, 4.9]], "segment_details": [{"start_frame": 4, "end_frame": 5, "start_frame_1based": 5, "end_frame_1based": 6, "start_s": 2.0, "end_s": 2.9, "last_observed_s": 2.4, "start_source_frame": 632, "end_source_frame": 638, "start_filename": "00000632.jpg", "end_filename": "00000638.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 7, "end_frame": 9, "start_frame_1based": 8, "end_frame_1based": 10, "start_s": 3.4, "end_s": 4.9, "last_observed_s": 4.4, "start_source_frame": 653, "end_source_frame": 668, "start_filename": "00000653.jpg", "end_filename": "00000668.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He extends toward the bags on his right, retracts, and then reaches down toward them again.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}, {"action_index": 2, "text": "gestures", "status": "visible", "segments_frames": [[10, 18]], "segments": [[5.0, 9.2]], "segment_details": [{"start_frame": 10, "end_frame": 18, "start_frame_1based": 11, "end_frame_1based": 19, "start_s": 5.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 677, "end_source_frame": 737, "start_filename": "00000677.jpg", "end_filename": "00000737.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He repeatedly raises and changes the position of his free gloved hand.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}, {"action_index": 3, "text": "moves", "status": "visible", "segments_frames": [[4, 15]], "segments": [[2.0, 7.9]], "segment_details": [{"start_frame": 4, "end_frame": 15, "start_frame_1based": 5, "end_frame_1based": 16, "start_s": 2.0, "end_s": 7.9, "last_observed_s": 7.4, "start_source_frame": 632, "end_source_frame": 713, "start_filename": "00000632.jpg", "end_filename": "00000713.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The orange-shirted man shifts the long tool along the left window area before lifting it away.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [602, 608, 617, 623, 632, 638, 647, 653, 662, 668, 677, 683, 692, 698, 707, 713, 722, 728, 737]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "826_0bniEAy5YfI", "split": "train", "caption": "Four players play badminton across a net indoors. They move sideways on the wooden floor during the rally. A man sits on a raised chair.", "mentions": [{"text": "Four players", "span": [0, 12], "occurrence": 1, "mark_ids": [7, 8, 9, 11], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a net", "span": [35, 40], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "They", "span": [50, 54], "occurrence": 1, "mark_ids": [7, 8, 9, 11], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "the wooden floor", "span": [72, 88], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "A man", "span": [107, 112], "occurrence": 1, "mark_ids": [10], "confidence": "high", "sentence_index": 2, "is_pronoun": false}, {"text": "a raised chair", "span": [121, 135], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 49], [50, 106], [107, 136]], "groups": [{"group_id": 0, "mark_id": 7, "category": "person", "isthing": 1, "mentions": [{"text": "Four players", "span": [0, 12], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "They", "span": [50, 54], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 8, "category": "person", "isthing": 1, "mentions": [{"text": "Four players", "span": [0, 12], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "They", "span": [50, 54], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 9, "category": "person", "isthing": 1, "mentions": [{"text": "Four players", "span": [0, 12], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "They", "span": [50, 54], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 11, "category": "person", "isthing": 1, "mentions": [{"text": "Four players", "span": [0, 12], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "They", "span": [50, 54], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 5, "category": "ball net", "isthing": 0, "mentions": [{"text": "a net", "span": [35, 40], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 5, "mark_id": 3, "category": "floor", "isthing": 0, "mentions": [{"text": "the wooden floor", "span": [72, 88], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 6, "mark_id": 10, "category": "person", "isthing": 1, "mentions": [{"text": "A man", "span": [107, 112], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 7, "mark_id": 6, "category": "shelf", "isthing": 0, "mentions": [{"text": "a raised chair", "span": [121, 135], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["disagree"], "chain_agreement": "disagree"}], "actions": [{"text": "play badminton", "span": [13, 27], "spans": [[13, 27]], "particle": null, "mentions": [{"text": "play badminton", "span": [13, 27], "spans": [[13, 27]], "particle": null, "sentence_index": 0}], "type": "nonvisual_object", "sentence_index": 0, "subject": {"text": "Four players", "span": [0, 12], "groups": [0, 1, 2, 3], "group_mentions": [0, 0, 0, 0], "group_match": "exact"}, "objects": [{"text": "a net", "span": [35, 40], "groups": [4], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "play badminton", "status": "visible", "segments_frames": [[4, 10]], "segments": [[2.0, 5.2]], "segment_details": [{"start_frame": 4, "end_frame": 10, "start_frame_1based": 5, "end_frame_1based": 11, "start_s": 2.0, "end_s": 5.2, "last_observed_s": 5.0, "start_source_frame": 107, "end_source_frame": 152, "start_filename": "00000107.jpg", "end_filename": "00000152.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The players exchange shots and react to the rally after initially holding ready positions.", "fps": 2.0, "input_frame_count": 11, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0], "source_frame_indices": [77, 83, 92, 98, 107, 113, 122, 128, 137, 143, 152]}}, {"text": "move", "span": [55, 59], "spans": [[55, 59]], "particle": null, "mentions": [{"text": "move", "span": [55, 59], "spans": [[55, 59]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "They", "span": [50, 54], "groups": [0, 1, 2, 3], "group_mentions": [1, 1, 1, 1], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 1, "text": "move", "status": "visible", "segments_frames": [[4, 10]], "segments": [[2.0, 5.2]], "segment_details": [{"start_frame": 4, "end_frame": 10, "start_frame_1based": 5, "end_frame_1based": 11, "start_s": 2.0, "end_s": 5.2, "last_observed_s": 5.0, "start_source_frame": 107, "end_source_frame": 152, "start_filename": "00000107.jpg", "end_filename": "00000152.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Both pairs reposition laterally during the rally, although their individual movements occur at different moments.", "fps": 2.0, "input_frame_count": 11, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0], "source_frame_indices": [77, 83, 92, 98, 107, 113, 122, 128, 137, 143, 152]}}, {"text": "sits", "span": [113, 117], "spans": [[113, 117]], "particle": null, "mentions": [{"text": "sits", "span": [113, 117], "spans": [[113, 117]], "particle": null, "sentence_index": 2}], "type": "posture", "sentence_index": 2, "subject": {"text": "A man", "span": [107, 112], "groups": [6], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a raised chair", "span": [121, 135], "groups": [7], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 2, "text": "sits", "status": "visible", "segments_frames": [[0, 10]], "segments": [[0.0, 5.2]], "segment_details": [{"start_frame": 0, "end_frame": 10, "start_frame_1based": 1, "end_frame_1based": 11, "start_s": 0.0, "end_s": 5.2, "last_observed_s": 5.0, "start_source_frame": 77, "end_source_frame": 152, "start_filename": "00000077.jpg", "end_filename": "00000152.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The man remains seated on the elevated chair throughout the sequence.", "fps": 2.0, "input_frame_count": 11, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0], "source_frame_indices": [77, 83, 92, 98, 107, 113, 122, 128, 137, 143, 152]}}], "coverage": {"marks_shown": 16, "marks_grounded": 8, "things_shown": 10, "things_grounded": 5, "stuff_shown": 6, "stuff_grounded": 3, "n_mentions": 12, "category_disagree": 1, "category_checked": 8}, "marks": [{"mark_id": 7, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 21, "max_rel_area": 0.05058, "mean_rel_area": 0.03702, "centroid": [0.624, 0.633], "motion": 0.095}, {"mark_id": 8, "category": "person", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 25, "max_rel_area": 0.03113, "mean_rel_area": 0.01781, "centroid": [0.489, 0.57], "motion": 0.118}, {"mark_id": 10, "category": "person", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 2, "max_rel_area": 0.00791, "mean_rel_area": 0.00728, "centroid": [0.87, 0.3], "motion": 0.039}, {"mark_id": 14, "category": "bag or package", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 14, "max_rel_area": 0.00465, "mean_rel_area": 0.00433, "centroid": [0.071, 0.647], "motion": 0.02}, {"mark_id": 13, "category": "bag or package", "isthing": 1, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 18, "max_rel_area": 0.00419, "mean_rel_area": 0.00395, "centroid": [0.782, 0.569], "motion": 0.037}, {"mark_id": 16, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 25, "max_rel_area": 0.00307, "mean_rel_area": 0.00288, "centroid": [0.084, 0.493], "motion": 0.02}, {"mark_id": 15, "category": "chair or seat", "isthing": 1, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 10, "max_rel_area": 0.00269, "mean_rel_area": 0.00224, "centroid": [0.014, 0.477], "motion": 0.019}, {"mark_id": 9, "category": "person", "isthing": 1, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 19, "max_rel_area": 0.00202, "mean_rel_area": 0.00118, "centroid": [0.399, 0.511], "motion": 0.112}, {"mark_id": 11, "category": "person", "isthing": 1, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 22, "first_frame": 0, "last_frame": 25, "best_frame": 25, "max_rel_area": 0.00204, "mean_rel_area": 0.00053, "centroid": [0.466, 0.37], "motion": 0.184}, {"mark_id": 12, "category": "person", "isthing": 1, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 1, "first_frame": 0, "last_frame": 0, "best_frame": 0, "max_rel_area": 8e-05, "mean_rel_area": 8e-05, "centroid": [0.532, 0.437], "motion": 0.0}, {"mark_id": 3, "category": "floor", "isthing": 0, "iscrowd": 0, "order": 10, "color": [139, 195, 74], "panel": 1, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 3, "max_rel_area": 0.44497, "mean_rel_area": 0.41641, "centroid": [0.5, 0.744], "motion": 0.016}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 11, "color": [255, 87, 34], "panel": 1, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 20, "max_rel_area": 0.29278, "mean_rel_area": 0.28657, "centroid": [0.5, 0.284], "motion": 0.018}, {"mark_id": 5, "category": "ball net", "isthing": 0, "iscrowd": 0, "order": 12, "color": [156, 39, 176], "panel": 1, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 17, "max_rel_area": 0.09738, "mean_rel_area": 0.08501, "centroid": [0.43, 0.505], "motion": 0.027}, {"mark_id": 2, "category": "ceiling", "isthing": 0, "iscrowd": 0, "order": 13, "color": [0, 188, 212], "panel": 1, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 0, "max_rel_area": 0.09218, "mean_rel_area": 0.07065, "centroid": [0.5, 0.054], "motion": 0.009}, {"mark_id": 6, "category": "shelf", "isthing": 0, "iscrowd": 0, "order": 14, "color": [121, 85, 72], "panel": 1, "n_present": 26, "first_frame": 0, "last_frame": 25, "best_frame": 5, "max_rel_area": 0.03631, "mean_rel_area": 0.03588, "centroid": [0.85, 0.436], "motion": 0.042}, {"mark_id": 4, "category": "lamp", "isthing": 0, "iscrowd": 0, "order": 15, "color": [233, 30, 99], "panel": 1, "n_present": 21, "first_frame": 0, "last_frame": 25, "best_frame": 0, "max_rel_area": 0.00224, "mean_rel_area": 0.00084, "centroid": [0.618, 0.015], "motion": 0.189}], "num_frames": 26, "duration_s": 5.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0, 4.2, 4.4, 4.6, 4.8, 5.0], "source_frame_indices": [77, 80, 83, 86, 89, 92, 95, 98, 101, 104, 107, 110, 113, 116, 119, 122, 125, 128, 131, 134, 137, 140, 143, 146, 149, 152], "timing_sha": "fdb0bb06e3f2", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 5, "unlinked": 0, "snapped": 0, "collective": 2, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 21225, "output_tokens": 1106, "thinking_tokens": 413, "total_tokens": 22331}, "seconds": 28.79, "response_id": "resp_0f98e74cdf600e1a006ac330164b4c87d0a33e2dc55b2c0c97", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 77, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 83, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 92, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 98, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 107, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 113, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 122, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 128, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 137, "time_s": 4.0}, {"role": "clean", "path": "clean/f022.jpg", "frame": 9, "source_frame": 143, "time_s": 4.4}, {"role": "clean", "path": "clean/f025.jpg", "frame": 10, "source_frame": 152, "time_s": 5.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 77, "time_s": 0.0, "mark_ids": [7, 8, 10, 14, 13, 16, 15, 9, 11, 12]}, {"role": "marks", "path": "marks/f000_p1.jpg", "frame": 0, "panel": 1, "source_frame": 77, "time_s": 0.0, "mark_ids": [3, 1, 5, 2, 6, 4]}, {"role": "marks", "path": "marks/f007_p0.jpg", "frame": 7, "panel": 0, "source_frame": 98, "time_s": 1.4, "mark_ids": [7, 8, 10, 14, 13, 16, 15, 9, 11]}, {"role": "marks", "path": "marks/f007_p1.jpg", "frame": 7, "panel": 1, "source_frame": 98, "time_s": 1.4, "mark_ids": [3, 1, 5, 2, 6, 4]}, {"role": "marks", "path": "marks/f017_p0.jpg", "frame": 17, "panel": 0, "source_frame": 128, "time_s": 3.4, "mark_ids": [7, 8, 10, 14, 13, 16, 15, 9, 11]}, {"role": "marks", "path": "marks/f017_p1.jpg", "frame": 17, "panel": 1, "source_frame": 128, "time_s": 3.4, "mark_ids": [3, 1, 5, 2, 6, 4]}, {"role": "marks", "path": "marks/f025_p0.jpg", "frame": 25, "panel": 0, "source_frame": 152, "time_s": 5.0, "mark_ids": [7, 8, 10, 14, 13, 16, 15, 9, 11]}, {"role": "marks", "path": "marks/f025_p1.jpg", "frame": 25, "panel": 1, "source_frame": 152, "time_s": 5.0, "mark_ids": [3, 1, 5, 2, 6, 4]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [7, 8, 10, 14, 13, 16, 15, 9, 11, 12, 3, 1, 5, 2, 6, 4]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "fdb0bb06e3f2", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 11, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0], "source_frame_indices": [77, 83, 92, 98, 107, 113, 122, 128, 137, 143, 152], "actions": [{"action_index": 0, "text": "play badminton", "status": "visible", "segments_frames": [[4, 10]], "segments": [[2.0, 5.2]], "segment_details": [{"start_frame": 4, "end_frame": 10, "start_frame_1based": 5, "end_frame_1based": 11, "start_s": 2.0, "end_s": 5.2, "last_observed_s": 5.0, "start_source_frame": 107, "end_source_frame": 152, "start_filename": "00000107.jpg", "end_filename": "00000152.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The players exchange shots and react to the rally after initially holding ready positions.", "fps": 2.0, "input_frame_count": 11, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0], "source_frame_indices": [77, 83, 92, 98, 107, 113, 122, 128, 137, 143, 152]}, {"action_index": 1, "text": "move", "status": "visible", "segments_frames": [[4, 10]], "segments": [[2.0, 5.2]], "segment_details": [{"start_frame": 4, "end_frame": 10, "start_frame_1based": 5, "end_frame_1based": 11, "start_s": 2.0, "end_s": 5.2, "last_observed_s": 5.0, "start_source_frame": 107, "end_source_frame": 152, "start_filename": "00000107.jpg", "end_filename": "00000152.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Both pairs reposition laterally during the rally, although their individual movements occur at different moments.", "fps": 2.0, "input_frame_count": 11, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0], "source_frame_indices": [77, 83, 92, 98, 107, 113, 122, 128, 137, 143, 152]}, {"action_index": 2, "text": "sits", "status": "visible", "segments_frames": [[0, 10]], "segments": [[0.0, 5.2]], "segment_details": [{"start_frame": 0, "end_frame": 10, "start_frame_1based": 1, "end_frame_1based": 11, "start_s": 0.0, "end_s": 5.2, "last_observed_s": 5.0, "start_source_frame": 77, "end_source_frame": 152, "start_filename": "00000077.jpg", "end_filename": "00000152.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The man remains seated on the elevated chair throughout the sequence.", "fps": 2.0, "input_frame_count": 11, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0], "source_frame_indices": [77, 83, 92, 98, 107, 113, 122, 128, 137, 143, 152]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "912_bGghFu3zO-c", "split": "train", "caption": "Four people sit in a red airplane-shaped ride outdoors. It moves right along an elevated track.", "mentions": [{"text": "Four people", "span": [0, 11], "occurrence": 1, "mark_ids": [10, 8, 11, 9], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a red airplane-shaped ride", "span": [19, 45], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "It", "span": [56, 58], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "an elevated track", "span": [77, 94], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 1, "is_pronoun": false}], "sentences": [[0, 55], [56, 95]], "groups": [{"group_id": 0, "mark_id": 10, "category": "person", "isthing": 1, "mentions": [{"text": "Four people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 8, "category": "person", "isthing": 1, "mentions": [{"text": "Four people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 11, "category": "person", "isthing": 1, "mentions": [{"text": "Four people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 9, "category": "person", "isthing": 1, "mentions": [{"text": "Four people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 6, "category": "toy", "isthing": 0, "mentions": [{"text": "a red airplane-shaped ride", "span": [19, 45], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "It", "span": [56, 58], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown", "pronoun"], "chain_agreement": "unknown"}, {"group_id": 5, "mark_id": 3, "category": "other construction", "isthing": 0, "mentions": [{"text": "an elevated track", "span": [77, 94], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["disagree"], "chain_agreement": "disagree"}], "actions": [{"text": "sit", "span": [12, 15], "spans": [[12, 15]], "particle": null, "mentions": [{"text": "sit", "span": [12, 15], "spans": [[12, 15]], "particle": null, "sentence_index": 0}], "type": "posture", "sentence_index": 0, "subject": {"text": "Four people", "span": [0, 11], "groups": [0, 1, 2, 3], "group_mentions": [0, 0, 0, 0], "group_match": "exact"}, "objects": [{"text": "a red airplane-shaped ride", "span": [19, 45], "groups": [4], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "sit", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 3896, "end_source_frame": 3938, "start_filename": "00003896.jpg", "end_filename": "00003938.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "All four riders remain seated throughout the sequence, although the rear riders are partly obscured.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3896, 3902, 3911, 3917, 3926, 3932, 3938]}}, {"text": "moves", "span": [59, 64], "spans": [[59, 64]], "particle": null, "mentions": [{"text": "moves", "span": [59, 64], "spans": [[59, 64]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "It", "span": [56, 58], "groups": [4], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "an elevated track", "span": [77, 94], "groups": [5], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 1, "text": "moves", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 3896, "end_source_frame": 3938, "start_filename": "00003896.jpg", "end_filename": "00003938.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The ride advances rightward along the track relative to the fixed supports and fence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3896, 3902, 3911, 3917, 3926, 3932, 3938]}}], "coverage": {"marks_shown": 11, "marks_grounded": 6, "things_shown": 4, "things_grounded": 4, "stuff_shown": 7, "stuff_grounded": 2, "n_mentions": 7, "category_disagree": 1, "category_checked": 5}, "marks": [{"mark_id": 10, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.00905, "mean_rel_area": 0.00805, "centroid": [0.292, 0.446], "motion": 0.082}, {"mark_id": 8, "category": "person", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.00655, "mean_rel_area": 0.00583, "centroid": [0.369, 0.463], "motion": 0.103}, {"mark_id": 11, "category": "person", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.00211, "mean_rel_area": 0.00173, "centroid": [0.337, 0.413], "motion": 0.081}, {"mark_id": 9, "category": "person", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 4, "max_rel_area": 0.00165, "mean_rel_area": 0.00121, "centroid": [0.394, 0.443], "motion": 0.099}, {"mark_id": 5, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.45942, "mean_rel_area": 0.43292, "centroid": [0.5, 0.5], "motion": 0.0}, {"mark_id": 3, "category": "other construction", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 9, "max_rel_area": 0.24749, "mean_rel_area": 0.24303, "centroid": [0.5, 0.586], "motion": 0.008}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.11157, "mean_rel_area": 0.10113, "centroid": [0.191, 0.849], "motion": 0.056}, {"mark_id": 6, "category": "toy", "isthing": 0, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 7, "max_rel_area": 0.08378, "mean_rel_area": 0.08018, "centroid": [0.262, 0.534], "motion": 0.167}, {"mark_id": 2, "category": "handrail or fence", "isthing": 0, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.049, "mean_rel_area": 0.04277, "centroid": [0.285, 0.512], "motion": 0.058}, {"mark_id": 7, "category": "other electronic product", "isthing": 0, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 13, "max_rel_area": 0.00461, "mean_rel_area": 0.0041, "centroid": [0.148, 0.347], "motion": 0.116}, {"mark_id": 4, "category": "lamp", "isthing": 0, "iscrowd": 0, "order": 10, "color": [139, 195, 74], "panel": 1, "n_present": 12, "first_frame": 0, "last_frame": 11, "best_frame": 4, "max_rel_area": 0.00311, "mean_rel_area": 0.00232, "centroid": [0.03, 0.924], "motion": 0.06}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [3896, 3899, 3902, 3905, 3908, 3911, 3914, 3917, 3920, 3923, 3926, 3929, 3932, 3935, 3938], "timing_sha": "80217a3560b8", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 4, "unlinked": 0, "snapped": 0, "collective": 1, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 19001, "output_tokens": 698, "thinking_tokens": 163, "total_tokens": 19699}, "seconds": 16.5, "response_id": "resp_04d4eddee56b0ec4006ac33019619487d09e97dd0e15c9f7a4", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 3896, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 3902, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 3911, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 3917, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 3926, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 3932, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 3938, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 3896, "time_s": 0.0, "mark_ids": [10, 8, 11, 9, 5, 3, 1, 6, 2, 7]}, {"role": "marks", "path": "marks/f000_p1.jpg", "frame": 0, "panel": 1, "source_frame": 3896, "time_s": 0.0, "mark_ids": [4]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 3911, "time_s": 1.0, "mark_ids": [10, 8, 11, 9, 5, 3, 1, 6, 2, 7]}, {"role": "marks", "path": "marks/f005_p1.jpg", "frame": 5, "panel": 1, "source_frame": 3911, "time_s": 1.0, "mark_ids": [4]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 3926, "time_s": 2.0, "mark_ids": [10, 8, 11, 9, 5, 3, 1, 6, 2, 7]}, {"role": "marks", "path": "marks/f010_p1.jpg", "frame": 10, "panel": 1, "source_frame": 3926, "time_s": 2.0, "mark_ids": [4]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 3938, "time_s": 2.8, "mark_ids": [10, 8, 11, 9, 5, 3, 1, 6, 2, 7]}, {"role": "marks", "path": "marks/f014_p1.jpg", "frame": 14, "panel": 1, "source_frame": 3938, "time_s": 2.8, "mark_ids": []}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [10, 8, 11, 9, 5, 3, 1, 6, 2, 7, 4]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "80217a3560b8", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3896, 3902, 3911, 3917, 3926, 3932, 3938], "actions": [{"action_index": 0, "text": "sit", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 3896, "end_source_frame": 3938, "start_filename": "00003896.jpg", "end_filename": "00003938.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "All four riders remain seated throughout the sequence, although the rear riders are partly obscured.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3896, 3902, 3911, 3917, 3926, 3932, 3938]}, {"action_index": 1, "text": "moves", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 3896, "end_source_frame": 3938, "start_filename": "00003896.jpg", "end_filename": "00003938.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The ride advances rightward along the track relative to the fixed supports and fence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3896, 3902, 3911, 3917, 3926, 3932, 3938]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "9_E0zfiF4DCt8", "split": "train", "caption": "A red-hooded skier gestures on a snowy slope. They remain standing. Three other skiers glide downhill.", "mentions": [{"text": "A red-hooded skier", "span": [0, 18], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a snowy slope", "span": [31, 44], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "They", "span": [46, 50], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "Three other skiers", "span": [68, 86], "occurrence": 1, "mark_ids": [7, 8, 9], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 45], [46, 67], [68, 102]], "groups": [{"group_id": 0, "mark_id": 6, "category": "person", "isthing": 1, "mentions": [{"text": "A red-hooded skier", "span": [0, 18], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "They", "span": [46, 50], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown", "pronoun"], "chain_agreement": "unknown"}, {"group_id": 1, "mark_id": 4, "category": "snowfield", "isthing": 0, "mentions": [{"text": "a snowy slope", "span": [31, 44], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["disagree"], "chain_agreement": "disagree"}, {"group_id": 2, "mark_id": 7, "category": "person", "isthing": 1, "mentions": [{"text": "Three other skiers", "span": [68, 86], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["unknown"], "chain_agreement": "unknown"}, {"group_id": 3, "mark_id": 8, "category": "person", "isthing": 1, "mentions": [{"text": "Three other skiers", "span": [68, 86], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["unknown"], "chain_agreement": "unknown"}, {"group_id": 4, "mark_id": 9, "category": "person", "isthing": 1, "mentions": [{"text": "Three other skiers", "span": [68, 86], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["unknown"], "chain_agreement": "unknown"}], "actions": [{"text": "gestures", "span": [19, 27], "spans": [[19, 27]], "particle": null, "mentions": [{"text": "gestures", "span": [19, 27], "spans": [[19, 27]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "A red-hooded skier", "span": [0, 18], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 0, "text": "gestures", "status": "visible", "segments_frames": [[0, 5]], "segments": [[0.0, 2.9]], "segment_details": [{"start_frame": 0, "end_frame": 5, "start_frame_1based": 1, "end_frame_1based": 6, "start_s": 0.0, "end_s": 2.9, "last_observed_s": 2.4, "start_source_frame": 152, "end_source_frame": 188, "start_filename": "00000152.jpg", "end_filename": "00000188.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The red-hooded skier moves their gloved hands from near their face to forward positions.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272, 278, 287]}}, {"text": "standing", "span": [58, 66], "spans": [[58, 66]], "particle": null, "mentions": [{"text": "standing", "span": [58, 66], "spans": [[58, 66]], "particle": null, "sentence_index": 1}], "type": "posture", "sentence_index": 1, "subject": {"text": "They", "span": [46, 50], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 1, "text": "standing", "status": "visible", "segments_frames": [[0, 18]], "segments": [[0.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 18, "start_frame_1based": 1, "end_frame_1based": 19, "start_s": 0.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 152, "end_source_frame": 287, "start_filename": "00000152.jpg", "end_filename": "00000287.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The red-hooded skier maintains an upright, slightly bent standing posture throughout.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272, 278, 287]}}, {"text": "glide", "span": [87, 92], "spans": [[87, 92]], "particle": null, "mentions": [{"text": "glide", "span": [87, 92], "spans": [[87, 92]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "Three other skiers", "span": [68, 86], "groups": [2, 3, 4], "group_mentions": [0, 0, 0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 2, "text": "glide", "status": "visible", "segments_frames": [[0, 18]], "segments": [[0.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 18, "start_frame_1based": 1, "end_frame_1based": 19, "start_s": 0.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 152, "end_source_frame": 287, "start_filename": "00000152.jpg", "end_filename": "00000287.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The skiers descend successively: the first is visible throughout, the second enters around frame 6, and the third appears around frame 14.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272, 278, 287]}}], "coverage": {"marks_shown": 9, "marks_grounded": 5, "things_shown": 4, "things_grounded": 4, "stuff_shown": 5, "stuff_grounded": 1, "n_mentions": 6, "category_disagree": 1, "category_checked": 1}, "marks": [{"mark_id": 6, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 13, "max_rel_area": 0.06561, "mean_rel_area": 0.05858, "centroid": [0.672, 0.468], "motion": 0.13}, {"mark_id": 7, "category": "person", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 4, "max_rel_area": 0.00795, "mean_rel_area": 0.00222, "centroid": [0.359, 0.248], "motion": 0.414}, {"mark_id": 8, "category": "person", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 32, "first_frame": 14, "last_frame": 45, "best_frame": 18, "max_rel_area": 0.01218, "mean_rel_area": 0.00399, "centroid": [0.3, 0.297], "motion": 0.458}, {"mark_id": 9, "category": "person", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 11, "first_frame": 35, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.0008, "mean_rel_area": 0.00065, "centroid": [0.071, 0.31], "motion": 0.133}, {"mark_id": 4, "category": "snowfield", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.74057, "mean_rel_area": 0.68521, "centroid": [0.5, 0.574], "motion": 0.024}, {"mark_id": 5, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 5, "max_rel_area": 0.17916, "mean_rel_area": 0.16862, "centroid": [0.5, 0.205], "motion": 0.018}, {"mark_id": 3, "category": "mountain", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 0, "max_rel_area": 0.07521, "mean_rel_area": 0.04843, "centroid": [0.423, 0.087], "motion": 0.178}, {"mark_id": 2, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 4, "max_rel_area": 0.06364, "mean_rel_area": 0.01888, "centroid": [0.458, 0.051], "motion": 0.396}, {"mark_id": 1, "category": "pillar", "isthing": 0, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 8, "max_rel_area": 0.00149, "mean_rel_area": 0.00123, "centroid": [0.32, 0.152], "motion": 0.147}], "num_frames": 46, "duration_s": 9.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0, 4.2, 4.4, 4.6, 4.8, 5.0, 5.2, 5.4, 5.6, 5.8, 6.0, 6.2, 6.4, 6.6, 6.8, 7.0, 7.2, 7.4, 7.6, 7.8, 8.0, 8.2, 8.4, 8.6, 8.8, 9.0], "source_frame_indices": [152, 155, 158, 161, 164, 167, 170, 173, 176, 179, 182, 185, 188, 191, 194, 197, 200, 203, 206, 209, 212, 215, 218, 221, 224, 227, 230, 233, 236, 239, 242, 245, 248, 251, 254, 257, 260, 263, 266, 269, 272, 275, 278, 281, 284, 287], "timing_sha": "2ea96d3c64f6", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 3, "unlinked": 0, "snapped": 0, "collective": 1, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 22262, "output_tokens": 955, "thinking_tokens": 333, "total_tokens": 23217}, "seconds": 20.35, "response_id": "resp_018ee69bcbc0c976006ac3302a030c87d08faf958df4cb608a", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 152, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 158, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 167, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 173, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 182, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 188, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 197, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 203, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 212, "time_s": 4.0}, {"role": "clean", "path": "clean/f022.jpg", "frame": 9, "source_frame": 218, "time_s": 4.4}, {"role": "clean", "path": "clean/f025.jpg", "frame": 10, "source_frame": 227, "time_s": 5.0}, {"role": "clean", "path": "clean/f027.jpg", "frame": 11, "source_frame": 233, "time_s": 5.4}, {"role": "clean", "path": "clean/f030.jpg", "frame": 12, "source_frame": 242, "time_s": 6.0}, {"role": "clean", "path": "clean/f032.jpg", "frame": 13, "source_frame": 248, "time_s": 6.4}, {"role": "clean", "path": "clean/f035.jpg", "frame": 14, "source_frame": 257, "time_s": 7.0}, {"role": "clean", "path": "clean/f037.jpg", "frame": 15, "source_frame": 263, "time_s": 7.4}, {"role": "clean", "path": "clean/f040.jpg", "frame": 16, "source_frame": 272, "time_s": 8.0}, {"role": "clean", "path": "clean/f042.jpg", "frame": 17, "source_frame": 278, "time_s": 8.4}, {"role": "clean", "path": "clean/f045.jpg", "frame": 18, "source_frame": 287, "time_s": 9.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 152, "time_s": 0.0, "mark_ids": [6, 7, 4, 5, 3, 2, 1]}, {"role": "marks", "path": "marks/f015_p0.jpg", "frame": 15, "panel": 0, "source_frame": 197, "time_s": 3.0, "mark_ids": [6, 7, 8, 4, 5, 3, 2, 1]}, {"role": "marks", "path": "marks/f030_p0.jpg", "frame": 30, "panel": 0, "source_frame": 242, "time_s": 6.0, "mark_ids": [6, 7, 8, 4, 5, 3, 2, 1]}, {"role": "marks", "path": "marks/f045_p0.jpg", "frame": 45, "panel": 0, "source_frame": 287, "time_s": 9.0, "mark_ids": [6, 7, 8, 9, 4, 5, 3, 2, 1]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [6, 7, 8, 9, 4, 5, 3, 2, 1]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "2ea96d3c64f6", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272, 278, 287], "actions": [{"action_index": 0, "text": "gestures", "status": "visible", "segments_frames": [[0, 5]], "segments": [[0.0, 2.9]], "segment_details": [{"start_frame": 0, "end_frame": 5, "start_frame_1based": 1, "end_frame_1based": 6, "start_s": 0.0, "end_s": 2.9, "last_observed_s": 2.4, "start_source_frame": 152, "end_source_frame": 188, "start_filename": "00000152.jpg", "end_filename": "00000188.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The red-hooded skier moves their gloved hands from near their face to forward positions.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272, 278, 287]}, {"action_index": 1, "text": "standing", "status": "visible", "segments_frames": [[0, 18]], "segments": [[0.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 18, "start_frame_1based": 1, "end_frame_1based": 19, "start_s": 0.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 152, "end_source_frame": 287, "start_filename": "00000152.jpg", "end_filename": "00000287.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The red-hooded skier maintains an upright, slightly bent standing posture throughout.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272, 278, 287]}, {"action_index": 2, "text": "glide", "status": "visible", "segments_frames": [[0, 18]], "segments": [[0.0, 9.2]], "segment_details": [{"start_frame": 0, "end_frame": 18, "start_frame_1based": 1, "end_frame_1based": 19, "start_s": 0.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 152, "end_source_frame": 287, "start_filename": "00000152.jpg", "end_filename": "00000287.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The skiers descend successively: the first is visible throughout, the second enters around frame 6, and the third appears around frame 14.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [152, 158, 167, 173, 182, 188, 197, 203, 212, 218, 227, 233, 242, 248, 257, 263, 272, 278, 287]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1039_QZROp4t0doU", "split": "train", "caption": "A motorcycle travels along a straight road between green fields and trees. It heads away into the distance.", "mentions": [{"text": "A motorcycle", "span": [0, 12], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a straight road", "span": [27, 42], "occurrence": 1, "mark_ids": [2], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "green fields", "span": [51, 63], "occurrence": 1, "mark_ids": [1], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "trees", "span": [68, 73], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "It", "span": [75, 77], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 1, "is_pronoun": true}], "sentences": [[0, 74], [75, 107]], "groups": [{"group_id": 0, "mark_id": 4, "category": "motorcycle", "isthing": 1, "mentions": [{"text": "A motorcycle", "span": [0, 12], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "It", "span": [75, 77], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 2, "category": "road", "isthing": 0, "mentions": [{"text": "a straight road", "span": [27, 42], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 1, "category": "grass", "isthing": 0, "mentions": [{"text": "green fields", "span": [51, 63], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 3, "category": "tree", "isthing": 0, "mentions": [{"text": "trees", "span": [68, 73], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "travels", "span": [13, 20], "spans": [[13, 20], [78, 83]], "particle": null, "mentions": [{"text": "travels", "span": [13, 20], "spans": [[13, 20]], "particle": null, "sentence_index": 0}, {"text": "heads", "span": [78, 83], "spans": [[78, 83]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 0, "subject": {"text": "A motorcycle", "span": [0, 12], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 0, "text": "travels", "status": "visible", "segments_frames": [[3, 18]], "segments": [[1.4, 9.2]], "segment_details": [{"start_frame": 3, "end_frame": 18, "start_frame_1based": 4, "end_frame_1based": 19, "start_s": 1.4, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 38, "end_source_frame": 152, "start_filename": "00000038.jpg", "end_filename": "00000152.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The motorcycle appears near the bottom of the road and progresses toward its distant end.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [17, 23, 32, 38, 47, 53, 62, 68, 77, 83, 92, 98, 107, 113, 122, 128, 137, 143, 152]}}], "coverage": {"marks_shown": 6, "marks_grounded": 4, "things_shown": 1, "things_grounded": 1, "stuff_shown": 5, "stuff_grounded": 3, "n_mentions": 5, "category_disagree": 0, "category_checked": 4}, "marks": [{"mark_id": 4, "category": "motorcycle", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 39, "first_frame": 7, "last_frame": 45, "best_frame": 7, "max_rel_area": 0.00243, "mean_rel_area": 0.00039, "centroid": [0.375, 0.385], "motion": 0.374}, {"mark_id": 1, "category": "grass", "isthing": 0, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 0, "max_rel_area": 0.87868, "mean_rel_area": 0.80078, "centroid": [0.5, 0.529], "motion": 0.07}, {"mark_id": 3, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 42, "max_rel_area": 0.14027, "mean_rel_area": 0.07337, "centroid": [0.295, 0.161], "motion": 0.348}, {"mark_id": 2, "category": "road", "isthing": 0, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 6, "max_rel_area": 0.09651, "mean_rel_area": 0.08177, "centroid": [0.298, 0.532], "motion": 0.276}, {"mark_id": 6, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 17, "first_frame": 29, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.24036, "mean_rel_area": 0.1177, "centroid": [0.545, 0.071], "motion": 0.327}, {"mark_id": 5, "category": "house", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 21, "first_frame": 24, "last_frame": 45, "best_frame": 28, "max_rel_area": 0.00063, "mean_rel_area": 0.00048, "centroid": [0.7, 0.141], "motion": 0.188}], "num_frames": 46, "duration_s": 9.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0, 4.2, 4.4, 4.6, 4.8, 5.0, 5.2, 5.4, 5.6, 5.8, 6.0, 6.2, 6.4, 6.6, 6.8, 7.0, 7.2, 7.4, 7.6, 7.8, 8.0, 8.2, 8.4, 8.6, 8.8, 9.0], "source_frame_indices": [17, 20, 23, 26, 29, 32, 35, 38, 41, 44, 47, 50, 53, 56, 59, 62, 65, 68, 71, 74, 77, 80, 83, 86, 89, 92, 95, 98, 101, 104, 107, 110, 113, 116, 119, 122, 125, 128, 131, 134, 137, 140, 143, 146, 149, 152], "timing_sha": "138c281dc236", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 1, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 21847, "output_tokens": 578, "thinking_tokens": 215, "total_tokens": 22425}, "seconds": 12.58, "response_id": "resp_09b6ec033efb9a97006ac330333db087d0b69bc70ae2ea04a0", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 17, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 23, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 32, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 38, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 47, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 53, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 62, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 68, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 77, "time_s": 4.0}, {"role": "clean", "path": "clean/f022.jpg", "frame": 9, "source_frame": 83, "time_s": 4.4}, {"role": "clean", "path": "clean/f025.jpg", "frame": 10, "source_frame": 92, "time_s": 5.0}, {"role": "clean", "path": "clean/f027.jpg", "frame": 11, "source_frame": 98, "time_s": 5.4}, {"role": "clean", "path": "clean/f030.jpg", "frame": 12, "source_frame": 107, "time_s": 6.0}, {"role": "clean", "path": "clean/f032.jpg", "frame": 13, "source_frame": 113, "time_s": 6.4}, {"role": "clean", "path": "clean/f035.jpg", "frame": 14, "source_frame": 122, "time_s": 7.0}, {"role": "clean", "path": "clean/f037.jpg", "frame": 15, "source_frame": 128, "time_s": 7.4}, {"role": "clean", "path": "clean/f040.jpg", "frame": 16, "source_frame": 137, "time_s": 8.0}, {"role": "clean", "path": "clean/f042.jpg", "frame": 17, "source_frame": 143, "time_s": 8.4}, {"role": "clean", "path": "clean/f045.jpg", "frame": 18, "source_frame": 152, "time_s": 9.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 17, "time_s": 0.0, "mark_ids": [1, 3, 2]}, {"role": "marks", "path": "marks/f015_p0.jpg", "frame": 15, "panel": 0, "source_frame": 62, "time_s": 3.0, "mark_ids": [4, 1, 3, 2]}, {"role": "marks", "path": "marks/f030_p0.jpg", "frame": 30, "panel": 0, "source_frame": 107, "time_s": 6.0, "mark_ids": [4, 1, 3, 2, 6, 5]}, {"role": "marks", "path": "marks/f045_p0.jpg", "frame": 45, "panel": 0, "source_frame": 152, "time_s": 9.0, "mark_ids": [4, 1, 3, 2, 6, 5]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [4, 1, 3, 2, 6, 5]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "138c281dc236", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [17, 23, 32, 38, 47, 53, 62, 68, 77, 83, 92, 98, 107, 113, 122, 128, 137, 143, 152], "actions": [{"action_index": 0, "text": "travels", "status": "visible", "segments_frames": [[3, 18]], "segments": [[1.4, 9.2]], "segment_details": [{"start_frame": 3, "end_frame": 18, "start_frame_1based": 4, "end_frame_1based": 19, "start_s": 1.4, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 38, "end_source_frame": 152, "start_filename": "00000038.jpg", "end_filename": "00000152.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The motorcycle appears near the bottom of the road and progresses toward its distant end.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [17, 23, 32, 38, 47, 53, 62, 68, 77, 83, 92, 98, 107, 113, 122, 128, 137, 143, 152]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1129_eM5eaNKgIHo", "split": "train", "caption": "A turkey walks across the grass beside gravestones and a small American flag. It looks left.", "mentions": [{"text": "A turkey", "span": [0, 8], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "the grass", "span": [22, 31], "occurrence": 1, "mark_ids": [1], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "gravestones", "span": [39, 50], "occurrence": 1, "mark_ids": [2], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a small American flag", "span": [55, 76], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "It", "span": [78, 80], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 1, "is_pronoun": true}], "sentences": [[0, 77], [78, 92]], "groups": [{"group_id": 0, "mark_id": 5, "category": "other animal", "isthing": 1, "mentions": [{"text": "A turkey", "span": [0, 8], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "It", "span": [78, 80], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown", "pronoun"], "chain_agreement": "unknown"}, {"group_id": 1, "mark_id": 1, "category": "grass", "isthing": 0, "mentions": [{"text": "the grass", "span": [22, 31], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown"], "chain_agreement": "unknown"}, {"group_id": 2, "mark_id": 2, "category": "stone", "isthing": 0, "mentions": [{"text": "gravestones", "span": [39, 50], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown"], "chain_agreement": "unknown"}, {"group_id": 3, "mark_id": 4, "category": "flag", "isthing": 1, "mentions": [{"text": "a small American flag", "span": [55, 76], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "walks", "span": [9, 14], "spans": [[9, 14]], "particle": null, "mentions": [{"text": "walks", "span": [9, 14], "spans": [[9, 14]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "A turkey", "span": [0, 8], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "gravestones", "span": [39, 50], "groups": [2], "group_mentions": [0], "group_match": "exact"}, {"text": "a small American flag", "span": [55, 76], "groups": [3], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "walks", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 243, "end_source_frame": 285, "start_filename": "00000243.jpg", "end_filename": "00000285.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The turkey takes successive steps across the grass throughout the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [243, 249, 258, 264, 273, 279, 285]}}, {"text": "looks", "span": [81, 86], "spans": [[81, 86]], "particle": null, "mentions": [{"text": "looks", "span": [81, 86], "spans": [[81, 86]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "It", "span": [78, 80], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [], "confidence": "medium", "temporal": {"action_index": 1, "text": "looks", "status": "visible", "segments_frames": [[5, 6]], "segments": [[2.4, 3.0]], "segment_details": [{"start_frame": 5, "end_frame": 6, "start_frame_1based": 6, "end_frame_1based": 7, "start_s": 2.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 279, "end_source_frame": 285, "start_filename": "00000279.jpg", "end_filename": "00000285.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The turkey visibly faces left in the final two frames.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [243, 249, 258, 264, 273, 279, 285]}}], "coverage": {"marks_shown": 6, "marks_grounded": 4, "things_shown": 2, "things_grounded": 2, "stuff_shown": 4, "stuff_grounded": 2, "n_mentions": 5, "category_disagree": 0, "category_checked": 1}, "marks": [{"mark_id": 5, "category": "other animal", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.06256, "mean_rel_area": 0.05196, "centroid": [0.645, 0.584], "motion": 0.144}, {"mark_id": 4, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 12, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.01187, "mean_rel_area": 0.0052, "centroid": [0.604, 0.448], "motion": 0.292}, {"mark_id": 1, "category": "grass", "isthing": 0, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 12, "max_rel_area": 0.67436, "mean_rel_area": 0.56004, "centroid": [0.5, 0.5], "motion": 0.0}, {"mark_id": 2, "category": "stone", "isthing": 0, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 6, "max_rel_area": 0.32972, "mean_rel_area": 0.28559, "centroid": [0.58, 0.308], "motion": 0.128}, {"mark_id": 3, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.15658, "mean_rel_area": 0.09606, "centroid": [0.269, 0.445], "motion": 0.139}, {"mark_id": 6, "category": "pole", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 8, "first_frame": 7, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.00249, "mean_rel_area": 0.00186, "centroid": [0.551, 0.322], "motion": 0.235}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [243, 246, 249, 252, 255, 258, 261, 264, 267, 270, 273, 276, 279, 282, 285], "timing_sha": "08c30d7b074a", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 4, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 16766, "output_tokens": 859, "thinking_tokens": 362, "total_tokens": 17625}, "seconds": 15.88, "response_id": "resp_0b09dc166d4145ff006ac3303e49a887d088278ce976226141", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 243, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 249, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 258, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 264, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 273, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 279, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 285, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 243, "time_s": 0.0, "mark_ids": [5, 4, 1, 2, 3]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 258, "time_s": 1.0, "mark_ids": [5, 1, 2, 3]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 273, "time_s": 2.0, "mark_ids": [5, 4, 1, 2, 3, 6]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 285, "time_s": 2.8, "mark_ids": [5, 4, 1, 2, 3, 6]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [5, 4, 1, 2, 3, 6]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "08c30d7b074a", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [243, 249, 258, 264, 273, 279, 285], "actions": [{"action_index": 0, "text": "walks", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 243, "end_source_frame": 285, "start_filename": "00000243.jpg", "end_filename": "00000285.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The turkey takes successive steps across the grass throughout the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [243, 249, 258, 264, 273, 279, 285]}, {"action_index": 1, "text": "looks", "status": "visible", "segments_frames": [[5, 6]], "segments": [[2.4, 3.0]], "segment_details": [{"start_frame": 5, "end_frame": 6, "start_frame_1based": 6, "end_frame_1based": 7, "start_s": 2.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 279, "end_source_frame": 285, "start_filename": "00000279.jpg", "end_filename": "00000285.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The turkey visibly faces left in the final two frames.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [243, 249, 258, 264, 273, 279, 285]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1215_NSw8hNtpSy0", "split": "train", "caption": "An orange sports car is on a glossy floor indoors. Two people in red stand behind it near a wall-mounted television.", "mentions": [{"text": "An orange sports car", "span": [0, 20], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a glossy floor", "span": [27, 41], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "Two people", "span": [51, 61], "occurrence": 1, "mark_ids": [8, 7], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "it", "span": [82, 84], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "a wall-mounted television", "span": [90, 115], "occurrence": 1, "mark_ids": [9], "confidence": "high", "sentence_index": 1, "is_pronoun": false}], "sentences": [[0, 50], [51, 116]], "groups": [{"group_id": 0, "mark_id": 6, "category": "car", "isthing": 1, "mentions": [{"text": "An orange sports car", "span": [0, 20], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "it", "span": [82, 84], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 4, "category": "floor", "isthing": 0, "mentions": [{"text": "a glossy floor", "span": [27, 41], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 8, "category": "person", "isthing": 1, "mentions": [{"text": "Two people", "span": [51, 61], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 7, "category": "person", "isthing": 1, "mentions": [{"text": "Two people", "span": [51, 61], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 9, "category": "screen or television", "isthing": 1, "mentions": [{"text": "a wall-mounted television", "span": [90, 115], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "stand", "span": [69, 74], "spans": [[69, 74]], "particle": null, "mentions": [{"text": "stand", "span": [69, 74], "spans": [[69, 74]], "particle": null, "sentence_index": 1}], "type": "posture", "sentence_index": 1, "subject": {"text": "Two people", "span": [51, 61], "groups": [2, 3], "group_mentions": [0, 0], "group_match": "exact"}, "objects": [{"text": "it", "span": [82, 84], "groups": [0], "group_mentions": [1], "group_match": "exact"}, {"text": "a wall-mounted television", "span": [90, 115], "groups": [4], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "stand", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.6]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.6, "last_observed_s": 1.4, "start_source_frame": 6031, "end_source_frame": 6052, "start_filename": "00006031.jpg", "end_filename": "00006052.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Both red-clad people remain upright behind the car throughout the sequence, although their lower bodies are obscured.", "fps": 2.0, "input_frame_count": 4, "frame_times_s": [0.0, 0.4, 1.0, 1.4], "source_frame_indices": [6031, 6037, 6046, 6052]}}], "coverage": {"marks_shown": 9, "marks_grounded": 5, "things_shown": 4, "things_grounded": 4, "stuff_shown": 5, "stuff_grounded": 1, "n_mentions": 6, "category_disagree": 0, "category_checked": 5}, "marks": [{"mark_id": 6, "category": "car", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 8, "first_frame": 0, "last_frame": 7, "best_frame": 0, "max_rel_area": 0.30063, "mean_rel_area": 0.24839, "centroid": [0.701, 0.517], "motion": 0.053}, {"mark_id": 9, "category": "screen or television", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 8, "first_frame": 0, "last_frame": 7, "best_frame": 0, "max_rel_area": 0.02383, "mean_rel_area": 0.01617, "centroid": [0.389, 0.103], "motion": 0.267}, {"mark_id": 8, "category": "person", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 8, "first_frame": 0, "last_frame": 7, "best_frame": 6, "max_rel_area": 0.00494, "mean_rel_area": 0.00456, "centroid": [0.524, 0.245], "motion": 0.101}, {"mark_id": 7, "category": "person", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 8, "first_frame": 0, "last_frame": 7, "best_frame": 5, "max_rel_area": 0.0027, "mean_rel_area": 0.00225, "centroid": [0.811, 0.228], "motion": 0.101}, {"mark_id": 4, "category": "floor", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 8, "first_frame": 0, "last_frame": 7, "best_frame": 7, "max_rel_area": 0.41607, "mean_rel_area": 0.39106, "centroid": [0.5, 0.692], "motion": 0.07}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 8, "first_frame": 0, "last_frame": 7, "best_frame": 7, "max_rel_area": 0.1782, "mean_rel_area": 0.15211, "centroid": [0.5, 0.218], "motion": 0.009}, {"mark_id": 3, "category": "pillar", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 8, "first_frame": 0, "last_frame": 7, "best_frame": 5, "max_rel_area": 0.12777, "mean_rel_area": 0.12115, "centroid": [0.523, 0.241], "motion": 0.138}, {"mark_id": 2, "category": "ceiling", "isthing": 0, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 8, "first_frame": 0, "last_frame": 7, "best_frame": 5, "max_rel_area": 0.03398, "mean_rel_area": 0.02972, "centroid": [0.672, 0.057], "motion": 0.025}, {"mark_id": 5, "category": "lamp", "isthing": 0, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 8, "first_frame": 0, "last_frame": 7, "best_frame": 3, "max_rel_area": 0.00381, "mean_rel_area": 0.00149, "centroid": [0.783, 0.049], "motion": 0.341}], "num_frames": 8, "duration_s": 1.6, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4], "source_frame_indices": [6031, 6034, 6037, 6040, 6043, 6046, 6049, 6052], "timing_sha": "a1a045839ab1", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 3, "unlinked": 0, "snapped": 0, "collective": 1, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 15915, "output_tokens": 709, "thinking_tokens": 254, "total_tokens": 16624}, "seconds": 17.96, "response_id": "resp_0c29e02762108a99006ac3303fa8dc87d09c93fd402fdf884a", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 6031, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 6037, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 6046, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 6052, "time_s": 1.4}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 6031, "time_s": 0.0, "mark_ids": [6, 9, 8, 7, 4, 1, 3, 2, 5]}, {"role": "marks", "path": "marks/f002_p0.jpg", "frame": 2, "panel": 0, "source_frame": 6037, "time_s": 0.4, "mark_ids": [6, 9, 8, 7, 4, 1, 3, 2, 5]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 6046, "time_s": 1.0, "mark_ids": [6, 9, 8, 7, 4, 1, 3, 2, 5]}, {"role": "marks", "path": "marks/f007_p0.jpg", "frame": 7, "panel": 0, "source_frame": 6052, "time_s": 1.4, "mark_ids": [6, 9, 8, 7, 4, 1, 3, 2, 5]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [6, 9, 8, 7, 4, 1, 3, 2, 5]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "a1a045839ab1", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 4, "frame_times_s": [0.0, 0.4, 1.0, 1.4], "source_frame_indices": [6031, 6037, 6046, 6052], "actions": [{"action_index": 0, "text": "stand", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.6]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.6, "last_observed_s": 1.4, "start_source_frame": 6031, "end_source_frame": 6052, "start_filename": "00006031.jpg", "end_filename": "00006052.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Both red-clad people remain upright behind the car throughout the sequence, although their lower bodies are obscured.", "fps": 2.0, "input_frame_count": 4, "frame_times_s": [0.0, 0.4, 1.0, 1.4], "source_frame_indices": [6031, 6037, 6046, 6052]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "12_1mWNahzcsAc", "split": "train", "caption": "A woman in pink strokes a dark horse. It bends down beside a woman in black. She holds a pale horse as it turns right.", "mentions": [{"text": "A woman", "span": [0, 7], "occurrence": 1, "mark_ids": [7], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a dark horse", "span": [24, 36], "occurrence": 1, "mark_ids": [10], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "It", "span": [38, 40], "occurrence": 1, "mark_ids": [10], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "a woman", "span": [59, 66], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "She", "span": [77, 80], "occurrence": 1, "mark_ids": [8], "confidence": "high", "sentence_index": 2, "is_pronoun": true}, {"text": "a pale horse", "span": [87, 99], "occurrence": 1, "mark_ids": [9], "confidence": "high", "sentence_index": 2, "is_pronoun": false}, {"text": "it", "span": [103, 105], "occurrence": 1, "mark_ids": [9], "confidence": "high", "sentence_index": 2, "is_pronoun": true}], "sentences": [[0, 37], [38, 76], [77, 118]], "groups": [{"group_id": 0, "mark_id": 7, "category": "person", "isthing": 1, "mentions": [{"text": "A woman", "span": [0, 7], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 10, "category": "horse", "isthing": 1, "mentions": [{"text": "a dark horse", "span": [24, 36], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "It", "span": [38, 40], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 8, "category": "person", "isthing": 1, "mentions": [{"text": "a woman", "span": [59, 66], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "She", "span": [77, 80], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 9, "category": "horse", "isthing": 1, "mentions": [{"text": "a pale horse", "span": [87, 99], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "it", "span": [103, 105], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}], "actions": [{"text": "strokes", "span": [16, 23], "spans": [[16, 23]], "particle": null, "mentions": [{"text": "strokes", "span": [16, 23], "spans": [[16, 23]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "A woman", "span": [0, 7], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "a dark horse", "span": [24, 36], "groups": [1], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "strokes", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.9]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 1255, "end_source_frame": 1276, "start_filename": "00001255.jpg", "end_filename": "00001276.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The woman moves her hand down the dark horse's face as it lowers.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1255, 1261, 1270, 1276, 1285, 1291, 1297]}}, {"text": "bends", "span": [41, 46], "spans": [[41, 46]], "particle": null, "mentions": [{"text": "bends", "span": [41, 46], "spans": [[41, 46]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "It", "span": [38, 40], "groups": [1], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "a woman", "span": [59, 66], "groups": [2], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 1, "text": "bends", "status": "visible", "segments_frames": [[0, 3], [5, 6]], "segments": [[0.0, 1.9], [2.4, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 1255, "end_source_frame": 1276, "start_filename": "00001255.jpg", "end_filename": "00001276.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 5, "end_frame": 6, "start_frame_1based": 6, "end_frame_1based": 7, "start_s": 2.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 1291, "end_source_frame": 1297, "start_filename": "00001291.jpg", "end_filename": "00001297.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The dark horse bends its neck downward, briefly raises it, then bends farther down near the end.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1255, 1261, 1270, 1276, 1285, 1291, 1297]}}, {"text": "holds", "span": [81, 86], "spans": [[81, 86]], "particle": null, "mentions": [{"text": "holds", "span": [81, 86], "spans": [[81, 86]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "She", "span": [77, 80], "groups": [2], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "a pale horse", "span": [87, 99], "groups": [3], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 2, "text": "holds", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 1255, "end_source_frame": 1297, "start_filename": "00001255.jpg", "end_filename": "00001297.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The woman maintains her grip near the pale horse's halter throughout the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1255, 1261, 1270, 1276, 1285, 1291, 1297]}}, {"text": "turns", "span": [106, 111], "spans": [[106, 111]], "particle": null, "mentions": [{"text": "turns", "span": [106, 111], "spans": [[106, 111]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "it", "span": [103, 105], "groups": [3], "group_mentions": [1], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 3, "text": "turns", "status": "visible", "segments_frames": [[3, 6]], "segments": [[1.4, 3.0]], "segment_details": [{"start_frame": 3, "end_frame": 6, "start_frame_1based": 4, "end_frame_1based": 7, "start_s": 1.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 1276, "end_source_frame": 1297, "start_filename": "00001276.jpg", "end_filename": "00001297.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The pale horse progressively turns its head toward the right side of the image.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1255, 1261, 1270, 1276, 1285, 1291, 1297]}}], "coverage": {"marks_shown": 10, "marks_grounded": 4, "things_shown": 4, "things_grounded": 4, "stuff_shown": 6, "stuff_grounded": 0, "n_mentions": 7, "category_disagree": 0, "category_checked": 4}, "marks": [{"mark_id": 9, "category": "horse", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.14087, "mean_rel_area": 0.11946, "centroid": [0.693, 0.622], "motion": 0.028}, {"mark_id": 7, "category": "person", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.13616, "mean_rel_area": 0.12854, "centroid": [0.238, 0.567], "motion": 0.044}, {"mark_id": 10, "category": "horse", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.12711, "mean_rel_area": 0.11351, "centroid": [0.341, 0.623], "motion": 0.075}, {"mark_id": 8, "category": "person", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.12213, "mean_rel_area": 0.10697, "centroid": [0.541, 0.581], "motion": 0.042}, {"mark_id": 1, "category": "handrail or fence", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 7, "max_rel_area": 0.28341, "mean_rel_area": 0.26855, "centroid": [0.5, 0.5], "motion": 0.0}, {"mark_id": 6, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 13, "max_rel_area": 0.17197, "mean_rel_area": 0.16256, "centroid": [0.424, 0.171], "motion": 0.002}, {"mark_id": 2, "category": "ground", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 11, "max_rel_area": 0.04564, "mean_rel_area": 0.04364, "centroid": [0.489, 0.931], "motion": 0.017}, {"mark_id": 4, "category": "other construction", "isthing": 0, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 6, "max_rel_area": 0.04175, "mean_rel_area": 0.03198, "centroid": [0.386, 0.268], "motion": 0.014}, {"mark_id": 3, "category": "sand", "isthing": 0, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.02053, "mean_rel_area": 0.01676, "centroid": [0.447, 0.801], "motion": 0.096}, {"mark_id": 5, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.00463, "mean_rel_area": 0.00408, "centroid": [0.665, 0.016], "motion": 0.002}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [1255, 1258, 1261, 1264, 1267, 1270, 1273, 1276, 1279, 1282, 1285, 1288, 1291, 1294, 1297], "timing_sha": "c8bd9276a77e", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 7, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 17246, "output_tokens": 1353, "thinking_tokens": 480, "total_tokens": 18599}, "seconds": 24.76, "response_id": "resp_0de305ce1b89d922006ac3304e07d087d09b7362d258edca18", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 1255, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 1261, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 1270, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 1276, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 1285, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 1291, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 1297, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 1255, "time_s": 0.0, "mark_ids": [9, 7, 10, 8, 1, 6, 2, 4, 3, 5]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 1270, "time_s": 1.0, "mark_ids": [9, 7, 10, 8, 1, 6, 2, 4, 3, 5]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 1285, "time_s": 2.0, "mark_ids": [9, 7, 10, 8, 1, 6, 2, 4, 3, 5]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 1297, "time_s": 2.8, "mark_ids": [9, 7, 10, 8, 1, 6, 2, 4, 3, 5]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [9, 7, 10, 8, 1, 6, 2, 4, 3, 5]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "c8bd9276a77e", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1255, 1261, 1270, 1276, 1285, 1291, 1297], "actions": [{"action_index": 0, "text": "strokes", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.9]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 1255, "end_source_frame": 1276, "start_filename": "00001255.jpg", "end_filename": "00001276.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The woman moves her hand down the dark horse's face as it lowers.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1255, 1261, 1270, 1276, 1285, 1291, 1297]}, {"action_index": 1, "text": "bends", "status": "visible", "segments_frames": [[0, 3], [5, 6]], "segments": [[0.0, 1.9], [2.4, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 1255, "end_source_frame": 1276, "start_filename": "00001255.jpg", "end_filename": "00001276.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 5, "end_frame": 6, "start_frame_1based": 6, "end_frame_1based": 7, "start_s": 2.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 1291, "end_source_frame": 1297, "start_filename": "00001291.jpg", "end_filename": "00001297.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The dark horse bends its neck downward, briefly raises it, then bends farther down near the end.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1255, 1261, 1270, 1276, 1285, 1291, 1297]}, {"action_index": 2, "text": "holds", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 1255, "end_source_frame": 1297, "start_filename": "00001255.jpg", "end_filename": "00001297.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The woman maintains her grip near the pale horse's halter throughout the sequence.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1255, 1261, 1270, 1276, 1285, 1291, 1297]}, {"action_index": 3, "text": "turns", "status": "visible", "segments_frames": [[3, 6]], "segments": [[1.4, 3.0]], "segment_details": [{"start_frame": 3, "end_frame": 6, "start_frame_1based": 4, "end_frame_1based": 7, "start_s": 1.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 1276, "end_source_frame": 1297, "start_filename": "00001276.jpg", "end_filename": "00001297.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The pale horse progressively turns its head toward the right side of the image.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [1255, 1261, 1270, 1276, 1285, 1291, 1297]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1409_44et30baPhs", "split": "train", "caption": "A man points toward books on a shelf. He turns forward. He then gestures toward the books.", "mentions": [{"text": "A man", "span": [0, 5], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "books", "span": [20, 25], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a shelf", "span": [29, 36], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "He", "span": [38, 40], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "He", "span": [56, 58], "occurrence": 2, "mark_ids": [6], "confidence": "high", "sentence_index": 2, "is_pronoun": true}, {"text": "the books", "span": [80, 89], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 37], [38, 55], [56, 90]], "groups": [{"group_id": 0, "mark_id": 6, "category": "person", "isthing": 1, "mentions": [{"text": "A man", "span": [0, 5], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "He", "span": [38, 40], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "He", "span": [56, 58], "occurrence": 2, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 5, "category": "book", "isthing": 0, "mentions": [{"text": "books", "span": [20, 25], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "the books", "span": [80, 89], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 4, "category": "cupboard or showcase or storage rack", "isthing": 0, "mentions": [{"text": "a shelf", "span": [29, 36], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["disagree"], "chain_agreement": "disagree"}], "actions": [{"text": "points", "span": [6, 12], "spans": [[6, 12]], "particle": null, "mentions": [{"text": "points", "span": [6, 12], "spans": [[6, 12]], "particle": null, "sentence_index": 0}], "type": "prepositional", "sentence_index": 0, "subject": {"text": "A man", "span": [0, 5], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "books", "span": [20, 25], "groups": [1], "group_mentions": [0], "group_match": "exact"}], "confidence": "medium", "temporal": {"action_index": 0, "text": "points", "status": "visible", "segments_frames": [[0, 4]], "segments": [[0.0, 2.4]], "segment_details": [{"start_frame": 0, "end_frame": 4, "start_frame_1based": 1, "end_frame_1based": 5, "start_s": 0.0, "end_s": 2.4, "last_observed_s": 2.0, "start_source_frame": 2208, "end_source_frame": 2238, "start_filename": "00002208.jpg", "end_filename": "00002238.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "His extended hand indicates the books, at times making contact with them.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2208, 2214, 2223, 2229, 2238, 2244, 2250]}}, {"text": "turns", "span": [41, 46], "spans": [[41, 46]], "particle": null, "mentions": [{"text": "turns", "span": [41, 46], "spans": [[41, 46]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "He", "span": [38, 40], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 1, "text": "turns", "status": "visible", "segments_frames": [[0, 2]], "segments": [[0.0, 1.4]], "segment_details": [{"start_frame": 0, "end_frame": 2, "start_frame_1based": 1, "end_frame_1based": 3, "start_s": 0.0, "end_s": 1.4, "last_observed_s": 1.0, "start_source_frame": 2208, "end_source_frame": 2223, "start_filename": "00002208.jpg", "end_filename": "00002223.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He changes from facing the shelves to facing forward.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2208, 2214, 2223, 2229, 2238, 2244, 2250]}}, {"text": "gestures", "span": [64, 72], "spans": [[64, 72]], "particle": null, "mentions": [{"text": "gestures", "span": [64, 72], "spans": [[64, 72]], "particle": null, "sentence_index": 2}], "type": "prepositional", "sentence_index": 2, "subject": {"text": "He", "span": [56, 58], "groups": [0], "group_mentions": [2], "group_match": "exact"}, "objects": [{"text": "the books", "span": [80, 89], "groups": [1], "group_mentions": [1], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 2, "text": "gestures", "status": "visible", "segments_frames": [[5, 6]], "segments": [[2.4, 3.0]], "segment_details": [{"start_frame": 5, "end_frame": 6, "start_frame_1based": 6, "end_frame_1based": 7, "start_s": 2.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 2244, "end_source_frame": 2250, "start_filename": "00002244.jpg", "end_filename": "00002250.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He opens his hand and extends it toward the books.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2208, 2214, 2223, 2229, 2238, 2244, 2250]}}], "coverage": {"marks_shown": 7, "marks_grounded": 3, "things_shown": 2, "things_grounded": 1, "stuff_shown": 5, "stuff_grounded": 2, "n_mentions": 6, "category_disagree": 1, "category_checked": 3}, "marks": [{"mark_id": 6, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.27069, "mean_rel_area": 0.20365, "centroid": [0.681, 0.579], "motion": 0.053}, {"mark_id": 7, "category": "painting or poster", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 2, "first_frame": 0, "last_frame": 1, "best_frame": 0, "max_rel_area": 0.00843, "mean_rel_area": 0.00484, "centroid": [0.248, 0.037], "motion": 0.025}, {"mark_id": 5, "category": "book", "isthing": 0, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 3, "max_rel_area": 0.72725, "mean_rel_area": 0.66634, "centroid": [0.5, 0.5], "motion": 0.0}, {"mark_id": 4, "category": "cupboard or showcase or storage rack", "isthing": 0, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 5, "max_rel_area": 0.10717, "mean_rel_area": 0.08632, "centroid": [0.5, 0.516], "motion": 0.051}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.07164, "mean_rel_area": 0.03177, "centroid": [0.615, 0.058], "motion": 0.109}, {"mark_id": 2, "category": "ceiling", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 3, "first_frame": 0, "last_frame": 2, "best_frame": 0, "max_rel_area": 0.07389, "mean_rel_area": 0.03755, "centroid": [0.515, 0.045], "motion": 0.226}, {"mark_id": 3, "category": "lamp", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 4, "first_frame": 0, "last_frame": 3, "best_frame": 1, "max_rel_area": 0.01896, "mean_rel_area": 0.01411, "centroid": [0.644, 0.052], "motion": 0.072}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [2208, 2211, 2214, 2217, 2220, 2223, 2226, 2229, 2232, 2235, 2238, 2241, 2244, 2247, 2250], "timing_sha": "28054371777c", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 5, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 16811, "output_tokens": 944, "thinking_tokens": 293, "total_tokens": 17755}, "seconds": 17.2, "response_id": "resp_097b8bfd1e4ead18006ac330519f9c87d0b81827fa561f3509", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 2208, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 2214, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 2223, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 2229, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 2238, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 2244, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 2250, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 2208, "time_s": 0.0, "mark_ids": [6, 7, 5, 4, 1, 2, 3]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 2223, "time_s": 1.0, "mark_ids": [6, 5, 4, 1]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 2238, "time_s": 2.0, "mark_ids": [6, 5, 4, 1]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 2250, "time_s": 2.8, "mark_ids": [6, 5, 4, 1]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [6, 7, 5, 4, 1, 2, 3]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "28054371777c", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2208, 2214, 2223, 2229, 2238, 2244, 2250], "actions": [{"action_index": 0, "text": "points", "status": "visible", "segments_frames": [[0, 4]], "segments": [[0.0, 2.4]], "segment_details": [{"start_frame": 0, "end_frame": 4, "start_frame_1based": 1, "end_frame_1based": 5, "start_s": 0.0, "end_s": 2.4, "last_observed_s": 2.0, "start_source_frame": 2208, "end_source_frame": 2238, "start_filename": "00002208.jpg", "end_filename": "00002238.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "His extended hand indicates the books, at times making contact with them.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2208, 2214, 2223, 2229, 2238, 2244, 2250]}, {"action_index": 1, "text": "turns", "status": "visible", "segments_frames": [[0, 2]], "segments": [[0.0, 1.4]], "segment_details": [{"start_frame": 0, "end_frame": 2, "start_frame_1based": 1, "end_frame_1based": 3, "start_s": 0.0, "end_s": 1.4, "last_observed_s": 1.0, "start_source_frame": 2208, "end_source_frame": 2223, "start_filename": "00002208.jpg", "end_filename": "00002223.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He changes from facing the shelves to facing forward.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2208, 2214, 2223, 2229, 2238, 2244, 2250]}, {"action_index": 2, "text": "gestures", "status": "visible", "segments_frames": [[5, 6]], "segments": [[2.4, 3.0]], "segment_details": [{"start_frame": 5, "end_frame": 6, "start_frame_1based": 6, "end_frame_1based": 7, "start_s": 2.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 2244, "end_source_frame": 2250, "start_filename": "00002244.jpg", "end_filename": "00002250.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He opens his hand and extends it toward the books.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [2208, 2214, 2223, 2229, 2238, 2244, 2250]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "155_GVuOmIPfHj0", "split": "train", "caption": "An orange excavator raises its arm behind a parked truck. It then rotates to the right on the dirt ground.", "mentions": [{"text": "An orange excavator", "span": [0, 19], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "its arm", "span": [27, 34], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a parked truck", "span": [42, 56], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "It", "span": [58, 60], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "the dirt ground", "span": [90, 105], "occurrence": 1, "mark_ids": [2], "confidence": "high", "sentence_index": 1, "is_pronoun": false}], "sentences": [[0, 57], [58, 106]], "groups": [{"group_id": 0, "mark_id": 4, "category": "wheeled machine", "isthing": 0, "mentions": [{"text": "An orange excavator", "span": [0, 19], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "its arm", "span": [27, 34], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "It", "span": [58, 60], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "disagree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 6, "category": "truck", "isthing": 1, "mentions": [{"text": "a parked truck", "span": [42, 56], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 2, "category": "ground", "isthing": 0, "mentions": [{"text": "the dirt ground", "span": [90, 105], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "raises", "span": [20, 26], "spans": [[20, 26]], "particle": null, "mentions": [{"text": "raises", "span": [20, 26], "spans": [[20, 26]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "An orange excavator", "span": [0, 19], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [{"text": "its arm", "span": [27, 34], "groups": [0], "group_mentions": [1], "group_match": "exact"}, {"text": "a parked truck", "span": [42, 56], "groups": [1], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "raises", "status": "visible", "segments_frames": [[0, 2], [8, 10], [14, 16]], "segments": [[0.0, 1.4], [4.0, 5.4], [7.0, 8.4]], "segment_details": [{"start_frame": 0, "end_frame": 2, "start_frame_1based": 1, "end_frame_1based": 3, "start_s": 0.0, "end_s": 1.4, "last_observed_s": 1.0, "start_source_frame": 1772, "end_source_frame": 1787, "start_filename": "00001772.jpg", "end_filename": "00001787.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 8, "end_frame": 10, "start_frame_1based": 9, "end_frame_1based": 11, "start_s": 4.0, "end_s": 5.4, "last_observed_s": 5.0, "start_source_frame": 1832, "end_source_frame": 1847, "start_filename": "00001832.jpg", "end_filename": "00001847.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 14, "end_frame": 16, "start_frame_1based": 15, "end_frame_1based": 17, "start_s": 7.0, "end_s": 8.4, "last_observed_s": 8.0, "start_source_frame": 1877, "end_source_frame": 1892, "start_filename": "00001877.jpg", "end_filename": "00001892.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The articulated arm rises initially and makes further upward movements after visible pauses.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1772, 1778, 1787, 1793, 1802, 1808, 1817, 1823, 1832, 1838, 1847, 1853, 1862, 1868, 1877, 1883, 1892, 1898, 1907]}}, {"text": "rotates", "span": [66, 73], "spans": [[66, 73]], "particle": null, "mentions": [{"text": "rotates", "span": [66, 73], "spans": [[66, 73]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "It", "span": [58, 60], "groups": [0], "group_mentions": [2], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 1, "text": "rotates", "status": "visible", "segments_frames": [[4, 6], [16, 18]], "segments": [[2.0, 3.4], [8.0, 9.2]], "segment_details": [{"start_frame": 4, "end_frame": 6, "start_frame_1based": 5, "end_frame_1based": 7, "start_s": 2.0, "end_s": 3.4, "last_observed_s": 3.0, "start_source_frame": 1802, "end_source_frame": 1817, "start_filename": "00001802.jpg", "end_filename": "00001817.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 16, "end_frame": 18, "start_frame_1based": 17, "end_frame_1based": 19, "start_s": 8.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 1892, "end_source_frame": 1907, "start_filename": "00001892.jpg", "end_filename": "00001907.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The excavator's upper body turns right, pauses, and turns farther right near the end.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1772, 1778, 1787, 1793, 1802, 1808, 1817, 1823, 1832, 1838, 1847, 1853, 1862, 1868, 1877, 1883, 1892, 1898, 1907]}}], "coverage": {"marks_shown": 6, "marks_grounded": 3, "things_shown": 1, "things_grounded": 1, "stuff_shown": 5, "stuff_grounded": 2, "n_mentions": 5, "category_disagree": 0, "category_checked": 3}, "marks": [{"mark_id": 6, "category": "truck", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 24, "max_rel_area": 0.17357, "mean_rel_area": 0.17301, "centroid": [0.5, 0.379], "motion": 0.002}, {"mark_id": 2, "category": "ground", "isthing": 0, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 45, "max_rel_area": 0.64871, "mean_rel_area": 0.63801, "centroid": [0.5, 0.526], "motion": 0.003}, {"mark_id": 5, "category": "other plant", "isthing": 0, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 12, "max_rel_area": 0.07401, "mean_rel_area": 0.07008, "centroid": [0.5, 0.11], "motion": 0.009}, {"mark_id": 4, "category": "wheeled machine", "isthing": 0, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 2, "max_rel_area": 0.07383, "mean_rel_area": 0.06378, "centroid": [0.772, 0.252], "motion": 0.062}, {"mark_id": 3, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 40, "max_rel_area": 0.05023, "mean_rel_area": 0.03949, "centroid": [0.575, 0.047], "motion": 0.086}, {"mark_id": 1, "category": "handrail or fence", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 46, "first_frame": 0, "last_frame": 45, "best_frame": 26, "max_rel_area": 0.01739, "mean_rel_area": 0.01548, "centroid": [0.736, 0.296], "motion": 0.003}], "num_frames": 46, "duration_s": 9.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0, 4.2, 4.4, 4.6, 4.8, 5.0, 5.2, 5.4, 5.6, 5.8, 6.0, 6.2, 6.4, 6.6, 6.8, 7.0, 7.2, 7.4, 7.6, 7.8, 8.0, 8.2, 8.4, 8.6, 8.8, 9.0], "source_frame_indices": [1772, 1775, 1778, 1781, 1784, 1787, 1790, 1793, 1796, 1799, 1802, 1805, 1808, 1811, 1814, 1817, 1820, 1823, 1826, 1829, 1832, 1835, 1838, 1841, 1844, 1847, 1850, 1853, 1856, 1859, 1862, 1865, 1868, 1871, 1874, 1877, 1880, 1883, 1886, 1889, 1892, 1895, 1898, 1901, 1904, 1907], "timing_sha": "34ee0eff5d25", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 4, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 21866, "output_tokens": 1045, "thinking_tokens": 478, "total_tokens": 22911}, "seconds": 21.11, "response_id": "resp_0c87dbfa30bc4e27006ac33062ef4087d0869ecadadabaa259", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 1772, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 1778, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 1787, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 1793, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 1802, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 1808, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 1817, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 1823, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 1832, "time_s": 4.0}, {"role": "clean", "path": "clean/f022.jpg", "frame": 9, "source_frame": 1838, "time_s": 4.4}, {"role": "clean", "path": "clean/f025.jpg", "frame": 10, "source_frame": 1847, "time_s": 5.0}, {"role": "clean", "path": "clean/f027.jpg", "frame": 11, "source_frame": 1853, "time_s": 5.4}, {"role": "clean", "path": "clean/f030.jpg", "frame": 12, "source_frame": 1862, "time_s": 6.0}, {"role": "clean", "path": "clean/f032.jpg", "frame": 13, "source_frame": 1868, "time_s": 6.4}, {"role": "clean", "path": "clean/f035.jpg", "frame": 14, "source_frame": 1877, "time_s": 7.0}, {"role": "clean", "path": "clean/f037.jpg", "frame": 15, "source_frame": 1883, "time_s": 7.4}, {"role": "clean", "path": "clean/f040.jpg", "frame": 16, "source_frame": 1892, "time_s": 8.0}, {"role": "clean", "path": "clean/f042.jpg", "frame": 17, "source_frame": 1898, "time_s": 8.4}, {"role": "clean", "path": "clean/f045.jpg", "frame": 18, "source_frame": 1907, "time_s": 9.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 1772, "time_s": 0.0, "mark_ids": [6, 2, 5, 4, 3, 1]}, {"role": "marks", "path": "marks/f015_p0.jpg", "frame": 15, "panel": 0, "source_frame": 1817, "time_s": 3.0, "mark_ids": [6, 2, 5, 4, 3, 1]}, {"role": "marks", "path": "marks/f030_p0.jpg", "frame": 30, "panel": 0, "source_frame": 1862, "time_s": 6.0, "mark_ids": [6, 2, 5, 4, 3, 1]}, {"role": "marks", "path": "marks/f045_p0.jpg", "frame": 45, "panel": 0, "source_frame": 1907, "time_s": 9.0, "mark_ids": [6, 2, 5, 4, 3, 1]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [6, 2, 5, 4, 3, 1]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "34ee0eff5d25", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1772, 1778, 1787, 1793, 1802, 1808, 1817, 1823, 1832, 1838, 1847, 1853, 1862, 1868, 1877, 1883, 1892, 1898, 1907], "actions": [{"action_index": 0, "text": "raises", "status": "visible", "segments_frames": [[0, 2], [8, 10], [14, 16]], "segments": [[0.0, 1.4], [4.0, 5.4], [7.0, 8.4]], "segment_details": [{"start_frame": 0, "end_frame": 2, "start_frame_1based": 1, "end_frame_1based": 3, "start_s": 0.0, "end_s": 1.4, "last_observed_s": 1.0, "start_source_frame": 1772, "end_source_frame": 1787, "start_filename": "00001772.jpg", "end_filename": "00001787.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 8, "end_frame": 10, "start_frame_1based": 9, "end_frame_1based": 11, "start_s": 4.0, "end_s": 5.4, "last_observed_s": 5.0, "start_source_frame": 1832, "end_source_frame": 1847, "start_filename": "00001832.jpg", "end_filename": "00001847.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 14, "end_frame": 16, "start_frame_1based": 15, "end_frame_1based": 17, "start_s": 7.0, "end_s": 8.4, "last_observed_s": 8.0, "start_source_frame": 1877, "end_source_frame": 1892, "start_filename": "00001877.jpg", "end_filename": "00001892.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The articulated arm rises initially and makes further upward movements after visible pauses.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1772, 1778, 1787, 1793, 1802, 1808, 1817, 1823, 1832, 1838, 1847, 1853, 1862, 1868, 1877, 1883, 1892, 1898, 1907]}, {"action_index": 1, "text": "rotates", "status": "visible", "segments_frames": [[4, 6], [16, 18]], "segments": [[2.0, 3.4], [8.0, 9.2]], "segment_details": [{"start_frame": 4, "end_frame": 6, "start_frame_1based": 5, "end_frame_1based": 7, "start_s": 2.0, "end_s": 3.4, "last_observed_s": 3.0, "start_source_frame": 1802, "end_source_frame": 1817, "start_filename": "00001802.jpg", "end_filename": "00001817.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}, {"start_frame": 16, "end_frame": 18, "start_frame_1based": 17, "end_frame_1based": 19, "start_s": 8.0, "end_s": 9.2, "last_observed_s": 9.0, "start_source_frame": 1892, "end_source_frame": 1907, "start_filename": "00001892.jpg", "end_filename": "00001907.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The excavator's upper body turns right, pauses, and turns farther right near the end.", "fps": 2.0, "input_frame_count": 19, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0, 4.4, 5.0, 5.4, 6.0, 6.4, 7.0, 7.4, 8.0, 8.4, 9.0], "source_frame_indices": [1772, 1778, 1787, 1793, 1802, 1808, 1817, 1823, 1832, 1838, 1847, 1853, 1862, 1868, 1877, 1883, 1892, 1898, 1907]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1688_ePhZYeMtsSU", "split": "train", "caption": "An orange lifeboat moves rightward along the river. A smaller white boat is visible farther behind it.", "mentions": [{"text": "An orange lifeboat", "span": [0, 18], "occurrence": 1, "mark_ids": [7], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "the river", "span": [41, 50], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "A smaller white boat", "span": [52, 72], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "it", "span": [99, 101], "occurrence": 2, "mark_ids": [7], "confidence": "high", "sentence_index": 1, "is_pronoun": true}], "sentences": [[0, 51], [52, 102]], "groups": [{"group_id": 0, "mark_id": 7, "category": "ship or boat", "isthing": 1, "mentions": [{"text": "An orange lifeboat", "span": [0, 18], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "it", "span": [99, 101], "occurrence": 2, "confidence": "high", "collective": false}], "agreement": ["unknown", "pronoun"], "chain_agreement": "unknown"}, {"group_id": 1, "mark_id": 4, "category": "river", "isthing": 0, "mentions": [{"text": "the river", "span": [41, 50], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 6, "category": "ship or boat", "isthing": 1, "mentions": [{"text": "A smaller white boat", "span": [52, 72], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "moves", "span": [19, 24], "spans": [[19, 24]], "particle": null, "mentions": [{"text": "moves", "span": [19, 24], "spans": [[19, 24]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "An orange lifeboat", "span": [0, 18], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 0, "text": "moves", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 3303, "end_source_frame": 3345, "start_filename": "00003303.jpg", "end_filename": "00003345.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The orange lifeboat advances rightward through the water throughout the sequence, trailing a visible wake.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3303, 3309, 3318, 3324, 3333, 3339, 3345]}}], "coverage": {"marks_shown": 7, "marks_grounded": 3, "things_shown": 2, "things_grounded": 2, "stuff_shown": 5, "stuff_grounded": 1, "n_mentions": 4, "category_disagree": 0, "category_checked": 2}, "marks": [{"mark_id": 7, "category": "ship or boat", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.02935, "mean_rel_area": 0.02457, "centroid": [0.332, 0.455], "motion": 0.041}, {"mark_id": 6, "category": "ship or boat", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 13, "max_rel_area": 0.00161, "mean_rel_area": 0.00147, "centroid": [0.521, 0.382], "motion": 0.046}, {"mark_id": 4, "category": "river", "isthing": 0, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.42172, "mean_rel_area": 0.41819, "centroid": [0.5, 0.689], "motion": 0.005}, {"mark_id": 3, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.28, "mean_rel_area": 0.2578, "centroid": [0.5, 0.142], "motion": 0.007}, {"mark_id": 1, "category": "handrail or fence", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 4, "max_rel_area": 0.22236, "mean_rel_area": 0.21275, "centroid": [0.5, 0.668], "motion": 0.007}, {"mark_id": 5, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 10, "max_rel_area": 0.0736, "mean_rel_area": 0.06996, "centroid": [0.551, 0.281], "motion": 0.065}, {"mark_id": 2, "category": "house", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 1, "max_rel_area": 0.01665, "mean_rel_area": 0.01448, "centroid": [0.122, 0.309], "motion": 0.027}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [3303, 3306, 3309, 3312, 3315, 3318, 3321, 3324, 3327, 3330, 3333, 3336, 3339, 3342, 3345], "timing_sha": "c0685d9fb1c4", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "repaired", "issues": ["entities: mentions[3]: re-anchored 'it' [64, 66] -> [99, 101] (word-bounded occurrence)"]}, "entity_validation": {"status": "repaired", "issues": ["mentions[3]: re-anchored 'it' [64, 66] -> [99, 101] (word-bounded occurrence)"]}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 1, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 16838, "output_tokens": 611, "thinking_tokens": 239, "total_tokens": 17449}, "seconds": 14.74, "response_id": "resp_0796767d46255a61006ac33066c99c87d0adaaabfe53752d50", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 3303, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 3309, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 3318, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 3324, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 3333, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 3339, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 3345, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 3303, "time_s": 0.0, "mark_ids": [7, 6, 4, 3, 1, 5, 2]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 3318, "time_s": 1.0, "mark_ids": [7, 6, 4, 3, 1, 5, 2]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 3333, "time_s": 2.0, "mark_ids": [7, 6, 4, 3, 1, 5, 2]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 3345, "time_s": 2.8, "mark_ids": [7, 6, 4, 3, 1, 5, 2]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [7, 6, 4, 3, 1, 5, 2]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "c0685d9fb1c4", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3303, 3309, 3318, 3324, 3333, 3339, 3345], "actions": [{"action_index": 0, "text": "moves", "status": "visible", "segments_frames": [[0, 6]], "segments": [[0.0, 3.0]], "segment_details": [{"start_frame": 0, "end_frame": 6, "start_frame_1based": 1, "end_frame_1based": 7, "start_s": 0.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 3303, "end_source_frame": 3345, "start_filename": "00003303.jpg", "end_filename": "00003345.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The orange lifeboat advances rightward through the water throughout the sequence, trailing a visible wake.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3303, 3309, 3318, 3324, 3333, 3339, 3345]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "1822_joLWlEd2bPM", "split": "train", "caption": "Furrowed ground and tall grass fill the foreground. A small white house is near dense trees beneath a pale sky.", "mentions": [{"text": "Furrowed ground", "span": [0, 15], "occurrence": 1, "mark_ids": [1], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "tall grass", "span": [20, 30], "occurrence": 1, "mark_ids": [2], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "A small white house", "span": [52, 71], "occurrence": 1, "mark_ids": [3], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "dense trees", "span": [80, 91], "occurrence": 1, "mark_ids": [5], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "a pale sky", "span": [100, 110], "occurrence": 1, "mark_ids": [4], "confidence": "high", "sentence_index": 1, "is_pronoun": false}], "sentences": [[0, 51], [52, 111]], "groups": [{"group_id": 0, "mark_id": 1, "category": "ground", "isthing": 0, "mentions": [{"text": "Furrowed ground", "span": [0, 15], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 2, "category": "grass", "isthing": 0, "mentions": [{"text": "tall grass", "span": [20, 30], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["unknown"], "chain_agreement": "unknown"}, {"group_id": 2, "mark_id": 3, "category": "house", "isthing": 0, "mentions": [{"text": "A small white house", "span": [52, 71], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 5, "category": "tree", "isthing": 0, "mentions": [{"text": "dense trees", "span": [80, 91], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 4, "category": "sky", "isthing": 0, "mentions": [{"text": "a pale sky", "span": [100, 110], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [], "coverage": {"marks_shown": 5, "marks_grounded": 5, "things_shown": 0, "things_grounded": 0, "stuff_shown": 5, "stuff_grounded": 5, "n_mentions": 5, "category_disagree": 0, "category_checked": 4}, "marks": [{"mark_id": 1, "category": "ground", "isthing": 0, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.44226, "mean_rel_area": 0.42173, "centroid": [0.5, 0.723], "motion": 0.01}, {"mark_id": 2, "category": "grass", "isthing": 0, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 1, "max_rel_area": 0.27225, "mean_rel_area": 0.25466, "centroid": [0.5, 0.487], "motion": 0.027}, {"mark_id": 5, "category": "tree", "isthing": 0, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.23383, "mean_rel_area": 0.21971, "centroid": [0.5, 0.17], "motion": 0.001}, {"mark_id": 4, "category": "sky", "isthing": 0, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.12231, "mean_rel_area": 0.10008, "centroid": [0.498, 0.103], "motion": 0.017}, {"mark_id": 3, "category": "house", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 9, "first_frame": 0, "last_frame": 8, "best_frame": 0, "max_rel_area": 0.01337, "mean_rel_area": 0.00635, "centroid": [0.952, 0.265], "motion": 0.078}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [3078, 3081, 3084, 3087, 3090, 3093, 3096, 3099, 3102, 3105, 3108, 3111, 3114, 3117, 3120], "timing_sha": "28e8b34bed85", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 0, "unlinked": 0, "snapped": 0, "collective": 0, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 16708, "output_tokens": 409, "thinking_tokens": 168, "total_tokens": 17117}, "seconds": 13.37, "response_id": "resp_0c3755d49314682c006ac33075968087d0abb8e9137bb2c783", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 3078, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 3084, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 3093, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 3099, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 3108, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 3114, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 3120, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 3078, "time_s": 0.0, "mark_ids": [1, 2, 5, 4, 3]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 3093, "time_s": 1.0, "mark_ids": [1, 2, 5, 4, 3]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 3108, "time_s": 2.0, "mark_ids": [1, 2, 5, 4]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 3120, "time_s": 2.8, "mark_ids": [1, 2, 5, 4]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [1, 2, 5, 4, 3]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "28e8b34bed85", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [3078, 3084, 3093, 3099, 3108, 3114, 3120], "actions": [], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "196_cS5frkjmPgA", "split": "train", "caption": "Many people skate or stand near a golden sculpture outdoors. Among them, a child glides away from a black-clad adult. Flags flutter overhead.", "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "mark_ids": [51, 46, 45, 25, 42, 43, 29, 44, 24, 32, 26, 39, 34, 33, 48, 20, 36, 31, 37, 38, 30, 35, 23, 40, 47, 21, 27], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "a golden sculpture", "span": [32, 50], "occurrence": 1, "mark_ids": [7], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "them", "span": [67, 71], "occurrence": 1, "mark_ids": [51, 46, 45, 25, 42, 43, 29, 44, 24, 32, 26, 39, 34, 33, 48, 20, 36, 31, 37, 38, 30, 35, 23, 40, 47, 21, 27], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "a child", "span": [73, 80], "occurrence": 1, "mark_ids": [24], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "a black-clad adult", "span": [98, 116], "occurrence": 1, "mark_ids": [25], "confidence": "high", "sentence_index": 1, "is_pronoun": false}, {"text": "Flags", "span": [118, 123], "occurrence": 1, "mark_ids": [19, 49, 8, 9, 12, 11, 10, 16, 18, 15, 14, 13], "confidence": "high", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 60], [61, 117], [118, 141]], "groups": [{"group_id": 0, "mark_id": 51, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 46, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 45, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 25, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "a black-clad adult", "span": [98, 116], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 4, "mark_id": 42, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 5, "mark_id": 43, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 6, "mark_id": 29, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 7, "mark_id": 44, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 8, "mark_id": 24, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "a child", "span": [73, 80], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun", "agree"], "chain_agreement": "agree"}, {"group_id": 9, "mark_id": 32, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 10, "mark_id": 26, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 11, "mark_id": 39, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 12, "mark_id": 34, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 13, "mark_id": 33, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 14, "mark_id": 48, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 15, "mark_id": 20, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 16, "mark_id": 36, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 17, "mark_id": 31, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 18, "mark_id": 37, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 19, "mark_id": 38, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 20, "mark_id": 30, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 21, "mark_id": 35, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 22, "mark_id": 23, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 23, "mark_id": 40, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 24, "mark_id": 47, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 25, "mark_id": 21, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 26, "mark_id": 27, "category": "person", "isthing": 1, "mentions": [{"text": "Many people", "span": [0, 11], "occurrence": 1, "confidence": "high", "collective": true}, {"text": "them", "span": [67, 71], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree", "pronoun"], "chain_agreement": "agree"}, {"group_id": 27, "mark_id": 7, "category": "sculpture", "isthing": 1, "mentions": [{"text": "a golden sculpture", "span": [32, 50], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 28, "mark_id": 19, "category": "flag", "isthing": 1, "mentions": [{"text": "Flags", "span": [118, 123], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 29, "mark_id": 49, "category": "flag", "isthing": 1, "mentions": [{"text": "Flags", "span": [118, 123], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 30, "mark_id": 8, "category": "flag", "isthing": 1, "mentions": [{"text": "Flags", "span": [118, 123], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 31, "mark_id": 9, "category": "flag", "isthing": 1, "mentions": [{"text": "Flags", "span": [118, 123], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 32, "mark_id": 12, "category": "flag", "isthing": 1, "mentions": [{"text": "Flags", "span": [118, 123], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 33, "mark_id": 11, "category": "flag", "isthing": 1, "mentions": [{"text": "Flags", "span": [118, 123], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 34, "mark_id": 10, "category": "flag", "isthing": 1, "mentions": [{"text": "Flags", "span": [118, 123], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 35, "mark_id": 16, "category": "flag", "isthing": 1, "mentions": [{"text": "Flags", "span": [118, 123], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 36, "mark_id": 18, "category": "flag", "isthing": 1, "mentions": [{"text": "Flags", "span": [118, 123], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 37, "mark_id": 15, "category": "flag", "isthing": 1, "mentions": [{"text": "Flags", "span": [118, 123], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 38, "mark_id": 14, "category": "flag", "isthing": 1, "mentions": [{"text": "Flags", "span": [118, 123], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 39, "mark_id": 13, "category": "flag", "isthing": 1, "mentions": [{"text": "Flags", "span": [118, 123], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "skate", "span": [12, 17], "spans": [[12, 17]], "particle": null, "mentions": [{"text": "skate", "span": [12, 17], "spans": [[12, 17]], "particle": null, "sentence_index": 0}], "type": "simple", "sentence_index": 0, "subject": {"text": "Many people", "span": [0, 11], "groups": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26], "group_mentions": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "group_match": "exact"}, "objects": [{"text": "a golden sculpture", "span": [32, 50], "groups": [27], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 0, "text": "skate", "status": "visible", "segments_frames": [[0, 8]], "segments": [[0.0, 4.2]], "segment_details": [{"start_frame": 0, "end_frame": 8, "start_frame_1based": 1, "end_frame_1based": 9, "start_s": 0.0, "end_s": 4.2, "last_observed_s": 4.0, "start_source_frame": 631, "end_source_frame": 691, "start_filename": "00000631.jpg", "end_filename": "00000691.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Multiple members skate throughout the sequence, while other members remain stationary.", "fps": 2.0, "input_frame_count": 9, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0], "source_frame_indices": [631, 637, 646, 652, 661, 667, 676, 682, 691]}}, {"text": "stand", "span": [21, 26], "spans": [[21, 26]], "particle": null, "mentions": [{"text": "stand", "span": [21, 26], "spans": [[21, 26]], "particle": null, "sentence_index": 0}], "type": "posture", "sentence_index": 0, "subject": {"text": "Many people", "span": [0, 11], "groups": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26], "group_mentions": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "group_match": "exact"}, "objects": [{"text": "a golden sculpture", "span": [32, 50], "groups": [27], "group_mentions": [0], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 1, "text": "stand", "status": "visible", "segments_frames": [[0, 8]], "segments": [[0.0, 4.2]], "segment_details": [{"start_frame": 0, "end_frame": 8, "start_frame_1based": 1, "end_frame_1based": 9, "start_s": 0.0, "end_s": 4.2, "last_observed_s": 4.0, "start_source_frame": 631, "end_source_frame": 691, "start_filename": "00000631.jpg", "end_filename": "00000691.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Several members hold upright stationary postures throughout, rather than every member standing still.", "fps": 2.0, "input_frame_count": 9, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0], "source_frame_indices": [631, 637, 646, 652, 661, 667, 676, 682, 691]}}, {"text": "glides", "span": [81, 87], "spans": [[81, 87]], "particle": null, "mentions": [{"text": "glides", "span": [81, 87], "spans": [[81, 87]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "a child", "span": [73, 80], "groups": [8], "group_mentions": [2], "group_match": "exact"}, "objects": [{"text": "a black-clad adult", "span": [98, 116], "groups": [3], "group_mentions": [2], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 2, "text": "glides", "status": "visible", "segments_frames": [[4, 8]], "segments": [[2.0, 4.2]], "segment_details": [{"start_frame": 4, "end_frame": 8, "start_frame_1based": 5, "end_frame_1based": 9, "start_s": 2.0, "end_s": 4.2, "last_observed_s": 4.0, "start_source_frame": 661, "end_source_frame": 691, "start_filename": "00000661.jpg", "end_filename": "00000691.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The child moves rightward and increasingly separates from the black-clad adult.", "fps": 2.0, "input_frame_count": 9, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0], "source_frame_indices": [631, 637, 646, 652, 661, 667, 676, 682, 691]}}, {"text": "flutter", "span": [124, 131], "spans": [[124, 131]], "particle": null, "mentions": [{"text": "flutter", "span": [124, 131], "spans": [[124, 131]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "Flags", "span": [118, 123], "groups": [28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39], "group_mentions": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "group_match": "exact"}, "objects": [], "confidence": "medium", "temporal": {"action_index": 3, "text": "flutter", "status": "visible", "segments_frames": [[0, 8]], "segments": [[0.0, 4.2]], "segment_details": [{"start_frame": 0, "end_frame": 8, "start_frame_1based": 1, "end_frame_1based": 9, "start_s": 0.0, "end_s": 4.2, "last_observed_s": 4.0, "start_source_frame": 631, "end_source_frame": 691, "start_filename": "00000631.jpg", "end_filename": "00000691.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Several flags visibly change their folds and outlines, although movement is subtle for some.", "fps": 2.0, "input_frame_count": 9, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0], "source_frame_indices": [631, 637, 646, 652, 661, 667, 676, 682, 691]}}], "coverage": {"marks_shown": 40, "marks_grounded": 40, "things_shown": 40, "things_grounded": 40, "stuff_shown": 0, "stuff_grounded": 0, "n_mentions": 69, "category_disagree": 0, "category_checked": 40}, "marks": [{"mark_id": 7, "category": "sculpture", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 19, "max_rel_area": 0.02849, "mean_rel_area": 0.02697, "centroid": [0.417, 0.45], "motion": 0.122}, {"mark_id": 51, "category": "person", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 19, "first_frame": 2, "last_frame": 20, "best_frame": 19, "max_rel_area": 0.00699, "mean_rel_area": 0.00389, "centroid": [0.662, 0.943], "motion": 0.401}, {"mark_id": 46, "category": "person", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 2, "max_rel_area": 0.00493, "mean_rel_area": 0.00343, "centroid": [0.816, 0.785], "motion": 0.397}, {"mark_id": 45, "category": "person", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 4, "max_rel_area": 0.00486, "mean_rel_area": 0.00274, "centroid": [0.556, 0.762], "motion": 0.479}, {"mark_id": 25, "category": "person", "isthing": 1, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 4, "max_rel_area": 0.00451, "mean_rel_area": 0.00378, "centroid": [0.16, 0.844], "motion": 0.214}, {"mark_id": 42, "category": "person", "isthing": 1, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 13, "max_rel_area": 0.00439, "mean_rel_area": 0.00374, "centroid": [0.773, 0.857], "motion": 0.136}, {"mark_id": 43, "category": "person", "isthing": 1, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 9, "max_rel_area": 0.00436, "mean_rel_area": 0.0036, "centroid": [0.747, 0.852], "motion": 0.135}, {"mark_id": 29, "category": "person", "isthing": 1, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 8, "max_rel_area": 0.00362, "mean_rel_area": 0.00233, "centroid": [0.294, 0.716], "motion": 0.327}, {"mark_id": 44, "category": "person", "isthing": 1, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 4, "max_rel_area": 0.00361, "mean_rel_area": 0.00248, "centroid": [0.653, 0.796], "motion": 0.293}, {"mark_id": 24, "category": "person", "isthing": 1, "iscrowd": 0, "order": 9, "color": [0, 140, 255], "panel": 0, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 19, "max_rel_area": 0.0035, "mean_rel_area": 0.00272, "centroid": [0.194, 0.886], "motion": 0.089}, {"mark_id": 32, "category": "person", "isthing": 1, "iscrowd": 0, "order": 10, "color": [139, 195, 74], "panel": 1, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 19, "max_rel_area": 0.00323, "mean_rel_area": 0.00272, "centroid": [0.346, 0.736], "motion": 0.385}, {"mark_id": 26, "category": "person", "isthing": 1, "iscrowd": 0, "order": 11, "color": [255, 87, 34], "panel": 1, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 9, "max_rel_area": 0.00316, "mean_rel_area": 0.00204, "centroid": [0.626, 0.912], "motion": 0.474}, {"mark_id": 39, "category": "person", "isthing": 1, "iscrowd": 0, "order": 12, "color": [156, 39, 176], "panel": 1, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 0, "max_rel_area": 0.00298, "mean_rel_area": 0.00184, "centroid": [0.925, 0.691], "motion": 0.153}, {"mark_id": 34, "category": "person", "isthing": 1, "iscrowd": 0, "order": 13, "color": [0, 188, 212], "panel": 1, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 12, "max_rel_area": 0.00295, "mean_rel_area": 0.00211, "centroid": [0.59, 0.673], "motion": 0.134}, {"mark_id": 19, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 14, "color": [121, 85, 72], "panel": 1, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 15, "max_rel_area": 0.00279, "mean_rel_area": 0.00161, "centroid": [0.714, 0.142], "motion": 0.134}, {"mark_id": 33, "category": "person", "isthing": 1, "iscrowd": 0, "order": 15, "color": [233, 30, 99], "panel": 1, "n_present": 19, "first_frame": 0, "last_frame": 20, "best_frame": 19, "max_rel_area": 0.0034, "mean_rel_area": 0.00172, "centroid": [0.402, 0.685], "motion": 0.292}, {"mark_id": 48, "category": "person", "isthing": 1, "iscrowd": 0, "order": 16, "color": [63, 81, 181], "panel": 1, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 14, "max_rel_area": 0.00277, "mean_rel_area": 0.00165, "centroid": [0.961, 0.72], "motion": 0.178}, {"mark_id": 20, "category": "person", "isthing": 1, "iscrowd": 0, "order": 17, "color": [205, 220, 57], "panel": 1, "n_present": 20, "first_frame": 0, "last_frame": 19, "best_frame": 17, "max_rel_area": 0.0029, "mean_rel_area": 0.00214, "centroid": [0.07, 0.642], "motion": 0.114}, {"mark_id": 49, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 18, "color": [255, 152, 0], "panel": 1, "n_present": 20, "first_frame": 1, "last_frame": 20, "best_frame": 17, "max_rel_area": 0.00286, "mean_rel_area": 0.00166, "centroid": [0.974, 0.143], "motion": 0.135}, {"mark_id": 36, "category": "person", "isthing": 1, "iscrowd": 0, "order": 19, "color": [96, 125, 139], "panel": 1, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 0, "max_rel_area": 0.00253, "mean_rel_area": 0.00187, "centroid": [0.661, 0.668], "motion": 0.137}, {"mark_id": 31, "category": "person", "isthing": 1, "iscrowd": 0, "order": 20, "color": [54, 67, 244], "panel": 2, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 18, "max_rel_area": 0.00245, "mean_rel_area": 0.00181, "centroid": [0.395, 0.687], "motion": 0.21}, {"mark_id": 8, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 21, "color": [34, 189, 255], "panel": 2, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 14, "max_rel_area": 0.00243, "mean_rel_area": 0.00191, "centroid": [0.931, 0.14], "motion": 0.138}, {"mark_id": 9, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 22, "color": [60, 200, 60], "panel": 2, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 7, "max_rel_area": 0.00226, "mean_rel_area": 0.00165, "centroid": [0.887, 0.142], "motion": 0.133}, {"mark_id": 12, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 23, "color": [240, 120, 40], "panel": 2, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 13, "max_rel_area": 0.00196, "mean_rel_area": 0.00149, "centroid": [0.758, 0.147], "motion": 0.144}, {"mark_id": 37, "category": "person", "isthing": 1, "iscrowd": 0, "order": 24, "color": [180, 60, 220], "panel": 2, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 1, "max_rel_area": 0.00194, "mean_rel_area": 0.00174, "centroid": [0.677, 0.623], "motion": 0.139}, {"mark_id": 38, "category": "person", "isthing": 1, "iscrowd": 0, "order": 25, "color": [0, 215, 255], "panel": 2, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 0, "max_rel_area": 0.00192, "mean_rel_area": 0.00159, "centroid": [0.73, 0.607], "motion": 0.139}, {"mark_id": 30, "category": "person", "isthing": 1, "iscrowd": 0, "order": 26, "color": [200, 90, 130], "panel": 2, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 20, "max_rel_area": 0.00182, "mean_rel_area": 0.00092, "centroid": [0.337, 0.662], "motion": 0.233}, {"mark_id": 11, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 27, "color": [80, 175, 76], "panel": 2, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 12, "max_rel_area": 0.00177, "mean_rel_area": 0.00131, "centroid": [0.799, 0.141], "motion": 0.14}, {"mark_id": 35, "category": "person", "isthing": 1, "iscrowd": 0, "order": 28, "color": [255, 100, 180], "panel": 2, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 0, "max_rel_area": 0.0017, "mean_rel_area": 0.00121, "centroid": [0.637, 0.685], "motion": 0.138}, {"mark_id": 10, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 29, "color": [0, 140, 255], "panel": 2, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 12, "max_rel_area": 0.00166, "mean_rel_area": 0.0013, "centroid": [0.843, 0.144], "motion": 0.137}, {"mark_id": 16, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 30, "color": [139, 195, 74], "panel": 3, "n_present": 20, "first_frame": 0, "last_frame": 19, "best_frame": 13, "max_rel_area": 0.0018, "mean_rel_area": 0.00146, "centroid": [0.1, 0.147], "motion": 0.136}, {"mark_id": 23, "category": "person", "isthing": 1, "iscrowd": 0, "order": 31, "color": [255, 87, 34], "panel": 3, "n_present": 17, "first_frame": 0, "last_frame": 16, "best_frame": 4, "max_rel_area": 0.00248, "mean_rel_area": 0.00182, "centroid": [0.116, 0.715], "motion": 0.183}, {"mark_id": 18, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 32, "color": [156, 39, 176], "panel": 3, "n_present": 21, "first_frame": 0, "last_frame": 20, "best_frame": 0, "max_rel_area": 0.00149, "mean_rel_area": 0.0013, "centroid": [0.13, 0.152], "motion": 0.153}, {"mark_id": 15, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 33, "color": [0, 188, 212], "panel": 3, "n_present": 19, "first_frame": 0, "last_frame": 18, "best_frame": 17, "max_rel_area": 0.00179, "mean_rel_area": 0.00157, "centroid": [0.075, 0.162], "motion": 0.122}, {"mark_id": 40, "category": "person", "isthing": 1, "iscrowd": 0, "order": 34, "color": [121, 85, 72], "panel": 3, "n_present": 20, "first_frame": 0, "last_frame": 20, "best_frame": 3, "max_rel_area": 0.00153, "mean_rel_area": 0.00112, "centroid": [0.839, 0.701], "motion": 0.15}, {"mark_id": 47, "category": "person", "isthing": 1, "iscrowd": 0, "order": 35, "color": [233, 30, 99], "panel": 3, "n_present": 19, "first_frame": 0, "last_frame": 18, "best_frame": 2, "max_rel_area": 0.00152, "mean_rel_area": 0.00089, "centroid": [0.055, 0.664], "motion": 0.106}, {"mark_id": 21, "category": "person", "isthing": 1, "iscrowd": 0, "order": 36, "color": [63, 81, 181], "panel": 3, "n_present": 20, "first_frame": 0, "last_frame": 19, "best_frame": 1, "max_rel_area": 0.00124, "mean_rel_area": 0.0009, "centroid": [0.097, 0.664], "motion": 0.124}, {"mark_id": 14, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 37, "color": [205, 220, 57], "panel": 3, "n_present": 17, "first_frame": 0, "last_frame": 16, "best_frame": 14, "max_rel_area": 0.00155, "mean_rel_area": 0.00118, "centroid": [0.041, 0.173], "motion": 0.102}, {"mark_id": 13, "category": "flag", "isthing": 1, "iscrowd": 0, "order": 38, "color": [255, 152, 0], "panel": 3, "n_present": 14, "first_frame": 0, "last_frame": 19, "best_frame": 2, "max_rel_area": 0.00188, "mean_rel_area": 0.00122, "centroid": [0.083, 0.192], "motion": 0.803}, {"mark_id": 27, "category": "person", "isthing": 1, "iscrowd": 0, "order": 39, "color": [96, 125, 139], "panel": 3, "n_present": 6, "first_frame": 0, "last_frame": 5, "best_frame": 3, "max_rel_area": 0.00231, "mean_rel_area": 0.00174, "centroid": [0.035, 0.757], "motion": 0.062}], "num_frames": 21, "duration_s": 4.2, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4, 3.6, 3.8, 4.0], "source_frame_indices": [631, 634, 637, 640, 643, 646, 649, 652, 655, 658, 661, 664, 667, 670, 673, 676, 679, 682, 685, 688, 691], "timing_sha": "715bd11d80ec", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 7, "unlinked": 0, "snapped": 0, "collective": 3, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 26763, "output_tokens": 1537, "thinking_tokens": 452, "total_tokens": 28300}, "seconds": 29.56, "response_id": "resp_0dbcb855c632be0c006ac3307813c487d0a320f83f766ea2f4", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 631, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 637, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 646, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 652, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 661, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 667, "time_s": 2.4}, {"role": "clean", "path": "clean/f015.jpg", "frame": 6, "source_frame": 676, "time_s": 3.0}, {"role": "clean", "path": "clean/f017.jpg", "frame": 7, "source_frame": 682, "time_s": 3.4}, {"role": "clean", "path": "clean/f020.jpg", "frame": 8, "source_frame": 691, "time_s": 4.0}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 631, "time_s": 0.0, "mark_ids": [7, 46, 45, 25, 42, 43, 29, 44, 24]}, {"role": "marks", "path": "marks/f000_p1.jpg", "frame": 0, "panel": 1, "source_frame": 631, "time_s": 0.0, "mark_ids": [32, 26, 39, 34, 19, 33, 48, 20, 36]}, {"role": "marks", "path": "marks/f000_p2.jpg", "frame": 0, "panel": 2, "source_frame": 631, "time_s": 0.0, "mark_ids": [31, 8, 9, 12, 37, 38, 30, 11, 35, 10]}, {"role": "marks", "path": "marks/f000_p3.jpg", "frame": 0, "panel": 3, "source_frame": 631, "time_s": 0.0, "mark_ids": [16, 23, 18, 15, 40, 47, 21, 14, 13, 27]}, {"role": "marks", "path": "marks/f007_p0.jpg", "frame": 7, "panel": 0, "source_frame": 652, "time_s": 1.4, "mark_ids": [7, 51, 46, 45, 25, 42, 43, 29, 44, 24]}, {"role": "marks", "path": "marks/f007_p1.jpg", "frame": 7, "panel": 1, "source_frame": 652, "time_s": 1.4, "mark_ids": [32, 26, 39, 34, 19, 33, 48, 20, 49, 36]}, {"role": "marks", "path": "marks/f007_p2.jpg", "frame": 7, "panel": 2, "source_frame": 652, "time_s": 1.4, "mark_ids": [31, 8, 9, 12, 37, 38, 30, 11, 35, 10]}, {"role": "marks", "path": "marks/f007_p3.jpg", "frame": 7, "panel": 3, "source_frame": 652, "time_s": 1.4, "mark_ids": [16, 23, 18, 15, 40, 47, 21, 14, 13]}, {"role": "marks", "path": "marks/f012_p0.jpg", "frame": 12, "panel": 0, "source_frame": 667, "time_s": 2.4, "mark_ids": [7, 51, 46, 45, 25, 42, 43, 29, 44, 24]}, {"role": "marks", "path": "marks/f012_p1.jpg", "frame": 12, "panel": 1, "source_frame": 667, "time_s": 2.4, "mark_ids": [32, 26, 39, 34, 19, 33, 48, 20, 49, 36]}, {"role": "marks", "path": "marks/f012_p2.jpg", "frame": 12, "panel": 2, "source_frame": 667, "time_s": 2.4, "mark_ids": [31, 8, 9, 12, 37, 38, 30, 11, 35, 10]}, {"role": "marks", "path": "marks/f012_p3.jpg", "frame": 12, "panel": 3, "source_frame": 667, "time_s": 2.4, "mark_ids": [16, 23, 18, 15, 40, 47, 21, 14, 13]}, {"role": "marks", "path": "marks/f020_p0.jpg", "frame": 20, "panel": 0, "source_frame": 691, "time_s": 4.0, "mark_ids": [7, 51, 46, 45, 25, 42, 43, 29, 44, 24]}, {"role": "marks", "path": "marks/f020_p1.jpg", "frame": 20, "panel": 1, "source_frame": 691, "time_s": 4.0, "mark_ids": [32, 26, 39, 34, 19, 33, 48, 49, 36]}, {"role": "marks", "path": "marks/f020_p2.jpg", "frame": 20, "panel": 2, "source_frame": 691, "time_s": 4.0, "mark_ids": [31, 8, 9, 12, 37, 38, 30, 11, 35, 10]}, {"role": "marks", "path": "marks/f020_p3.jpg", "frame": 20, "panel": 3, "source_frame": 691, "time_s": 4.0, "mark_ids": [18, 40]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [7, 51, 46, 45, 25, 42, 43, 29, 44, 24, 32, 26, 39, 34, 19, 33]}, {"role": "ids", "path": "ids/sheet1.jpg", "mark_ids": [48, 20, 49, 36, 31, 8, 9, 12, 37, 38, 30, 11, 35, 10, 16, 23]}, {"role": "ids", "path": "ids/sheet2.jpg", "mark_ids": [18, 15, 40, 47, 21, 14, 13, 27]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "715bd11d80ec", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 9, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0], "source_frame_indices": [631, 637, 646, 652, 661, 667, 676, 682, 691], "actions": [{"action_index": 0, "text": "skate", "status": "visible", "segments_frames": [[0, 8]], "segments": [[0.0, 4.2]], "segment_details": [{"start_frame": 0, "end_frame": 8, "start_frame_1based": 1, "end_frame_1based": 9, "start_s": 0.0, "end_s": 4.2, "last_observed_s": 4.0, "start_source_frame": 631, "end_source_frame": 691, "start_filename": "00000631.jpg", "end_filename": "00000691.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Multiple members skate throughout the sequence, while other members remain stationary.", "fps": 2.0, "input_frame_count": 9, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0], "source_frame_indices": [631, 637, 646, 652, 661, 667, 676, 682, 691]}, {"action_index": 1, "text": "stand", "status": "visible", "segments_frames": [[0, 8]], "segments": [[0.0, 4.2]], "segment_details": [{"start_frame": 0, "end_frame": 8, "start_frame_1based": 1, "end_frame_1based": 9, "start_s": 0.0, "end_s": 4.2, "last_observed_s": 4.0, "start_source_frame": 631, "end_source_frame": 691, "start_filename": "00000631.jpg", "end_filename": "00000691.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Several members hold upright stationary postures throughout, rather than every member standing still.", "fps": 2.0, "input_frame_count": 9, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0], "source_frame_indices": [631, 637, 646, 652, 661, 667, 676, 682, 691]}, {"action_index": 2, "text": "glides", "status": "visible", "segments_frames": [[4, 8]], "segments": [[2.0, 4.2]], "segment_details": [{"start_frame": 4, "end_frame": 8, "start_frame_1based": 5, "end_frame_1based": 9, "start_s": 2.0, "end_s": 4.2, "last_observed_s": 4.0, "start_source_frame": 661, "end_source_frame": 691, "start_filename": "00000661.jpg", "end_filename": "00000691.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The child moves rightward and increasingly separates from the black-clad adult.", "fps": 2.0, "input_frame_count": 9, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0], "source_frame_indices": [631, 637, 646, 652, 661, 667, 676, 682, 691]}, {"action_index": 3, "text": "flutter", "status": "visible", "segments_frames": [[0, 8]], "segments": [[0.0, 4.2]], "segment_details": [{"start_frame": 0, "end_frame": 8, "start_frame_1based": 1, "end_frame_1based": 9, "start_s": 0.0, "end_s": 4.2, "last_observed_s": 4.0, "start_source_frame": 631, "end_source_frame": 691, "start_filename": "00000631.jpg", "end_filename": "00000691.jpg", "touches_clip_start": true, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Several flags visibly change their folds and outlines, although movement is subtle for some.", "fps": 2.0, "input_frame_count": 9, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 3.0, 3.4, 4.0], "source_frame_indices": [631, 637, 646, 652, 661, 667, 676, 682, 691]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}
{"schema": "viposeg_caption_actions_temporal/unified-v1", "video_id": "2112_N3YcO7YTOBE", "split": "train", "caption": "A man holds a handstand indoors. He lowers himself and stands upright. Two people descend on the left, and another person moves on the right.", "mentions": [{"text": "A man", "span": [0, 5], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 0, "is_pronoun": false}, {"text": "He", "span": [33, 35], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "himself", "span": [43, 50], "occurrence": 1, "mark_ids": [6], "confidence": "high", "sentence_index": 1, "is_pronoun": true}, {"text": "Two people", "span": [71, 81], "occurrence": 1, "mark_ids": [8, 7], "confidence": "high", "sentence_index": 2, "is_pronoun": false}, {"text": "another person", "span": [107, 121], "occurrence": 1, "mark_ids": [9], "confidence": "medium", "sentence_index": 2, "is_pronoun": false}], "sentences": [[0, 32], [33, 70], [71, 141]], "groups": [{"group_id": 0, "mark_id": 6, "category": "person", "isthing": 1, "mentions": [{"text": "A man", "span": [0, 5], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "He", "span": [33, 35], "occurrence": 1, "confidence": "high", "collective": false}, {"text": "himself", "span": [43, 50], "occurrence": 1, "confidence": "high", "collective": false}], "agreement": ["agree", "pronoun", "pronoun"], "chain_agreement": "agree"}, {"group_id": 1, "mark_id": 8, "category": "person", "isthing": 1, "mentions": [{"text": "Two people", "span": [71, 81], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 2, "mark_id": 7, "category": "person", "isthing": 1, "mentions": [{"text": "Two people", "span": [71, 81], "occurrence": 1, "confidence": "high", "collective": true}], "agreement": ["agree"], "chain_agreement": "agree"}, {"group_id": 3, "mark_id": 9, "category": "person", "isthing": 1, "mentions": [{"text": "another person", "span": [107, 121], "occurrence": 1, "confidence": "medium", "collective": false}], "agreement": ["agree"], "chain_agreement": "agree"}], "actions": [{"text": "holds a handstand", "span": [6, 23], "spans": [[6, 23]], "particle": null, "mentions": [{"text": "holds a handstand", "span": [6, 23], "spans": [[6, 23]], "particle": null, "sentence_index": 0}], "type": "light_verb", "sentence_index": 0, "subject": {"text": "A man", "span": [0, 5], "groups": [0], "group_mentions": [0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 0, "text": "holds a handstand", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.9]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 483, "end_source_frame": 504, "start_filename": "00000483.jpg", "end_filename": "00000504.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The man supports himself upside down through frame 3, when one leg begins dropping.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [483, 489, 498, 504, 513, 519, 525]}}, {"text": "lowers", "span": [36, 42], "spans": [[36, 42]], "particle": null, "mentions": [{"text": "lowers", "span": [36, 42], "spans": [[36, 42]], "particle": null, "sentence_index": 1}], "type": "simple", "sentence_index": 1, "subject": {"text": "He", "span": [33, 35], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [{"text": "himself", "span": [43, 50], "groups": [0], "group_mentions": [2], "group_match": "exact"}], "confidence": "high", "temporal": {"action_index": 1, "text": "lowers", "status": "visible", "segments_frames": [[3, 4]], "segments": [[1.4, 2.4]], "segment_details": [{"start_frame": 3, "end_frame": 4, "start_frame_1based": 4, "end_frame_1based": 5, "start_s": 1.4, "end_s": 2.4, "last_observed_s": 2.0, "start_source_frame": 504, "end_source_frame": 513, "start_filename": "00000504.jpg", "end_filename": "00000513.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He brings his legs down from the handstand and returns to his feet.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [483, 489, 498, 504, 513, 519, 525]}}, {"text": "stands", "span": [55, 61], "spans": [[55, 61]], "particle": null, "mentions": [{"text": "stands", "span": [55, 61], "spans": [[55, 61]], "particle": null, "sentence_index": 1}], "type": "posture", "sentence_index": 1, "subject": {"text": "He", "span": [33, 35], "groups": [0], "group_mentions": [1], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 2, "text": "stands", "status": "visible", "segments_frames": [[5, 6]], "segments": [[2.4, 3.0]], "segment_details": [{"start_frame": 5, "end_frame": 6, "start_frame_1based": 6, "end_frame_1based": 7, "start_s": 2.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 519, "end_source_frame": 525, "start_filename": "00000519.jpg", "end_filename": "00000525.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He is back on both feet and becomes upright in the final frames.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [483, 489, 498, 504, 513, 519, 525]}}, {"text": "descend", "span": [82, 89], "spans": [[82, 89]], "particle": null, "mentions": [{"text": "descend", "span": [82, 89], "spans": [[82, 89]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "Two people", "span": [71, 81], "groups": [1, 2], "group_mentions": [0, 0], "group_match": "exact"}, "objects": [], "confidence": "high", "temporal": {"action_index": 3, "text": "descend", "status": "visible", "segments_frames": [[2, 6]], "segments": [[1.0, 3.0]], "segment_details": [{"start_frame": 2, "end_frame": 6, "start_frame_1based": 3, "end_frame_1based": 7, "start_s": 1.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 498, "end_source_frame": 525, "start_filename": "00000498.jpg", "end_filename": "00000525.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Both people on the left become progressively lower and larger as they descend.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [483, 489, 498, 504, 513, 519, 525]}}, {"text": "moves", "span": [122, 127], "spans": [[122, 127]], "particle": null, "mentions": [{"text": "moves", "span": [122, 127], "spans": [[122, 127]], "particle": null, "sentence_index": 2}], "type": "simple", "sentence_index": 2, "subject": {"text": "another person", "span": [107, 121], "groups": [3], "group_mentions": [0], "group_match": "exact"}, "objects": [], "confidence": "medium", "temporal": {"action_index": 4, "text": "moves", "status": "visible", "segments_frames": [[3, 6]], "segments": [[1.4, 3.0]], "segment_details": [{"start_frame": 3, "end_frame": 6, "start_frame_1based": 4, "end_frame_1based": 7, "start_s": 1.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 504, "end_source_frame": 525, "start_filename": "00000504.jpg", "end_filename": "00000525.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The small person on the right shifts rightward across the later frames.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [483, 489, 498, 504, 513, 519, 525]}}], "coverage": {"marks_shown": 9, "marks_grounded": 4, "things_shown": 4, "things_grounded": 4, "stuff_shown": 5, "stuff_grounded": 0, "n_mentions": 6, "category_disagree": 0, "category_checked": 4}, "marks": [{"mark_id": 6, "category": "person", "isthing": 1, "iscrowd": 0, "order": 0, "color": [54, 67, 244], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.05837, "mean_rel_area": 0.05102, "centroid": [0.532, 0.428], "motion": 0.045}, {"mark_id": 8, "category": "person", "isthing": 1, "iscrowd": 0, "order": 1, "color": [34, 189, 255], "panel": 0, "n_present": 11, "first_frame": 4, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.01741, "mean_rel_area": 0.00824, "centroid": [0.3, 0.286], "motion": 0.084}, {"mark_id": 7, "category": "person", "isthing": 1, "iscrowd": 0, "order": 2, "color": [60, 200, 60], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 14, "max_rel_area": 0.00883, "mean_rel_area": 0.00319, "centroid": [0.366, 0.296], "motion": 0.07}, {"mark_id": 9, "category": "person", "isthing": 1, "iscrowd": 0, "order": 3, "color": [240, 120, 40], "panel": 0, "n_present": 9, "first_frame": 6, "last_frame": 14, "best_frame": 9, "max_rel_area": 0.00195, "mean_rel_area": 0.00147, "centroid": [0.872, 0.5], "motion": 0.125}, {"mark_id": 2, "category": "ceiling", "isthing": 0, "iscrowd": 0, "order": 4, "color": [180, 60, 220], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 2, "max_rel_area": 0.33625, "mean_rel_area": 0.28286, "centroid": [0.5, 0.28], "motion": 0.008}, {"mark_id": 1, "category": "wall", "isthing": 0, "iscrowd": 0, "order": 5, "color": [0, 215, 255], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 0, "max_rel_area": 0.05704, "mean_rel_area": 0.04818, "centroid": [0.5, 0.463], "motion": 0.105}, {"mark_id": 4, "category": "billboard or Bulletin Board", "isthing": 0, "iscrowd": 0, "order": 6, "color": [200, 90, 130], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 9, "max_rel_area": 0.03507, "mean_rel_area": 0.02395, "centroid": [0.457, 0.25], "motion": 0.048}, {"mark_id": 5, "category": "lamp", "isthing": 0, "iscrowd": 0, "order": 7, "color": [80, 175, 76], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 3, "max_rel_area": 0.03005, "mean_rel_area": 0.02242, "centroid": [0.476, 0.187], "motion": 0.034}, {"mark_id": 3, "category": "handrail or fence", "isthing": 0, "iscrowd": 0, "order": 8, "color": [255, 100, 180], "panel": 0, "n_present": 15, "first_frame": 0, "last_frame": 14, "best_frame": 3, "max_rel_area": 0.0288, "mean_rel_area": 0.0226, "centroid": [0.906, 0.635], "motion": 0.031}], "num_frames": 15, "duration_s": 3.0, "fps": 5.0, "sample_fps": 2.0, "frame_times_s": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8], "source_frame_indices": [483, 486, 489, 492, 495, 498, 501, 504, 507, 510, 513, 516, 519, 522, 525], "timing_sha": "d659ada47120", "timing_policy": "viposeg-vspw15-vipseg5-v2", "validation": {"status": "ok", "issues": []}, "entity_validation": {"status": "ok", "issues": []}, "action_validation": {"status": "ok", "issues": []}, "temporal_validation": {"status": "ok", "issues": [], "warnings": []}, "linking": {"linked": 6, "unlinked": 0, "snapped": 0, "collective": 1, "ambiguous": 0, "crossing": 0, "notes": []}, "model": "gpt-6.1-sol", "provider": "openai", "effort": "medium", "detail": "high", "prompt_sha": "58e38fadcafc6fd0", "usage": {"input_tokens": 17183, "output_tokens": 1201, "thinking_tokens": 290, "total_tokens": 18384}, "seconds": 26.97, "response_id": "resp_035c0931d78350d5006ac33082ea6887d0997465c4d126761f", "api_status": "completed", "model_calls": 1, "images": [{"role": "clean", "path": "clean/f000.jpg", "frame": 0, "source_frame": 483, "time_s": 0.0}, {"role": "clean", "path": "clean/f002.jpg", "frame": 1, "source_frame": 489, "time_s": 0.4}, {"role": "clean", "path": "clean/f005.jpg", "frame": 2, "source_frame": 498, "time_s": 1.0}, {"role": "clean", "path": "clean/f007.jpg", "frame": 3, "source_frame": 504, "time_s": 1.4}, {"role": "clean", "path": "clean/f010.jpg", "frame": 4, "source_frame": 513, "time_s": 2.0}, {"role": "clean", "path": "clean/f012.jpg", "frame": 5, "source_frame": 519, "time_s": 2.4}, {"role": "clean", "path": "clean/f014.jpg", "frame": 6, "source_frame": 525, "time_s": 2.8}, {"role": "marks", "path": "marks/f000_p0.jpg", "frame": 0, "panel": 0, "source_frame": 483, "time_s": 0.0, "mark_ids": [6, 7, 2, 1, 4, 5, 3]}, {"role": "marks", "path": "marks/f005_p0.jpg", "frame": 5, "panel": 0, "source_frame": 498, "time_s": 1.0, "mark_ids": [6, 8, 7, 2, 1, 4, 5, 3]}, {"role": "marks", "path": "marks/f010_p0.jpg", "frame": 10, "panel": 0, "source_frame": 513, "time_s": 2.0, "mark_ids": [6, 8, 7, 9, 2, 1, 4, 5, 3]}, {"role": "marks", "path": "marks/f014_p0.jpg", "frame": 14, "panel": 0, "source_frame": 525, "time_s": 2.8, "mark_ids": [6, 8, 7, 9, 2, 1, 4, 5, 3]}, {"role": "ids", "path": "ids/sheet0.jpg", "mark_ids": [6, 8, 7, 9, 2, 1, 4, 5, 3]}], "marks_params_sha": "3bbaf9c8aec7", "config": {"prompt_sha": "58e38fadcafc6fd0", "model": "gpt-6.1-sol", "timing_sha": "d659ada47120", "schema_sha": "d9e2707412f3c4d4", "output_shape": "one-call-caption-action-temporal-v1"}, "temporal": {"schema": "viposeg_temporal_actions/embedded-v1", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [483, 489, 498, 504, 513, 519, 525], "actions": [{"action_index": 0, "text": "holds a handstand", "status": "visible", "segments_frames": [[0, 3]], "segments": [[0.0, 1.9]], "segment_details": [{"start_frame": 0, "end_frame": 3, "start_frame_1based": 1, "end_frame_1based": 4, "start_s": 0.0, "end_s": 1.9, "last_observed_s": 1.4, "start_source_frame": 483, "end_source_frame": 504, "start_filename": "00000483.jpg", "end_filename": "00000504.jpg", "touches_clip_start": true, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "The man supports himself upside down through frame 3, when one leg begins dropping.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [483, 489, 498, 504, 513, 519, 525]}, {"action_index": 1, "text": "lowers", "status": "visible", "segments_frames": [[3, 4]], "segments": [[1.4, 2.4]], "segment_details": [{"start_frame": 3, "end_frame": 4, "start_frame_1based": 4, "end_frame_1based": 5, "start_s": 1.4, "end_s": 2.4, "last_observed_s": 2.0, "start_source_frame": 504, "end_source_frame": 513, "start_filename": "00000504.jpg", "end_filename": "00000513.jpg", "touches_clip_start": false, "touches_clip_end": false, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He brings his legs down from the handstand and returns to his feet.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [483, 489, 498, 504, 513, 519, 525]}, {"action_index": 2, "text": "stands", "status": "visible", "segments_frames": [[5, 6]], "segments": [[2.4, 3.0]], "segment_details": [{"start_frame": 5, "end_frame": 6, "start_frame_1based": 6, "end_frame_1based": 7, "start_s": 2.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 519, "end_source_frame": 525, "start_filename": "00000519.jpg", "end_filename": "00000525.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "high", "reason": "He is back on both feet and becomes upright in the final frames.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [483, 489, 498, 504, 513, 519, 525]}, {"action_index": 3, "text": "descend", "status": "visible", "segments_frames": [[2, 6]], "segments": [[1.0, 3.0]], "segment_details": [{"start_frame": 2, "end_frame": 6, "start_frame_1based": 3, "end_frame_1based": 7, "start_s": 1.0, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 498, "end_source_frame": 525, "start_filename": "00000498.jpg", "end_filename": "00000525.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "Both people on the left become progressively lower and larger as they descend.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [483, 489, 498, 504, 513, 519, 525]}, {"action_index": 4, "text": "moves", "status": "visible", "segments_frames": [[3, 6]], "segments": [[1.4, 3.0]], "segment_details": [{"start_frame": 3, "end_frame": 6, "start_frame_1based": 4, "end_frame_1based": 7, "start_s": 1.4, "end_s": 3.0, "last_observed_s": 2.8, "start_source_frame": 504, "end_source_frame": 525, "start_filename": "00000504.jpg", "end_filename": "00000525.jpg", "touches_clip_start": false, "touches_clip_end": true, "sampling_gap_after_frames": []}], "confidence": "medium", "reason": "The small person on the right shifts rightward across the later frames.", "fps": 2.0, "input_frame_count": 7, "frame_times_s": [0.0, 0.4, 1.0, 1.4, 2.0, 2.4, 2.8], "source_frame_indices": [483, 489, 498, 504, 513, 519, 525]}], "model_calls": 0}, "integrity": {"status": "ok", "errors": []}}