From 249222f29b199aa32eaac586486f33edde0fefaa Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Wed, 29 Apr 2026 05:48:39 -0400 Subject: [PATCH 1/2] Add interactive segmentation, text query, and stereo (desktop) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brings in the SAM2/SAM3-based interactive segmentation feature, the SAM3 text-query workflow, and the desktop interactive stereo mode. Web-girder paths are intentionally untouched for now — web support will come in a follow-up. - New segmentation point-click recipe + EditorMenu wiring; SAM2/SAM3 models loaded via VIAME install configs. - Desktop backend: viame_segmentation_service-backed IPC handlers and matching frontend API for segmentationInitialize/Predict/SetImage/ ClearImage/Shutdown/IsReady, textQuery/refineDetections/ runTextQueryPipeline, and stereoEnable/Disable/SetFrame/GetStatus/ TransferLine/TransferPoints/SetCalibration/IsEnabled, plus disparity ready/error event hooks. - EditAnnotationLayer: track shift-key state and right-click for Point mode, propagate background flag for negative SAM points. - Sidebar / ViewerLoader / Viewer: stereo annotation mode UI, error dialog when seg or text-query model fails to load, dot-only-on-source -frame fix. - useModeManager / EditAnnotationLayer / recipes: keep existing geometry type when current editing mode already matches; right-click in Point creation finalises and deselects. --- client/bundle.css | 477 ++++++++++ client/dive-common/apispec.ts | 116 +++ .../dive-common/components/DeleteControls.vue | 3 + client/dive-common/components/EditorMenu.vue | 243 ++++- client/dive-common/components/Sidebar.vue | 213 ++++- .../components/TrackSettingsPanel.vue | 46 + client/dive-common/components/Viewer.vue | 645 +++++++++++++- client/dive-common/recipes/headtail.ts | 55 +- .../recipes/segmentationpointclick.ts | 759 ++++++++++++++++ client/dive-common/store/settings.ts | 27 +- client/dive-common/use/useModeManager.ts | 402 ++++++++- client/platform/desktop/backend/ipcService.ts | 173 ++++ .../desktop/backend/native/segmentation.ts | 535 +++++++++++ .../platform/desktop/backend/native/stereo.ts | 668 ++++++++++++++ client/platform/desktop/frontend/api.ts | 207 +++++ .../frontend/components/ViewerLoader.vue | 840 ++++++++++++++++-- client/src/components/LayerManager.vue | 184 +++- .../src/layers/AnnotationLayers/LineLayer.ts | 19 +- .../SegmentationPointsLayer.ts | 76 ++ client/src/layers/EditAnnotationLayer.ts | 180 +++- client/src/provides.ts | 23 + client/src/recipe.ts | 4 + 22 files changed, 5799 insertions(+), 96 deletions(-) create mode 100644 client/bundle.css create mode 100644 client/dive-common/recipes/segmentationpointclick.ts create mode 100644 client/platform/desktop/backend/native/segmentation.ts create mode 100644 client/platform/desktop/backend/native/stereo.ts create mode 100644 client/src/layers/AnnotationLayers/SegmentationPointsLayer.ts diff --git a/client/bundle.css b/client/bundle.css new file mode 100644 index 000000000..b2f0abd65 --- /dev/null +++ b/client/bundle.css @@ -0,0 +1,477 @@ +.event-chart { + position: relative; + height: calc(100% - 10px); + margin: 5px 0; + overflow-y: auto; + overflow-x: hidden; +} +.event-chart .tooltip { + position: absolute; + background: black; + border: 1px solid white; + padding: 0px 5px; + font-size: 14px; + z-index: 2; +} + +.controls { + position: absolute; +} + +.video-annotator { + position: relative; + left: 0; + right: 0; + top: 0; + bottom: 0; + z-index: 0; + display: flex; + flex-direction: column; +} +.video-annotator .geojs-map { + margin: 2px; +} +.video-annotator .geojs-map.geojs-map:focus { + outline: none; +} +.video-annotator .playback-container { + flex: 1; +} +.video-annotator .loadingSpinnerContainer { + z-index: 20; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); +} +.video-annotator .geojs-map.annotation-input { + cursor: inherit; +} + +.selected-camera { + box-sizing: content-box; +} +.selected-camera .geojs-map { + outline: 3px cyan dashed; +} +.selected-camera .geojs-map.geojs-map:focus { + outline: 3px cyan dashed; +} + +.imageCursor { + z-index: 10; + position: fixed; + backface-visibility: hidden; + top: 0; + left: 0; + pointer-events: none; +} + +.controls { + bottom: 0; +} + +.controls { + position: absolute; +} + +.video-annotator { + position: relative; + left: 0; + right: 0; + top: 0; + bottom: 0; + z-index: 0; + display: flex; + flex-direction: column; +} +.video-annotator .geojs-map { + margin: 2px; +} +.video-annotator .geojs-map.geojs-map:focus { + outline: none; +} +.video-annotator .playback-container { + flex: 1; +} +.video-annotator .loadingSpinnerContainer { + z-index: 20; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); +} +.video-annotator .geojs-map.annotation-input { + cursor: inherit; +} + +.selected-camera { + box-sizing: content-box; +} +.selected-camera .geojs-map { + outline: 3px cyan dashed; +} +.selected-camera .geojs-map.geojs-map:focus { + outline: 3px cyan dashed; +} + +.imageCursor { + z-index: 10; + position: fixed; + backface-visibility: hidden; + top: 0; + left: 0; + pointer-events: none; +} + +.controls { + bottom: 0; +} + +.border-radius[data-v-77dee125] { + border: 1px solid #888888; + padding: 2px 5px; + border-radius: 5px; +} + +.line-chart { + height: 100%; +} +.line-chart .line { + fill: none; + stroke-width: 1.5px; +} +.line-chart .axis-y { + font-size: 12px; +} +.line-chart .axis-y g:first-of-type, +.line-chart .axis-y g:last-of-type { + display: none; +} +.line-chart .tooltip { + position: absolute; + background: black; + border: 1px solid white; + padding: 0px 5px; + font-size: 14px; +} + +.timeline .tick { + shape-rendering: crispEdges; + font-size: 12px; + stroke-opacity: 0.5; + stroke-dasharray: 2, 2; +} + +.timeline[data-v-0d0fe2ba] { + min-height: 175px; + position: relative; + display: flex; + flex-direction: column; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] { + flex: 1; + position: relative; + overflow: hidden; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .hand[data-v-0d0fe2ba] { + position: absolute; + top: 0; + width: 0; + height: 100%; + border-left: 1px solid #299be3; + z-index: 10; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-line[data-v-0d0fe2ba] { + position: absolute; + top: 0; + width: 0; + height: 100%; + z-index: 2; + cursor: col-resize; + pointer-events: auto; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-tooltip[data-v-0d0fe2ba] { + position: absolute; + top: 30px; + transform: translateX(-50%); + background-color: rgba(0, 0, 0, 0.8); + color: white; + padding: 4px 8px; + border-radius: 4px; + font-size: 12px; + white-space: nowrap; + pointer-events: none; + z-index: 20; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-start-line[data-v-0d0fe2ba] { + border-left: 3px solid #4caf50; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-end-line[data-v-0d0fe2ba] { + border-left: 3px solid #f44336; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-dimming[data-v-0d0fe2ba] { + position: absolute; + top: 0; + height: 100%; + background-color: rgba(0, 0, 0, 0.3); + pointer-events: none; + z-index: 1; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .child[data-v-0d0fe2ba] { + position: absolute; + top: 0; + bottom: 17px; + left: 0; + right: 0; + z-index: 0; +} +.timeline[data-v-0d0fe2ba] .minimap[data-v-0d0fe2ba] { + height: 10px; +} +.timeline[data-v-0d0fe2ba] .minimap[data-v-0d0fe2ba] .fill[data-v-0d0fe2ba] { + position: relative; + height: 100%; + background-color: #80c6e8; +} + +.controls { + position: absolute; +} + +.video-annotator { + position: relative; + left: 0; + right: 0; + top: 0; + bottom: 0; + z-index: 0; + display: flex; + flex-direction: column; +} +.video-annotator .geojs-map { + margin: 2px; +} +.video-annotator .geojs-map.geojs-map:focus { + outline: none; +} +.video-annotator .playback-container { + flex: 1; +} +.video-annotator .loadingSpinnerContainer { + z-index: 20; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); +} +.video-annotator .geojs-map.annotation-input { + cursor: inherit; +} + +.selected-camera { + box-sizing: content-box; +} +.selected-camera .geojs-map { + outline: 3px cyan dashed; +} +.selected-camera .geojs-map.geojs-map:focus { + outline: 3px cyan dashed; +} + +.imageCursor { + z-index: 10; + position: fixed; + backface-visibility: hidden; + top: 0; + left: 0; + pointer-events: none; +} + +.controls { + bottom: 0; +} + +.input-box { + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + padding: 0 6px; + color: white; +} + +.trackNumber { + font-family: monospace; + max-width: 80px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} +.trackNumber:hover { + cursor: pointer; + font-weight: bolder; + text-decoration: underline; +} + +.border-highlight[data-v-0d46f934] { + border-bottom: 1px solid gray; +} + +.type-checkbox[data-v-0d46f934] { + max-width: 80%; + overflow-wrap: anywhere; +} + +.hover-show-parent[data-v-0d46f934] .hover-show-child[data-v-0d46f934] { + display: none; +} +.hover-show-parent[data-v-0d46f934][data-v-0d46f934]:hover .hover-show-child[data-v-0d46f934] { + display: inherit; +} + +.outlined[data-v-0d46f934] { + background-color: gray; + color: #222; + font-weight: 600; + border-radius: 6px; + padding: 0 5px; + font-size: 12px; +} + +.input-box { + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + padding: 0 6px; + color: white; +} + +.trackNumber { + font-family: monospace; + max-width: 80px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} +.trackNumber:hover { + cursor: pointer; + font-weight: bolder; + text-decoration: underline; +} + +.freeform-input[data-v-d679c59c] { + width: 150px; +} + +.groups[data-v-c26ed586] { + overflow-y: auto; + overflow-x: hidden; +} + +.input-box { + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + padding: 0 6px; + color: white; +} + +.trackNumber { + font-family: monospace; + max-width: 80px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} +.trackNumber:hover { + cursor: pointer; + font-weight: bolder; + text-decoration: underline; +} + +.track-item[data-v-7a688bfe] { + border-radius: inherit; +} +.track-item[data-v-7a688bfe] .item-row[data-v-7a688bfe] { + width: 100%; +} +.track-item[data-v-7a688bfe] .type-color-box[data-v-7a688bfe] { + margin: 7px; + margin-top: 4px; + min-width: 15px; + max-width: 15px; + min-height: 15px; + max-height: 15px; +} + +.strcoller { + height: 100%; +} + +.trackHeader { + height: auto; +} + +.tracks { + overflow-y: auto; + overflow-x: hidden; +} +.tracks .v-input--checkbox label { + white-space: pre-wrap; +} + +.nowrap[data-v-a4da19c6] { + white-space: nowrap; + overflow: hidden; + max-width: var(--content-width); + text-overflow: ellipsis; +} + +.hover-show-parent[data-v-a4da19c6] .hover-show-child[data-v-a4da19c6] { + display: none; +} +.hover-show-parent[data-v-a4da19c6][data-v-a4da19c6]:hover .hover-show-child[data-v-a4da19c6] { + display: inherit; +} + +.outlined[data-v-a4da19c6] { + background-color: gray; + color: #222; + font-weight: 600; + border-radius: 6px; + padding: 0 5px; + font-size: 12px; +} + +.input-box { + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + padding: 0 6px; + color: white; +} + +.trackNumber { + font-family: monospace; + max-width: 80px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} +.trackNumber:hover { + cursor: pointer; + font-weight: bolder; + text-decoration: underline; +} + +.freeform-input[data-v-07a75698] { + width: 135px; +} + +.select-input[data-v-07a75698] { + width: 120px; + background-color: #1e1e1e; + appearance: menulist; +} \ No newline at end of file diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index d8888c6b1..036ca7eb6 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -238,6 +238,122 @@ function useApi() { return use>(ApiSymbol); } +/** + * Interactive Segmentation Types + */ +export interface SegmentationPredictRequest { + /** Path to the image file */ + imagePath: string; + /** Point coordinates as [x, y] pairs */ + points: [number, number][]; + /** Point labels: 1 for foreground, 0 for background */ + pointLabels: number[]; + /** Optional low-res mask from previous prediction for refinement */ + maskInput?: number[][]; + /** Whether to return multiple mask options */ + multimaskOutput?: boolean; +} + +export interface SegmentationPredictResponse { + /** Whether the prediction succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** Polygon coordinates as [x, y] pairs */ + polygon?: [number, number][]; + /** Bounding box [x_min, y_min, x_max, y_max] */ + bounds?: [number, number, number, number]; + /** Quality score from segmentation model */ + score?: number; + /** Low-res mask for subsequent refinement */ + lowResMask?: number[][]; + /** Mask dimensions [height, width] */ + maskShape?: [number, number]; + /** RLE-encoded full-resolution mask for display: [[value, count], ...] */ + rleMask?: [number, number][]; +} + +export interface SegmentationStatusResponse { + /** Whether segmentation is available */ + available: boolean; + /** Whether the model is currently loaded */ + loaded?: boolean; + /** Whether the service is ready for predictions */ + ready?: boolean; +} + +/** + * Text Query Types for open-vocabulary detection/segmentation + */ + +/** A single detection returned from a text query */ +export interface TextQueryDetection { + /** Bounding box [x1, y1, x2, y2] */ + box: [number, number, number, number]; + /** Polygon coordinates as [x, y] pairs */ + polygon?: [number, number][]; + /** Confidence score */ + score: number; + /** Label/class name (often the query text) */ + label: string; + /** Low-res mask for refinement (optional) */ + lowResMask?: number[][]; +} + +export interface TextQueryRequest { + /** Path to the image file */ + imagePath: string; + /** Text query describing what to find (e.g., "fish", "person swimming") */ + text: string; + /** Confidence threshold for detections (default: 0.3) */ + boxThreshold?: number; + /** Maximum number of detections to return (default: 10) */ + maxDetections?: number; + /** Optional boxes to refine [x1, y1, x2, y2][] */ + boxes?: [number, number, number, number][]; + /** Optional keypoints for refinement [x, y][] */ + points?: [number, number][]; + /** Labels for points: 1 for foreground, 0 for background */ + pointLabels?: number[]; + /** Optional masks to refine */ + masks?: number[][][]; +} + +export interface TextQueryResponse { + /** Whether the query succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** List of detections found */ + detections?: TextQueryDetection[]; + /** The original query text */ + query?: string; + /** Whether fallback method was used (no native text support) */ + fallback?: boolean; +} + +export interface RefineDetectionsRequest { + /** Path to the image file */ + imagePath: string; + /** Detections to refine */ + detections: TextQueryDetection[]; + /** Optional additional keypoints for refinement [x, y][] */ + points?: [number, number][]; + /** Labels for additional points: 1 for foreground, 0 for background */ + pointLabels?: number[]; + /** Whether to include refined masks in response */ + refineMasks?: boolean; +} + +export interface RefineDetectionsResponse { + /** Whether the refinement succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** Refined detections */ + detections?: TextQueryDetection[]; +} + export { provideApi, useApi, diff --git a/client/dive-common/components/DeleteControls.vue b/client/dive-common/components/DeleteControls.vue index 50ecdd08a..f6022ac8a 100644 --- a/client/dive-common/components/DeleteControls.vue +++ b/client/dive-common/components/DeleteControls.vue @@ -24,6 +24,9 @@ export default Vue.extend({ if (this.editingMode === 'rectangle') { return true; // deleting rectangle is unsupported } + if (this.editingMode === 'Point') { + return true; // Point mode uses reset instead of delete + } return false; }, }, diff --git a/client/dive-common/components/EditorMenu.vue b/client/dive-common/components/EditorMenu.vue index a7bea7bb8..5bf223e42 100644 --- a/client/dive-common/components/EditorMenu.vue +++ b/client/dive-common/components/EditorMenu.vue @@ -11,6 +11,7 @@ import { flatten } from 'lodash'; import { Mousetrap } from 'vue-media-annotator/types'; import { EditAnnotationTypes, VisibleAnnotationTypes } from 'vue-media-annotator/layers'; import Recipe from 'vue-media-annotator/recipe'; +import SegmentationPointClick from 'dive-common/recipes/segmentationpointclick'; import AnnotationVisibilityMenu from './AnnotationVisibilityMenu.vue'; @@ -19,6 +20,7 @@ interface ButtonData { icon: string; type?: VisibleAnnotationTypes; active: boolean; + loading?: boolean; mousetrap?: Mousetrap[]; description: string; click: () => void; @@ -67,7 +69,14 @@ export default defineComponent({ default: true, }, }, - emits: ['set-annotation-state', 'update:tail-settings', 'update:show-user-created-icon'], + emits: [ + 'set-annotation-state', + 'update:tail-settings', + 'update:show-user-created-icon', + 'text-query-init', + 'text-query', + 'text-query-all-frames', + ], setup(props, { emit }) { const toolTimeTimeout = ref(null); const STORAGE_KEY = 'editorMenu.editButtonsExpanded'; @@ -85,6 +94,59 @@ export default defineComponent({ localStorage.setItem(STORAGE_KEY, String(value)); }); + // Text query state + const textQueryDialogOpen = ref(false); + const textQueryInput = ref(''); + const textQueryLoading = ref(false); + const textQueryThreshold = ref(0.3); + const textQueryInitializing = ref(false); + const textQueryServiceError = ref(''); + const textQueryAllFrames = ref(false); + + const openTextQueryDialog = () => { + textQueryDialogOpen.value = true; + textQueryInput.value = ''; + textQueryServiceError.value = ''; + textQueryAllFrames.value = false; + textQueryInitializing.value = true; + emit('text-query-init'); + }; + + const closeTextQueryDialog = () => { + textQueryDialogOpen.value = false; + textQueryInput.value = ''; + textQueryServiceError.value = ''; + textQueryInitializing.value = false; + textQueryAllFrames.value = false; + }; + + const onTextQueryServiceReady = (success: boolean, error?: string) => { + textQueryInitializing.value = false; + if (!success) { + textQueryServiceError.value = error || 'Text query service is not available'; + } + }; + + const submitTextQuery = () => { + if (!textQueryInput.value.trim()) { + return; + } + textQueryLoading.value = true; + if (textQueryAllFrames.value) { + emit('text-query-all-frames', { + text: textQueryInput.value.trim(), + boxThreshold: textQueryThreshold.value, + }); + } else { + emit('text-query', { + text: textQueryInput.value.trim(), + boxThreshold: textQueryThreshold.value, + }); + } + closeTextQueryDialog(); + textQueryLoading.value = false; + }; + const modeToolTips = { Creating: { rectangle: 'Drag to draw rectangle. Press ESC to exit.', @@ -121,6 +183,7 @@ export default defineComponent({ id: r.name, icon: r.icon.value || 'mdi-pencil', active: props.editingTrack && r.active.value, + loading: r.loading?.value ?? false, description: r.name, click: () => r.activate(), mousetrap: [ @@ -134,7 +197,13 @@ export default defineComponent({ ]; }); - const mousetrap = computed((): Mousetrap[] => flatten(editButtons.value.map((b) => b.mousetrap || []))); + const mousetrap = computed((): Mousetrap[] => [ + ...flatten(editButtons.value.map((b) => b.mousetrap || [])), + { + bind: 't', + handler: () => openTextQueryDialog(), + }, + ]); const activeEditButton = computed(() => editButtons.value.find((b) => b.active) || editButtons.value[0]); @@ -161,6 +230,13 @@ export default defineComponent({ return { text: 'Not editing', icon: 'mdi-pencil-off-outline', color: '' }; }); + const activeSegmentationRecipe = computed((): SegmentationPointClick | null => { + const segRecipe = props.recipes.find( + (r) => r instanceof SegmentationPointClick && r.active.value, + ) as SegmentationPointClick | undefined; + return segRecipe || null; + }); + const editingTooltip = computed(() => { if (props.editingDetails === 'disabled' || !props.editingMode || typeof props.editingMode !== 'string') { return ''; @@ -194,6 +270,19 @@ export default defineComponent({ toggleEditButtonsExpanded, activeEditButton, editButtonsMenuKey, + activeSegmentationRecipe, + // Text query + textQueryDialogOpen, + textQueryInput, + textQueryLoading, + textQueryThreshold, + textQueryInitializing, + textQueryServiceError, + textQueryAllFrames, + openTextQueryDialog, + closeTextQueryDialog, + onTextQueryServiceReady, + submitTextQuery, }; }, }); @@ -244,7 +333,7 @@ export default defineComponent({ + + + 
T:
+ mdi-text-search + + + - + + @@ -335,6 +463,103 @@ export default defineComponent({ @update:show-user-created-icon="$emit('update:show-user-created-icon', $event)" /> + + + + + + + mdi-text-search + + Text Query + + + +
+ +

+ Loading text query model... +

+
+ +
+ + mdi-alert-circle + +

+ {{ textQueryServiceError }} +

+
+ + + + + + + {{ textQueryServiceError ? 'Close' : 'Cancel' }} + + + Search + + + + diff --git a/client/dive-common/components/Sidebar.vue b/client/dive-common/components/Sidebar.vue index d7b277dc9..7642c1fdd 100644 --- a/client/dive-common/components/Sidebar.vue +++ b/client/dive-common/components/Sidebar.vue @@ -18,6 +18,7 @@ import { } from 'vue-media-annotator/provides'; import { clientSettings } from 'dive-common/store/settings'; +import ConfidenceFilter from 'dive-common/components/ConfidenceFilter.vue'; import TrackDetailsPanel from 'dive-common/components/TrackDetailsPanel.vue'; import TrackSettingsPanel from 'dive-common/components/TrackSettingsPanel.vue'; import TypeSettingsPanel from 'dive-common/components/TypeSettingsPanel.vue'; @@ -27,6 +28,7 @@ import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; export default defineComponent({ components: { + ConfidenceFilter, StackedVirtualSidebarContainer, TrackDetailsPanel, TrackSettingsPanel, @@ -43,6 +45,14 @@ export default defineComponent({ type: Boolean, default: true, }, + horizontal: { + type: Boolean, + default: false, + }, + isStereoDataset: { + type: Boolean, + default: false, + }, }, setup() { @@ -63,7 +73,9 @@ export default defineComponent({ const styleManager = useTrackStyleManager(); const data = reactive({ - currentTab: 'tracks' as 'tracks' | 'attributes', + currentTab: 'tracks' as 'tracks' | 'attributes' | 'types', + // For horizontal mode, cycle through 3 tabs + horizontalTab: 'tracks' as 'tracks' | 'attributes' | 'types', }); function swapTabs() { @@ -74,6 +86,28 @@ export default defineComponent({ } } + function cycleHorizontalTabs() { + if (data.horizontalTab === 'tracks') { + data.horizontalTab = 'attributes'; + } else if (data.horizontalTab === 'attributes') { + data.horizontalTab = 'types'; + } else { + data.horizontalTab = 'tracks'; + } + } + + const horizontalTabIcon = computed(() => { + if (data.horizontalTab === 'tracks') return 'mdi-format-list-bulleted'; + if (data.horizontalTab === 'attributes') return 'mdi-card-text'; + return 'mdi-filter-variant'; + }); + + const horizontalTabTooltip = computed(() => { + if (data.horizontalTab === 'tracks') return 'Detection List (click to cycle)'; + if (data.horizontalTab === 'attributes') return 'Detection Details (click to cycle)'; + return 'Type Filters (click to cycle)'; + }); + function doToggleMerge() { if (toggleMerge().length) { data.currentTab = 'attributes'; @@ -121,17 +155,23 @@ export default defineComponent({ readOnlyMode, styleManager, disableAnnotationFilters: trackFilterControls.disableAnnotationFilters, + confidenceFilters: trackFilterControls.confidenceFilters, visible, + horizontalTabIcon, + horizontalTabTooltip, /* methods */ doToggleMerge, swapTabs, + cycleHorizontalTabs, }; }, }); + + +
+ + + {{ horizontalTabTooltip }} + + +
+ +
+ +
+ +
+ + + +
+
+ +
+ +
+ +
+ + + +
+
diff --git a/client/dive-common/components/TrackSettingsPanel.vue b/client/dive-common/components/TrackSettingsPanel.vue index 71b1ec1b2..ba8672680 100644 --- a/client/dive-common/components/TrackSettingsPanel.vue +++ b/client/dive-common/components/TrackSettingsPanel.vue @@ -16,6 +16,10 @@ export default defineComponent({ type: Array as PropType>, required: true, }, + isStereoDataset: { + type: Boolean, + default: false, + }, }, setup(props) { @@ -33,6 +37,7 @@ export default defineComponent({ filterTracksByFrame: 'Filter the track list by those with detections in the current frame', autoZoom: 'Automatically zoom to the track when selected', showMultiCamToolbar: 'Show multi-camera tools in the top toolbar when a track is selected', + stereoInteractiveMode: 'When enabled, annotations created on one camera are automatically warped to the other camera using stereo disparity', }); const modes = ref(['Track', 'Detection']); // Add unknown as the default type to the typeList @@ -362,6 +367,47 @@ export default defineComponent({ + diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 8d562df15..44bbd20ba 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -1,6 +1,6 @@ + + diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 675dcdcd5..4dd235079 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -39,7 +39,9 @@ import { useSelectedCamera, useAttributes, useComparisonSets, + useSegmentationPoints, } from '../provides'; +import SegmentationPointsLayer from '../layers/AnnotationLayers/SegmentationPointsLayer'; /** LayerManager is a component intended to be used as a child of an Annotator. * It provides logic for switching which layers are visible, but more importantly @@ -89,7 +91,8 @@ export default defineComponent({ return trackStyleManager.typeStyling.value; }); - const annotator = injectAggregateController().value.getController(props.camera); + const aggregateController = injectAggregateController(); + const annotator = aggregateController.value.getController(props.camera); const frameNumberRef = annotator.frame; const flickNumberRef = annotator.flick; @@ -154,6 +157,20 @@ export default defineComponent({ type: 'rectangle', }); + // Segmentation points layer for displaying prompt points during point-click segmentation + const segmentationPointsRef = useSegmentationPoints(); + const segmentationPointsLayer = new SegmentationPointsLayer(annotator); + + // Watch for segmentation points updates - only show points for current frame + watch([segmentationPointsRef, frameNumberRef], ([newPoints, currentFrame]) => { + // Only display points if they belong to the current frame + if (newPoints.points.length > 0 && newPoints.frameNum === currentFrame) { + segmentationPointsLayer.updatePoints(newPoints.points, newPoints.labels); + } else { + segmentationPointsLayer.clear(); + } + }, { deep: true }); + const updateAttributes = () => { const newList = attributes.value.filter((item) => item.render).sort((a, b) => { if (a.render && b.render) { @@ -388,6 +405,7 @@ export default defineComponent({ typeStylingRef, toRef(props, 'colorBy'), selectedCamera, + selectedKeyRef, ], () => { updateLayers( @@ -435,6 +453,23 @@ export default defineComponent({ ); }); + /** Watch for resize events to redraw layers after view mode changes */ + watch( + () => aggregateController.value.resizeTrigger.value, + () => { + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + }, + ); + const Clicked = (trackId: number, editing: boolean, modifiers?: {ctrl: boolean}) => { // If the camera isn't selected yet we ignore the click if (selectedCamera.value !== props.camera) { @@ -443,20 +478,108 @@ export default defineComponent({ //So we only want to pass the click whjen not in creation mode or editing mode for features if (editAnnotationLayer.getMode() !== 'creation') { editAnnotationLayer.disable(); - handler.trackSelect(trackId, editing, modifiers); + // When entering editing mode (right-click), use trackEdit so the + // geometry type is auto-detected (e.g. LineString vs rectangle). + if (editing && trackId !== null) { + handler.trackEdit(trackId); + } else { + handler.trackSelect(trackId, editing, modifiers); + } + } else if (editing && trackId !== null) { + // Right-click on another detection while in creation mode: + // cancel creation and switch to editing the clicked detection + editAnnotationLayer.disable(); + handler.trackEdit(trackId); } }; //Sync of internal geoJS state with the application - editAnnotationLayer.bus.$on('editing-annotation-sync', (editing: boolean) => { - handler.trackSelect(selectedTrackIdRef.value, editing); + editAnnotationLayer.bus.$on('editing-annotation-sync', (editing: boolean, deselect?: boolean) => { + if (deselect) { + handler.trackSelect(null, false); + } else { + handler.trackSelect(selectedTrackIdRef.value, editing); + } + }); + // Handle right-click to confirm/lock annotation in Point mode (segmentation) + editAnnotationLayer.bus.$on('confirm-annotation', () => { + handler.confirmRecipe(); }); rectAnnotationLayer.bus.$on('annotation-clicked', Clicked); rectAnnotationLayer.bus.$on('annotation-right-clicked', Clicked); rectAnnotationLayer.bus.$on('annotation-ctrl-clicked', Clicked); polyAnnotationLayer.bus.$on('annotation-clicked', Clicked); polyAnnotationLayer.bus.$on('annotation-right-clicked', Clicked); + // Handle right-click polygon selection for multi-polygon support + polyAnnotationLayer.bus.$on('polygon-right-clicked', (_trackId: number, polygonKey: string) => { + // If in creation mode, cancel it first so we can select the polygon + if (editAnnotationLayer.getMode() === 'creation') { + handler.cancelCreation(); + } + // Set the polygon key for the right-clicked polygon + handler.selectFeatureHandle(-1, polygonKey); + // Force layer update to load the selected polygon + // This is especially important when already editing the same track + // since annotation-right-clicked won't be emitted in that case + window.setTimeout(() => { + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + }, 0); + }); polyAnnotationLayer.bus.$on('annotation-ctrl-clicked', Clicked); + lineLayer.bus.$on('annotation-clicked', Clicked); + lineLayer.bus.$on('annotation-right-clicked', Clicked); + // Handle polygon selection for multi-polygon support + polyAnnotationLayer.bus.$on('polygon-clicked', (_trackId: number, polygonKey: string) => { + // If in creation mode, don't interrupt - let the edit layer handle clicks for placing points + // This is important for hole drawing where left-clicks place hole vertices + if (editAnnotationLayer.getMode() === 'creation') { + return; + } + handler.selectFeatureHandle(-1, polygonKey); + // Force layer update to load the newly selected polygon + // Use nextTick to ensure the selectedKey ref has been updated + window.setTimeout(() => { + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + }, 0); + }); + // Handle right-click outside polygons to finalize/cancel creation + polyAnnotationLayer.bus.$on('polygon-right-clicked-outside', () => { + if (editAnnotationLayer.getMode() === 'creation') { + // Cancel creation and go back to editing the default polygon + handler.cancelCreation(); + handler.selectFeatureHandle(-1, ''); + window.setTimeout(() => { + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + }, 0); + } + }); editAnnotationLayer.bus.$on('update:geojson', ( mode: 'in-progress' | 'editing', geometryCompleteEvent: boolean, @@ -492,8 +615,59 @@ export default defineComponent({ }); editAnnotationLayer.bus.$on( 'update:selectedIndex', - (index: number, _type: EditAnnotationTypes, key = '') => handler.selectFeatureHandle(index, key), + (index: number, _type: EditAnnotationTypes, key?: string) => { + // When deselecting (index -1), don't change the key - it may have been + // set by polygon-right-clicked/polygon-clicked for multi-polygon selection + if (index >= 0 && key !== undefined) { + handler.selectFeatureHandle(index, key); + } else { + // Just update the handle index, preserve the current key + handler.selectFeatureHandle(index, selectedKeyRef.value); + } + }, ); + // Handle clicks outside the edit polygon to allow selecting other polygons + editAnnotationLayer.bus.$on('click-outside-edit', (geo: { x: number; y: number }) => { + // Check which polygon was clicked by iterating through formatted data + const point: [number, number] = [geo.x, geo.y]; + const polygonData = polyAnnotationLayer.formattedData; + + // Find the polygon that contains the click point + const clickedPolygon = polygonData.find((data) => { + const coords = data.polygon.coordinates[0] as [number, number][]; + // Ray casting algorithm + let inside = false; + for (let i = 0, j = coords.length - 1; i < coords.length; j = i, i += 1) { + const xi = coords[i][0]; + const yi = coords[i][1]; + const xj = coords[j][0]; + const yj = coords[j][1]; + const intersect = ((yi > point[1]) !== (yj > point[1])) + && (point[0] < ((xj - xi) * (point[1] - yi)) / (yj - yi) + xi); + if (intersect) inside = !inside; + } + return inside; + }); + + if (clickedPolygon) { + const polygonKey = clickedPolygon.polygonKey || ''; + // Select the clicked polygon + handler.selectFeatureHandle(-1, polygonKey); + // Force layer update to load the newly selected polygon + window.setTimeout(() => { + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + }, 0); + } + }); const annotationHoverTooltip = ( found: { styleType: [string, number]; diff --git a/client/src/layers/AnnotationLayers/LineLayer.ts b/client/src/layers/AnnotationLayers/LineLayer.ts index d4c2516fa..8f814003b 100644 --- a/client/src/layers/AnnotationLayers/LineLayer.ts +++ b/client/src/layers/AnnotationLayers/LineLayer.ts @@ -1,4 +1,5 @@ /* eslint-disable class-methods-use-this */ +import geo, { GeoEvent } from 'geojs'; import { cloneDeep } from 'lodash'; import BaseLayer, { LayerStyle, BaseLayerParams } from '../BaseLayer'; @@ -24,7 +25,23 @@ export default class LineLayer extends BaseLayer { const layer = this.annotator.geoViewerRef.value.createLayer('feature', { features: ['point', 'line'], }); - this.featureLayer = layer.createFeature('line'); + this.featureLayer = layer + .createFeature('line', { selectionAPI: true }) + .geoOn(geo.event.feature.mouseclick, (e: GeoEvent) => { + if (e.mouse.buttonsDown.left) { + if (!e.data.editing || (e.data.editing && !e.data.selected)) { + this.bus.$emit('annotation-clicked', e.data.trackId, false); + } + } else if (e.mouse.buttonsDown.right) { + if (!e.data.editing || (e.data.editing && !e.data.selected)) { + this.bus.$emit('annotation-right-clicked', e.data.trackId, true); + } + } + }); + this.featureLayer.geoOn( + geo.event.feature.mouseclick_order, + this.featureLayer.mouseOverOrderClosestBorder, + ); super.initialize(); } diff --git a/client/src/layers/AnnotationLayers/SegmentationPointsLayer.ts b/client/src/layers/AnnotationLayers/SegmentationPointsLayer.ts new file mode 100644 index 000000000..09db4c186 --- /dev/null +++ b/client/src/layers/AnnotationLayers/SegmentationPointsLayer.ts @@ -0,0 +1,76 @@ +import { MediaController } from '../../components/annotators/mediaControllerType'; + +interface SegmentationPointData { + x: number; + y: number; + label: number; // 1=foreground, 0=background +} + +/** + * Layer for displaying segmentation prompt points (green=foreground, red=background) + * This is a simple layer that doesn't follow the BaseLayer pattern since it's + * not tied to track data - it's UI feedback during the segmentation process. + */ +export default class SegmentationPointsLayer { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private featureLayer: any; + + private annotator: MediaController; + + private points: SegmentationPointData[] = []; + + constructor(annotator: MediaController) { + this.annotator = annotator; + this.initialize(); + } + + private initialize() { + const layer = this.annotator.geoViewerRef.value.createLayer('feature', { + features: ['point'], + }); + this.featureLayer = layer.createFeature('point'); + this.featureLayer.style({ + radius: 8, + strokeWidth: 2, + strokeColor: (data: SegmentationPointData) => (data.label === 1 ? '#00FF00' : '#FF0000'), + fillColor: (data: SegmentationPointData) => (data.label === 1 ? '#00FF00' : '#FF0000'), + fillOpacity: 0.6, + strokeOpacity: 1, + }); + this.featureLayer.position((data: SegmentationPointData) => ({ + x: data.x, + y: data.y, + })); + } + + /** + * Update the displayed points + * @param points - Array of [x, y] coordinates + * @param labels - Array of labels (1=foreground, 0=background) + */ + updatePoints(points: [number, number][], labels: number[]) { + this.points = points.map((p, i) => ({ + x: p[0], + y: p[1], + label: labels[i] ?? 1, + })); + this.redraw(); + } + + /** + * Clear all displayed points + */ + clear() { + this.points = []; + this.redraw(); + } + + private redraw() { + this.featureLayer.data(this.points).draw(); + } + + disable() { + this.points = []; + this.featureLayer.data([]).draw(); + } +} diff --git a/client/src/layers/EditAnnotationLayer.ts b/client/src/layers/EditAnnotationLayer.ts index 4c1204a1a..17d7f9ece 100644 --- a/client/src/layers/EditAnnotationLayer.ts +++ b/client/src/layers/EditAnnotationLayer.ts @@ -85,6 +85,17 @@ export default class EditAnnotationLayer extends BaseLayer { unrotatedGeoJSONCoords: GeoJSON.Position[] | null; + /* Track if the last click was a right-click or shift-click for Point mode */ + lastClickWasBackground: boolean; + + /* Track shift key state from native DOM events (more reliable than GeoJS events) */ + lastShiftKeyState: boolean; + + /* Bound event handlers for cleanup */ + private boundTrackShiftKey: ((e: MouseEvent) => void) | null = null; + + private boundHandleContextMenu: ((e: MouseEvent) => void) | null = null; + constructor(params: BaseLayerParams & EditAnnotationLayerParams) { super(params); this.skipNextExternalUpdate = false; @@ -97,11 +108,83 @@ export default class EditAnnotationLayer extends BaseLayer { this.disableModeSync = false; this.leftButtonCheckTimeout = -1; this.unrotatedGeoJSONCoords = null; + this.lastClickWasBackground = false; + this.lastShiftKeyState = false; + + // Bind event handlers once (listeners are added/removed dynamically based on type) + this.boundTrackShiftKey = this.trackShiftKey.bind(this); + this.boundHandleContextMenu = this.handleContextMenu.bind(this); + + // Add listeners if starting in Point mode + if (this.type === 'Point') { + this.addPointModeListeners(); + } //Only initialize once, prevents recreating Layer each edit this.initialize(); } + /** + * Add event listeners needed for Point mode (segmentation). + */ + private addPointModeListeners() { + if (this.boundTrackShiftKey) { + document.addEventListener('mousedown', this.boundTrackShiftKey, true); + } + if (this.boundHandleContextMenu) { + document.addEventListener('contextmenu', this.boundHandleContextMenu, true); + } + } + + /** + * Remove event listeners used for Point mode. + */ + private removePointModeListeners() { + if (this.boundTrackShiftKey) { + document.removeEventListener('mousedown', this.boundTrackShiftKey, true); + } + if (this.boundHandleContextMenu) { + document.removeEventListener('contextmenu', this.boundHandleContextMenu, true); + } + } + + /** + * Track shift key state from native DOM mousedown events. + * This is more reliable than GeoJS events for detecting shift+click. + */ + trackShiftKey(e: MouseEvent) { + this.lastShiftKeyState = e.shiftKey; + // Also track middle-click (button 1) from native events for background points + if (e.button === 1 && this.type === 'Point' && this.getMode() === 'creation') { + this.lastClickWasBackground = true; + } + } + + /** + * Handle right-click context menu in Point mode. + * In segmentation mode, right-click confirms/locks the annotation. + * Prevents browser context menu. + */ + handleContextMenu(e: MouseEvent) { + if (this.type === 'Point' && this.getMode() === 'creation') { + // Prevent the browser context menu + e.preventDefault(); + e.stopPropagation(); + + // Emit confirm event to lock the annotation (like other edit modes) + this.bus.$emit('confirm-annotation'); + } + } + + /** + * Clean up event listeners when the layer is destroyed. + */ + destroy() { + this.removePointModeListeners(); + this.boundTrackShiftKey = null; + this.boundHandleContextMenu = null; + } + /** * Initialization of the layer should only be done once for edit layers * Handlers for edit_action and state which will emit data when necessary @@ -130,6 +213,13 @@ export default class EditAnnotationLayer extends BaseLayer { (e: GeoEvent) => this.hoverEditHandle(e), ); this.featureLayer.geoOn(geo.event.mouseclick, (e: GeoEvent) => { + // Right-click in creation mode (non-Point): cancel and fully deselect. + // Point mode has its own right-click handler (handleContextMenu). + if (e.buttonsDown.right && this.getMode() === 'creation' && this.type !== 'Point') { + this.shapeInProgress = null; + this.bus.$emit('editing-annotation-sync', false, true); + return; + } //Used to sync clicks that kick out of editing mode with application //This prevents that pseudo Edit state when left clicking on a object in edit mode if (!this.disableModeSync && (e.buttonsDown.left) @@ -184,10 +274,56 @@ export default class EditAnnotationLayer extends BaseLayer { * shape that GeoJS is keeps internally. Emit the shape as update:in-progress-geojson */ setShapeInProgress(e: GeoEvent) { - // Allow middle click movement when placing points - if (e.mouse.buttons.middle && !e.propogated) { + // Allow middle click movement when placing points (except in Point mode where it creates background points) + if (e.mouse.buttons.middle && !e.propogated && this.type !== 'Point') { return; } + // Right-click should never add vertices - cancel/confirm is handled by + // mouseclick (line/polygon cancel) and contextmenu (Point confirm) + if (e.mouse.buttons.right) { + return; + } + + // Track if this is a background point (shift+click or middle-click) for Point mode + // Check both GeoJS event modifiers and our native DOM event tracking for reliability + // Preserve the value if it was already set to true by trackShiftKey (native event) + if (this.type === 'Point' && this.getMode() === 'creation') { + this.lastClickWasBackground = this.lastClickWasBackground + || e.mouse.buttons.middle + || e.mouse.modifiers.shift + || this.lastShiftKeyState; + } + + // Handle middle-click in Point mode - GeoJS doesn't create points on middle-click, + // so we need to manually create the point and emit the event + if (this.type === 'Point' && this.getMode() === 'creation' && e.mouse.buttons.middle) { + const pointGeojson: GeoJSON.Feature = { + type: 'Feature', + geometry: { + type: 'Point', + coordinates: [Math.round(e.mouse.geo.x), Math.round(e.mouse.geo.y)], + }, + properties: { + background: true, + }, + }; + + // Emit the point creation event directly + this.bus.$emit( + 'update:geojson', + 'editing', + true, // geometryCompleteEvent - point is complete + pointGeojson, + this.type, + this.selectedKey, + this.skipNextFunc(), + ); + + // Reset background flag for next point + this.lastClickWasBackground = false; + return; + } + if (this.getMode() === 'creation' && ['LineString', 'Polygon'].includes(this.type)) { if (this.shapeInProgress === null) { // Initialize a new in-progress shape @@ -205,6 +341,25 @@ export default class EditAnnotationLayer extends BaseLayer { } else { const coords = this.shapeInProgress?.coordinates as GeoJSON.Position[]; coords.push(newPoint); + // Auto-complete LineString after 2 points (simple line with 2 endpoints) + if (coords.length >= 2) { + const feature: GeoJSON.Feature = { + type: 'Feature', + geometry: this.shapeInProgress!, + properties: {}, + }; + this.shapeInProgress = null; + this.disableModeSync = true; + this.bus.$emit( + 'update:geojson', + 'editing', + true, // geometryCompleteEvent - line is complete + feature, + this.type, + this.selectedKey, + ); + return; + } } this.bus.$emit( 'update:geojson', @@ -281,9 +436,19 @@ export default class EditAnnotationLayer extends BaseLayer { /** * Set the current Editing type for switching between editing polygons or rects. - * */ + * Also manages event listeners that are only needed for Point mode (segmentation). + */ setType(type: EditAnnotationTypes) { + const wasPoint = this.type === 'Point'; + const isPoint = type === 'Point'; this.type = type; + + // Add or remove Point mode listeners based on type change + if (!wasPoint && isPoint) { + this.addPointModeListeners(); + } else if (wasPoint && !isPoint) { + this.removePointModeListeners(); + } } setKey(key: string) { @@ -479,6 +644,15 @@ export default class EditAnnotationLayer extends BaseLayer { const geoJSONData = [e.annotation.geojson()]; this.unrotatedGeoJSONCoords = geoJSONData[0].geometry.coordinates[0] as GeoJSON.Position[]; + + // For Point mode, add background property if it was a right-click or shift-click + if (this.type === 'Point' && this.lastClickWasBackground) { + geoJSONData[0].properties = { + ...geoJSONData[0].properties, + background: true, + }; + this.lastClickWasBackground = false; // Reset for next point + } this.formattedData = geoJSONData; // The new annotation is in a state without styling, so apply local stypes this.applyStylesToAnnotations(); diff --git a/client/src/provides.ts b/client/src/provides.ts index 078b8ae5f..0197bd7ba 100644 --- a/client/src/provides.ts +++ b/client/src/provides.ts @@ -8,6 +8,7 @@ import type { EditAnnotationTypes } from './layers/EditAnnotationLayer'; import type { AnnotationId, StringKeyObject } from './BaseAnnotation'; import type { VisibleAnnotationTypes } from './layers'; import type { RectBounds } from './utils'; +import type { TrackSupportedFeature } from './track'; import type { Attribute, AttributeFilter, @@ -54,6 +55,9 @@ type EditingModeType = Readonly>; const MultiSelectSymbol = Symbol('multiSelect'); type MultiSelectType = Readonly>; +const SegmentationPointsSymbol = Symbol('segmentationPoints'); +type SegmentationPointsType = Readonly>; + const PendingSaveCountSymbol = Symbol('pendingSaveCount'); type pendingSaveCountType = Readonly>; @@ -119,6 +123,8 @@ export interface Handler { seekFrame(frame: number): void; /* Toggle editing mode for track */ trackEdit(AnnotationId: AnnotationId): void; + /* Confirm/lock the current annotation for active recipes */ + confirmRecipe(): void; /* toggle selection mode for track */ trackSelect(AnnotationId: AnnotationId | null, edit: boolean, modifiers?: { ctrl: boolean }): void; /* select next track in the list */ @@ -134,6 +140,13 @@ export interface Handler { bounds: RectBounds, rotation?: number, ): void; + /* Set a feature on the selected track with proper interpolation handling */ + setTrackFeature( + frameNum: number, + bounds: RectBounds, + geometry: GeoJSON.Feature[], + runAfterLogic?: boolean, + ): void; /* update geojson for track */ updateGeoJSON( eventType: 'in-progress' | 'editing', @@ -200,11 +213,13 @@ function dummyHandler(handle: (name: string, args: unknown[]) => void): Handler trackSeek(...args) { handle('trackSeek', args); }, seekFrame(...args) { handle('seekFrame', args); }, trackEdit(...args) { handle('trackEdit', args); }, + confirmRecipe(...args) { handle('confirmRecipe', args); }, trackSelect(...args) { handle('trackSelect', args); }, trackSelectNext(...args) { handle('trackSelectNext', args); }, trackSplit(...args) { handle('trackSplit', args); }, trackAdd(...args) { handle('trackAdd', args); return 0; }, updateRectBounds(...args) { handle('updateRectBounds', args); }, + setTrackFeature(...args) { handle('setTrackFeature', args); }, updateGeoJSON(...args) { handle('updateGeoJSON', args); }, removeTrack(...args) { handle('removeTrack', args); }, removeGroup(...args) { handle('removeGroup', args); }, @@ -252,6 +267,7 @@ export interface State { annotationSet: AnnotationSetType; annotationSets: AnnotationSetsType; comparisonSets: ComparisonSetsType; + segmentationPoints: SegmentationPointsType; selectedCamera: SelectedCameraType; selectedKey: SelectedKeyType; selectedTrackId: SelectedTrackIdType; @@ -318,6 +334,7 @@ function dummyState(): State { comparisonSets: ref([]), groupFilters: groupFilterControls, groupStyleManager: new StyleManager({ markChangesPending }), + segmentationPoints: ref({ points: [], labels: [], frameNum: -1 }), selectedCamera: ref('singleCam'), selectedKey: ref(''), selectedTrackId: ref(null), @@ -367,6 +384,7 @@ function provideAnnotator(state: State, handler: Handler, attributesFilters: Att provide(AnnotationSetSymbol, state.annotationSet); provide(AnnotationSetsSymbol, state.annotationSets); provide(ComparisonSetsSymbol, state.comparisonSets); + provide(SegmentationPointsSymbol, state.segmentationPoints); provide(TrackFilterControlsSymbol, state.trackFilters); provide(TrackStyleManagerSymbol, state.trackStyleManager); provide(SelectedCameraSymbol, state.selectedCamera); @@ -499,6 +517,10 @@ function useImageEnhancements() { return use(ImageEnhancementsSymbol); } +function useSegmentationPoints() { + return use(SegmentationPointsSymbol); +} + export { dummyHandler, dummyState, @@ -531,4 +553,5 @@ export { useReadOnlyMode, useImageEnhancements, useAttributesFilters, + useSegmentationPoints, }; diff --git a/client/src/recipe.ts b/client/src/recipe.ts index efd5635b7..bd259fa08 100644 --- a/client/src/recipe.ts +++ b/client/src/recipe.ts @@ -28,6 +28,8 @@ interface Recipe { icon: Ref; active: Ref; toggleable: Ref; + /** Whether the recipe is currently loading (e.g., initializing models) */ + loading?: Ref; bus: Vue; update: ( mode: 'in-progress' | 'editing', @@ -52,6 +54,8 @@ interface Recipe { activate: () => unknown; mousetrap: () => Mousetrap[]; deactivate: () => void; + /** Optional method to confirm/lock the current annotation (e.g., for segmentation) */ + confirm?: () => void; } export default Recipe; From b55bd2020f1211c1cc5ec9c2aa5b2d459be9f101 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Wed, 29 Apr 2026 05:50:12 -0400 Subject: [PATCH 2/2] Multi-polygon support with holes A track-frame's polygon now expands to a list of polygons each with their own keys, and each polygon supports holes. - Server CSV (de)serializer: emit polygon-key column per polygon, support holes in the geoJSON FeatureCollection; auto_key path to append a new polygon to an existing track frame. - Client recipes / useModeManager: handleAddHole / handleAddPolygon / handleCancelCreation; PolygonLayer emits polygon-clicked. - Hole drawing reuses the polygon edit pipeline (left-click places a hole vertex without exiting creation mode). - Test fixtures cover multi-polygon and polygons-with-holes round-trip. --- .../dive-common/components/DeleteControls.vue | 122 +++- client/dive-common/components/Viewer.vue | 645 +----------------- client/dive-common/recipes/polygonbase.ts | 201 +++++- client/dive-common/use/useModeManager.ts | 79 ++- .../desktop/backend/native/segmentation.ts | 4 +- .../platform/desktop/backend/native/stereo.ts | 4 +- .../desktop/backend/serializers/viame.ts | 88 ++- .../layers/AnnotationLayers/PolygonLayer.ts | 132 +++- client/src/layers/EditAnnotationLayer.ts | 179 ++++- client/src/provides.ts | 9 + client/src/track.ts | 123 +++- server/dive_utils/models.py | 3 +- server/dive_utils/serializers/viame.py | 105 ++- server/tests/test_serialize_viame_csv.py | 123 ++++ testutils/viame.spec.json | 197 ++++++ 15 files changed, 1278 insertions(+), 736 deletions(-) diff --git a/client/dive-common/components/DeleteControls.vue b/client/dive-common/components/DeleteControls.vue index f6022ac8a..3d10226e5 100644 --- a/client/dive-common/components/DeleteControls.vue +++ b/client/dive-common/components/DeleteControls.vue @@ -29,6 +29,15 @@ export default Vue.extend({ } return false; }, + isPolygonMode(): boolean { + return this.editingMode === 'Polygon'; + }, + editModeIcon(): string { + if (this.editingMode === 'Polygon') return 'mdi-vector-polygon'; + if (this.editingMode === 'LineString') return 'mdi-vector-line'; + if (this.editingMode === 'rectangle') return 'mdi-vector-square'; + return 'mdi-shape'; + }, }, methods: { @@ -42,33 +51,104 @@ export default Vue.extend({ this.$emit('delete-annotation'); } }, + addHole() { + this.$emit('add-hole'); + }, + addPolygon() { + this.$emit('add-polygon'); + }, }, }); diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 44bbd20ba..8d562df15 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -1,6 +1,6 @@