Skip to content

Commit 245aff8

Browse files
BryonLewisBryonLewis
authored
GeoJS Texture Testing (#289)
* GeoJS Texture Testing * support for both RLE and not RLE for comparisons * editing of RLE masks * sam2 rle_mask live updating * adding support for luminance and luminance alpha conversion from rlemasks * RLE worker mask creation * testing multi-worker concurrency * worker updates * new geoJS release version with webGL texture support * resolving some conflicts and linting * saving masks * fix: re-render RLE mask after done editing * fix: swap svg filters by trackId to filters by type for performance * fix: default coloring for new tracks based on type * fix: client deleting of existing masks * update the loading and displaying of textures * add mask caching seconds * add mask caching seconds * mask loading percentage bar * adding cacheMaxSeconds and tooManyMasks * pause on loading frames * cloning mask support * linting/formatting * add documentation about Converting binary mask to COCO RLE
1 parent b02990f commit 245aff8

File tree

29 files changed

+2883
-1356
lines changed

29 files changed

+2883
-1356
lines changed

‎client/dive-common/components/EditorMenu.vue‎

Lines changed: 93 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ export default defineComponent({
194194
type: 'Mask',
195195
active: isVisible('Mask'),
196196
icon: 'mdi-draw',
197-
tooltip: 'Segementation Masks',
197+
tooltip: 'Segmentation Masks',
198198
click: () => toggleVisible('Mask'),
199199
},
200200
{
@@ -297,12 +297,66 @@ export default defineComponent({
297297
const updateMaskOpacity = (event: Event) => {
298298
const target = event.target as HTMLInputElement;
299299
const value = Number.parseFloat(target.value);
300-
editorFunctions.setEditorOptions({ opactiy: value });
300+
editorFunctions.setEditorOptions({ opacity: value });
301+
};
302+
303+
const updateMaskCacheSeconds = (event: Event) => {
304+
const target = event.target as HTMLInputElement;
305+
const value = Number.parseFloat(target.value);
306+
editorFunctions.setEditorOptions({ maskCacheSeconds: value });
301307
};
302308
303309
const maskOpacity = computed(() => editorOptions.opacity.value);
310+
const maskCacheSeconds = computed(() => editorOptions.maskCacheSeconds.value);
311+
const maxCacheSeconds = computed(() => editorOptions.maskMaxCacheSeconds.value);
312+
const maskLoadingPercent = computed(() => editorOptions.maskLoadingPercent?.value || 0);
313+
const tooManyMasks = computed(() => editorOptions.tooManyMasks.value);
304314
const loadingFrame = computed(() => editorOptions.loadingFrame.value);
315+
const pauseOnLoading = computed({
316+
get() {
317+
return editorOptions.pauseOnLoading.value;
318+
},
319+
set(val: boolean) {
320+
editorFunctions.setEditorOptions({ pauseOnLoading: val });
321+
},
322+
});
305323
324+
const maskIcon = computed(() => {
325+
if (tooManyMasks.value) {
326+
return {
327+
icon: 'mdi-alert-circle-outline',
328+
color: 'error',
329+
value: true,
330+
content: undefined,
331+
offsetX: 25,
332+
};
333+
}
334+
if (loadingFrame.value) {
335+
return {
336+
icon: 'mdi-spin mdi-sync',
337+
color: 'primary',
338+
value: true,
339+
content: undefined,
340+
offsetX: 25,
341+
};
342+
}
343+
if (maskLoadingPercent.value && maskLoadingPercent.value < 100) {
344+
return {
345+
icon: undefined,
346+
color: 'warning',
347+
value: maskLoadingPercent.value && maskLoadingPercent.value < 100,
348+
content: `${Math.round(maskLoadingPercent.value)}%`,
349+
offsetX: 35,
350+
};
351+
}
352+
return {
353+
icon: undefined,
354+
color: undefined,
355+
value: false,
356+
content: undefined,
357+
offsetX: 25,
358+
};
359+
});
306360
return {
307361
toolTipForce,
308362
editButtons,
@@ -316,6 +370,13 @@ export default defineComponent({
316370
modeToolTips,
317371
updateMaskOpacity,
318372
maskOpacity,
373+
maskCacheSeconds,
374+
maxCacheSeconds,
375+
maskLoadingPercent,
376+
pauseOnLoading,
377+
tooManyMasks,
378+
maskIcon,
379+
updateMaskCacheSeconds,
319380
loadingFrame,
320381
buttonOptions,
321382
menuOptions,
@@ -412,11 +473,11 @@ export default defineComponent({
412473
v-if="button.id === 'Mask'"
413474
overlap
414475
bottom
415-
:color="loadingFrame ? 'primary' : undefined"
416-
:content="!loadingFrame ? '' : undefined"
417-
:icon="loadingFrame ? 'mdi-spin mdi-sync' : undefined"
418-
:value="loadingFrame ? true : false"
419-
offset-x="25"
476+
:color="maskIcon.color"
477+
:content="maskIcon.content"
478+
:icon="maskIcon.icon"
479+
:value="maskIcon.value"
480+
:offset-x="maskIcon.offsetX"
420481
offset-y="18"
421482
>
422483

@@ -444,6 +505,13 @@ export default defineComponent({
444505
outlined
445506
>
446507
<v-card-text>Segementation Masks</v-card-text>
508+
<v-progress-linear
509+
v-if="maskLoadingPercent && maskLoadingPercent < 100"
510+
:value="maskLoadingPercent"
511+
height="10"
512+
class="mb-4"
513+
color="primary"
514+
/>
447515
<label for="frames-before">Opacity: {{ maskOpacity }}</label>
448516
<input
449517
id="frames-before"
@@ -456,6 +524,24 @@ export default defineComponent({
456524
:value="maskOpacity"
457525
@input="updateMaskOpacity($event)"
458526
>
527+
<label for="cache-seconds">Cache Seconds: {{ maskCacheSeconds }}</label>
528+
<input
529+
id="cache-seconds"
530+
type="range"
531+
name="cache-seconds"
532+
class="tail-slider-width"
533+
label
534+
min="0.1"
535+
:max="maxCacheSeconds"
536+
step="0.1"
537+
:value="maskCacheSeconds"
538+
@input="updateMaskCacheSeconds($event)"
539+
>
540+
<v-checkbox
541+
v-model="pauseOnLoading"
542+
label="Pause on Loading Masks"
543+
hide-details
544+
/>
459545
</v-card>
460546
</v-menu>
461547
</v-badge>

‎client/dive-common/components/MaskEditor.vue‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ export default defineComponent({
2323
const existingMask = ref(false);
2424
2525
const handleMouseScroll = (event: WheelEvent) => {
26-
if (!event.ctrlKey || event.metaKey) {
26+
if (!event.shiftKey || event.metaKey) {
2727
return;
2828
}
2929
event.preventDefault();

‎client/dive-common/components/MaskTracking.vue‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import type { GirderJob } from '@girder/components/src';
1414
import girderRest from 'platform/web-girder/plugins/girder';
1515
import { all } from '@girder/components/src/components/Job/status';
1616
import Track from 'vue-media-annotator/track';
17-
import { MaskItem } from 'vue-media-annotator/use/useMasks';
17+
import { MaskSAM2UpdateItem } from 'vue-media-annotator/use/useMasks';
1818
1919
const JobStatus = all();
2020
const NonRunningStates = [
@@ -28,7 +28,7 @@ export interface MaskUpdate {
2828
trackId: number;
2929
currentFrame: number;
3030
trackFeatures: Track['features'];
31-
masks: MaskItem[];
31+
masks: MaskSAM2UpdateItem[];
3232
}
3333
3434
export default defineComponent({
@@ -108,6 +108,7 @@ export default defineComponent({
108108
handler.seekFrame(maskUpdate.currentFrame);
109109
}
110110
};
111+
111112
girderRest.$on('message:progress', jobTracker);
112113
girderRest.$on('message:mask_update', ({ data: maskUpdate }: { data: MaskUpdate }) => maskUpdateProcessor(maskUpdate));
113114
girderRest.$on('message:job_status', ({ data: job }: { data: GirderJob }) => {

‎client/dive-common/components/Viewer.vue‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ export default defineComponent({
182182
removeCamera: removeSaveCamera,
183183
configurationId,
184184
setConfigurationId,
185-
} = useSave(datasetId, readonlyState);
185+
} = useSave(datasetId, readonlyState, undefined);
186186
187187
const {
188188
imageEnhancements,
@@ -295,6 +295,7 @@ export default defineComponent({
295295
selectedCamera,
296296
editingTrack,
297297
diveMetadataRootId,
298+
setDeleteLocalMasks,
298299
} = useModeManager({
299300
recipes,
300301
trackFilterControls: trackFilters,
@@ -493,6 +494,14 @@ export default defineComponent({
493494
async function save() {
494495
// If editing the track, disable editing mode before save
495496
saveInProgress.value = true;
497+
if (editingMode.value === 'Mask') {
498+
await prompt({
499+
title: 'In Mask Editing Mode',
500+
text: 'Please exit mask editing mode and save or Cancel before saving your changes.',
501+
});
502+
saveInProgress.value = false;
503+
return;
504+
}
496505
if (editingTrack.value) {
497506
handler.trackSelect(selectedTrackId.value, false);
498507
}
@@ -727,7 +736,9 @@ export default defineComponent({
727736
setFrameRate(meta.fps);
728737
initializeMaskData({ masks: mediaMasks.value });
729738
}
730-
getFolderRLEMasks(cameraId);
739+
if (editorOptions.useRLE.value) {
740+
getFolderRLEMasks(cameraId);
741+
}
731742
// eslint-disable-next-line no-await-in-loop
732743
const { tracks, groups } = await loadDetections(cameraId, props.revision);
733744
progress.total = tracks.length + groups.length;
@@ -925,15 +936,20 @@ export default defineComponent({
925936
setConfigurationId,
926937
};
927938
939+
const annotationModeVisible = computed(() => visibleModes.value.includes('Mask'));
940+
const visibleMaskIds = computed(() => trackFilters.enabledAnnotations.value.filter((t) => t.context.hasMasks).map((t) => t.annotation.id));
928941
const {
929942
initializeMaskData,
930943
setFrameRate,
931944
getMask,
945+
getRLEMask,
946+
getRLELuminanceMask,
932947
getFolderRLEMasks,
933948
editorFunctions,
934949
editorOptions,
935-
} = useMasks(time.frame, time.flick, datasetId, globalHandler);
936-
950+
deleteLocalMasks,
951+
} = useMasks(time.frame, time.flick, datasetId, globalHandler, annotationModeVisible, visibleMaskIds, aggregateController);
952+
setDeleteLocalMasks(deleteLocalMasks);
937953
const useAttributeFilters = {
938954
attributeFilters,
939955
addAttributeFilter,
@@ -986,8 +1002,10 @@ export default defineComponent({
9861002
visibleModes,
9871003
readOnlyMode: readonlyState,
9881004
imageEnhancements,
1005+
masks: {
1006+
getMask, getRLEMask, getRLELuminanceMask, editorFunctions, editorOptions,
1007+
},
9891008
diveMetadataRootId,
990-
masks: { getMask, editorFunctions, editorOptions },
9911009
},
9921010
globalHandler,
9931011
useAttributeFilters,

‎client/dive-common/use/useModeManager.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ export default function useModeManager({
7575
let creating = false;
7676
const { prompt, inputValue } = usePrompt();
7777
const annotationModes = reactive({
78-
visible: ['rectangle', 'Polygon', 'LineString', 'text', 'Time', 'Mask'] as VisibleAnnotationTypes[],
78+
visible: ['rectangle', 'Polygon', 'LineString', 'text', 'Time'] as VisibleAnnotationTypes[],
7979
editing: 'rectangle' as EditAnnotationTypes,
8080
});
8181
const trackSettings = toRef(clientSettings, 'trackSettings');
@@ -168,6 +168,13 @@ export default function useModeManager({
168168
}
169169
return 'disabled';
170170
});
171+
172+
let deleteLocalMasks: (((trackId: AnnotationId, frameList: number[]) => void) | null) = null;
173+
174+
function setDeleteLocalMasks(func: (trackId: AnnotationId, frameList: number[]) => void) {
175+
deleteLocalMasks = func;
176+
}
177+
171178
// which types are currently visible, always including the editingType
172179
const visibleModes = computed(() => (
173180
uniq(annotationModes.visible.concat(editingMode.value || []))
@@ -629,6 +636,10 @@ export default function useModeManager({
629636
}
630637
}
631638
trackIds.forEach((trackId) => {
639+
const track = cameraStore.getPossibleTrack(trackId, selectedCamera.value);
640+
if (track?.hasAnyMask() && deleteLocalMasks) {
641+
deleteLocalMasks(trackId, track.getMaskFrameList());
642+
}
632643
cameraStore.remove(trackId, cameraName);
633644
});
634645
handleUnstageFromMerge(trackIds);
@@ -977,6 +988,7 @@ export default function useModeManager({
977988
selectedCamera,
978989
diveMetadataRootId,
979990
selectNextTrack,
991+
setDeleteLocalMasks,
980992
handler: {
981993
commitMerge: handleCommitMerge,
982994
groupAdd: handleAddGroup,

‎client/dive-common/use/useSave.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import {
1010
import { useApi, DatasetMetaMutable, SaveStylingArgs } from 'dive-common/apispec';
1111
import { AnnotationId } from 'vue-media-annotator/BaseAnnotation';
1212
import Group from 'vue-media-annotator/Group';
13+
import { usePrompt } from 'dive-common/vue-utilities/prompt-service';
14+
import { EditAnnotationTypes } from 'vue-media-annotator/layers';
1315

1416
interface ChangeMap {
1517
upsert: Map<TrackId, Track>;
@@ -73,6 +75,8 @@ export default function useSave(
7375
saveDetections, saveMetadata, saveAttributes, saveTimelines, saveFilters, saveSwimlanes, saveStyling,
7476
} = useApi();
7577

78+
const { prompt } = usePrompt();
79+
7680
async function save(
7781
datasetMeta?: DatasetMetaMutable,
7882
) {

‎client/package.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@
1818
"lint:templates": "vtc --workspace . --srcDir src/",
1919
"test": "vue-cli-service test:unit src/ dive-common/ platform/"
2020
},
21-
"resolutions": {},
21+
"resolutions": {
22+
"@types/babel__traverse": "7.18.2"
23+
},
2224
"files": [
2325
"/dist"
2426
],
@@ -34,7 +36,7 @@
3436
"core-js": "^3.22.2",
3537
"csv-stringify": "^5.6.0",
3638
"d3": "^5.12.0",
37-
"geojs": "~1.6.2",
39+
"geojs": "^1.15.1",
3840
"glob-to-regexp": "^0.4.1",
3941
"js-cookie": "^3.0.5",
4042
"lodash": "^4.17.19",

‎client/platform/web-girder/api/annotation.service.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ async function updateRLEMasks(folderId: string, trackFrameList?: [number, number
116116
return uploadMetadata;
117117
}
118118

119-
async function getRLEMask(folderId: string) {
119+
async function getRLEMaskData(folderId: string) {
120120
const response = await girderRest.get<RLETrackFrameData>('dive_annotation/rle_mask', {
121121
params: {
122122
folderId,
@@ -153,7 +153,7 @@ export {
153153
saveDetections,
154154
uploadMask,
155155
updateRLEMasks,
156-
getRLEMask,
156+
getRLEMaskData,
157157
deleteMask,
158158

159159
};

‎client/platform/web-girder/views/Home.vue‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ export default defineComponent({
159159
/>
160160
<CreateDIVEMetadata
161161
v-bind="{ buttonOptions, menuOptions }"
162-
:dataset-id="location._id || null"
162+
:dataset-id="location?._id || null"
163163
/>
164164
165165
<v-btn

‎client/src/@types/geojs.d.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,14 @@ declare module 'geojs' {
88
const geojs: any;
99
export default geojs;
1010
}
11+
12+
declare module 'geojs/src/vgl' {
13+
export class texture {
14+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
15+
m_textureHandle: any;
16+
}
17+
18+
const vgl: {
19+
texture: typeof texture;
20+
};
21+
}

0 commit comments

Comments
 (0)