From 8ab4038f02ca0d2c0931c794887716582e11f126 Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Fri, 6 Jan 2023 16:04:31 -0500 Subject: [PATCH 01/46] added appspot and state share config bumped appengine runtime to python312 since "Runtime python27 is end of support and no longer allowed" removed theadsafe - "This field is not supported with runtime [python312] and can safely be removed." removed skip_files (no longer supported) "api_version" field is not allowed in runtime python312. forgot to add .gcloudignore (replacement of skip_files removed "on pull request" --- .github/workflows/build.yml | 369 ++++------------------------ .github/workflows/build_preview.yml | 29 --- appengine/frontend/.gcloudignore | 5 + appengine/frontend/app.yaml | 19 ++ config/state_servers.json | 6 + 5 files changed, 83 insertions(+), 345 deletions(-) delete mode 100644 .github/workflows/build_preview.yml create mode 100644 appengine/frontend/.gcloudignore create mode 100644 appengine/frontend/app.yaml create mode 100644 config/state_servers.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 18b060b1cc..79a0bb1a2a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,26 +3,20 @@ name: Build on: push: branches: - - master + - spelunker tags: - v** - pull_request: - workflow_dispatch: - inputs: - debug_enabled: - type: boolean - description: "Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)" - required: false - default: false jobs: - client: + build-and-deploy: + permissions: + contents: "read" + id-token: "write" + deployments: "write" strategy: matrix: os: - "ubuntu-latest" - - "windows-latest" - - "macos-latest" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -35,313 +29,56 @@ jobs: cache: "npm" cache-dependency-path: | package-lock.json - # uv required for javascript tests - - uses: astral-sh/setup-uv@v5 - with: - enable-cache: false - # go needed for fake_gcs_server used by the javascript tests - - name: Setup go - uses: actions/setup-go@v5 - with: - go-version: "stable" - - run: npm ci - - run: npm run format:fix - - name: Check for dirty working directory - run: git diff --exit-code - - run: npm run lint:check + examples/**/package-lock.json + - run: npm install - name: Typecheck with TypeScript run: npm run typecheck - - name: Build client bundles - run: | - build_info="{'tag':'$(git describe --always --tags)', 'url':'https://github.com/google/neuroglancer/commit/$(git rev-parse HEAD)', 'timestamp':'$(date)'}" - npm run build -- --no-typecheck --no-lint --define NEUROGLANCER_BUILD_INFO="${build_info}" - echo $build_info > ./dist/client/version.json + - name: Get branch name (merge) + if: github.event_name != 'pull_request' shell: bash - - name: Build Python client bundles - run: npm run build-python -- --no-typecheck --no-lint - - run: npm run build-package - - run: npm publish --dry-run - working-directory: dist/package - - name: Run JavaScript tests (including WebGL) - run: npm test - if: ${{ runner.os != 'macOS' }} - - name: Run JavaScript tests (excluding WebGL) - run: npm test -- --project node - if: ${{ runner.os == 'macOS' }} - - name: Run JavaScript benchmarks - run: npm run benchmark - - name: Upload NPM package as artifact - uses: actions/upload-artifact@v4 - with: - name: npm-package - path: dist/package - if: ${{ runner.os == 'Linux' }} - - name: Upload client as artifact - uses: actions/upload-artifact@v4 - with: - name: client - path: dist/client - if: ${{ runner.os == 'Linux' }} - example-project-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install pnpm - uses: pnpm/action-setup@v4 - with: - version: 10 - - uses: actions/setup-node@v4 - with: - node-version: 22.x - cache: "pnpm" - cache-dependency-path: | - examples/**/pnpm-lock.yaml - - uses: actions/setup-node@v4 - with: - node-version: 22.x - cache: "npm" - cache-dependency-path: | - package-lock.json - - run: npm ci - - run: npm run example-project-test -- --reporter=html - - name: Upload report and built clients - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: example-project-test-results - path: | - playwright-report/ - examples/*/*/dist/ - - # Builds Python package and runs Python tests - # - # On ubuntu-latest, this also runs browser-based tests. On Mac OS and - # Windows, this only runs tests that do not require a browser, since a working - # headless WebGL2 implementation is not available on Github actions. - python-tests: - strategy: - matrix: - python-version: - - "3.9" - - "3.12" - os: - - "ubuntu-latest" - - "windows-latest" - - "macos-latest" - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - # Need full history to determine version number. - fetch-depth: 0 - - uses: actions/setup-node@v4 - with: - node-version: 22.x - - uses: astral-sh/setup-uv@v5 - with: - enable-cache: false - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - uses: ./.github/actions/setup-firefox - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - - run: uvx nox -s lint format mypy - - name: Check for dirty working directory - run: git diff --exit-code - - name: Run python tests (skip browser tests) - run: uvx nox -s test -- --skip-browser-tests - if: ${{ runner.os != 'Linux' }} - - name: Run python tests (include browser tests) - run: uvx nox -s test_xvfb -- --browser firefox - if: ${{ runner.os == 'Linux' }} - # Verify that editable install works - - name: Test in editable form - run: uvx nox -s test_editable - - python-build-package: - strategy: - matrix: - include: - - os: "ubuntu-latest" - cibw_build: "*" - wheel_identifier: "linux" - - os: "windows-latest" - cibw_build: "*" - wheel_identifier: "windows" - - os: "macos-14" - cibw_build: "*_x86_64" - wheel_identifier: "macos_x86_64" - - os: "macos-14" - cibw_build: "*_arm64" - wheel_identifier: "macos_arm64" - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - # Need full history to determine version number. - fetch-depth: 0 - - uses: actions/setup-node@v4 - with: - node-version: 22.x - cache: "npm" - - uses: astral-sh/setup-uv@v5 - with: - enable-cache: false - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.x - - name: Get uv cache dir - id: uv-cache - run: | - echo "dir=$(uv cache dir)" >> "$GITHUB_OUTPUT" - - run: npm ci - - run: | - build_info="{'tag':'$(git describe --always --tags)', 'url':'https://github.com/google/neuroglancer/commit/$(git rev-parse HEAD)', 'timestamp':'$(date)'}" - npm run build-python -- --no-typecheck --no-lint --define NEUROGLANCER_BUILD_INFO="${build_info}" + run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV + - name: Get branch name (pull request) + if: github.event_name == 'pull_request' + shell: bash + run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV + - run: echo "BRANCH_NAME_URL=$(echo ${{ env.BRANCH_NAME }} | tr / - | tr _ -)" >> $GITHUB_ENV + - name: Get build info + run: echo "BUILD_INFO={\"tag\":\"$(git describe --always --tags)\", \"url\":\"https://github.com/${{github.repository}}/commit/$(git rev-parse HEAD)\", \"timestamp\":\"$(date)\", \"branch\":\"${{github.repository}}/${{env.BRANCH_NAME}}\"}" >> $GITHUB_ENV shell: bash - name: Check for dirty working directory run: git diff --exit-code - - name: Build Python source distribution (sdist) - run: uv build --sdist - if: ${{ runner.os == 'Linux' }} - - name: Build Python wheels - run: uvx nox -s cibuildwheel - env: - # On Linux, share uv cache with manylinux docker containers - CIBW_ENVIRONMENT_LINUX: UV_CACHE_DIR=/host${{ steps.uv-cache.outputs.dir }} - CIBW_BEFORE_ALL_LINUX: /project/python/build_tools/cibuildwheel_linux_cache_setup.sh /host${{ steps.uv-cache.outputs.dir }} - CIBW_BUILD: ${{ matrix.cibw_build }} - - name: Upload wheels as artifacts - uses: actions/upload-artifact@v4 - with: - name: python-wheels-${{ matrix.wheel_identifier }} - path: | - dist/*.whl - dist/*.tar.gz - - docs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 - with: - enable-cache: false - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Setup Graphviz - uses: ts-graphviz/setup-graphviz@b1de5da23ed0a6d14e0aeee8ed52fdd87af2363c # v2.0.2 - with: - macos-skip-brew-update: "true" - - name: Build docs - run: uvx nox -s docs - - name: Upload docs as artifact - uses: actions/upload-artifact@v4 - with: - name: docs - path: | - dist/docs - - publish-package: - # Only publish package on push to tag or default branch. - if: ${{ github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master') }} - runs-on: ubuntu-latest - needs: - - "client" - - "python-build-package" - - "docs" - steps: - - uses: actions/checkout@v4 - - name: Use Node.js - uses: actions/setup-node@v4 - with: - node-version: 22.x - registry-url: "https://registry.npmjs.org" - - uses: actions/download-artifact@v4 - with: - pattern: python-wheels-* - path: dist - merge-multiple: true - - uses: actions/download-artifact@v4 - with: - name: npm-package - path: npm-package - # - name: Publish to PyPI (test server) - # uses: pypa/gh-action-pypi-publish@54b39fb9371c0b3a6f9f14bb8a67394defc7a806 # 2020-09-25 - # with: - # user: __token__ - # password: ${{ secrets.pypi_test_token }} - - name: Publish to PyPI (main server) - uses: pypa/gh-action-pypi-publish@54b39fb9371c0b3a6f9f14bb8a67394defc7a806 # 2020-09-25 - with: - user: __token__ - password: ${{ secrets.pypi_token }} - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - - name: Publish to NPM registry - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - run: npm publish - working-directory: npm-package - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - # Download dist/client after publishing to PyPI, because PyPI publish - # action fails if dist/client directory is present. - - uses: actions/download-artifact@v4 - with: - name: client - path: dist/client - - name: Publish client to Firebase hosting - uses: FirebaseExtended/action-hosting-deploy@v0 - with: - firebaseServiceAccount: "${{ secrets.FIREBASE_HOSTING_SERVICE_ACCOUNT_KEY }}" - projectId: neuroglancer-demo - channelId: live - target: app - # Download dist/docs after publishing to PyPI, because PyPI publish - # action fails if dist/docs directory is present. - - uses: actions/download-artifact@v4 - with: - name: docs - path: dist/docs - - name: Publish docs to Firebase hosting - uses: FirebaseExtended/action-hosting-deploy@v0 - with: - firebaseServiceAccount: "${{ secrets.FIREBASE_HOSTING_SERVICE_ACCOUNT_KEY }}" - projectId: neuroglancer-demo - channelId: live - target: docs - - ngauth: - strategy: - matrix: - os: - - ubuntu-latest - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Setup go - uses: actions/setup-go@v5 - with: - go-version-file: ngauth_server/go.mod - cache-dependency-path: ngauth_server/go.sum - - run: go build . - working-directory: ngauth_server - wasm: - # Ensures that .wasm files are reproducible. - strategy: - matrix: - os: - - ubuntu-latest - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - run: ./src/mesh/draco/build.sh - - run: ./src/sliceview/compresso/build.sh - - run: ./src/sliceview/png/build.sh - - run: ./src/sliceview/jxl/build.sh - # Check that there are no differences. - - run: git diff --exit-code + - name: Build client bundles + run: npm run build -- --no-typecheck --no-lint --define STATE_SERVERS=$(cat config/state_servers.json | tr -d " \t\n\r") --define NEUROGLANCER_BUILD_INFO='${{ env.BUILD_INFO }}' --define CUSTOM_BINDINGS=$(cat config/custom-keybinds.json | tr -d " \t\n\r") + - name: Write build info + run: echo $BUILD_INFO > ./dist/client/version.json + shell: bash + - run: cp -r ./dist/client appengine/frontend/static/ + - name: start deployment + uses: bobheadxi/deployments@v1 + id: deployment + with: + step: start + token: ${{ secrets.GITHUB_TOKEN }} + env: ${{ env.BRANCH_NAME }} + desc: Setting up staging deployment for ${{ env.BRANCH_NAME }} + - id: "auth" + uses: "google-github-actions/auth@v1" + with: + workload_identity_provider: "projects/483670036293/locations/global/workloadIdentityPools/neuroglancer-github/providers/github" + service_account: "chris-apps-deploy@seung-lab.iam.gserviceaccount.com" + - id: deploy + uses: google-github-actions/deploy-appengine@main + with: + version: ${{ env.GITHUB_SHA }} + deliverables: appengine/frontend/app.yaml + promote: true + - name: update deployment status + uses: bobheadxi/deployments@v1 + if: always() + with: + step: finish + token: ${{ secrets.GITHUB_TOKEN }} + env: ${{ steps.deployment.outputs.env }} + env_url: ${{ steps.deploy.outputs.url }} + status: ${{ job.status }} + deployment_id: ${{ steps.deployment.outputs.deployment_id }} diff --git a/.github/workflows/build_preview.yml b/.github/workflows/build_preview.yml deleted file mode 100644 index 7a6c997612..0000000000 --- a/.github/workflows/build_preview.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Build preview - -on: - pull_request: - -jobs: - upload: - strategy: - matrix: - node-version: - - "22.x" - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - run: npm install - - run: | - build_info="{'tag':'$(git describe --always --tags)', 'url':'https://github.com/google/neuroglancer/commit/$(git rev-parse HEAD)', 'timestamp':'$(date)'}" - npm run build -- --no-typecheck --no-lint --define NEUROGLANCER_BUILD_INFO="${build_info}" - - name: Upload client as artifact - uses: actions/upload-artifact@v4 - with: - name: client - path: | - dist/client/* diff --git a/appengine/frontend/.gcloudignore b/appengine/frontend/.gcloudignore new file mode 100644 index 0000000000..c2edb00e92 --- /dev/null +++ b/appengine/frontend/.gcloudignore @@ -0,0 +1,5 @@ +.gcloudignore +.envrc +.git +.gitignore +node_modules/ \ No newline at end of file diff --git a/appengine/frontend/app.yaml b/appengine/frontend/app.yaml new file mode 100644 index 0000000000..7d375d9d1b --- /dev/null +++ b/appengine/frontend/app.yaml @@ -0,0 +1,19 @@ +runtime: python312 + +service: neuroglancer + +handlers: + # Handle the main page by serving the index page. + # Note the $ to specify the end of the path, since app.yaml does prefix matching. + - url: /$ + static_files: static/index.html + upload: static/index.html + login: optional + secure: always + redirect_http_response_code: 301 + + - url: / + static_dir: static + login: optional + secure: always + redirect_http_response_code: 301 diff --git a/config/state_servers.json b/config/state_servers.json new file mode 100644 index 0000000000..279a63557d --- /dev/null +++ b/config/state_servers.json @@ -0,0 +1,6 @@ +{ + "cave": { + "url": "middleauth+https://global.daf-apis.com/nglstate/api/v1/post", + "default": true + } +} From 7117555f265c024f51f38e7e9a724ea75d72a940 Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Fri, 5 May 2023 09:06:07 -0400 Subject: [PATCH 02/46] specific app config for cave-explorer --- appengine/frontend/app.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/frontend/app.yaml b/appengine/frontend/app.yaml index 7d375d9d1b..9e6dfcfb07 100644 --- a/appengine/frontend/app.yaml +++ b/appengine/frontend/app.yaml @@ -1,6 +1,6 @@ runtime: python312 -service: neuroglancer +service: base-cave handlers: # Handle the main page by serving the index page. From 8ac681cda4a65cf30ad1a94252488ab93ad8a08a Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Mon, 15 May 2023 12:52:41 -0400 Subject: [PATCH 03/46] specific app config for spelunker --- appengine/frontend/app.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/frontend/app.yaml b/appengine/frontend/app.yaml index 9e6dfcfb07..9d82ed9370 100644 --- a/appengine/frontend/app.yaml +++ b/appengine/frontend/app.yaml @@ -1,6 +1,6 @@ runtime: python312 -service: base-cave +service: spelunker handlers: # Handle the main page by serving the index page. From 3acf4b8e9217e48e5c35ccabe7958cc9310a05c0 Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Tue, 16 May 2023 11:13:46 -0400 Subject: [PATCH 04/46] trying to reduce flicker --- src/chunk_manager/frontend.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/chunk_manager/frontend.ts b/src/chunk_manager/frontend.ts index 3df76ce075..f1e87239ac 100644 --- a/src/chunk_manager/frontend.ts +++ b/src/chunk_manager/frontend.ts @@ -266,9 +266,10 @@ export class ChunkQueueManager extends SharedObject { if (newState !== oldState) { switch (newState) { case ChunkState.GPU_MEMORY: - // console.log("Copying to GPU", chunk); chunk.copyToGPU(this.gl); - visibleChunksChanged = true; + if (chunk.constructor.name !== "ManifestChunk") { + visibleChunksChanged = true; + } break; case ChunkState.SYSTEM_MEMORY: if (oldState === ChunkState.GPU_MEMORY) { From bca340ad874679a3d1cd81af1296ab978070f816 Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Tue, 6 Dec 2022 16:51:02 -0500 Subject: [PATCH 05/46] added graphene find path tool using l2cache when available for fast and more accurate find path check for l2cache existing is now cached and only checked when necessary --- src/datasource/graphene/frontend.ts | 893 ++++++++++++++++++++++----- src/datasource/graphene/graphene.css | 39 +- src/ui/annotations.ts | 293 +++++---- src/util/json.ts | 10 + 4 files changed, 929 insertions(+), 306 deletions(-) diff --git a/src/datasource/graphene/frontend.ts b/src/datasource/graphene/frontend.ts index d3fc71fa18..63227572eb 100644 --- a/src/datasource/graphene/frontend.ts +++ b/src/datasource/graphene/frontend.ts @@ -20,8 +20,11 @@ import { AnnotationDisplayState, AnnotationLayerState, } from "#src/annotation/annotation_layer_state.js"; +import type { MultiscaleAnnotationSource } from "#src/annotation/frontend_source.js"; import type { + Annotation, AnnotationReference, + AnnotationSource, Line, Point, } from "#src/annotation/index.js"; @@ -147,9 +150,11 @@ import { } from "#src/trackable_value.js"; import { AnnotationLayerView, + makeAnnotationListElement, MergedAnnotationStates, PlaceLineTool, } from "#src/ui/annotations.js"; +import { getDefaultAnnotationListBindings } from "#src/ui/default_input_event_bindings.js"; import type { ToolActivation } from "#src/ui/tool.js"; import { LayerTool, @@ -158,15 +163,16 @@ import { registerLegacyTool, registerTool, } from "#src/ui/tool.js"; -import type { Uint64Set } from "#src/uint64_set.js"; +import { Uint64Set } from "#src/uint64_set.js"; import { packColor } from "#src/util/color.js"; import type { Owned } from "#src/util/disposable.js"; import { RefCounted } from "#src/util/disposable.js"; +import { removeChildren } from "#src/util/dom.js"; import type { ValueOrError } from "#src/util/error.js"; import { makeValueOrError, valueOrThrow } from "#src/util/error.js"; import { EventActionMap } from "#src/util/event_action_map.js"; import { mat4, vec3, vec4 } from "#src/util/geom.js"; -import { HttpError, isNotFoundError } from "#src/util/http_request.js"; +import { fetchOk, HttpError, isNotFoundError } from "#src/util/http_request.js"; import { parseArray, parseFixedLengthArray, @@ -176,7 +182,9 @@ import { verifyEnumString, verifyFiniteFloat, verifyFinitePositiveFloat, + verifyFloatArray, verifyInt, + verifyIntegerArray, verifyNonnegativeInt, verifyObject, verifyObjectProperty, @@ -184,7 +192,9 @@ import { verifyOptionalString, verifyPositiveInt, verifyString, + verifyStringArray, } from "#src/util/json.js"; +import { MouseEventBinder } from "#src/util/mouse_bindings.js"; import type { ProgressOptions } from "#src/util/progress_listener.js"; import { ProgressSpan } from "#src/util/progress_listener.js"; import { NullarySignal } from "#src/util/signal.js"; @@ -210,6 +220,7 @@ const RED_COLOR_SEGMENT_PACKED = BigInt(packColor(RED_COLOR_SEGMENT)); const BLUE_COLOR_SEGMENT_PACKED = BigInt(packColor(BLUE_COLOR_SEGMENT)); const TRANSPARENT_COLOR_PACKED = BigInt(packColor(TRANSPARENT_COLOR)); const MULTICUT_OFF_COLOR = vec4.fromValues(0, 0, 0, 0.5); +const WHITE_COLOR = vec3.fromValues(1, 1, 1); class GrapheneMeshSource extends WithParameters( WithSharedKvStoreContext(MeshSource), @@ -224,18 +235,23 @@ class GrapheneMeshSource extends WithParameters( class AppInfo { segmentationUrl: string; meshingUrl: string; + l2CacheUrl: string; + table: string; supported_api_versions: number[]; constructor(infoUrl: string, obj: any) { // .../1.0/... is the legacy link style // .../table/... is the current, version agnostic link style (for retrieving the info file) const linkStyle = - /^((?:middleauth\+)?https?:\/\/[.\w:\-/]+)\/segmentation\/(?:1\.0|table)\/([^/]+)\/?$/; + /^((?:middleauth\+)?)(https?:\/\/[.\w:\-/]+)\/segmentation\/(?:1\.0|table)\/([^/]+)\/?$/; const match = infoUrl.match(linkStyle); if (match === null) { throw Error(`Graph URL invalid: ${infoUrl}`); } - this.segmentationUrl = `${match[1]}/segmentation/api/v${PYCG_APP_VERSION}/table/${match[2]}`; - this.meshingUrl = `${match[1]}/meshing/api/v${PYCG_APP_VERSION}/table/${match[2]}`; + this.table = match[3]; + const { table } = this; + this.segmentationUrl = `${match[1]}${match[2]}/segmentation/api/v${PYCG_APP_VERSION}/table/${table}`; + this.meshingUrl = `${match[1]}${match[2]}/meshing/api/v${PYCG_APP_VERSION}/table/${table}`; + this.l2CacheUrl = `${match[2]}/l2cache/api/v${PYCG_APP_VERSION}`; try { verifyObject(obj); @@ -807,47 +823,81 @@ function restoreSegmentSelection(obj: any): SegmentSelection { }; } +const segmentSelectionToJSON = (x: SegmentSelection) => { + return { + [SEGMENT_ID_JSON_KEY]: x.segmentId.toString(), + [ROOT_ID_JSON_KEY]: x.rootId.toString(), + [POSITION_JSON_KEY]: [...x.position], + }; +}; + const ID_JSON_KEY = "id"; -const ERROR_JSON_KEY = "error"; +const SEGMENT_ID_JSON_KEY = "segmentId"; +const ROOT_ID_JSON_KEY = "rootId"; +const POSITION_JSON_KEY = "position"; +const SINK_JSON_KEY = "sink"; +const SOURCE_JSON_KEY = "source"; + const MULTICUT_JSON_KEY = "multicut"; const FOCUS_SEGMENT_JSON_KEY = "focusSegment"; const SINKS_JSON_KEY = "sinks"; const SOURCES_JSON_KEY = "sources"; -const SEGMENT_ID_JSON_KEY = "segmentId"; -const ROOT_ID_JSON_KEY = "rootId"; -const POSITION_JSON_KEY = "position"; + const MERGE_JSON_KEY = "merge"; const MERGES_JSON_KEY = "merges"; const AUTOSUBMIT_JSON_KEY = "autosubmit"; -const SINK_JSON_KEY = "sink"; -const SOURCE_JSON_KEY = "source"; -const MERGED_ROOT_JSON_KEY = "mergedRoot"; const LOCKED_JSON_KEY = "locked"; +const MERGED_ROOT_JSON_KEY = "mergedRoot"; +const ERROR_JSON_KEY = "error"; -class GrapheneState implements Trackable { +const FIND_PATH_JSON_KEY = "findPath"; +const TARGET_JSON_KEY = "target"; +const CENTROIDS_JSON_KEY = "centroids"; +const PRECISION_MODE_JSON_KEY = "precision"; + +class GrapheneState extends RefCounted implements Trackable { changed = new NullarySignal(); public multicutState = new MulticutState(); public mergeState = new MergeState(); + public findPathState = new FindPathState(); constructor() { - this.multicutState.changed.add(() => { - this.changed.dispatch(); - }); - this.mergeState.changed.add(() => { - this.changed.dispatch(); - }); + super(); + this.registerDisposer( + this.multicutState.changed.add(() => { + this.changed.dispatch(); + }), + ); + this.registerDisposer( + this.mergeState.changed.add(() => { + this.changed.dispatch(); + }), + ); + this.registerDisposer( + this.findPathState.changed.add(() => { + this.changed.dispatch(); + }), + ); + } + + replaceSegments(oldValues: Uint64Set, newValues: Uint64Set) { + this.multicutState.replaceSegments(oldValues, newValues); + this.mergeState.replaceSegments(oldValues, newValues); + this.findPathState.replaceSegments(oldValues, newValues); } reset() { this.multicutState.reset(); this.mergeState.reset(); + this.findPathState.reset(); } toJSON() { return { [MULTICUT_JSON_KEY]: this.multicutState.toJSON(), [MERGE_JSON_KEY]: this.mergeState.toJSON(), + [FIND_PATH_JSON_KEY]: this.findPathState.toJSON(), }; } @@ -858,6 +908,9 @@ class GrapheneState implements Trackable { verifyOptionalObjectProperty(x, MERGE_JSON_KEY, (value) => { this.mergeState.restoreState(value); }); + verifyOptionalObjectProperty(x, FIND_PATH_JSON_KEY, (value) => { + this.findPathState.restoreState(value); + }); } } @@ -870,7 +923,6 @@ export interface SegmentSelection { class MergeState extends RefCounted implements Trackable { changed = new NullarySignal(); - merges = new WatchableValue([]); autoSubmit = new TrackableBoolean(false); @@ -879,6 +931,31 @@ class MergeState extends RefCounted implements Trackable { this.registerDisposer(this.merges.changed.add(this.changed.dispatch)); } + replaceSegments(oldValues: Uint64Set, newValues: Uint64Set) { + const { + merges: { value: merges }, + } = this; + const newValue = newValues.size === 1 ? [...newValues][0] : undefined; + for (const merge of merges) { + if (merge.source && oldValues.has(merge.source.rootId)) { + if (newValue) { + merge.source.rootId = newValue; + } else { + this.reset(); + return; + } + } + if (merge.sink && oldValues.has(merge.sink.rootId)) { + if (newValue) { + merge.sink.rootId = newValue; + } else { + this.reset(); + return; + } + } + } + } + reset() { this.merges.value = []; this.autoSubmit.reset(); @@ -887,14 +964,6 @@ class MergeState extends RefCounted implements Trackable { toJSON() { const { merges, autoSubmit } = this; - const segmentSelectionToJSON = (x: SegmentSelection) => { - return { - [SEGMENT_ID_JSON_KEY]: x.segmentId.toString(), - [ROOT_ID_JSON_KEY]: x.rootId.toString(), - [POSITION_JSON_KEY]: [...x.position], - }; - }; - const mergeToJSON = (x: MergeSubmission) => { const res: any = { [ID_JSON_KEY]: x.id, @@ -902,17 +971,14 @@ class MergeState extends RefCounted implements Trackable { [SINK_JSON_KEY]: segmentSelectionToJSON(x.sink), [SOURCE_JSON_KEY]: segmentSelectionToJSON(x.source!), }; - if (x.mergedRoot) { res[MERGED_ROOT_JSON_KEY] = x.mergedRoot.toString(); } if (x.error) { res[ERROR_JSON_KEY] = x.error; } - return res; }; - return { [MERGES_JSON_KEY]: merges.value.filter((x) => x.source).map(mergeToJSON), [AUTOSUBMIT_JSON_KEY]: autoSubmit.toJSON(), @@ -958,6 +1024,146 @@ class MergeState extends RefCounted implements Trackable { } } +class FindPathState extends RefCounted implements Trackable { + changed = new NullarySignal(); + triggerPathUpdate = new NullarySignal(); + source = new TrackableValue( + undefined, + (x) => x, + ); + target = new TrackableValue( + undefined, + (x) => x, + ); + centroids = new TrackableValue([], (x) => x); + precisionMode = new TrackableBoolean(true); + + constructor() { + super(); + this.registerDisposer( + this.source.changed.add(() => { + this.centroids.reset(); + this.changed.dispatch(); + }), + ); + this.registerDisposer( + this.target.changed.add(() => { + this.centroids.reset(); + this.changed.dispatch(); + }), + ); + this.registerDisposer(this.centroids.changed.add(this.changed.dispatch)); + } + + get path() { + const path: Line[] = []; + const { + source: { value: source }, + target: { value: target }, + centroids: { value: centroids }, + } = this; + if (!source || !target || centroids.length === 0) { + return path; + } + for (let i = 0; i < centroids.length - 1; i++) { + const pointA = centroids[i]; + const pointB = centroids[i + 1]; + const line: Line = { + pointA: vec3.fromValues(pointA[0], pointA[1], pointA[2]), + pointB: vec3.fromValues(pointB[0], pointB[1], pointB[2]), + id: "", + type: AnnotationType.LINE, + properties: [], + }; + path.push(line); + } + const firstLine: Line = { + pointA: source.position, + pointB: path[0].pointA, + id: "", + type: AnnotationType.LINE, + properties: [], + }; + const lastLine: Line = { + pointA: path[path.length - 1].pointB, + pointB: target.position, + id: "", + type: AnnotationType.LINE, + properties: [], + }; + + return [firstLine, ...path, lastLine]; + } + + replaceSegments(oldValues: Uint64Set, newValues: Uint64Set) { + const { + source: { value: source }, + target: { value: target }, + } = this; + const newValue = newValues.size === 1 ? [...newValues][0] : undefined; + const sourceChanged = !!source && oldValues.has(source.rootId); + const targetChanged = !!target && oldValues.has(target.rootId); + if (newValue) { + if (sourceChanged) { + source.rootId = newValue; + } + if (targetChanged) { + target.rootId = newValue; + } + // don't want to fire off multiple changed + if (sourceChanged || targetChanged) { + if (this.centroids.value.length) { + this.centroids.reset(); + this.triggerPathUpdate.dispatch(); + } else { + this.changed.dispatch(); + } + } + } else { + if (sourceChanged || targetChanged) { + this.reset(); + } + } + } + + reset() { + this.source.reset(); + this.target.reset(); + this.centroids.reset(); + this.precisionMode.reset(); + } + + toJSON() { + const { + source: { value: source }, + target: { value: target }, + centroids, + precisionMode, + } = this; + return { + [SOURCE_JSON_KEY]: source ? segmentSelectionToJSON(source) : undefined, + [TARGET_JSON_KEY]: target ? segmentSelectionToJSON(target) : undefined, + [CENTROIDS_JSON_KEY]: centroids.toJSON(), + [PRECISION_MODE_JSON_KEY]: precisionMode.toJSON(), + }; + } + + restoreState(x: any) { + verifyOptionalObjectProperty(x, SOURCE_JSON_KEY, (value) => { + this.source.restoreState(restoreSegmentSelection(value)); + }); + verifyOptionalObjectProperty(x, TARGET_JSON_KEY, (value) => { + this.target.restoreState(restoreSegmentSelection(value)); + }); + verifyOptionalObjectProperty(x, CENTROIDS_JSON_KEY, (value) => { + this.centroids.restoreState(value); + }); + verifyOptionalObjectProperty(x, PRECISION_MODE_JSON_KEY, (value) => { + this.precisionMode.restoreState(value); + }); + } +} + class MulticutState extends RefCounted implements Trackable { changed = new NullarySignal(); @@ -988,8 +1194,29 @@ class MulticutState extends RefCounted implements Trackable { this.registerDisposer(this.sources.changed.add(this.changed.dispatch)); } + replaceSegments(oldValues: Uint64Set, newValues: Uint64Set) { + const newValue = newValues.size === 1 ? [...newValues][0] : undefined; + const { + focusSegment: { value: focusSegment }, + } = this; + if (focusSegment && oldValues.has(focusSegment)) { + if (newValue) { + this.focusSegment.value = newValue; + for (const sink of this.sinks) { + sink.rootId = newValue; + } + for (const source of this.sources) { + source.rootId = newValue; + } + this.changed.dispatch(); + } else { + this.reset(); + } + } + } + reset() { - this.focusSegment.value = undefined; + this.focusSegment.reset(); this.blueGroup.value = false; this.sinks.clear(); this.sources.clear(); @@ -997,15 +1224,6 @@ class MulticutState extends RefCounted implements Trackable { toJSON() { const { focusSegment, sinks, sources } = this; - - const segmentSelectionToJSON = (x: SegmentSelection) => { - return { - [SEGMENT_ID_JSON_KEY]: x.segmentId.toString(), - [ROOT_ID_JSON_KEY]: x.rootId.toString(), - [POSITION_JSON_KEY]: [...x.position], - }; - }; - return { [FOCUS_SEGMENT_JSON_KEY]: focusSegment.toJSON()?.toString(), [SINKS_JSON_KEY]: [...sinks].map(segmentSelectionToJSON), @@ -1072,6 +1290,7 @@ class MulticutState extends RefCounted implements Trackable { class GraphConnection extends SegmentationGraphSourceConnection { public annotationLayerStates: AnnotationLayerState[] = []; public mergeAnnotationState: AnnotationLayerState; + public findPathAnnotationState: AnnotationLayerState; constructor( public graph: GrapheneGraphSource, @@ -1090,7 +1309,6 @@ class GraphConnection extends SegmentationGraphSourceConnection { this.selectedSegmentsChanged(segmentIds, add); }, ); - segmentsState.visibleSegments.changed.add( (segmentIds: bigint[] | bigint | null, add: boolean) => { if (segmentIds !== null) { @@ -1100,10 +1318,9 @@ class GraphConnection extends SegmentationGraphSourceConnection { this.visibleSegmentsChanged(segmentIds, add); }, ); - const { annotationLayerStates, - state: { multicutState }, + state: { multicutState, findPathState }, } = this; const loadedSubsource = getGraphLoadedSubsource(layer)!; const redGroup = makeColoredAnnotationState( @@ -1179,19 +1396,19 @@ class GraphConnection extends SegmentationGraphSourceConnection { const annotation = ref.value as Line | undefined; if (annotation) { const relatedSegments = annotation.relatedSegments![0]; - const visibles = Array.from(relatedSegments, (x) => - visibleSegments.has(x), - ); if (relatedSegments.length < 4) { mergeAnnotationState.source.delete(ref); StatusMessage.showTemporaryMessage( - "Cannot merge segment with itself.", + `Cannot merge segment with itself.`, ); } + const visibles: boolean[] = Array.from(relatedSegments, (x) => + visibleSegments.has(x), + ); if (visibles[2] === false) { mergeAnnotationState.source.delete(ref); StatusMessage.showTemporaryMessage( - "Cannot merge a hidden segment.", + `Cannot merge a hidden segment.`, ); } const existingSubmission = merges.value.find((x) => x.id === ref.id); @@ -1208,7 +1425,6 @@ class GraphConnection extends SegmentationGraphSourceConnection { } ref.dispose(); }); - mergeAnnotationState.source.childDeleted.add((id) => { let changed = false; const filtered = merges.value.filter((x) => { @@ -1223,6 +1439,66 @@ class GraphConnection extends SegmentationGraphSourceConnection { } }); } + + const findPathGroup = makeColoredAnnotationState( + layer, + loadedSubsource, + "findpath", + WHITE_COLOR, + ); + this.findPathAnnotationState = findPathGroup; + findPathGroup.source.childDeleted.add((annotationId) => { + if ( + findPathState.source.value?.annotationReference?.id === annotationId + ) { + findPathState.source.value = undefined; + } + if ( + findPathState.target.value?.annotationReference?.id === annotationId + ) { + findPathState.target.value = undefined; + } + }); + const findPathChanged = () => { + const { path, source, target } = findPathState; + const annotationSource = findPathGroup.source; + if (source.value && !source.value.annotationReference) { + addSelection(annotationSource, source.value, "find path source"); + } + if (target.value && !target.value.annotationReference) { + addSelection(annotationSource, target.value, "find path target"); + } + for (const annotation of annotationSource) { + if ( + annotation.id !== source.value?.annotationReference?.id && + annotation.id !== target.value?.annotationReference?.id + ) { + annotationSource.delete(annotationSource.getReference(annotation.id)); + } + } + for (const line of path) { + // line.id = ''; // TODO, is it a bug that this is necessary? annotationMap is empty if I + // step through it but logging shows it isn't empty + annotationSource.add(line); + } + }; + this.registerDisposer(findPathState.changed.add(findPathChanged)); + this.registerDisposer( + findPathState.triggerPathUpdate.add(() => { + const loadedSubsource = getGraphLoadedSubsource(this.layer)!; + const annotationToNanometers = + loadedSubsource.loadedDataSource.transform.inputSpace.value.scales.map( + (x) => x / 1e-9, + ); + this.submitFindPath( + findPathState.precisionMode.value, + annotationToNanometers, + ).then((success) => { + success; + }); + }), + ); + findPathChanged(); // initial state } createRenderLayers( @@ -1342,7 +1618,7 @@ class GraphConnection extends SegmentationGraphSourceConnection { const graphSubsource = subsources.filter( (subsource) => subsource.id === "graph", )[0]; - if (graphSubsource?.subsource.segmentationGraph) { + if (graphSubsource && graphSubsource.subsource.segmentationGraph) { if (graphSubsource.subsource.segmentationGraph !== this.graph) { continue; } @@ -1381,27 +1657,35 @@ class GraphConnection extends SegmentationGraphSourceConnection { 7000, ); return false; + } else { + const splitRoots = await this.graph.graphServer.splitSegments( + [...sinks].map((x) => selectionInNanometers(x, annotationToNanometers)), + [...sources].map((x) => + selectionInNanometers(x, annotationToNanometers), + ), + ); + if (splitRoots.length === 0) { + StatusMessage.showTemporaryMessage(`No split found.`, 3000); + return false; + } else { + const focusSegment = multicutState.focusSegment.value!; + multicutState.reset(); // need to clear the focus segment before deleting the multicut segment + const { segmentsState } = this; + segmentsState.selectedSegments.delete(focusSegment); + for (const segment of [...sinks, ...sources]) { + segmentsState.selectedSegments.delete(segment.rootId); + } + this.meshAddNewSegments(splitRoots); + segmentsState.selectedSegments.add(splitRoots); + segmentsState.visibleSegments.add(splitRoots); + const oldValues = new Uint64Set(); + oldValues.add(focusSegment); + const newValues = new Uint64Set(); + newValues.add(splitRoots); + this.state.replaceSegments(oldValues, newValues); + return true; + } } - const splitRoots = await this.graph.graphServer.splitSegments( - [...sinks], - [...sources], - annotationToNanometers, - ); - if (splitRoots.length === 0) { - StatusMessage.showTemporaryMessage("No split found.", 3000); - return false; - } - const focusSegment = multicutState.focusSegment.value!; - multicutState.reset(); // need to clear the focus segment before deleting the multicut segment - const { segmentsState } = this; - segmentsState.selectedSegments.delete(focusSegment); - for (const segment of [...sinks, ...sources]) { - segmentsState.selectedSegments.delete(segment.rootId); - } - this.meshAddNewSegments(splitRoots); - segmentsState.selectedSegments.add(splitRoots); - segmentsState.visibleSegments.add(splitRoots); - return true; } deleteMergeSubmission = (submission: MergeSubmission) => { @@ -1425,11 +1709,17 @@ class GraphConnection extends SegmentationGraphSourceConnection { submission.error = undefined; for (let i = 1; i <= attempts; i++) { try { - return await this.graph.graphServer.mergeSegments( - submission.sink, - submission.source!, - annotationToNanometers, + const newRoot = await this.graph.graphServer.mergeSegments( + selectionInNanometers(submission.sink, annotationToNanometers), + selectionInNanometers(submission.source!, annotationToNanometers), ); + const oldValues = new Uint64Set(); + oldValues.add(submission.sink.rootId); + oldValues.add(submission.source!.rootId); + const newValues = new Uint64Set(); + newValues.add(newRoot); + this.state.replaceSegments(oldValues, newValues); + return newRoot; } catch (err) { if (i === attempts) { submission.error = err.message || "unknown"; @@ -1452,17 +1742,6 @@ class GraphConnection extends SegmentationGraphSourceConnection { return; } const segmentsToRemove: bigint[] = []; - const replaceSegment = (a: bigint, b: bigint) => { - segmentsToRemove.push(a); - for (const submission of submissions) { - if (submission.source && submission.source.rootId === a) { - submission.source.rootId = b; - } - if (submission.sink.rootId === a) { - submission.sink.rootId = b; - } - } - }; let completed = 0; let activeLoops = 0; const loop = (completedAt: number, pending: MergeSubmission[]) => { @@ -1483,10 +1762,13 @@ class GraphConnection extends SegmentationGraphSourceConnection { submission.locked = true; submission.status = "trying..."; merges.changed.dispatch(); + const segments = [ + submission.source!.rootId, + submission.sink.rootId, + ]; this.submitMerge(submission, 3) .then((mergedRoot) => { - replaceSegment(submission.source!.rootId, mergedRoot); - replaceSegment(submission.sink.rootId, mergedRoot); + segmentsToRemove.push(...segments); submission.status = "done"; submission.mergedRoot = mergedRoot; merges.changed.dispatch(); @@ -1523,17 +1805,46 @@ class GraphConnection extends SegmentationGraphSourceConnection { } else if (submission.mergedRoot) { segmentsToAdd.push(submission.mergedRoot); } + const latestRoots = + await this.graph.graphServer.filterLatestRoots(segmentsToAdd); + const segmentsState = + this.layer.displayState.segmentationGroupState.value; + const { visibleSegments, selectedSegments } = segmentsState; + selectedSegments.delete(segmentsToRemove); + this.meshAddNewSegments(latestRoots); + selectedSegments.add(latestRoots); + visibleSegments.add(latestRoots); + merges.changed.dispatch(); } - const latestRoots = - await this.graph.graphServer.filterLatestRoots(segmentsToAdd); const segmentsState = this.layer.displayState.segmentationGroupState.value; const { visibleSegments, selectedSegments } = segmentsState; selectedSegments.delete(segmentsToRemove); - this.meshAddNewSegments(latestRoots); + const latestRoots = + await this.graph.graphServer.filterLatestRoots(segmentsToAdd); selectedSegments.add(latestRoots); visibleSegments.add(latestRoots); merges.changed.dispatch(); } + + async submitFindPath( + precisionMode: boolean, + annotationToNanometers: Float64Array, + ): Promise { + const { + state: { findPathState }, + } = this; + const { source, target } = findPathState; + if (!source.value || !target.value) return false; + const centroids = await this.graph.findPath( + source.value, + target.value, + precisionMode, + annotationToNanometers, + ); + StatusMessage.showTemporaryMessage("Path found!", 5000); + findPathState.centroids.value = centroids; + return true; + } } async function withErrorMessageHTTP( @@ -1557,33 +1868,36 @@ async function withErrorMessageHTTP( } catch (e) { if (e instanceof HttpError && e.response) { const { errorPrefix = "" } = options; - const msg = await parseGrapheneError(e); - if (msg) { - if (!status) { - status = new StatusMessage(true); - } - status.setErrorMessage(errorPrefix + msg); - status.setVisible(true); - throw new Error(`[${e.response.status}] ${errorPrefix}${msg}`); + const msg = (await parseGrapheneError(e)) || "unknown error"; + if (!status) { + status = new StatusMessage(true); } + status.setErrorMessage(errorPrefix + msg); + status.setVisible(true); + throw new Error(`[${e.response.status}] ${errorPrefix}${msg}`); } throw e; } } -export const GRAPH_SERVER_NOT_SPECIFIED = Symbol("Graph Server Not Specified."); +const selectionInNanometers = ( + selection: SegmentSelection, + annotationToNanometers: Float64Array, +): SegmentSelection => { + const { rootId, segmentId, position } = selection; + return { + rootId, + segmentId, + position: position.map((val, i) => val * annotationToNanometers[i]), + }; +}; class GrapheneGraphServerInterface { - private httpSource: HttpSource; - constructor(sharedKvStoreContext: SharedKvStoreContext, url: string) { - this.httpSource = getHttpSource(sharedKvStoreContext.kvStoreContext, url); - } + constructor(private httpSource: HttpSource) {} async getRoot(segment: bigint, timestamp = "") { const timestampEpoch = new Date(timestamp).valueOf() / 1000; - const { fetchOkImpl, baseUrl } = this.httpSource; - const jsonResp = await withErrorMessageHTTP( fetchOkImpl( `${baseUrl}/node/${String(segment)}/root?int64_as_str=1${ @@ -1602,23 +1916,15 @@ class GrapheneGraphServerInterface { async mergeSegments( first: SegmentSelection, second: SegmentSelection, - annotationToNanometers: Float64Array, ): Promise { const { fetchOkImpl, baseUrl } = this.httpSource; const promise = fetchOkImpl(`${baseUrl}/merge?int64_as_str=1`, { method: "POST", body: JSON.stringify([ - [ - String(first.segmentId), - ...first.position.map((val, i) => val * annotationToNanometers[i]), - ], - [ - String(second.segmentId), - ...second.position.map((val, i) => val * annotationToNanometers[i]), - ], + [String(first.segmentId), ...first.position], + [String(second.segmentId), ...second.position], ]), }); - try { const response = await promise; const jsonResp = await response.json(); @@ -1635,23 +1941,15 @@ class GrapheneGraphServerInterface { async splitSegments( first: SegmentSelection[], second: SegmentSelection[], - annotationToNanometers: Float64Array, ): Promise { const { fetchOkImpl, baseUrl } = this.httpSource; const promise = fetchOkImpl(`${baseUrl}/split?int64_as_str=1`, { method: "POST", body: JSON.stringify({ - sources: first.map((x) => [ - String(x.segmentId), - ...x.position.map((val, i) => val * annotationToNanometers[i]), - ]), - sinks: second.map((x) => [ - String(x.segmentId), - ...x.position.map((val, i) => val * annotationToNanometers[i]), - ]), + sources: first.map((x) => [String(x.segmentId), ...x.position]), + sinks: second.map((x) => [String(x.segmentId), ...x.position]), }), }); - const response = await withErrorMessageHTTP(promise, { initialMessage: `Splitting ${first.length} sources from ${second.length} sinks`, errorPrefix: "Split failed: ", @@ -1667,21 +1965,18 @@ class GrapheneGraphServerInterface { async filterLatestRoots(segments: bigint[]): Promise { const { fetchOkImpl, baseUrl } = this.httpSource; const url = `${baseUrl}/is_latest_roots`; - const promise = fetchOkImpl(url, { method: "POST", body: JSON.stringify({ node_ids: segments.map((x) => x.toString()), }), }); - const jsonResp = await withErrorMessageHTTP( promise.then((response) => response.json()), { errorPrefix: "Could not check latest: ", }, ); - const res: bigint[] = []; for (const [i, isLatest] of jsonResp.is_latest.entries()) { if (isLatest) { @@ -1690,11 +1985,62 @@ class GrapheneGraphServerInterface { } return res; } + + async findPath( + first: SegmentSelection, + second: SegmentSelection, + precisionMode: boolean, + ) { + const { fetchOkImpl, baseUrl } = this.httpSource; + const promise = fetchOkImpl( + `${baseUrl}/graph/find_path?int64_as_str=1&precision_mode=${Number( + precisionMode, + )}`, + { + method: "POST", + body: JSON.stringify([ + [String(first.rootId), ...first.position], + [String(second.rootId), ...second.position], + ]), + }, + ); + const jsonResp = await withErrorMessageHTTP( + promise.then((response) => response.json()), + { + initialMessage: `Finding path between ${first.segmentId} and ${second.segmentId}`, + errorPrefix: "Path finding failed: ", + }, + ); + const supervoxelCentroidsKey = "centroids_list"; + const centroids = verifyObjectProperty( + jsonResp, + supervoxelCentroidsKey, + (x) => parseArray(x, verifyFloatArray), + ); + const missingL2IdsKey = "failed_l2_ids"; + const missingL2Ids = jsonResp[missingL2IdsKey]; + if (missingL2Ids && missingL2Ids.length > 0) { + StatusMessage.showTemporaryMessage( + "Some level 2 meshes are missing, so the path shown may have a poor level of detail.", + ); + } + const l2_path = verifyOptionalObjectProperty( + jsonResp, + "l2_path", + verifyStringArray, + ); + return { + centroids, + l2_path, + }; + } } class GrapheneGraphSource extends SegmentationGraphSource { private connections = new Set(); public graphServer: GrapheneGraphServerInterface; + private l2CacheAvailable: boolean | undefined = undefined; + private httpSource: HttpSource; constructor( public info: GrapheneMultiscaleVolumeInfo, @@ -1702,10 +2048,12 @@ class GrapheneGraphSource extends SegmentationGraphSource { public state: GrapheneState, ) { super(); - this.graphServer = new GrapheneGraphServerInterface( - chunkSource.sharedKvStoreContext, - info.app!.segmentationUrl, + const url = info.app!.segmentationUrl; + this.httpSource = getHttpSource( + chunkSource.sharedKvStoreContext.kvStoreContext, + url, ); + this.graphServer = new GrapheneGraphServerInterface(this.httpSource); } connect( @@ -1737,6 +2085,83 @@ class GrapheneGraphSource extends SegmentationGraphSource { return this.graphServer.getRoot(segment); } + async isL2CacheUrlAvailable() { + if (this.l2CacheAvailable !== undefined) { + return this.l2CacheAvailable; + } + try { + const { l2CacheUrl, table } = this.info.app; + const tableMapping = await fetchOk(`${l2CacheUrl}/table_mapping`).then( + (response) => response.json(), + ); + verifyObject(tableMapping); + this.l2CacheAvailable = !!(tableMapping && tableMapping[table]); + return this.l2CacheAvailable; + } catch (e) { + console.error("e", e); + return false; + } + } + + async getAttributesForL2Ids( + l2CacheUrl: string, + table: string, + l2Ids: string[], + ) { + const { fetchOkImpl } = this.httpSource; + const repCoordinatesUrl = `${l2CacheUrl}/table/${table}/attributes`; + const promise = fetchOkImpl(repCoordinatesUrl, { + method: "POST", + body: JSON.stringify({ + l2_ids: l2Ids, + }), + }).then((response) => response.json()); + return verifyObject(promise); + } + + async findPath( + first: SegmentSelection, + second: SegmentSelection, + precisionMode: boolean, + annotationToNanometers: Float64Array, + ): Promise { + const { l2CacheUrl, table } = this.info.app; + const l2CacheAvailable = + precisionMode && (await this.isL2CacheUrlAvailable()); + let { centroids, l2_path } = await this.graphServer.findPath( + selectionInNanometers(first, annotationToNanometers), + selectionInNanometers(second, annotationToNanometers), + precisionMode && !l2CacheAvailable, + ); + if (precisionMode && l2CacheAvailable && l2_path) { + try { + const attributes = await this.getAttributesForL2Ids( + l2CacheUrl, + table, + l2_path, + ); + // many reasons why an l2 id might not have info + // l2 cache has a process that takes time for new ids (even hours) + // maybe a small fraction have no info + // sometime l2 is so small (single voxel), it is ignored by l2 + // best to just drop those points + centroids = l2_path + .map((id) => { + return verifyOptionalObjectProperty(attributes, id, (x) => { + return verifyIntegerArray(x["rep_coord_nm"]); + }); + }) + .filter((x): x is number[] => x !== undefined); + } catch (e) { + console.log("e", e); + } + } + const centroidsTransformed = centroids.map((point: number[]) => { + return point.map((val, i) => val / annotationToNanometers[i]); + }); + return centroidsTransformed; + } + tabContents( layer: SegmentationUserLayer, context: DependentViewContext, @@ -1760,6 +2185,13 @@ class GrapheneGraphSource extends SegmentationGraphSource { title: "Merge segments", }), ); + toolbox.appendChild( + makeToolButton(context, layer.toolBinder, { + toolJson: GRAPHENE_FIND_PATH_TOOL_ID, + label: "Find Path", + title: "Find Path", + }), + ); parent.appendChild(toolbox); parent.appendChild( context.registerDisposer( @@ -1800,6 +2232,15 @@ class ChunkedGraphChunkSource { declare spec: ChunkedGraphChunkSpecification; declare OPTIONS: { spec: ChunkedGraphChunkSpecification }; + + constructor( + chunkManager: ChunkManager, + options: { + spec: ChunkedGraphChunkSpecification; + }, + ) { + super(chunkManager, options); + } } class GrapheneChunkedGraphChunkSource extends WithParameters( @@ -1947,6 +2388,7 @@ class SliceViewPanelChunkedGraphLayer extends SliceViewPanelRenderLayer { const GRAPHENE_MULTICUT_SEGMENTS_TOOL_ID = "grapheneMulticutSegments"; const GRAPHENE_MERGE_SEGMENTS_TOOL_ID = "grapheneMergeSegments"; +const GRAPHENE_FIND_PATH_TOOL_ID = "grapheneFindPath"; class MulticutAnnotationLayerView extends AnnotationLayerView { declare private _annotationStates: MergedAnnotationStates; @@ -1975,12 +2417,28 @@ class MulticutAnnotationLayerView extends AnnotationLayerView { } } +const addSelection = ( + source: AnnotationSource | MultiscaleAnnotationSource, + selection: SegmentSelection, + description?: string, +) => { + const annotation: Point = { + id: "", + point: selection.position, + type: AnnotationType.POINT, + properties: [], + relatedSegments: [BigUint64Array.of(selection.segmentId, selection.rootId)], + description, + }; + const ref = source.add(annotation); + selection.annotationReference = ref; +}; + const synchronizeAnnotationSource = ( source: WatchableSet, state: AnnotationLayerState, ) => { const annotationSource = state.source; - annotationSource.childDeleted.add((annotationId) => { const selection = [...source].find( (selection) => selection.annotationReference?.id === annotationId, @@ -1988,40 +2446,22 @@ const synchronizeAnnotationSource = ( if (selection) source.delete(selection); }); - const addSelection = (selection: SegmentSelection) => { - const annotation: Point = { - id: "", - point: selection.position, - type: AnnotationType.POINT, - properties: [], - relatedSegments: [ - BigUint64Array.of(selection.segmentId, selection.rootId), - ], - }; - const ref = annotationSource.add(annotation); - selection.annotationReference = ref; - }; - source.changed.add((x, add) => { if (x === null) { for (const annotation of annotationSource) { - // using .clear does not remove annotations from the list - // (this.blueGroupAnnotationState.source as LocalAnnotationSource).clear(); annotationSource.delete(annotationSource.getReference(annotation.id)); } return; } - if (add) { - addSelection(x); + addSelection(annotationSource, x); } else if (x.annotationReference) { annotationSource.delete(x.annotationReference); } }); - // load initial state for (const selection of source) { - addSelection(selection); + addSelection(annotationSource, selection); } }; @@ -2092,7 +2532,7 @@ class MulticutSegmentsTool extends LayerTool { const { body, header } = makeToolActivationStatusMessageWithHeader(activation); header.textContent = "Multicut segments"; - body.classList.add("graphene-multicut-status"); + body.classList.add("graphene-tool-status", "graphene-multicut"); body.appendChild( makeIcon({ text: "Swap", @@ -2378,8 +2818,6 @@ function mergeToLine(submission: MergeSubmission): Line { const MAX_MERGE_COUNT = 10; -// on error, copy (also clean up error message) - const MERGE_SEGMENTS_INPUT_EVENT_MAP = EventActionMap.fromObject({ "at:shift?+enter": { action: "submit" }, }); @@ -2409,7 +2847,7 @@ class MergeSegmentsTool extends LayerTool { const { body, header } = makeToolActivationStatusMessageWithHeader(activation); header.textContent = "Merge segments"; - body.classList.add("graphene-merge-segments-status"); + body.classList.add("graphene-tool-status", "graphene-merge-segments"); activation.bindInputEventMap(MERGE_SEGMENTS_INPUT_EVENT_MAP); const submitAction = async () => { if (merges.value.filter((x) => x.locked).length) return; @@ -2524,6 +2962,143 @@ class MergeSegmentsTool extends LayerTool { } } +const FIND_PATH_INPUT_EVENT_MAP = EventActionMap.fromObject({ + "at:shift?+enter": { action: "submit" }, + "at:shift?+control+mousedown0": { action: "add-point" }, +}); + +class FindPathTool extends LayerTool { + activate(activation: ToolActivation) { + const { layer } = this; + const { + graphConnection: { value: graphConnection }, + } = layer; + if (!graphConnection || !(graphConnection instanceof GraphConnection)) + return; + const { + state: { findPathState }, + findPathAnnotationState, + } = graphConnection; + const { source, target, precisionMode } = findPathState; + // Ensure we use the same segmentationGroupState while activated. + const segmentationGroupState = + this.layer.displayState.segmentationGroupState.value; + const { body, header } = + makeToolActivationStatusMessageWithHeader(activation); + header.textContent = "Find Path"; + body.classList.add("graphene-tool-status", "graphene-find-path"); + const submitAction = () => { + findPathState.triggerPathUpdate.dispatch(); + }; + body.appendChild( + makeIcon({ + text: "Submit", + title: "Submit Find Path", + onClick: () => { + submitAction(); + }, + }), + ); + body.appendChild( + makeIcon({ + text: "Clear", + title: "Clear Find Path", + onClick: () => { + findPathState.source.reset(); + findPathState.target.reset(); + findPathState.centroids.reset(); + }, + }), + ); + const checkbox = activation.registerDisposer( + new TrackableBooleanCheckbox(precisionMode), + ); + const label = document.createElement("label"); + const labelText = document.createElement("span"); + labelText.textContent = "Precision mode: "; + label.appendChild(labelText); + label.title = + "Precision mode returns a more accurate path, but takes longer."; + label.appendChild(checkbox.element); + body.appendChild(label); + const annotationElements = document.createElement("div"); + annotationElements.classList.add("find-path-annotations"); + body.appendChild(annotationElements); + const bindings = getDefaultAnnotationListBindings(); + this.registerDisposer(new MouseEventBinder(annotationElements, bindings)); + const updateAnnotationElements = () => { + removeChildren(annotationElements); + const maxColumnWidths = [0, 0, 0]; + const globalDimensionIndices = [0, 1, 2]; + const localDimensionIndices: number[] = []; + const template = + "[symbol] 2ch [dim] var(--neuroglancer-column-0-width) [dim] var(--neuroglancer-column-1-width) [dim] var(--neuroglancer-column-2-width) [delete] min-content"; + const endpoints = [source, target]; + const endpointAnnotations = endpoints + .map((x) => x.value?.annotationReference?.value) + .filter((x) => x) as Annotation[]; + for (const annotation of endpointAnnotations) { + const [element, elementColumnWidths] = makeAnnotationListElement( + this.layer, + annotation, + findPathAnnotationState, + template, + globalDimensionIndices, + localDimensionIndices, + ); + for (const [column, width] of elementColumnWidths.entries()) { + maxColumnWidths[column] = width; + } + annotationElements.appendChild(element); + } + for (const [column, width] of maxColumnWidths.entries()) { + annotationElements.style.setProperty( + `--neuroglancer-column-${column}-width`, + `${width + 2}ch`, + ); + } + }; + findPathState.changed.add(updateAnnotationElements); + updateAnnotationElements(); + activation.bindInputEventMap(FIND_PATH_INPUT_EVENT_MAP); + activation.bindAction("submit", (event) => { + event.stopPropagation(); + submitAction(); + }); + activation.bindAction("add-point", (event) => { + event.stopPropagation(); + (async () => { + if (!source.value) { + // first selection + const selection = maybeGetSelection( + this, + segmentationGroupState.visibleSegments, + ); + if (selection) { + source.value = selection; + } + } else if (!target.value) { + const selection = maybeGetSelection( + this, + segmentationGroupState.visibleSegments, + ); + if (selection) { + target.value = selection; + } + } + })(); + }); + } + + toJSON() { + return GRAPHENE_FIND_PATH_TOOL_ID; + } + + get description() { + return "find path"; + } +} + registerTool( SegmentationUserLayer, GRAPHENE_MULTICUT_SEGMENTS_TOOL_ID, @@ -2540,6 +3115,10 @@ registerTool( }, ); +registerTool(SegmentationUserLayer, GRAPHENE_FIND_PATH_TOOL_ID, (layer) => { + return new FindPathTool(layer, true); +}); + const ANNOTATE_MERGE_LINE_TOOL_ID = "annotateMergeLine"; registerLegacyTool( diff --git a/src/datasource/graphene/graphene.css b/src/datasource/graphene/graphene.css index 7de9f02ad2..9696385327 100644 --- a/src/datasource/graphene/graphene.css +++ b/src/datasource/graphene/graphene.css @@ -14,12 +14,12 @@ color: #4444ff; } -.graphene-multicut-status { +.graphene-tool-status { display: flex; - flex-direction: row; + gap: 10px; } -.graphene-multicut-status > .activeGroupIndicator { +.graphene-multicut > .activeGroupIndicator { padding: 2px; margin: auto 0; background-color: red; @@ -27,15 +27,15 @@ font-weight: 900; } -.graphene-multicut-status > .activeGroupIndicator.blueGroup { +.graphene-multicut > .activeGroupIndicator.blueGroup { background-color: blue; } -.graphene-multicut-status > .activeGroupIndicator::after { +.graphene-multicut > .activeGroupIndicator::after { content: "Red"; } -.graphene-multicut-status > .activeGroupIndicator.blueGroup::after { +.graphene-multicut > .activeGroupIndicator.blueGroup::after { content: "Blue"; } @@ -49,13 +49,7 @@ gap: 10px; } -.graphene-merge-segments-status { - display: flex; - gap: 10px; -} - -.graphene-merge-segments-status .neuroglancer-icon, -.graphene-multicut-status .neuroglancer-icon { +.graphene-tool-status .neuroglancer-icon { height: 100%; } @@ -78,3 +72,22 @@ .graphene-merge-segments-point .neuroglancer-segment-list-entry-copy-container { display: none; } + +.graphene-find-path > label { + display: flex; +} + +.graphene-find-path > label > span { + display: flex; + align-content: center; + flex-wrap: wrap; +} + +.find-path-annotations { + display: flex; + gap: 10px; +} + +.find-path-annotations > .neuroglancer-annotation-list-entry { + background-color: black; +} diff --git a/src/ui/annotations.ts b/src/ui/annotations.ts index 2185673158..11051deca3 100644 --- a/src/ui/annotations.ts +++ b/src/ui/annotations.ts @@ -628,7 +628,39 @@ export class AnnotationLayerView extends Tab { private render(index: number) { const { annotation, state } = this.listElements[index]; - return this.makeAnnotationListElement(annotation, state); + const { + layer, + gridTemplate, + globalDimensionIndices, + localDimensionIndices, + } = this; + const [element, elementColumnWidths] = makeAnnotationListElement( + layer, + annotation, + state, + gridTemplate, + globalDimensionIndices, + localDimensionIndices, + ); + for (const [column, width] of elementColumnWidths.entries()) { + this.setColumnWidth(column, width); + } + element.addEventListener("mouseenter", () => { + this.displayState.hoverState.value = { + id: annotation.id, + partIndex: 0, + annotationLayerState: state, + }; + }); + const selectionState = this.selectedAnnotationState.value; + if ( + selectionState !== undefined && + selectionState.annotationLayerState === state && + selectionState.annotationId === annotation.id + ) { + element.classList.add("neuroglancer-annotation-selected"); + } + return element; } private setColumnWidth(column: number, width: number) { @@ -848,141 +880,6 @@ export class AnnotationLayerView extends Tab { this.updateHoverView(); this.updateSelectionView(); } - - private makeAnnotationListElement( - annotation: Annotation, - state: AnnotationLayerState, - ) { - const chunkTransform = state.chunkTransform - .value as ChunkTransformParameters; - const element = document.createElement("div"); - element.classList.add("neuroglancer-annotation-list-entry"); - element.dataset.color = state.displayState.color.toString(); - element.style.gridTemplateColumns = this.gridTemplate; - const icon = document.createElement("div"); - icon.className = "neuroglancer-annotation-icon"; - icon.textContent = annotationTypeHandlers[annotation.type].icon; - element.appendChild(icon); - - let deleteButton: HTMLElement | undefined; - - const maybeAddDeleteButton = () => { - if (state.source.readonly) return; - if (deleteButton !== undefined) return; - deleteButton = makeDeleteButton({ - title: "Delete annotation", - onClick: (event) => { - event.stopPropagation(); - event.preventDefault(); - const ref = state.source.getReference(annotation.id); - try { - state.source.delete(ref); - } finally { - ref.dispose(); - } - }, - }); - deleteButton.classList.add("neuroglancer-annotation-list-entry-delete"); - element.appendChild(deleteButton); - }; - - let numRows = 0; - visitTransformedAnnotationGeometry( - annotation, - chunkTransform, - (layerPosition, isVector) => { - isVector; - ++numRows; - const position = document.createElement("div"); - position.className = "neuroglancer-annotation-position"; - element.appendChild(position); - let i = 0; - const addDims = ( - viewDimensionIndices: readonly number[], - layerDimensionIndices: readonly number[], - ) => { - for (const viewDim of viewDimensionIndices) { - const layerDim = layerDimensionIndices[viewDim]; - if (layerDim !== -1) { - const coord = Math.floor(layerPosition[layerDim]); - const coordElement = document.createElement("div"); - const text = coord.toString(); - coordElement.textContent = text; - coordElement.classList.add("neuroglancer-annotation-coordinate"); - coordElement.style.gridColumn = `dim ${i + 1}`; - this.setColumnWidth(i, text.length); - position.appendChild(coordElement); - } - ++i; - } - }; - addDims( - this.globalDimensionIndices, - chunkTransform.modelTransform.globalToRenderLayerDimensions, - ); - addDims( - this.localDimensionIndices, - chunkTransform.modelTransform.localToRenderLayerDimensions, - ); - maybeAddDeleteButton(); - }, - ); - if (annotation.description) { - ++numRows; - const description = document.createElement("div"); - description.classList.add("neuroglancer-annotation-description"); - description.textContent = annotation.description; - element.appendChild(description); - } - icon.style.gridRow = `span ${numRows}`; - if (deleteButton !== undefined) { - deleteButton.style.gridRow = `span ${numRows}`; - } - element.addEventListener("mouseenter", () => { - this.displayState.hoverState.value = { - id: annotation.id, - partIndex: 0, - annotationLayerState: state, - }; - this.layer.selectAnnotation(state, annotation.id, false); - }); - element.addEventListener("action:select-position", (event) => { - event.stopPropagation(); - this.layer.selectAnnotation(state, annotation.id, "toggle"); - }); - - element.addEventListener("action:pin-annotation", (event) => { - event.stopPropagation(); - this.layer.selectAnnotation(state, annotation.id, true); - }); - - element.addEventListener("action:move-to-annotation", (event) => { - event.stopPropagation(); - event.preventDefault(); - const { layerRank } = chunkTransform; - const chunkPosition = new Float32Array(layerRank); - const layerPosition = new Float32Array(layerRank); - getCenterPosition(chunkPosition, annotation); - matrix.transformPoint( - layerPosition, - chunkTransform.chunkToLayerTransform, - layerRank + 1, - chunkPosition, - layerRank, - ); - setLayerPosition(this.layer, chunkTransform, layerPosition); - }); - - const selectionState = this.selectedAnnotationState.value; - if ( - selectionState !== undefined && - selectionState.annotationLayerState === state && - selectionState.annotationId === annotation.id - ) { - element.classList.add("neuroglancer-annotation-selected"); - } - return element; - } } export class AnnotationTab extends Tab { @@ -2178,3 +2075,127 @@ type UserLayerWithAnnotationsClass = ReturnType< export type UserLayerWithAnnotations = InstanceType; + +export function makeAnnotationListElement( + layer: UserLayerWithAnnotations, + annotation: Annotation, + state: AnnotationLayerState, + gridTemplate: string, + globalDimensionIndices: number[], + localDimensionIndices: number[], +): [HTMLDivElement, number[]] { + const chunkTransform = state.chunkTransform.value as ChunkTransformParameters; + const element = document.createElement("div"); + element.classList.add("neuroglancer-annotation-list-entry"); + element.dataset.color = state.displayState.color.toString(); + element.style.gridTemplateColumns = gridTemplate; + const icon = document.createElement("div"); + icon.className = "neuroglancer-annotation-icon"; + icon.textContent = annotationTypeHandlers[annotation.type].icon; + element.appendChild(icon); + + let deleteButton: HTMLElement | undefined; + + const maybeAddDeleteButton = () => { + if (state.source.readonly) return; + if (deleteButton !== undefined) return; + deleteButton = makeDeleteButton({ + title: "Delete annotation", + onClick: (event) => { + event.stopPropagation(); + event.preventDefault(); + const ref = state.source.getReference(annotation.id); + try { + state.source.delete(ref); + } finally { + ref.dispose(); + } + }, + }); + deleteButton.classList.add("neuroglancer-annotation-list-entry-delete"); + element.appendChild(deleteButton); + }; + + const columnWidths: number[] = []; + let numRows = 0; + visitTransformedAnnotationGeometry( + annotation, + chunkTransform, + (layerPosition, isVector) => { + isVector; + ++numRows; + const position = document.createElement("div"); + position.className = "neuroglancer-annotation-position"; + element.appendChild(position); + let i = 0; + + const addDims = ( + viewDimensionIndices: readonly number[], + layerDimensionIndices: readonly number[], + ) => { + for (const viewDim of viewDimensionIndices) { + const layerDim = layerDimensionIndices[viewDim]; + if (layerDim !== -1) { + const coord = Math.floor(layerPosition[layerDim]); + const coordElement = document.createElement("div"); + const text = coord.toString(); + coordElement.textContent = text; + coordElement.classList.add("neuroglancer-annotation-coordinate"); + coordElement.style.gridColumn = `dim ${i + 1}`; + columnWidths[i] = Math.max(columnWidths[i] || 0, text.length); + position.appendChild(coordElement); + } + ++i; + } + }; + addDims( + globalDimensionIndices, + chunkTransform.modelTransform.globalToRenderLayerDimensions, + ); + addDims( + localDimensionIndices, + chunkTransform.modelTransform.localToRenderLayerDimensions, + ); + maybeAddDeleteButton(); + }, + ); + if (annotation.description) { + ++numRows; + const description = document.createElement("div"); + description.classList.add("neuroglancer-annotation-description"); + description.textContent = annotation.description; + element.appendChild(description); + } + icon.style.gridRow = `span ${numRows}`; + if (deleteButton !== undefined) { + deleteButton.style.gridRow = `span ${numRows}`; + } + element.addEventListener("mouseenter", () => { + layer.selectAnnotation(state, annotation.id, false); + }); + element.addEventListener("action:select-position", (event) => { + event.stopPropagation(); + layer.selectAnnotation(state, annotation.id, "toggle"); + }); + element.addEventListener("action:pin-annotation", (event) => { + event.stopPropagation(); + layer.selectAnnotation(state, annotation.id, true); + }); + element.addEventListener("action:move-to-annotation", (event) => { + event.stopPropagation(); + event.preventDefault(); + const { layerRank } = chunkTransform; + const chunkPosition = new Float32Array(layerRank); + const layerPosition = new Float32Array(layerRank); + getCenterPosition(chunkPosition, annotation); + matrix.transformPoint( + layerPosition, + chunkTransform.chunkToLayerTransform, + layerRank + 1, + chunkPosition, + layerRank, + ); + setLayerPosition(layer, chunkTransform, layerPosition); + }); + return [element, columnWidths]; +} diff --git a/src/util/json.ts b/src/util/json.ts index 5c4d21563f..b8e0826690 100644 --- a/src/util/json.ts +++ b/src/util/json.ts @@ -706,6 +706,16 @@ export function verifyIntegerArray(a: unknown) { return a; } +export function verifyFloatArray(a: unknown) { + if (!Array.isArray(a)) { + throw new Error(`Expected array, received: ${JSON.stringify(a)}.`); + } + for (const x of a) { + verifyFloat(x); + } + return a; +} + export function verifyBoolean(x: any) { if (typeof x !== "boolean") { throw new Error(`Expected boolean, received: ${JSON.stringify(x)}`); From 51f408381cbbe2641598e264cb1dd507f52d7fcb Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Fri, 12 Jan 2024 15:01:49 -0500 Subject: [PATCH 06/46] added custom error handler support for credentials provider so middleauth can catch terms of service errors and prompt the user to agree to the terms of service fixed lint errors and minor cleanup --- src/credentials_provider/http_request.ts | 9 +- src/credentials_provider/index.ts | 23 +++ .../middleauth/credentials_provider.ts | 187 ++++++++++++++---- 3 files changed, 176 insertions(+), 43 deletions(-) diff --git a/src/credentials_provider/http_request.ts b/src/credentials_provider/http_request.ts index bdf8bec3f0..fda9ef6702 100644 --- a/src/credentials_provider/http_request.ts +++ b/src/credentials_provider/http_request.ts @@ -32,7 +32,10 @@ export async function fetchOkWithCredentials( credentials: Credentials, requestInit: RequestInit & { progressListener?: ProgressListener }, ) => RequestInit & { progressListener?: ProgressListener }, - errorHandler: (httpError: HttpError, credentials: Credentials) => "refresh", + errorHandler: ( + httpError: HttpError, + credentials: Credentials, + ) => "refresh" | Promise<"refresh">, ): Promise { let credentials: CredentialsWithGeneration | undefined; for (let credentialsAttempt = 0; ; ) { @@ -56,7 +59,9 @@ export async function fetchOkWithCredentials( ); } catch (error) { if (error instanceof HttpError) { - if (errorHandler(error, credentials.credentials) === "refresh") { + if ( + (await errorHandler(error, credentials.credentials)) === "refresh" + ) { if (++credentialsAttempt === maxCredentialsAttempts) throw error; continue; } diff --git a/src/credentials_provider/index.ts b/src/credentials_provider/index.ts index 1290e24036..7f2f04a0d1 100644 --- a/src/credentials_provider/index.ts +++ b/src/credentials_provider/index.ts @@ -18,8 +18,10 @@ * @file Generic facility for providing authentication/authorization credentials. */ +import type { OAuth2Credentials } from "#src/credentials_provider/oauth2.js"; import type { Owned } from "#src/util/disposable.js"; import { RefCounted } from "#src/util/disposable.js"; +import type { HttpError } from "#src/util/http_request.js"; import type { AsyncMemoizeWithProgress } from "#src/util/memoize.js"; import { asyncMemoizeWithProgress, StringMemoize } from "#src/util/memoize.js"; import type { ProgressOptions } from "#src/util/progress_listener.js"; @@ -45,6 +47,27 @@ export abstract class CredentialsProvider extends RefCounted { invalidCredentials?: CredentialsWithGeneration, options?: Partial, ) => Promise>; + + errorHandler? = async ( + error: HttpError, + credentials: OAuth2Credentials, + ): Promise<"refresh"> => { + const { status } = error; + if (status === 401) { + // 401: Authorization needed. OAuth2 token may have expired. + return "refresh"; + } + if (status === 403 && !credentials.accessToken) { + // Anonymous access denied. Request credentials. + return "refresh"; + } + if (error instanceof Error && credentials.email !== undefined) { + error.message += ` (Using credentials for ${JSON.stringify( + credentials.email, + )})`; + } + throw error; + }; } export function makeCachedCredentialsGetter( diff --git a/src/kvstore/middleauth/credentials_provider.ts b/src/kvstore/middleauth/credentials_provider.ts index 25c5126051..3e25dcb79d 100644 --- a/src/kvstore/middleauth/credentials_provider.ts +++ b/src/kvstore/middleauth/credentials_provider.ts @@ -26,8 +26,10 @@ import { getCredentialsWithStatus, monitorAuthPopupWindow, } from "#src/credentials_provider/interactive_credentials_provider.js"; +import type { OAuth2Credentials } from "#src/credentials_provider/oauth2.js"; import { StatusMessage } from "#src/status.js"; import { raceWithAbort } from "#src/util/abort.js"; +import type { HttpError } from "#src/util/http_request.js"; import { verifyObject, verifyObjectProperty, @@ -57,46 +59,83 @@ function openPopupCenter(url: string, width: number, height: number) { ); } -function waitForAuthResponseMessage( - serverUrl: string, +function waitForPopupResponseMessage( source: Window, signal: AbortSignal, -): Promise { - return new Promise((resolve, reject) => { +): Promise { + return new Promise((resolve) => { window.addEventListener( "message", (event) => { if (event.source !== source) return; - try { - const obj = verifyObject(event.data); - const accessToken = verifyObjectProperty(obj, "token", verifyString); - const appUrls = verifyObjectProperty( - obj, - "app_urls", - verifyStringArray, - ); - - const token: MiddleAuthToken = { - tokenType: "Bearer", - accessToken, - url: serverUrl, - appUrls, - }; - resolve(token); - } catch (parseError) { - reject( - new Error( - `Received unexpected authentication response: ${parseError.message}`, - ), - ); - console.error("Response received: ", event.data); - } + resolve(event.data); }, - { signal: signal }, + { signal }, ); }); } +async function waitForRemoteFlow( + url: string, + startMessage: string, + startAction: string, + retryMessage: string, + closedMessage: string, +): Promise { + const status = new StatusMessage(/*delay=*/ false, /*modal=*/ true); + const res: Promise = new Promise((f) => { + function writeStatus(message: string, buttonMessage: string) { + status.element.textContent = message + " "; + const button = document.createElement("button"); + button.textContent = buttonMessage; + status.element.appendChild(button); + + button.addEventListener("click", () => { + writeStatus(retryMessage, "Retry"); + const popup = openPopupCenter(url, 400, 650); + const closePopup = () => { + popup?.close(); + }; + window.addEventListener("beforeunload", closePopup); + const checkClosed = setInterval(() => { + if (popup?.closed) { + clearInterval(checkClosed); + writeStatus(closedMessage, "Retry"); + } + }, 1000); + + const messageListener = async (ev: MessageEvent) => { + if (ev.source === popup) { + clearInterval(checkClosed); + window.removeEventListener("message", messageListener); + window.removeEventListener("beforeunload", closePopup); + closePopup(); + f(ev.data); + } + }; + window.addEventListener("message", messageListener); + }); + } + writeStatus(startMessage, startAction); + }); + try { + return await res; + } finally { + status.dispose(); + } +} + +async function showTosForm(url: string, tosName: string) { + const data = await waitForRemoteFlow( + url, + `Before you can access ${tosName}, you need to accept its Terms of Service.`, + "Open", + "Waiting for Terms of Service agreement...", + `Terms of Service closed for ${tosName}.`, + ); + return data === "success"; +} + async function waitForLogin( serverUrl: string, signal: AbortSignal, @@ -114,7 +153,33 @@ async function waitForLogin( } monitorAuthPopupWindow(newWindow, abortController); return await raceWithAbort( - waitForAuthResponseMessage(serverUrl, newWindow, abortController.signal), + waitForPopupResponseMessage(newWindow, abortController.signal).then( + async (data) => { + try { + const obj = verifyObject(data); + const accessToken = verifyObjectProperty( + obj, + "token", + verifyString, + ); + const appUrls = verifyObjectProperty( + obj, + "app_urls", + verifyStringArray, + ); + return { + tokenType: "Bearer", + accessToken, + url: serverUrl, + appUrls, + } satisfies MiddleAuthToken; + } catch (parseError) { + throw new Error( + `Received unexpected authentication response: ${parseError.message}`, + ); + } + }, + ), signal, ); } finally { @@ -182,6 +247,7 @@ export class UnverifiedApp extends Error { export class MiddleAuthAppCredentialsProvider extends CredentialsProvider { private credentials: CredentialsWithGeneration | undefined = undefined; + private agreedToTos = false; constructor( private serverUrl: string, @@ -191,23 +257,23 @@ export class MiddleAuthAppCredentialsProvider extends CredentialsProvider { - let authInfo: any; - { - using _span = new ProgressSpan(options.progressListener, { - message: `Determining authentication server for ${this.serverUrl}`, - }); - const response = await fetch(`${this.serverUrl}/auth_info`, { - signal: options.signal, - }); - authInfo = await response.json(); + if (this.credentials && this.agreedToTos) { + return this.credentials.credentials; } + this.agreedToTos = false; + const { progressListener, signal } = options; + using _span = new ProgressSpan(progressListener, { + message: `Determining authentication server for ${this.serverUrl}`, + }); + const response = await fetch(`${this.serverUrl}/auth_info`, { + signal, + }); + const authInfo = await response.json(); const provider = this.credentialsManager.getCredentialsProvider( "middleauth", authInfo.login_url, ) as MiddleAuthCredentialsProvider; - this.credentials = await provider.get(this.credentials, options); - if (this.credentials.credentials.appUrls.includes(this.serverUrl)) { return this.credentials.credentials; } @@ -215,4 +281,43 @@ export class MiddleAuthAppCredentialsProvider extends CredentialsProvider => { + const { status } = error; + if (status === 401) { + // 401: Authorization needed. OAuth2 token may have expired. + return "refresh"; + } + if (status === 403) { + const { response } = error; + if (response) { + const { headers } = response; + const contentType = headers.get("content-type"); + if (contentType === "application/json") { + const json = await response.json(); + if (json.error && json.error === "missing_tos") { + // Missing terms of service agreement. Prompt user. + const url = new URL(json.data.tos_form_url); + url.searchParams.set("client", "ng"); + const success = await showTosForm( + url.toString(), + json.data.tos_name, + ); + if (success) { + this.agreedToTos = true; + return "refresh"; + } + } + } + } + if (!credentials.accessToken) { + // Anonymous access denied. Request credentials. + return "refresh"; + } + } + throw error; + }; } From 6197a37be4b92af742ca1aa505bff127b2f60c0e Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Wed, 21 Feb 2024 19:15:02 -0500 Subject: [PATCH 07/46] feat(segmentation_user_layer): add individual segment color picker tool --- src/layer/segmentation/style.css | 16 ++++++ src/segmentation_display_state/frontend.ts | 66 ++++++++++++++++++++-- src/widget/color.ts | 29 +++++++--- 3 files changed, 98 insertions(+), 13 deletions(-) diff --git a/src/layer/segmentation/style.css b/src/layer/segmentation/style.css index dd92be0194..359f45ee7b 100644 --- a/src/layer/segmentation/style.css +++ b/src/layer/segmentation/style.css @@ -136,3 +136,19 @@ + .neuroglancer-tool-button { margin-left: 1em; } + +.neuroglancer-segment-list-entry .neuroglancer-color-widget { + border: none; + border-color: transparent; + appearance: none; + background-color: transparent; + padding: 0; + margin: 0; + margin-left: 3px; + height: 19px; + width: 20px; +} + +.neuroglancer-segment-list-entry .neuroglancer-color-widget.overridden { + background-color: white; +} diff --git a/src/segmentation_display_state/frontend.ts b/src/segmentation_display_state/frontend.ts index f9397cae52..28f9ee99b3 100644 --- a/src/segmentation_display_state/frontend.ts +++ b/src/segmentation_display_state/frontend.ts @@ -48,14 +48,19 @@ import { isWithinSelectionPanel } from "#src/ui/selection_details.js"; import type { Uint64Map } from "#src/uint64_map.js"; import { wrapSigned32BitIntegerToUint64 } from "#src/util/bigint.js"; import { setClipboard } from "#src/util/clipboard.js"; -import { useWhiteBackground } from "#src/util/color.js"; +import { + packColor, + serializeColor, + TrackableRGB, + useWhiteBackground, +} from "#src/util/color.js"; import { RefCounted } from "#src/util/disposable.js"; import { measureElementClone } from "#src/util/dom.js"; -import type { vec3 } from "#src/util/geom.js"; -import { kOneVec, vec4 } from "#src/util/geom.js"; +import { kOneVec, vec3, vec4 } from "#src/util/geom.js"; import { parseUint64 } from "#src/util/json.js"; import { NullarySignal } from "#src/util/signal.js"; import { withSharedVisibility } from "#src/visibility_priority/frontend.js"; +import { ColorWidget } from "#src/widget/color.js"; import { makeCopyButton } from "#src/widget/copy_button.js"; import { makeEyeButton } from "#src/widget/eye_button.js"; import { makeFilterButton } from "#src/widget/filter_button.js"; @@ -337,6 +342,8 @@ const segmentWidgetTemplate = (() => { filterElement.classList.add("neuroglancer-segment-list-entry-filter"); const filterIndex = template.childElementCount; template.appendChild(filterElement); + const colorWidgetIndex = template.childElementCount; + template.appendChild(ColorWidget.template()); return { template, copyContainerIndex, @@ -347,6 +354,7 @@ const segmentWidgetTemplate = (() => { labelIndex, filterIndex, starIndex, + colorWidgetIndex, unmappedIdIndex: -1, unmappedCopyIndex: -1, }; @@ -528,6 +536,28 @@ function makeRegisterSegmentWidgetEventHandlers( const { selectedSegments } = displayState.segmentationGroupState.value; selectedSegments.set(id, !selectedSegments.has(id)); }); + + const trackableRGB = new TrackableRGB(vec3.fromValues(0, 0, 0)); + trackableRGB.changed.add(() => { + const idString = element.dataset.id!; + const id = BigInt(idString); + const color = BigInt(packColor(trackableRGB.value)); + displayState.segmentStatedColors.value.delete(id); + displayState.segmentStatedColors.value.set(id, color); + }); + + // TODO, need to register disposer? + new ColorWidget( + trackableRGB, + undefined, + children[template.colorWidgetIndex] as HTMLInputElement, + () => { + const idString = element.dataset.id!; + const id = BigInt(idString); + displayState.segmentStatedColors.value.delete(id); + }, + false, + ); }; } @@ -653,14 +683,21 @@ export class SegmentWidgetFactory