diff --git a/src/main/kotlin/BdvNotifier.kt b/src/main/kotlin/BdvNotifier.kt index 72ced99..22211e0 100644 --- a/src/main/kotlin/BdvNotifier.kt +++ b/src/main/kotlin/BdvNotifier.kt @@ -1,11 +1,10 @@ -package org.mastodon.mamut - import bdv.viewer.TimePointListener import bdv.viewer.TransformListener import graphics.scenery.utils.lazyLogger import net.imglib2.realtransform.AffineTransform3D import org.mastodon.graph.GraphChangeListener import org.mastodon.graph.GraphListener +import org.mastodon.mamut.ProjectModel import org.mastodon.mamut.model.Link import org.mastodon.mamut.model.Spot import org.mastodon.mamut.views.bdv.MamutViewBdv diff --git a/src/main/kotlin/SciviewBridge.kt b/src/main/kotlin/Manvr3dMain.kt similarity index 56% rename from src/main/kotlin/SciviewBridge.kt rename to src/main/kotlin/Manvr3dMain.kt index 816004a..f643048 100644 --- a/src/main/kotlin/SciviewBridge.kt +++ b/src/main/kotlin/Manvr3dMain.kt @@ -1,18 +1,12 @@ @file:Suppress("UNCHECKED_CAST") -package org.mastodon.mamut - import bdv.viewer.Source import bdv.viewer.SourceAndConverter import graphics.scenery.* -import graphics.scenery.attribute.spatial.DefaultSpatial -import graphics.scenery.attribute.spatial.Spatial import graphics.scenery.backends.RenderConfigReader import graphics.scenery.controls.OpenVRHMD import graphics.scenery.controls.behaviours.SelectCommand import graphics.scenery.controls.behaviours.WithCameraDelegateBase -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard import graphics.scenery.utils.extensions.minus import graphics.scenery.utils.extensions.plus import graphics.scenery.utils.extensions.times @@ -38,6 +32,8 @@ import org.mastodon.adapter.TimepointModelAdapter import org.mastodon.collection.RefCollections import org.mastodon.mamut.model.Link import org.mastodon.mamut.model.Spot +import org.mastodon.mamut.ui.Manvr3dUIMig +import org.mastodon.mamut.util.DataAxes import org.mastodon.mamut.views.bdv.MamutViewBdv import org.mastodon.model.tag.TagSetStructure import org.mastodon.ui.coloring.DefaultGraphColorGenerator @@ -48,11 +44,11 @@ import org.scijava.ui.behaviour.ClickBehaviour import org.scijava.ui.behaviour.DragBehaviour import org.scijava.ui.behaviour.util.Actions import sc.iview.SciView -import sc.iview.commands.analysis.CellTrackingBase -import sc.iview.commands.analysis.TimepointObserver -import sc.iview.commands.analysis.EyeTracking -import util.SphereLinkNodes -import util.updateInstanceBuffers +import org.mastodon.mamut.ProjectModel +import graphics.scenery.utils.TimepointObserver +import vr.EyeTracking +import util.GeometryHandler +import vr.CellTrackingBase import java.util.concurrent.Executors import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.TimeUnit @@ -64,8 +60,7 @@ import kotlin.math.* import kotlin.time.Duration.Companion.seconds import kotlin.time.TimeSource - -class SciviewBridge: TimepointObserver { +class Manvr3dMain: TimepointObserver { private val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) //data source stuff val mastodon: ProjectModel @@ -87,7 +82,7 @@ class SciviewBridge: TimepointObserver { var updateVolAutomatically = true override fun toString(): String { - val sb = StringBuilder("Mastodon-sciview bridge internal settings:\n") + val sb = StringBuilder("Manvr3d internal settings:\n") sb.append(" SOURCE_ID = $sourceID\n") sb.append(" SOURCE_USED_RES_LEVEL = $volumeMipmapLevel\n") sb.append(" INTENSITY_CONTRAST = ${intensity.contrast}\n") @@ -101,18 +96,17 @@ class SciviewBridge: TimepointObserver { //data sink stuff val sciviewWin: SciView - val sphereLinkNodes: SphereLinkNodes + val geometryHandler: GeometryHandler //sink scene graph structuring nodes - val axesParent: Node? + val axesParent: DataAxes // Worker queue for async 3D updating private val updateQueue = LinkedBlockingQueue<() -> Unit>() private val workerExecutor = Executors.newSingleThreadExecutor { thread -> - Thread(thread, "SphereLinkUpdateWorker").apply { isDaemon = true } + Thread(thread, "GeometryHandlerUpdateWorker").apply { isDaemon = true } } var volumeNode: Volume - val volumeTPWidget = TextBoard() var spimSource: Source // the source and converter that contains our volume data var sac: SourceAndConverter<*> @@ -123,8 +117,18 @@ class SciviewBridge: TimepointObserver { // the event watcher for BDV, needed here for the lock handling to prevent BDV from // triggering the event watcher while a spot is edited in Sciview var bdvNotifier: BdvNotifier? = null + lateinit var bdvWindow: MamutViewBdv + var currentTimepoint: Int = 0 + private set + var minTimepoint: Int = 0 + private set + var maxTimepoint: Int = 0 + private set + val currentColorizer: GraphColorGenerator + get() = getCurrentColorizer(bdvWindow) + var moveSpotInSciview: (Spot?) -> Unit? - var associatedUI: SciviewBridgeUIMig? = null + var associatedUI: Manvr3dUIMig? = null var uiFrame: JFrame? = null private var isRunning = true var isVRactive = false @@ -137,21 +141,12 @@ class SciviewBridge: TimepointObserver { var defaultVolumeRotation: Quaternionf lateinit var vrTracking: CellTrackingBase - private var adjacentEdges = mutableListOf() - private var moveInstanceVRInit: (Vector3f) -> Unit - private var moveInstanceVRDrag: (Vector3f) -> Unit - private var moveInstanceVREnd: (Vector3f) -> Unit private val pluginActions: Actions - private val predictSpotsAction: Action - private val predictSpotsCallback: ((all: Boolean) -> Unit) - private val trainSpotsAction: Action - private val trainsSpotsCallback: (() -> Unit) -// private val trainFlowAction: Action -// private val trainFlowCallback: (() -> Unit) - private val neighborLinkingAction: Action - private val neighborLinkingCallback: (() -> Unit) - private val stageSpotsCallback: (() -> Unit) + private var predictSpotsAction: Action? = null + private var trainSpotsAction: Action? = null + private val trainFlowAction: Action? = null + private var neighborLinkingAction: Action? = null constructor( mastodonMainWindow: ProjectModel, @@ -167,10 +162,9 @@ class SciviewBridge: TimepointObserver { mastodon = mastodonMainWindow sciviewWin = targetSciviewWindow sciviewWin.setPushMode(true) - detachedDPP_withOwnTime = DPP_DetachedOwnTime( - mastodon.minTimepoint, - mastodon.maxTimepoint - ) + minTimepoint = mastodon.minTimepoint + maxTimepoint = mastodon.maxTimepoint + currentTimepoint = minTimepoint //adjust the default scene's settings sciviewWin.applicationName = ("sciview for Mastodon: " + mastodon.projectName) @@ -185,7 +179,7 @@ class SciviewBridge: TimepointObserver { sciviewWin.addNode(AmbientLight(0.05f, Vector3f(1f, 1f, 1f))) //add "root" with data axes - axesParent = constructDataAxes() + axesParent = DataAxes() sciviewWin.addNode(axesParent, activePublish = false) //get necessary metadata - from image data @@ -235,160 +229,126 @@ class SciviewBridge: TimepointObserver { logger.info("volume size is ${volumeNode.boundingBox!!.max - volumeNode.boundingBox!!.min}") //add the sciview-side displaying handler for the spots - sphereLinkNodes = SphereLinkNodes(sciviewWin, this, updateQueue, mastodon, volumeNode, volumeNode) + geometryHandler = GeometryHandler(sciviewWin, this, updateQueue, mastodon, volumeNode, volumeNode) - sphereLinkNodes.showInstancedSpots(0, noTSColorizer) - sphereLinkNodes.showInstancedLinks(SphereLinkNodes.ColorMode.LUT, colorizer = noTSColorizer) + geometryHandler.showInstancedSpots(0, noTSColorizer) + geometryHandler.showInstancedLinks(GeometryHandler.ColorMode.LUT, colorizer = noTSColorizer) // lambda function that is passed to the event handler and called // when a vertex position change occurs on the BDV side moveSpotInSciview = { spot: Spot? -> spot?.let { selectedSpotInstances.clear() - sphereLinkNodes.findInstanceFromSpot(spot)?.let { selectedSpotInstances.add(it) } - sphereLinkNodes.moveAndScaleSpotInSciview(spot) } + geometryHandler.findInstanceFromSpot(spot)?.let { selectedSpotInstances.add(it) } + geometryHandler.moveAndScaleSpotInSciview(spot) } } defaultVolumePosition = volumeNode.spatial().position defaultVolumeScale = volumeNode.spatial().scale defaultVolumeRotation = volumeNode.spatial().rotation - var currentControllerPos = Vector3f() + pluginActions = mastodon.plugins.pluginActions - // Three lambdas that are passed to the sciview class to handle the three drag behavior stages with controllers - moveInstanceVRInit = fun (pos: Vector3f) { - if (mastodon.selectionModel.selectedVertices == null) { - selectedSpotInstances.clear() - return - } else { - bdvNotifier?.lockUpdates = true - selectedSpotInstances.forEach { inst -> - logger.debug("selected spot instance is $inst") - val spot = sphereLinkNodes.findSpotFromInstance(inst) - val selectedTP = spot?.timepoint ?: -1 - if (selectedTP != volumeNode.currentTimepoint) { - selectedSpotInstances.clear() - logger.warn("Tried to move a spot that was outside the current timepoint. Aborting.") - return - } else { - bdvNotifier?.lockUpdates = true - currentControllerPos = sciviewToMastodonCoords(pos) - spot?.let { s -> - adjacentEdges.addAll(s.edges().map { it.internalPoolIndex }) - logger.debug("Moving edges $adjacentEdges for spot ${spot.internalPoolIndex}.") - } - } - } - } - } + openSyncedBDV() - moveInstanceVRDrag = fun (pos: Vector3f) { - val newPos = sciviewToMastodonCoords(pos) - val movement = newPos - currentControllerPos - selectedSpotInstances.forEach { - it.spatial { - position += movement - } - sphereLinkNodes.moveSpotInBDV(it, movement) - } - sphereLinkNodes.mainSpotInstance?.updateInstanceBuffers() - sphereLinkNodes.updateLinkTransforms(adjacentEdges) - currentControllerPos = newPos - } + registerKeyboardHandlers() - moveInstanceVREnd = fun (pos: Vector3f) { - bdvNotifier?.lockUpdates = false - sphereLinkNodes.showInstancedSpots(detachedDPP_showsLastTimepoint.timepoint, - detachedDPP_showsLastTimepoint.colorizer) - adjacentEdges.clear() - bdvNotifier?.lockUpdates = false - } + submitToTaskExecutor() + } - pluginActions = mastodon.plugins.pluginActions + val eventService: EventService? + get() = sciviewWin.scijavaContext?.getService(EventService::class.java) - predictSpotsAction = pluginActions.actionMap.get("[elephant] predict spots") - predictSpotsCallback = { predictAll -> - // Limitation of Elephant: we can only predict X number of frames in the past - // So we have to temporarily move to the last TP and set the time range to the size of all TPs - val settings = ElephantMainSettingsManager.getInstance().forwardDefaultStyle - settings.timeRange = if (predictAll) volumeNode.timepointCount else 1 - logger.info("Elephant settings.timeRange was set to ${settings.timeRange}.") - val start = TimeSource.Monotonic.markNow() - val currentTP = detachedDPP_showsLastTimepoint.timepoint - val groupHandle = mastodon.groupManager.createGroupHandle() - groupHandle.groupId = 0 - val tpAdapter = TimepointModelAdapter(groupHandle.getModel(mastodon.TIMEPOINT)) - - if (predictAll) { - tpAdapter.timepoint = volumeNode.timepointCount - } - (predictSpotsAction as PredictSpotsAction).run() - logger.info("Predicting spots took ${start.elapsedNow()} ms") - if (predictAll) { - tpAdapter.timepoint = currentTP - } - sphereLinkNodes.showInstancedSpots(detachedDPP_showsLastTimepoint.timepoint, - detachedDPP_showsLastTimepoint.colorizer) - sciviewWin.camera?.showMessage("Prediction took ${start.elapsedNow()} ms", 2f, 0.2f, centered = true) + /** Train the ELEPHANT model on all timepoints. */ + fun trainSpots() { + if (trainSpotsAction == null) { + trainSpotsAction = pluginActions.actionMap.get("[elephant] train detection model (all timepoints)") } + val start = TimeSource.Monotonic.markNow() + logger.info("Training spots from all timepoints...") + (trainSpotsAction as TrainDetectionAction).run() + logger.info("Training spots took ${start.elapsedNow()} ms") + sciviewWin.camera?.showMessage("Training took ${start.elapsedNow()} ms", 2f, 0.2f, centered = true) + } - trainSpotsAction = pluginActions.actionMap.get("[elephant] train detection model (all timepoints)") - trainsSpotsCallback = { - val start = TimeSource.Monotonic.markNow() - logger.info("Training spots from all timepoints...") - (trainSpotsAction as TrainDetectionAction).run() - logger.info("Training spots took ${start.elapsedNow()} ms") - sciviewWin.camera?.showMessage("Training took ${start.elapsedNow()} ms", 2f, 0.2f, centered = true) + /** Predict spots with ELEPHANT. If [predictAll] is true, all timepoints will be predicted. + * Otherwise just the current timepoint will be predicted. */ + fun preditSpots(predictAll: Boolean) { + if (predictSpotsAction == null) { + predictSpotsAction = pluginActions.actionMap.get("[elephant] predict spots") } - - neighborLinkingAction = pluginActions.actionMap.get("[elephant] nearest neighbor linking") - neighborLinkingCallback = { - logger.info("Linking nearest neighbors...") - // Setting the NN linking range to always include the whole time range - val settings = ElephantMainSettingsManager.getInstance().forwardDefaultStyle - settings.timeRange = volumeNode.timepointCount - // Store current TP so we can revert to it after the linking - val currentTP = detachedDPP_showsLastTimepoint.timepoint - // Get the group handle and move its TP to the last TP - val groupHandle = mastodon.groupManager.createGroupHandle() - groupHandle.groupId = 0 - val tpAdapter = TimepointModelAdapter(groupHandle.getModel(mastodon.TIMEPOINT)) - tpAdapter.timepoint = volumeNode.timepointCount - (neighborLinkingAction as NearestNeighborLinkingAction).run() - // Revert to the previous TP - tpAdapter.timepoint = currentTP - sciviewWin.camera?.showMessage("Linked nearest neighbors.", 2f, 0.2f, centered = true) + // Limitation of Elephant: we can only predict X number of frames in the past + // So we have to temporarily move to the last TP and set the time range to the size of all TPs + val settings = ElephantMainSettingsManager.getInstance().forwardDefaultStyle + settings.timeRange = if (predictAll) volumeNode.timepointCount else 1 + logger.info("Elephant settings.timeRange was set to ${settings.timeRange}.") + val start = TimeSource.Monotonic.markNow() + val currentTP = currentTimepoint + val groupHandle = mastodon.groupManager.createGroupHandle() + groupHandle.groupId = 0 + val tpAdapter = TimepointModelAdapter(groupHandle.getModel(mastodon.TIMEPOINT)) + + if (predictAll) { + tpAdapter.timepoint = volumeNode.timepointCount } - - stageSpotsCallback = { - logger.info("Adding all spots to the true positive tag set...") - val tagResult = sphereLinkNodes.applyTagToAllSpots("Detection", "tp") - if (!tagResult) { - logger.warn("Could not find tag or tag set! Please ensure both exist.") - } else { - sphereLinkNodes.showInstancedSpots( - detachedDPP_showsLastTimepoint.timepoint, - detachedDPP_showsLastTimepoint.colorizer) - } + (predictSpotsAction as PredictSpotsAction).run() + logger.info("Predicting spots took ${start.elapsedNow()} ms") + if (predictAll) { + tpAdapter.timepoint = currentTP } + geometryHandler.showInstancedSpots(currentTimepoint, + currentColorizer) + sciviewWin.camera?.showMessage("Prediction took ${start.elapsedNow()} ms", 2f, 0.2f, centered = true) - openSyncedBDV() + } - registerKeyboardHandlers() + /** Prepare all spots in the scene for ELEPHANT training. */ + fun stageSpots() { + logger.info("Adding all spots to the true positive tag set...") + val tagResult = geometryHandler.applyTagToAllSpots("Detection", "tp") + if (!tagResult) { + logger.warn("Could not find tag or tag set! Please ensure both exist.") + } else { + geometryHandler.showInstancedSpots( + currentTimepoint, + currentColorizer) + } + } - submitToTaskExecutor() + /** Use the nearest-neighbor linking algorithm in ELEPHANT to connect all spots in the scene. */ + fun linkNearestNeighbors() { + if (neighborLinkingAction == null) { + neighborLinkingAction = pluginActions.actionMap.get("[elephant] nearest neighbor linking") + } + logger.info("Linking nearest neighbors...") + // Setting the NN linking range to always include the whole time range + val settings = ElephantMainSettingsManager.getInstance().forwardDefaultStyle + settings.timeRange = volumeNode.timepointCount + // Store current TP so we can revert to it after the linking + val currentTP = currentTimepoint + // Get the group handle and move its TP to the last TP + val groupHandle = mastodon.groupManager.createGroupHandle() + groupHandle.groupId = 0 + val tpAdapter = TimepointModelAdapter(groupHandle.getModel(mastodon.TIMEPOINT)) + tpAdapter.timepoint = volumeNode.timepointCount + (neighborLinkingAction as NearestNeighborLinkingAction).run() + // Revert to the previous TP + tpAdapter.timepoint = currentTP + sciviewWin.camera?.showMessage("Linked nearest neighbors.", 2f, 0.2f, centered = true) } - val eventService: EventService? - get() = sciviewWin.scijavaContext?.getService(EventService::class.java) + /** Train the ELEPHANT flow model on the scene. */ + fun trainFlow() { + // TODO + } - /** Sets the [vrResolutionScale]. Changes are only applied once [SciviewBridge.launchVR] is executed. */ + /** Sets the [vrResolutionScale]. Changes are only applied once [Manvr3dMain.launchVR] is executed. */ fun setVrResolutionScale(scale: Float) { vrResolutionScale = scale } - /** Launches a worker that sequentially executes queued spot and link updates from [SphereLinkNodes]. */ + /** Launches a worker that sequentially executes queued spot and link updates from [GeometryHandler]. */ private fun submitToTaskExecutor() { workerExecutor.submit { while (isRunning && !Thread.currentThread().isInterrupted) { @@ -410,38 +370,29 @@ class SciviewBridge: TimepointObserver { /** Centers the camera on the volume and adjusts its distance to fully fit the volume into the camera's FOV. */ private fun centerCameraOnVolume() { - // get the extent of the volume in sciview coordinates - val volSize = (volumeNode.boundingBox!!.max - volumeNode.boundingBox!!.min) * volumeNode.pixelToWorldRatio * sceneScale - val hFOVRad = Math.toRadians((sciviewWin.camera?.fov ?: 70f).toDouble()) - val aspectRatio = sciviewWin.camera?.aspectRatio() ?: 1f - val vFOVRad = 2 * atan(tan(hFOVRad / 2.0) / aspectRatio) - // calculate the maximum distances for vertical and horizontal FOV - val distanceHeight = (volSize.y / 2f) / tan(vFOVRad / 2.0) - val distanceWidth = (volSize.x / 2f) / tan(hFOVRad / 2.0) - val maxDistance = max(distanceWidth, distanceHeight) * 1.2f // add a little margin - - sciviewWin.camera?.spatial { - rotation = Quaternionf().lookAlong(Vector3f(0f, 0f, 1f), Vector3f(0f, 1f, 0f)) - position = Vector3f(0f, 0f, maxDistance.toFloat()) - } + sciviewWin.camera?.centerOnNode( + node = volumeNode, + sceneScale = volumeNode.pixelToWorldRatio * sceneScale, + resetPosition = true + ) } fun close() { stopAndDetachUI() deregisterKeyboardHandlers() - logger.info("Mastodon-sciview Bridge closing procedure: UI and keyboard handlers are removed now") - sciviewWin.setActiveNode(axesParent) - logger.info("Mastodon-sciview Bridge closing procedure: focus shifted away from our nodes") + logger.info("Manvr3d closing procedure: UI and keyboard handlers are removed now") +// sciviewWin.setActiveNode(axesParent) + logger.info("Manvr3d closing procedure: focus shifted away from our nodes") val updateGraceTime = 100L // in ms try { sciviewWin.deleteNode(volumeNode, true) - logger.debug("Mastodon-sciview Bridge closing procedure: red volume removed") + logger.debug("Manvr3d closing procedure: red volume removed") Thread.sleep(updateGraceTime) // sciviewWin.deleteNode(sphereParent, true) - logger.debug("Mastodon-sciview Bridge closing procedure: spots were removed") + logger.debug("Manvr3d closing procedure: spots were removed") } catch (e: InterruptedException) { /* do nothing */ } - sciviewWin.deleteNode(axesParent, true) +// sciviewWin.deleteNode(axesParent, true) } /** Convert a [Vector3f] from sciview space into Mastodon's voxel coordinate space, @@ -582,150 +533,100 @@ class SciviewBridge: TimepointObserver { /** Overload that implicitly uses the existing [spimSource] for [volumeIntensityProcessing] */ fun volumeIntensityProcessing() { - val srcImg = spimSource.getSource(detachedDPP_withOwnTime.timepoint, volumeMipmapLevel) as RandomAccessibleInterval + val srcImg = spimSource.getSource(currentTimepoint, volumeMipmapLevel) as RandomAccessibleInterval volumeIntensityProcessing(srcImg) } - private var bdvWinParamsProvider: DisplayParamsProvider? = null - /** Create a BDV window and launch a [BdvNotifier] instance to synchronize time point and viewing direction. */ fun openSyncedBDV() { - val bdvWin = mastodon.windowManager.createView(MamutViewBdv::class.java) - bdvWin.frame.setTitle("BDV linked to ${sciviewWin.getName()}") - //initial spots content: - bdvWinParamsProvider = DPP_BdvAdapter(bdvWin) - bdvWinParamsProvider?.let { - updateSciviewContent(it) - bdvNotifier = BdvNotifier( - // time point processor - { updateSciviewContent(it) }, - // view update processor - { updateSciviewCameraFromBDV(bdvWin) }, - // vertex update processor - moveSpotInSciview as (Spot?) -> Unit, - // graph update processor: redraws track segments and spots - { - sphereLinkNodes.showInstancedLinks(sphereLinkNodes.currentColorMode, it.colorizer) - sphereLinkNodes.showInstancedSpots(it.timepoint, it.colorizer) - }, - mastodon, - bdvWin - ) - } - } + bdvWindow = mastodon.windowManager.createView(MamutViewBdv::class.java) + bdvWindow.frame.setTitle("BDV linked to ${sciviewWin.getName()}") + + updateSciviewContent() + bdvNotifier = BdvNotifier( + { updateSciviewContent() }, + { updateSciviewCameraFromBDV() }, + moveSpotInSciview as (Spot?) -> Unit, + { + geometryHandler.showInstancedLinks(geometryHandler.currentColorMode, currentColorizer) + geometryHandler.showInstancedSpots(currentTimepoint, currentColorizer) + }, + mastodon, + bdvWindow + ) + } private var recentTagSet: TagSetStructure.TagSet? = null var recentColorizer: GraphColorGenerator? = null val noTSColorizer = DefaultGraphColorGenerator() private fun getCurrentColorizer(forThisBdv: MamutViewBdv): GraphColorGenerator { - //NB: trying to avoid re-creating of new TagSetGraphColorGenerator objs with every new content rending - val colorizer: GraphColorGenerator + //NB: trying to avoid re-creating of new TagSetGraphColorGenerator objs with every new content rendering val ts = forThisBdv.coloringModel.tagSet if (ts != null) { if (ts !== recentTagSet) { recentColorizer = TagSetGraphColorGenerator(mastodon.model.tagSetModel, ts) + recentTagSet = ts } - colorizer = recentColorizer!! - } else { - colorizer = noTSColorizer + return recentColorizer!! } - recentTagSet = ts - return colorizer + return noTSColorizer } - interface DisplayParamsProvider { - val timepoint: Int - val colorizer: GraphColorGenerator + fun setTimepoint(tp: Int) { + currentTimepoint = max(minTimepoint.toDouble(), min(maxTimepoint.toDouble(), tp.toDouble())).toInt() } - internal inner class DPP_BdvAdapter(ofThisBdv: MamutViewBdv) : DisplayParamsProvider { - val bdv: MamutViewBdv = ofThisBdv - override val timepoint: Int - get() = bdv.viewerPanelMamut.state().currentTimepoint - override val colorizer: GraphColorGenerator - get() = getCurrentColorizer(bdv) + fun nextTimepoint() { + setTimepoint(currentTimepoint + 1) } - internal inner class DPP_Detached : DisplayParamsProvider { - override val timepoint: Int - get() = lastUpdatedSciviewTP - override val colorizer: GraphColorGenerator - get() = recentColorizer ?: noTSColorizer + fun prevTimepoint() { + setTimepoint(currentTimepoint - 1) } - inner class DPP_DetachedOwnTime(val min: Int, val max: Int) : DisplayParamsProvider { - - override var timepoint = 0 - set(value) { - field = max(min.toDouble(), min(max.toDouble(), value.toDouble())).toInt() - } - - fun prevTimepoint() { - timepoint = max(min.toDouble(), (timepoint - 1).toDouble()).toInt() - } - - fun nextTimepoint() { - timepoint = min(max.toDouble(), (timepoint + 1).toDouble()).toInt() - } - - override val colorizer: GraphColorGenerator - get() = recentColorizer ?: noTSColorizer - } - - /** Calls [updateSciviewTimepointFromBDV] and [SphereLinkNodes.showInstancedSpots] to update the current volume and corresponding spots. */ - fun updateSciviewContent(forThisBdv: DisplayParamsProvider) { - logger.debug("Called updateSciviewContent") - val needsUpdate = updateSciviewTimepointFromBDV(forThisBdv) - if (needsUpdate) { - sphereLinkNodes.showInstancedSpots(forThisBdv.timepoint, forThisBdv.colorizer) - sphereLinkNodes.updateLinkVisibility(forThisBdv.timepoint) - sphereLinkNodes.updateLinkColors(forThisBdv.colorizer) - } + /** Calls [updateSciviewTimepointFromBDV] and [GeometryHandler.showInstancedSpots] to update the current volume and corresponding spots. */ + fun updateSciviewContent() { + volumeNode.goToTimepoint(currentTimepoint) + geometryHandler.showInstancedSpots(currentTimepoint, currentColorizer) + geometryHandler.updateSegmentVisibility(currentTimepoint) + geometryHandler.updateLinkColors(currentColorizer) } /** Uses the current [bdvWinParamsProvider] to update the sciview spots of the current timepoint. */ fun redrawSciviewSpots() { - bdvWinParamsProvider?.let { - sphereLinkNodes.showInstancedSpots(it.timepoint, it.colorizer) - } + geometryHandler.showInstancedSpots(currentTimepoint, currentColorizer) + } + + /** Rebuild all geometry on the sciview side for the default [bdvWinParamsProvider]. */ + fun rebuildGeometry() { + logger.debug("Called rebuildGeometryCallback") + geometryHandler.showInstancedSpots(currentTimepoint, currentColorizer) + geometryHandler.showInstancedLinks(geometryHandler.currentColorMode, currentColorizer) } /** Takes a timepoint and updates the current BDV window's time accordingly. */ fun updateBDV_TimepointFromSciview(tp: Int) { - logger.debug("Updated BDV timepoint from sciview") - if (bdvWinParamsProvider != null) { - (bdvWinParamsProvider as DPP_BdvAdapter).bdv.viewerPanelMamut.state().currentTimepoint = tp - } else { - logger.warn("BDV window was likely not initialized, can't synchronize sciview timepoint to BDV window!") - } + logger.debug("Updated BDV timepoint from sciview to $tp") + bdvWindow.viewerPanelMamut.state().currentTimepoint = tp } - var lastUpdatedSciviewTP = 0 - val detachedDPP_showsLastTimepoint: DisplayParamsProvider = DPP_Detached() - /** Update the sciview content based on the timepoint from the BDV window. * Returns true if the content was updated. */ @JvmOverloads - fun updateSciviewTimepointFromBDV( - forThisBdv: DisplayParamsProvider = detachedDPP_showsLastTimepoint, - force: Boolean = false - ): Boolean { - + fun updateSciviewTimepointFromBDV(force: Boolean = false): Boolean { if (updateVolAutomatically || force) { - val currTP = forThisBdv.timepoint - if (currTP != lastUpdatedSciviewTP) { - lastUpdatedSciviewTP = currTP - val tp = forThisBdv.timepoint - volumeNode.goToTimepoint(tp) - // Only return true + val bdvTP = bdvWindow.viewerPanelMamut?.state()?.currentTimepoint ?: currentTimepoint + if (bdvTP != currentTimepoint) { + currentTimepoint = bdvTP + volumeNode.goToTimepoint(currentTimepoint) return true } } return false } - private fun updateSciviewCameraFromBDV(forThisBdv: MamutViewBdv) { + private fun updateSciviewCameraFromBDV() { // Let's not move the camera around when the user is in VR if (isVRactive) { return @@ -733,7 +634,7 @@ class SciviewBridge: TimepointObserver { val auxTransform = AffineTransform3D() val viewMatrix = Matrix4f() val viewRotation = Quaternionf() - forThisBdv.viewerPanelMamut.state().getViewerTransform(auxTransform) + bdvWindow.viewerPanelMamut.state().getViewerTransform(auxTransform) for (r in 0..2) for (c in 0..3) viewMatrix[c, r] = auxTransform[r, c].toFloat() viewMatrix.getUnnormalizedRotation(viewRotation) val camSpatial = sciviewWin.camera?.spatial() ?: return @@ -745,9 +646,9 @@ class SciviewBridge: TimepointObserver { } fun setVolumeOnlyVisibility(state: Boolean) { - val spots = sphereLinkNodes.mainSpotInstance + val spots = geometryHandler.mainSpotInstance val spotVis = spots?.visible ?: false - val links = sphereLinkNodes.mainLinkInstance + val links = geometryHandler.mainLinkInstance val linksVis = links?.visible ?: false volumeNode.visible = state @@ -765,20 +666,10 @@ class SciviewBridge: TimepointObserver { volumeNode.multiResolutionLevelLimits = level to level + 1 } - val detachedDPP_withOwnTime: DPP_DetachedOwnTime - fun showTimepoint(timepoint: Int) { - val maxTP = detachedDPP_withOwnTime.max - // if we play backwards, start with the highest TP once we reach below 0, otherwise play forward and wrap at maxTP - detachedDPP_withOwnTime.timepoint = when { - timepoint < 0 -> maxTP - timepoint > maxTP -> 0 - else -> timepoint - } - // Clear the selection between time points, otherwise we might run into problems - sphereLinkNodes.clearSelection() - updateSciviewContent(detachedDPP_withOwnTime) - vrTracking.volumeTimepointWidget.text = detachedDPP_withOwnTime.timepoint.toString() + geometryHandler.clearSelection() + updateSciviewContent() + vrTracking.volumeTimepointWidget.text = currentTimepoint.toString() } private fun registerKeyboardHandlers() { @@ -788,20 +679,18 @@ class SciviewBridge: TimepointObserver { val handler = sciviewWin.sceneryInputHandler ?: throw IllegalStateException("Could not find input handler!") val behaviourCollection = arrayOf( - BehaviourTriple(desc_DEC_SPH, key_DEC_SPH, { _, _ -> sphereLinkNodes.decreaseSphereInstanceScale(); updateUI() }), - BehaviourTriple(desc_INC_SPH, key_INC_SPH, { _, _ -> sphereLinkNodes.increaseSphereInstanceScale(); updateUI() }), - BehaviourTriple(desc_DEC_LINK, key_DEC_LINK, { _, _ -> sphereLinkNodes.decreaseLinkScale(); updateUI() }), - BehaviourTriple(desc_INC_LINK, key_INC_LINK, { _, _ -> sphereLinkNodes.increaseLinkScale(); updateUI() }), + BehaviourTriple(desc_DEC_SPH, key_DEC_SPH, { _, _ -> geometryHandler.decreaseSphereInstanceScale(); updateUI() }), + BehaviourTriple(desc_INC_SPH, key_INC_SPH, { _, _ -> geometryHandler.increaseSphereInstanceScale(); updateUI() }), + BehaviourTriple(desc_DEC_LINK, key_DEC_LINK, { _, _ -> geometryHandler.decreaseLinkScale(); updateUI() }), + BehaviourTriple(desc_INC_LINK, key_INC_LINK, { _, _ -> geometryHandler.increaseLinkScale(); updateUI() }), BehaviourTriple(desc_CTRL_WIN, key_CTRL_WIN, { _, _ -> createAndShowControllingUI() }), BehaviourTriple(desc_CTRL_INFO, key_CTRL_INFO, { _, _ -> logger.info(this.toString()) }), - BehaviourTriple(desc_PREV_TP, key_PREV_TP, { _, _ -> detachedDPP_withOwnTime.prevTimepoint() - updateSciviewContent(detachedDPP_withOwnTime) }), - BehaviourTriple(desc_NEXT_TP, key_NEXT_TP, { _, _ -> detachedDPP_withOwnTime.nextTimepoint() - updateSciviewContent(detachedDPP_withOwnTime) }), + BehaviourTriple(desc_PREV_TP, key_PREV_TP, { _, _ -> prevTimepoint(); updateSciviewContent() }), + BehaviourTriple(desc_NEXT_TP, key_NEXT_TP, { _, _ -> nextTimepoint(); updateSciviewContent() }), BehaviourTriple("Scale Instance Up", "ctrl E", - {_, _ -> sphereLinkNodes.changeSpotRadius(selectedSpotInstances, 1.1f)}), + {_, _ -> geometryHandler.changeSpotRadius(selectedSpotInstances, 1.1f)}), BehaviourTriple("Scale Instance Down", "ctrl Q", - {_, _ -> sphereLinkNodes.changeSpotRadius(selectedSpotInstances, 0.9f)}), + {_, _ -> geometryHandler.changeSpotRadius(selectedSpotInstances, 0.9f)}), ) behaviourCollection.forEach { @@ -818,19 +707,17 @@ class SciviewBridge: TimepointObserver { action = { result, _, _ -> if (result.matches.isNotEmpty()) { // Remove previous selections first - sphereLinkNodes.clearSelection() + geometryHandler.clearSelection() // Try to cast the result to an instance, or clear the existing selection if it fails selectedSpotInstances.add(result.matches.first().node as InstancedNode.Instance) logger.debug("selected instance {}", selectedSpotInstances) selectedSpotInstances.forEach { s -> - sphereLinkNodes.selectSpot2D(s) - sphereLinkNodes.showInstancedSpots( - detachedDPP_showsLastTimepoint.timepoint, - detachedDPP_showsLastTimepoint.colorizer - ) + geometryHandler.selectSpot2D(s) + geometryHandler.showInstancedSpots(currentTimepoint, currentColorizer) } } else { - sphereLinkNodes.clearSelection() + geometryHandler.clearSelection() + geometryHandler.showInstancedSpots(currentTimepoint, currentColorizer) } } ) @@ -863,7 +750,7 @@ class SciviewBridge: TimepointObserver { if (selectedSpotInstances.isNotEmpty()) { distance = cam.spatial().position.distance(selectedSpotInstances.first().spatial().position) currentHit = rayStart + rayDir * distance - val spot = sphereLinkNodes.findSpotFromInstance(selectedSpotInstances.first()) + val spot = geometryHandler.findSpotFromInstance(selectedSpotInstances.first()) spot?.let { edges.addAll(it.edges().map { it.internalPoolIndex }) } @@ -890,9 +777,9 @@ class SciviewBridge: TimepointObserver { currentHit = newHit it.instancedParent.updateInstanceBuffers() } - sphereLinkNodes.moveSpotInBDV(it, movement) - sphereLinkNodes.updateLinkTransforms(edges) - sphereLinkNodes.links.values + geometryHandler.moveSpotInBDV(it, movement) + geometryHandler.updateLinkTransforms(edges) + geometryHandler.links.values } } @@ -902,11 +789,75 @@ class SciviewBridge: TimepointObserver { override fun end(x: Int, y: Int) { edges.clear() bdvNotifier?.lockUpdates = false - sphereLinkNodes.showInstancedSpots(detachedDPP_showsLastTimepoint.timepoint, - detachedDPP_showsLastTimepoint.colorizer) + geometryHandler.showInstancedSpots(currentTimepoint, + currentColorizer) } } + var timeSinceUndo = TimeSource.Monotonic.markNow() + + /** Reverts to the point previously saved by Mastodon's undo recorder. + * Performs redo events if [redo] is set to true. */ + fun undoRedo(redo: Boolean = false) { + val now = TimeSource.Monotonic.markNow() + if (now.minus(timeSinceUndo) > 0.5.seconds) { + if (redo) { + mastodon.model.redo() + logger.info("Redid last change.") + } else { + mastodon.model.undo() + logger.info("Undid last change.") + } + timeSinceUndo = now + } + } + + /** */ + fun mergeSelectionAndUpdate() { + val spots = RefCollections.createRefList(mastodon.model.graph.vertices()) + spots.addAll(selectedSpotInstances.map { geometryHandler.findSpotFromInstance(it) }.distinct()) + geometryHandler.mergeSpots(spots) + geometryHandler.clearSelection() + geometryHandler.showInstancedSpots(currentTimepoint, currentColorizer) + } + + fun mergeOverlapsAndUpdate(tp: Int = volumeNode.currentTimepoint) { + geometryHandler.mergeOverlappingSpots(tp) + geometryHandler.showInstancedSpots(currentTimepoint, currentColorizer) + } + + /** Deletes the whole graph and updates the geometry. */ + fun deleteGraphAndUpdate() { + val spots = RefCollections.createRefSet(mastodon.model.graph.vertices()) + spots.addAll(mastodon.model.graph.vertices()) + geometryHandler.deleteSpots(spots) + rebuildGeometry() + } + + /** Deletes all annotations from this timepoint. */ + fun deleteTimepointAndUpdate(tp: Int) { + val tp = mastodon.model.spatioTemporalIndex.getSpatialIndex(tp) + val spots = RefCollections.createRefSet(mastodon.model.graph.vertices()) + spots.addAll(tp) + geometryHandler.deleteSpots(spots) + rebuildGeometry() + } + + /** Recenter and set default scaling for the volume, then center camera on the volume. */ + fun resetView() { + volumeNode.spatial { + position = Vector3f(0f) + scale = defaultVolumeScale + rotation = defaultVolumeRotation + needsUpdate = true + needsUpdateWorld = true + } + centerCameraOnVolume() + // TODO this is a hacky workaround for the geometry not updating properly when resetting the volume + rebuildGeometry() + rebuildGeometry() + } + /** Starts the sciview VR environment and optionally the eye tracking environment, * depending on the user's selection in the UI. Sends spot and track manipulation callbacks to the VR environment. */ fun launchVR(wantEyeTracking: Boolean = true): Boolean { @@ -925,123 +876,19 @@ class SciviewBridge: TimepointObserver { thread { vrTracking = if (useEyeTrackers) { - val eyeTracking = EyeTracking(sciviewWin, vrResolutionScale) + val eyeTracking = EyeTracking(sciviewWin, this, geometryHandler, vrResolutionScale) if (eyeTracking.establishEyeTrackerConnection()) { eyeTracking } else { useEyeTrackers = false - CellTrackingBase(sciviewWin, vrResolutionScale) + CellTrackingBase(sciviewWin, this, geometryHandler, vrResolutionScale) } } else { - CellTrackingBase(sciviewWin, vrResolutionScale) + CellTrackingBase(sciviewWin, this, geometryHandler, vrResolutionScale) } sciviewWin.getSceneryRenderer()?.setRenderingQuality(RenderConfigReader.RenderingQuality.Low) - // Pass track and spot handling callbacks to sciview - vrTracking.trackCreationCallback = sphereLinkNodes.addTrackToMastodon - vrTracking.spotCreateDeleteCallback = sphereLinkNodes.addOrRemoveSpots - vrTracking.spotSelectCallback = sphereLinkNodes.selectClosestSpotsVR - vrTracking.spotMoveInitCallback = moveInstanceVRInit - vrTracking.spotMoveDragCallback = moveInstanceVRDrag - vrTracking.spotMoveEndCallback = moveInstanceVREnd - vrTracking.singleLinkTrackedCallback = sphereLinkNodes.addTrackedPoint - vrTracking.toggleTrackingPreviewCallback = sphereLinkNodes.toggleLinkPreviews - vrTracking.rebuildGeometryCallback = { - logger.debug("Called rebuildGeometryCallback") - sphereLinkNodes.showInstancedSpots( - detachedDPP_showsLastTimepoint.timepoint, - detachedDPP_showsLastTimepoint.colorizer - ) - sphereLinkNodes.showInstancedLinks( - sphereLinkNodes.currentColorMode, - detachedDPP_showsLastTimepoint.colorizer - ) - } - vrTracking.predictSpotsCallback = predictSpotsCallback - vrTracking.trainSpotsCallback = trainsSpotsCallback - vrTracking.trainFlowCallback = null - vrTracking.neighborLinkingCallback = neighborLinkingCallback - vrTracking.stageSpotsCallback = stageSpotsCallback - vrTracking.getSelectionCallback = { - selectedSpotInstances.toList() - } - vrTracking.scaleSpotsCallback = { factor, update -> - sphereLinkNodes.changeSpotRadius(selectedSpotInstances, factor, update) - } - - var timeSinceUndo = TimeSource.Monotonic.markNow() - vrTracking.mastodonUndoRedoCallback = { undo -> - val now = TimeSource.Monotonic.markNow() - if (now.minus(timeSinceUndo) > 0.5.seconds) { - if (undo) { - mastodon.model.undo() - } else { - mastodon.model.redo() - } - logger.info("Undid last change.") - timeSinceUndo = now - } - } - - vrTracking.setSpotVisCallback = { state -> - sphereLinkNodes.mainSpotInstance?.let { - it.visible = state - it.updateInstanceBuffers() - } - } - vrTracking.setTrackVisCallback = { state -> - sphereLinkNodes.mainLinkInstance?.let { - it.visible = state - it.updateInstanceBuffers() - } - } - vrTracking.setVolumeVisCallback = { state -> - setVolumeOnlyVisibility(state) - } - vrTracking.mergeOverlapsCallback = { tp -> - sphereLinkNodes.mergeOverlappingSpots(tp) - sphereLinkNodes.showInstancedSpots( - detachedDPP_showsLastTimepoint.timepoint, - detachedDPP_showsLastTimepoint.colorizer - ) - } - vrTracking.mergeSelectedCallback = { - val spots = RefCollections.createRefList(mastodon.model.graph.vertices()) - spots.addAll(selectedSpotInstances.map { sphereLinkNodes.findSpotFromInstance(it) }) - sphereLinkNodes.mergeSpots(spots) - sphereLinkNodes.clearSelection() - sphereLinkNodes.showInstancedSpots( - detachedDPP_showsLastTimepoint.timepoint, - detachedDPP_showsLastTimepoint.colorizer - ) - } - vrTracking.deleteGraphCallback = { - val spots = RefCollections.createRefSet(mastodon.model.graph.vertices()) - spots.addAll(mastodon.model.graph.vertices()) - sphereLinkNodes.deleteSpots(spots) - vrTracking.rebuildGeometryCallback?.invoke() - } - vrTracking.deleteTimepointCallback = { - val tp = mastodon.model.spatioTemporalIndex.getSpatialIndex(detachedDPP_showsLastTimepoint.timepoint) - val spots = RefCollections.createRefSet(mastodon.model.graph.vertices()) - spots.addAll(tp) - sphereLinkNodes.deleteSpots(spots) - vrTracking.rebuildGeometryCallback?.invoke() - } - vrTracking.resetViewCallback = { - volumeNode.spatial { - position = Vector3f(0f) - scale = defaultVolumeScale - rotation = defaultVolumeRotation - needsUpdate = true - needsUpdateWorld = true - } - centerCameraOnVolume() - // TODO this is a hacky workaround for the geometry not updating properly when resetting the volume - vrTracking.rebuildGeometryCallback?.invoke() - vrTracking.rebuildGeometryCallback?.invoke() - } - // register the bridge as an observer to the timepoint changes by the user in VR, + // register manvr3d as an observer to the timepoint changes by the user in VR, // allowing us to get updates via the onTimepointChanged() function vrTracking.registerObserver(this) @@ -1078,30 +925,20 @@ class SciviewBridge: TimepointObserver { /** Implementation of the [TimepointObserver] interface; this method is called whenever the VR user triggers * a timepoint change or plays the animation */ override fun onTimePointChanged(timepoint: Int) { - logger.debug("Called onTimepointChanged") - updateBDV_TimepointFromSciview(timepoint) - showTimepoint(timepoint) + logger.debug("Called onTimepointChanged with $timepoint") + setTimepoint(when { + timepoint < 0 -> maxTimepoint + timepoint > maxTimepoint -> 0 + else -> timepoint + }) + updateBDV_TimepointFromSciview(currentTimepoint) + showTimepoint(currentTimepoint) } /** Quickly flashes the volume's bounding grid to indicate the borders of the volume. */ - fun flashBoundingGrid(flashColor: Vector3f = Vector3f(0.95f, 0.25f, 0.15f)) { + fun flashVolumeGrid() { val bg = volumeNode.children.filterIsInstance() - bg.firstOrNull()?.let { grid -> - val initVisibility = grid.visible - val initColor = grid.gridColor - thread { - grid.gridColor = flashColor - var count = 7 - while (count > 0) { - grid.visible = !grid.visible - grid.spatial().needsUpdate = true - Thread.sleep(100) - count -= 1 - } - grid.visible = initVisibility - grid.gridColor = initColor - } - } + bg.firstOrNull()?.flashGrid() } private fun deregisterKeyboardHandlers() { @@ -1128,7 +965,7 @@ class SciviewBridge: TimepointObserver { val panel = JPanel() add(panel) setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE) - associatedUI = SciviewBridgeUIMig(this@SciviewBridge, panel) + associatedUI = Manvr3dUIMig(this@Manvr3dMain, panel) pack() isVisible = true } @@ -1137,7 +974,7 @@ class SciviewBridge: TimepointObserver { fun stopAndDetachUI() { isRunning = false workerExecutor.shutdownNow() - logger.info("Stopped bridge worker queue.") + logger.info("Stopped manvr3d worker queue.") try { // Wait for graceful termination if (!workerExecutor.awaitTermination(5, TimeUnit.SECONDS)) { @@ -1180,35 +1017,6 @@ class SciviewBridge: TimepointObserver { hl.spatial().rotation = Quaternionf().rotateY(Math.PI.toFloat()) } - fun constructDataAxes(): Node { - //add the data axes - val AXES_LINE_WIDTHS = 0.01f - val AXES_LINE_LENGTHS = 0.1f - // - val axesParent = Group() - axesParent.name = "Data Axes" - // - var c = Cylinder(AXES_LINE_WIDTHS / 2.0f, AXES_LINE_LENGTHS, 12) - c.name = "Data x axis" - c.material().diffuse = Vector3f(1f, 0f, 0f) - val halfPI = Math.PI.toFloat() / 2.0f - c.spatial().rotation = Quaternionf().rotateLocalZ(-halfPI) - axesParent.addChild(c) - // - c = Cylinder(AXES_LINE_WIDTHS / 2.0f, AXES_LINE_LENGTHS, 12) - c.name = "Data y axis" - c.material().diffuse = Vector3f(0f, 1f, 0f) - c.spatial().rotation = Quaternionf().rotateLocalZ(Math.PI.toFloat()) - axesParent.addChild(c) - // - c = Cylinder(AXES_LINE_WIDTHS / 2.0f, AXES_LINE_LENGTHS, 12) - c.name = "Data z axis" - c.material().diffuse = Vector3f(0f, 0f, 1f) - c.spatial().rotation = Quaternionf().rotateLocalX(-halfPI) - axesParent.addChild(c) - return axesParent - } - // -------------------------------------------------------------------------- const val key_DEC_SPH = "O" const val key_INC_SPH = "shift O" diff --git a/src/main/kotlin/analysis/HedgehogAnalysis.kt b/src/main/kotlin/analysis/HedgehogAnalysis.kt new file mode 100644 index 0000000..fd84959 --- /dev/null +++ b/src/main/kotlin/analysis/HedgehogAnalysis.kt @@ -0,0 +1,413 @@ +package analysis + +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.utils.gaussSmoothing +import graphics.scenery.utils.lazyLogger +import graphics.scenery.utils.localMaxima +import graphics.scenery.utils.stdDev +import org.joml.Matrix4f +import org.joml.Quaternionf +import org.joml.Vector3f +import org.slf4j.LoggerFactory +import util.SpineMetadata +import java.io.File +import kotlin.collections.iterator +import kotlin.math.sqrt + +/** + * Performs analysis over a collection of eye-tracking spines (aka hedgehog). Extracts a list of local maxima from + * the sampled volume, removes statistical outliers and performs a graph optimization over the remaining maxima to + * extract the likeliest path of the cell. The companion object contains methods to load CSV files. + * @author Ulrik Günther + */ +class HedgehogAnalysis(val spines: List, val localToWorld: Matrix4f) { + + private val logger by lazyLogger() + + val timepoints = LinkedHashMap>() + + var avgConfidence = 0.0f + private set + var totalSampleCount = 0 + private set + + /** Data class for collecting track points, consisting of positions and a [SpineGraphVertex], and the averaged + * confidences of all spines. Returned by [kotlin.run]. */ + data class Track( + val points: List>, + val confidence: Float + ) + + init { + logger.info("Starting analysis with ${spines.size} spines") + + spines.forEach { spine -> + val timepoint = spine.timepoint + val current = timepoints[timepoint] + + if(current == null) { + timepoints[timepoint] = arrayListOf(spine) + } else { + current.add(spine) + } + + avgConfidence += spine.confidence + totalSampleCount++ + } + + avgConfidence /= totalSampleCount + } + + + + /** Cell positions extracted from gaze analysis are collected in this data class together with other information + * such as the volume [value] at this point, and the [previous] and [next] vertices. */ + data class SpineGraphVertex(val timepoint: Int, + val position: Vector3f, + val worldPosition: Vector3f, + val index: Int, + val value: Float, + val metadata : SpineMetadata? = null, + var previous: SpineGraphVertex? = null, + var next: SpineGraphVertex? = null) { + + fun distance(): Float { + val n = next + return if(n != null) { + val t = (n.worldPosition - this.worldPosition) + sqrt(t.x * t.x + t.y * t.y + t.z * t.z) + } else { + 0.0f + } + } + + fun drop() { + previous?.next = next + next?.previous = previous + } + + override fun toString() : String { + return "SpineGraphVertex for t=$timepoint, pos=$position,index=$index, worldPos=$worldPosition, value=$value" + } + } + + data class VertexWithDistance(val vertex: SpineGraphVertex, val distance: Float) + + fun run(): Track? { + + // Adapt thresholds based on data from the first spine + val startingThreshold = timepoints.entries.first().value.first.samples.min() * 2f + 0.002f + val localMaxThreshold = timepoints.entries.first().value.first.samples.max() * 0.2f + val zscoreThreshold = 2.0f + val removeTooFarThreshold = 5.0f + + if(timepoints.isEmpty()) { + return null + } + + + //step1: find the startingPoint by using startingThreshold + val startingPoint = timepoints.entries.firstOrNull { entry -> + entry.value.any { metadata -> metadata.samples.any { it > startingThreshold } } + } ?: return null + + logger.info("Starting point is ${startingPoint.key}/${timepoints.size} (threshold=$startingThreshold), localMayThreshold=$localMaxThreshold") + + // filter timepoints, remove all before the starting point + timepoints.filter { it.key > startingPoint.key } + .forEach { timepoints.remove(it.key) } + + // Stop timepoints after reaching 0 + val result = mutableMapOf>() + var foundZero = false + + for ((time, value) in timepoints) { + if (foundZero) { + break + } + result[time] = value + if (time == 0) { + foundZero = true + } + } + timepoints.clear() + timepoints.putAll(result) + + logger.info("${timepoints.size} timepoints left") + + // step2: find the maxIndices along the spine + // this will be a list of lists, where each entry in the first-level list + // corresponds to a time point, which then contains a list of vertices within that timepoint. + val candidates: List> = timepoints.map { tp -> + val vs = tp.value.mapIndexedNotNull { i, spine -> + // First apply a subtle smoothing kernel to prevent many close/similar local maxima + val smoothedSamples = gaussSmoothing(spine.samples, 4) + // determine local maxima (and their indices) along the spine, aka, actual things the user might have + // seen when looking into the direction of the spine + val maxIndices = localMaxima(smoothedSamples) + logger.debug("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") + + // if there actually are local maxima, generate a graph vertex for them with all the necessary metadata + if(maxIndices.isNotEmpty()) { + //maxIndices. +// filter the maxIndices which are too far away, which can be removed + //filter { it.first <1200}. + maxIndices.map { index -> + logger.debug("Generating vertex at index $index") + // get the position of the current index along the spine + val position = spine.samplePosList[index.first] + val worldPosition = localToWorld.transform((Vector3f(position)).xyzw()).xyz() + SpineGraphVertex(tp.key, + position, + worldPosition, + index.first, + index.second, + spine) + } + } else { + null + } + } + vs + }.flatten() + + logger.info("SpineGraphVertices extracted") + + // step3: connect localMaximal points between 2 candidate spines according to the shortest path principle + // get the initial vertex, this one is assumed to always be in front, and have a local maximum - aka, what + // the user looks at first is assumed to be the actual cell they want to track + val initial = candidates.first().first { it.value > startingThreshold } + var current = initial + var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> + // calculate world-space distances between current point, and all candidate + // vertices, sorting them by distance + val vertices = vs + .filter { it.value > localMaxThreshold } + .map { vertex -> + val t = current.worldPosition - vertex.worldPosition + val distance = t.length() + VertexWithDistance(vertex, distance) + } + .sortedBy { it.distance } + + val closest = vertices.firstOrNull() + if(closest != null && closest.distance > 0) { + // create a linked list between current and closest vertices + current.next = closest.vertex + closest.vertex.previous = current + current = closest.vertex + current + } else { + null + } + }.toMutableList() + + // calculate average path lengths over all + val beforeCount = shortestPath.size + var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() + var stdDevPathLength = shortestPath.map { it.distance() }.stdDev() + logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") + + fun zScore(value: Float, m: Float, sd: Float) = ((value - m)/sd) + + //step4: if some path is longer than multiple average length, it should be removed + // TODO Don't remove vertices along the path, as that doesn't translate well to Mastodon tracks. Find a different way? +// while (shortestPath.any { it.distance() >= removeTooFarThreshold * avgPathLength }) { +// shortestPath = shortestPath.filter { it.distance() < removeTooFarThreshold * avgPathLength }.toMutableList() +// shortestPath.windowed(3, 1, partialWindows = true).forEach { +// // this reconnects the neighbors after the offending vertex has been removed +// it.getOrNull(0)?.next = it.getOrNull(1) +// it.getOrNull(1)?.previous = it.getOrNull(0) +// it.getOrNull(1)?.next = it.getOrNull(2) +// it.getOrNull(2)?.previous = it.getOrNull(1) +// } +// } + + // recalculate statistics after offending vertex removal + avgPathLength = shortestPath.map { it.distance() }.average().toFloat() + stdDevPathLength = shortestPath.map { it.distance() }.stdDev().toFloat() + + //step5: remove some vertices according to zscoreThreshold +// var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } +// logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") +// while(remaining > 0) { +// val outliers = shortestPath +// .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } +// .map { +// val idx = shortestPath.indexOf(it) +// listOf(idx-1,idx,idx+1) +// }.flatten() +// +// shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() +// remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } +// +// shortestPath.windowed(3, 1, partialWindows = true).forEach { +// it.getOrNull(0)?.next = it.getOrNull(1) +// it.getOrNull(1)?.previous = it.getOrNull(0) +// it.getOrNull(1)?.next = it.getOrNull(2) +// it.getOrNull(2)?.previous = it.getOrNull(1) +// } +// logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") +// } + +// val afterCount = shortestPath.size +// logger.info("Pruned ${beforeCount - afterCount} vertices due to path length") + val singlePoints = shortestPath + .groupBy { it.timepoint } + .mapNotNull { vs -> vs.value.maxByOrNull{ it.metadata?.confidence ?: 0f } } + .filter { + (it.metadata?.direction?.dot(it.previous!!.metadata?.direction) ?: 0f) > 0.5f + } + + + logger.info("Returning ${singlePoints.size} points") + + + return Track(singlePoints.map { it.position to it}, avgConfidence) + } + + companion object { + private val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) + + fun fromIncompleteCSV(csv: File, separator: String = ","): HedgehogAnalysis { + logger.info("Loading spines from incomplete CSV at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val confidence = tokens[1].toFloat() + val samples = tokens.subList(2, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + Vector3f(0.0f), + Vector3f(0.0f), + 0.0f, + Vector3f(0.0f), + Vector3f(0.0f), + Vector3f(0.0f), + Vector3f(0.0f), + Quaternionf(), + Vector3f(0.0f), + confidence, + samples + ) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, Matrix4f()) + } + + private fun String.toVector3f(): Vector3f { + val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} + + if (array[0] == "+Inf" || array[0] == "-Inf") + return Vector3f(0.0f,0.0f,0.0f) + + return Vector3f(array[0].toFloat(),array[1].toFloat(),array[2].toFloat()) + } + + private fun String.toQuaternionf(): Quaternionf { + val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} + return Quaternionf(array[0].toFloat(), array[1].toFloat(), array[2].toFloat(), array[3].toFloat()) + } + fun fromCSVWithMatrix(csv: File, matrix4f: Matrix4f,separator: String = ";"): HedgehogAnalysis { + logger.info("Loading spines from complete CSV with Matrix at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + logger.info("lines number: " + lines.size) + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val origin = tokens[1].toVector3f() + val direction = tokens[2].toVector3f() + val localEntry = tokens[3].toVector3f() + val localExit = tokens[4].toVector3f() + val localDirection = tokens[5].toVector3f() + val headPosition = tokens[6].toVector3f() + val headOrientation = tokens[7].toQuaternionf() + val position = tokens[8].toVector3f() + val confidence = tokens[9].toFloat() + val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + origin, + direction, + 0.0f, + localEntry, + localExit, + localDirection, + headPosition, + headOrientation, + position, + confidence, + samples + ) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, matrix4f) + } + + fun fromCSV(csv: File, separator: String = ";"): HedgehogAnalysis { + logger.info("Loading spines from complete CSV at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val origin = tokens[1].toVector3f() + val direction = tokens[2].toVector3f() + val localEntry = tokens[3].toVector3f() + val localExit = tokens[4].toVector3f() + val localDirection = tokens[5].toVector3f() + val headPosition = tokens[6].toVector3f() + val headOrientation = tokens[7].toQuaternionf() + val position = tokens[8].toVector3f() + val confidence = tokens[9].toFloat() + val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + origin, + direction, + 0.0f, + localEntry, + localExit, + localDirection, + headPosition, + headOrientation, + position, + confidence, + samples + ) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, Matrix4f()) + } + } +} + +fun main(args: Array) { + val logger = LoggerFactory.getLogger("HedgehogAnalysisMain") + // main should only be called for testing purposes + if (args.isNotEmpty()) { + val file = File(args[0]) + val analysis = HedgehogAnalysis.fromCSV(file) + val results = analysis.run() + logger.info("Results: \n$results") + } +} \ No newline at end of file diff --git a/src/main/kotlin/plugins/SciviewPlugin.kt b/src/main/kotlin/plugins/SciviewPlugin.kt index 393bcb1..1a488e5 100644 --- a/src/main/kotlin/plugins/SciviewPlugin.kt +++ b/src/main/kotlin/plugins/SciviewPlugin.kt @@ -1,6 +1,6 @@ package plugins -import org.mastodon.mamut.SciviewBridge +import Manvr3dMain import org.scijava.command.Command import org.scijava.plugin.Menu import org.scijava.plugin.Plugin @@ -16,11 +16,11 @@ import javax.swing.JLabel @Plugin( type = Command::class, menuRoot = "SciView", - menu = [Menu(label = "Help", weight = HELP), Menu(label = "Mastodon Bridge", weight = HELP_HELP)] + menu = [Menu(label = "Help", weight = HELP), Menu(label = "manvr3d", weight = HELP_HELP)] ) class SciviewPlugin : Command { override fun run() { - val panel = JFrame("Mastodon-sciview Bridge Keys Overview") + val panel = JFrame("manvr3d Keys Overview") panel.setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE) val gridBagLayout = GridBagLayout() panel.layout = gridBagLayout @@ -58,12 +58,12 @@ class SciviewPlugin : Command { private fun collectRowsOfText(): Collection> { val rows: MutableCollection> = ArrayList(15) - rows.add(arrayOf(SciviewBridge.key_DEC_SPH, SciviewBridge.desc_DEC_SPH)) - rows.add(arrayOf(SciviewBridge.key_INC_SPH, SciviewBridge.desc_INC_SPH)) - rows.add(arrayOf(SciviewBridge.key_PREV_TP, SciviewBridge.desc_PREV_TP)) - rows.add(arrayOf(SciviewBridge.key_NEXT_TP, SciviewBridge.desc_NEXT_TP)) - rows.add(arrayOf(SciviewBridge.key_CTRL_WIN, SciviewBridge.desc_CTRL_WIN)) - rows.add(arrayOf(SciviewBridge.key_CTRL_INFO, SciviewBridge.desc_CTRL_INFO)) + rows.add(arrayOf(Manvr3dMain.key_DEC_SPH, Manvr3dMain.desc_DEC_SPH)) + rows.add(arrayOf(Manvr3dMain.key_INC_SPH, Manvr3dMain.desc_INC_SPH)) + rows.add(arrayOf(Manvr3dMain.key_PREV_TP, Manvr3dMain.desc_PREV_TP)) + rows.add(arrayOf(Manvr3dMain.key_NEXT_TP, Manvr3dMain.desc_NEXT_TP)) + rows.add(arrayOf(Manvr3dMain.key_CTRL_WIN, Manvr3dMain.desc_CTRL_WIN)) + rows.add(arrayOf(Manvr3dMain.key_CTRL_INFO, Manvr3dMain.desc_CTRL_INFO)) return rows } } diff --git a/src/main/kotlin/plugins/scijava/MastodonSidePlugin.kt b/src/main/kotlin/plugins/scijava/MastodonSidePlugin.kt index b8a46ed..b049dcc 100644 --- a/src/main/kotlin/plugins/scijava/MastodonSidePlugin.kt +++ b/src/main/kotlin/plugins/scijava/MastodonSidePlugin.kt @@ -3,11 +3,10 @@ package plugins.scijava import graphics.scenery.utils.lazyLogger import org.mastodon.mamut.CloseListener import org.mastodon.mamut.ProjectModel -import org.mastodon.mamut.SciviewBridge +import Manvr3dMain import org.scijava.command.Command import org.scijava.command.CommandService import org.scijava.command.DynamicCommand -import org.scijava.log.LogService import org.scijava.plugin.Parameter import org.scijava.plugin.Plugin import sc.iview.SciViewService @@ -21,7 +20,7 @@ class MastodonSidePlugin : DynamicCommand() { var tryToReuseExistingSciviewWindow = true @Parameter(label = "Also open the controlling window right away:") - var openBridgeUI = true + var openManvr3dUI = true @Parameter(label = "Choose image data channel:", choices = [], initializer = "volumeParams") var useThisChannel = "default first channel" @@ -50,7 +49,7 @@ class MastodonSidePlugin : DynamicCommand() { "mastodon", mastodon, "channelIdx", chosenChannel, "tryToReuseExistingSciviewWindow", tryToReuseExistingSciviewWindow, - "openBridgeUI", openBridgeUI + "openManvr3dUI", this@MastodonSidePlugin.openManvr3dUI ) } @@ -72,7 +71,7 @@ class MastodonSidePlugin : DynamicCommand() { var tryToReuseExistingSciviewWindow = true @Parameter(persist = false) - var openBridgeUI = true + var openManvr3dUI = true @Parameter(label = "Choose resolution level:", choices = [], initializer = "levelParams") var useThisResolutionDownscale = "[1,1,1]" @@ -107,11 +106,11 @@ class MastodonSidePlugin : DynamicCommand() { try { if (!tryToReuseExistingSciviewWindow) sciViewService.createSciView() val sv = sciViewService.getOrCreateActiveSciView() - val bridge = SciviewBridge(mastodon, channelIdx, chosenLevel, sv) - if (openBridgeUI) bridge.createAndShowControllingUI() + val manvr3d = Manvr3dMain(mastodon, channelIdx, chosenLevel, sv) + if (openManvr3dUI) manvr3d.createAndShowControllingUI() mastodon.projectClosedListeners().add(CloseListener { logger.debug("Mastodon project was closed, cleaning up in sciview:") - bridge.close() //calls also bridge.detachControllingUI(); + manvr3d.close() //calls also manvr3dMain.detachControllingUI(); }) } catch (e: Exception) { logger.error("MastodonSciview plugin error: " + e.message) diff --git a/src/main/kotlin/util/AbstractAdjustableSliderBasedControl.kt b/src/main/kotlin/ui/AbstractAdjustableSliderBasedControl.kt similarity index 98% rename from src/main/kotlin/util/AbstractAdjustableSliderBasedControl.kt rename to src/main/kotlin/ui/AbstractAdjustableSliderBasedControl.kt index 737448d..b5e6e62 100644 --- a/src/main/kotlin/util/AbstractAdjustableSliderBasedControl.kt +++ b/src/main/kotlin/ui/AbstractAdjustableSliderBasedControl.kt @@ -1,7 +1,10 @@ -package util +package ui -import java.awt.event.* -import java.util.function.Consumer +import java.awt.event.KeyEvent +import java.awt.event.KeyListener +import java.awt.event.MouseEvent +import java.awt.event.MouseListener +import java.awt.event.MouseMotionListener import javax.swing.JLabel import javax.swing.JSlider import javax.swing.JSpinner @@ -253,4 +256,4 @@ abstract class AbstractAdjustableSliderBasedControl(// ========================= const val MIN_BOUND_LIMIT = 0 const val MAX_BOUND_LIMIT = 65535 } -} +} \ No newline at end of file diff --git a/src/main/kotlin/util/AdjustableBoundsRangeSlider.kt b/src/main/kotlin/ui/AdjustableBoundsRangeSlider.kt similarity index 96% rename from src/main/kotlin/util/AdjustableBoundsRangeSlider.kt rename to src/main/kotlin/ui/AdjustableBoundsRangeSlider.kt index 3ddbd72..361f6b6 100644 --- a/src/main/kotlin/util/AdjustableBoundsRangeSlider.kt +++ b/src/main/kotlin/ui/AdjustableBoundsRangeSlider.kt @@ -1,9 +1,13 @@ -package util +package ui import bdv.ui.rangeslider.RangeSlider import net.miginfocom.swing.MigLayout -import java.awt.* -import javax.swing.* +import java.awt.Font +import javax.swing.JLabel +import javax.swing.JPanel +import javax.swing.JSpinner +import javax.swing.SpinnerNumberModel +import javax.swing.UIManager import kotlin.math.max import kotlin.math.min diff --git a/src/main/kotlin/util/AdjustableBoundsSlider.kt b/src/main/kotlin/ui/AdjustableBoundsSlider.kt similarity index 95% rename from src/main/kotlin/util/AdjustableBoundsSlider.kt rename to src/main/kotlin/ui/AdjustableBoundsSlider.kt index 8526077..940a3c4 100644 --- a/src/main/kotlin/util/AdjustableBoundsSlider.kt +++ b/src/main/kotlin/ui/AdjustableBoundsSlider.kt @@ -1,6 +1,10 @@ -package util +package ui -import java.awt.* +import java.awt.Container +import java.awt.Font +import java.awt.GridBagConstraints +import java.awt.GridBagLayout +import java.awt.Insets import javax.swing.JLabel import javax.swing.JSlider import javax.swing.JSpinner @@ -65,4 +69,4 @@ class AdjustableBoundsSlider( return AdjustableBoundsSlider(slider, spinner, lowBoundInformer, highBoundInformer) } } -} +} \ No newline at end of file diff --git a/src/main/kotlin/SciviewBridgeUI.kt b/src/main/kotlin/ui/Manvr3dUI.kt similarity index 75% rename from src/main/kotlin/SciviewBridgeUI.kt rename to src/main/kotlin/ui/Manvr3dUI.kt index 935a8ee..ad06424 100644 --- a/src/main/kotlin/SciviewBridgeUI.kt +++ b/src/main/kotlin/ui/Manvr3dUI.kt @@ -1,18 +1,34 @@ -package org.mastodon.mamut +package org.mastodon.mamut.ui import graphics.scenery.utils.lazyLogger -import util.AdjustableBoundsRangeSlider +import Manvr3dMain +import ui.AdjustableBoundsRangeSlider import util.GroupLocksHandling -import util.SphereLinkNodes -import java.awt.* +import util.GeometryHandler +import java.awt.Container +import java.awt.Dimension +import java.awt.GridBagConstraints +import java.awt.GridBagLayout +import java.awt.Insets +import java.awt.Label import java.awt.event.ActionListener import java.util.function.Consumer -import javax.swing.* +import javax.swing.JButton +import javax.swing.JCheckBox +import javax.swing.JComboBox +import javax.swing.JComponent +import javax.swing.JLabel +import javax.swing.JPanel +import javax.swing.JSeparator +import javax.swing.JSpinner +import javax.swing.JToggleButton +import javax.swing.SpinnerModel +import javax.swing.SpinnerNumberModel import javax.swing.event.ChangeEvent import javax.swing.event.ChangeListener -class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Container) { - var controlledBridge: SciviewBridge? +class Manvr3dUI(manvr3dContext: Manvr3dMain, populateThisContainer: Container) { + var manvr3dContext: Manvr3dMain? val controlsWindowPanel: Container private val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) @@ -39,7 +55,7 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co // ------------------------------------------------------------------------------------------- private fun populatePane() { - val bridge = this.controlledBridge ?: throw IllegalStateException("The passed bridge cannot be null.") + val manvr3d = this.manvr3dContext ?: throw IllegalStateException("The passed manvr3d instance cannot be null.") controlsWindowPanel.setLayout(GridBagLayout()) val c = GridBagConstraints() @@ -53,12 +69,12 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co mc.insets = Insets(0, 0, 0, 0) mc.weightx = 0.2 mc.gridx = 0 - lockGroupHandler = GroupLocksHandling(bridge, bridge.mastodon) + lockGroupHandler = GroupLocksHandling(manvr3d, manvr3d.mastodon) mastodonRowPlaceHolder.add(lockGroupHandler.createAndActivate()!!, mc) mc.weightx = 0.6 mc.gridx = 1 val openBdvBtn = JButton("Open synced Mastodon BDV") - openBdvBtn.addActionListener { bridge.openSyncedBDV() } + openBdvBtn.addActionListener { manvr3d.openSyncedBDV() } mastodonRowPlaceHolder.add(openBdvBtn, mc) // c.gridy = 0 @@ -82,21 +98,21 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co // c.gridx = 1 INTENSITY_CONTRAST = SpinnerNumberModel(1.0, -100.0, 100.0, 0.5) - insertSpinner(INTENSITY_CONTRAST, { f: Float -> bridge.intensity.contrast = f }, c) + insertSpinner(INTENSITY_CONTRAST, { f: Float -> manvr3d.intensity.contrast = f }, c) c.gridy++ c.gridx = 0 insertLabel("Apply on Volume this shifting bias:", c) // c.gridx = 1 INTENSITY_SHIFT = SpinnerNumberModel(0.0, -65535.0, 65535.0, 50.0) - insertSpinner(INTENSITY_SHIFT, { f: Float -> bridge.intensity.shift = f }, c) + insertSpinner(INTENSITY_SHIFT, { f: Float -> manvr3d.intensity.shift = f }, c) c.gridy++ c.gridx = 0 insertLabel("Apply on Volume this gamma level:", c) // c.gridx = 1 INTENSITY_GAMMA = SpinnerNumberModel(1.0, 0.1, 3.0, 0.1) - insertSpinner(INTENSITY_GAMMA, { f: Float -> bridge.intensity.gamma = f }, c) + insertSpinner(INTENSITY_GAMMA, { f: Float -> manvr3d.intensity.gamma = f }, c) c.gridy++ c.gridx = 0 insertLabel("Clamp all voxels so that their values are not above:", c) @@ -105,7 +121,7 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co INTENSITY_CLAMP_AT_TOP = SpinnerNumberModel(700.0, 0.0, 65535.0, 50.0) insertSpinner( INTENSITY_CLAMP_AT_TOP, - { f: Float -> bridge.intensity.clampTop = f }, + { f: Float -> manvr3d.intensity.clampTop = f }, c ) @@ -116,7 +132,7 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co c.gridx = 1 MIPMAP_LEVEL = SpinnerNumberModel(0, 0, 6, 1) insertSpinner(MIPMAP_LEVEL, { level: Float -> - controlledBridge?.let { + manvr3dContext?.let { // update the UI spinner to allow spinning up to the mipmap level found in the volume // subtract 1 to go from range 0 to max setMaxMipmapLevel(it.sac.spimSource.numMipmapLevels - 1) @@ -135,10 +151,10 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co c.gridwidth = 2 controlsWindowPanel.add(sliderBarPlaceHolder, c) // - INTENSITY_RANGE_MINMAX_CTRL_GUI_ELEM = AdjustableBoundsRangeSlider.createAndPlaceHere( + INTENSITY_RANGE_MINMAX_CTRL_GUI_ELEM = AdjustableBoundsRangeSlider.Companion.createAndPlaceHere( sliderBarPlaceHolder, - bridge.intensity.rangeMin.toInt(), - bridge.intensity.rangeMax.toInt(), + manvr3d.intensity.rangeMin.toInt(), + manvr3d.intensity.rangeMax.toInt(), 0, 10000 ) @@ -150,12 +166,12 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co c.gridx = 0 insertLabel("Link window range backwards", c) c.gridx = 1 - linkRangeBackwards = SpinnerNumberModel(bridge.mastodon.maxTimepoint, 0, bridge.mastodon.maxTimepoint, 1) + linkRangeBackwards = SpinnerNumberModel(manvr3d.mastodon.maxTimepoint, 0, manvr3d.mastodon.maxTimepoint, 1) insertSpinner( linkRangeBackwards, { - f: Float -> bridge.sphereLinkNodes.linkBackwardRange = f.toInt() - bridge.sphereLinkNodes.updateLinkVisibility(bridge.lastUpdatedSciviewTP) + f: Float -> manvr3d.geometryHandler.linkBackwardRange = f.toInt() + manvr3d.geometryHandler.updateSegmentVisibility(manvr3d.currentTimepoint) }, c) @@ -163,12 +179,12 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co c.gridx = 0 insertLabel("Link window range forwards:", c) c.gridx = 1 - linkRangeForwards = SpinnerNumberModel(bridge.mastodon.maxTimepoint, 0, bridge.mastodon.maxTimepoint, 1) + linkRangeForwards = SpinnerNumberModel(manvr3d.mastodon.maxTimepoint, 0, manvr3d.mastodon.maxTimepoint, 1) insertSpinner( linkRangeForwards, { - f: Float -> bridge.sphereLinkNodes.linkForwardRange = f.toInt() - bridge.sphereLinkNodes.updateLinkVisibility(bridge.lastUpdatedSciviewTP) + f: Float -> manvr3d.geometryHandler.linkForwardRange = f.toInt() + manvr3d.geometryHandler.updateSegmentVisibility(manvr3d.currentTimepoint) }, c ) @@ -189,7 +205,7 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co // add the first choice of the list manually val linkColorChoices = mutableListOf("By Spot") // get the rest of the LUTs from sciview and clean up their names - val availableLUTs = bridge.sciviewWin.getAvailableLUTs() as MutableList + val availableLUTs = manvr3d.sciviewWin.getAvailableLUTs() as MutableList for (i in availableLUTs.indices) { availableLUTs[i] = availableLUTs[i].removeSuffix(".lut") } @@ -220,7 +236,7 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co visToggleTracks = JButton("Toggle tracks") visToggleTracks.addActionListener(toggleTrackVisivility) - autoIntensityBtn = JToggleButton("Auto Intensity", bridge.isVolumeAutoAdjust) + autoIntensityBtn = JToggleButton("Auto Intensity", manvr3d.isVolumeAutoAdjust) autoIntensityBtn.addActionListener(autoAdjustIntensity) // val visButtonsPlaceholder = JPanel() @@ -248,9 +264,9 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co c.gridy++ startEyeTracking = JButton("Start Eye Tracking") - startEyeTracking.addActionListener { bridge.launchVR() } + startEyeTracking.addActionListener { manvr3d.launchVR() } stopEyeTracking = JButton("Stop Eye Tracking") - stopEyeTracking.addActionListener { bridge.stopVR() } + stopEyeTracking.addActionListener { manvr3d.stopVR() } val trackingBtnPlaceholder = JPanel() controlsWindowPanel.add(trackingBtnPlaceholder, c) @@ -269,7 +285,7 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co c.gridx = 1 c.gridwidth = 1 val closeBtn = JButton("Close") - closeBtn.addActionListener { bridge.stopAndDetachUI() } + closeBtn.addActionListener { manvr3d.stopAndDetachUI() } c.insets = Insets(0, 0, 15, 15) controlsWindowPanel.add(closeBtn, c) } @@ -358,10 +374,10 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co ) : ChangeListener { // override fun stateChanged(changeEvent: ChangeEvent) { - val bridge = this@SciviewBridgeUI.controlledBridge ?: throw IllegalStateException("Bridge is null.") + val manvr3d = this@Manvr3dUI.manvr3dContext ?: throw IllegalStateException("Manvr3d context is null.") val s = changeEvent.source as SpinnerNumberModel pushChangeToHere.accept(s.number.toFloat()) - if (bridge.updateVolAutomatically) bridge.updateSciviewTimepointFromBDV() + if (manvr3d.updateVolAutomatically) manvr3d.updateSciviewTimepointFromBDV() } } @@ -371,46 +387,46 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co val chooseLinkColormap = ActionListener { _ -> when (linkColorSelector.selectedItem) { "By Spot" -> { - controlledBridge.sphereLinkNodes.currentColorMode = SphereLinkNodes.ColorMode.SPOT + manvr3dContext.geometryHandler.currentColorMode = GeometryHandler.ColorMode.SPOT logger.info("Coloring links by spot color") } else -> { - controlledBridge.sphereLinkNodes.currentColorMode = SphereLinkNodes.ColorMode.LUT - controlledBridge.sphereLinkNodes.setLUT("${linkColorSelector.selectedItem}.lut") + manvr3dContext.geometryHandler.currentColorMode = GeometryHandler.ColorMode.LUT + manvr3dContext.geometryHandler.setLUT("${linkColorSelector.selectedItem}.lut") logger.info("Coloring links with LUT ${linkColorSelector.selectedItem}") } } - controlledBridge.sphereLinkNodes.updateLinkColors(controlledBridge.recentColorizer ?: controlledBridge.noTSColorizer) + manvr3dContext.geometryHandler.updateLinkColors(manvr3dContext.recentColorizer ?: manvr3dContext.noTSColorizer) } val chooseVolumeColormap = ActionListener { - controlledBridge.sciviewWin.setColormap(controlledBridge.volumeNode, "${volumeColorSelector.selectedItem}.lut") + manvr3dContext.sciviewWin.setColormap(manvr3dContext.volumeNode, "${volumeColorSelector.selectedItem}.lut") logger.info("Coloring volume with LUT ${volumeColorSelector.selectedItem}") } val toggleSpotsVisibility = ActionListener { - val spots = controlledBridge.volumeNode.getChildrenByName("SpotInstance").first() + val spots = manvr3dContext.volumeNode.getChildrenByName("SpotInstance").first() val newState = !spots.visible spots.visible = newState } val toggleVolumeVisibility = ActionListener { - val spots = controlledBridge.volumeNode.getChildrenByName("SpotInstance").first() + val spots = manvr3dContext.volumeNode.getChildrenByName("SpotInstance").first() val spotVis = spots.visible - val links = controlledBridge.volumeNode.getChildrenByName("LinkInstance").first() + val links = manvr3dContext.volumeNode.getChildrenByName("LinkInstance").first() val linksVis = links.visible - val newState = !controlledBridge.volumeNode.visible - controlledBridge.setVolumeOnlyVisibility(newState) + val newState = !manvr3dContext.volumeNode.visible + manvr3dContext.setVolumeOnlyVisibility(newState) spots.visible = spotVis links.visible = linksVis } val toggleTrackVisivility = ActionListener { - val links = controlledBridge.volumeNode.getChildrenByName("LinkInstance").first() + val links = manvr3dContext.volumeNode.getChildrenByName("LinkInstance").first() val newState = !links.visible links.visible = newState } val autoAdjustIntensity = ActionListener { - controlledBridge.autoAdjustIntensity() + manvr3dContext.autoAdjustIntensity() } /** @@ -430,49 +446,49 @@ class SciviewBridgeUI(controlledBridge: SciviewBridge, populateThisContainer: Co visToggleSpots.removeActionListener(toggleSpotsVisibility) visToggleVols.removeActionListener(toggleVolumeVisibility) autoIntensityBtn.removeActionListener(autoAdjustIntensity) - controlledBridge = null + manvr3dContext = null } fun updatePaneValues() { - val bridge = this.controlledBridge ?: throw IllegalStateException("Bridge is null.") - val updVolAutoBackup = bridge.updateVolAutomatically + val manvr3d = this.manvr3dContext ?: throw IllegalStateException("Manvr3d context is null.") + val updVolAutoBackup = manvr3d.updateVolAutomatically //temporarily disable because setting the controls trigger their listeners //that trigger (not all of them) the expensive volume updating - bridge.updateVolAutomatically = false - INTENSITY_CONTRAST.value = bridge.intensity.contrast - INTENSITY_SHIFT.value = bridge.intensity.shift - INTENSITY_CLAMP_AT_TOP.value = bridge.intensity.clampTop - INTENSITY_GAMMA.value = bridge.intensity.gamma - val upperValBackup = bridge.intensity.rangeMax + manvr3d.updateVolAutomatically = false + INTENSITY_CONTRAST.value = manvr3d.intensity.contrast + INTENSITY_SHIFT.value = manvr3d.intensity.shift + INTENSITY_CLAMP_AT_TOP.value = manvr3d.intensity.clampTop + INTENSITY_GAMMA.value = manvr3d.intensity.gamma + val upperValBackup = manvr3d.intensity.rangeMax INTENSITY_RANGE_MINMAX_CTRL_GUI_ELEM .rangeSlider - .value = bridge.intensity.rangeMin.toInt() + .value = manvr3d.intensity.rangeMin.toInt() //NB: this triggers a "value changed listener" which updates _both_ the value and upperValue, // which resets the value with the new one (so no change in the end) but clears upperValue // to the value the dialog was left with (forgets the new upperValue effectively) - bridge.intensity.rangeMax = upperValBackup + manvr3d.intensity.rangeMax = upperValBackup INTENSITY_RANGE_MINMAX_CTRL_GUI_ELEM .rangeSlider - .upperValue = bridge.intensity.rangeMax.toInt() + .upperValue = manvr3d.intensity.rangeMax.toInt() - autoIntensityBtn.isSelected = bridge.isVolumeAutoAdjust - bridge.updateVolAutomatically = updVolAutoBackup + autoIntensityBtn.isSelected = manvr3d.isVolumeAutoAdjust + manvr3d.updateVolAutomatically = updVolAutoBackup } val rangeSliderListener = ChangeListener { - controlledBridge.intensity.rangeMin = INTENSITY_RANGE_MINMAX_CTRL_GUI_ELEM.value.toFloat() - controlledBridge.intensity.rangeMax = INTENSITY_RANGE_MINMAX_CTRL_GUI_ELEM.upperValue.toFloat() - controlledBridge.volumeNode.minDisplayRange = controlledBridge.intensity.rangeMin - controlledBridge.volumeNode.maxDisplayRange = controlledBridge.intensity.rangeMax + manvr3dContext.intensity.rangeMin = INTENSITY_RANGE_MINMAX_CTRL_GUI_ELEM.value.toFloat() + manvr3dContext.intensity.rangeMax = INTENSITY_RANGE_MINMAX_CTRL_GUI_ELEM.upperValue.toFloat() + manvr3dContext.volumeNode.minDisplayRange = manvr3dContext.intensity.rangeMin + manvr3dContext.volumeNode.maxDisplayRange = manvr3dContext.intensity.rangeMax } init { - this.controlledBridge = controlledBridge + this.manvr3dContext = manvr3dContext controlsWindowPanel = populateThisContainer populatePane() updatePaneValues() diff --git a/src/main/kotlin/SciviewBridgeUIMig.kt b/src/main/kotlin/ui/Manvr3dUIMig.kt similarity index 65% rename from src/main/kotlin/SciviewBridgeUIMig.kt rename to src/main/kotlin/ui/Manvr3dUIMig.kt index e8b55ad..8819835 100644 --- a/src/main/kotlin/SciviewBridgeUIMig.kt +++ b/src/main/kotlin/ui/Manvr3dUIMig.kt @@ -1,16 +1,25 @@ -package org.mastodon.mamut +package org.mastodon.mamut.ui import graphics.scenery.utils.lazyLogger import net.miginfocom.swing.MigLayout -import util.AdjustableBoundsRangeSlider +import Manvr3dMain +import ui.AdjustableBoundsRangeSlider import util.GroupLocksHandling -import util.SphereLinkNodes +import util.GeometryHandler import java.awt.event.ActionListener -import javax.swing.* +import javax.swing.JButton +import javax.swing.JCheckBox +import javax.swing.JComboBox +import javax.swing.JLabel +import javax.swing.JPanel +import javax.swing.JSpinner +import javax.swing.JToggleButton +import javax.swing.SpinnerModel +import javax.swing.SpinnerNumberModel import javax.swing.event.ChangeListener -class SciviewBridgeUIMig(controlledBridge: SciviewBridge, populateThisContainer: JPanel) { - var controlledBridge: SciviewBridge? +class Manvr3dUIMig(manvr3dContext: Manvr3dMain, populateThisContainer: JPanel) { + var manvr3dContext: Manvr3dMain? val windowPanel: JPanel private val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) @@ -31,32 +40,32 @@ class SciviewBridgeUIMig(controlledBridge: SciviewBridge, populateThisContainer: lateinit var vrResolutionScale: SpinnerModel private fun populatePane() { - val bridge = this.controlledBridge ?: throw IllegalStateException("The passed bridge cannot be null.") + val manvr3d = this.manvr3dContext ?: throw IllegalStateException("The passed manvr3d instance cannot be null.") windowPanel.layout = MigLayout("insets 15", "[grow,leading] [grow]", "") // Lock Group Handling and Mastodon - lockGroupHandler = GroupLocksHandling(bridge, bridge.mastodon) + lockGroupHandler = GroupLocksHandling(manvr3d, manvr3d.mastodon) windowPanel.add(lockGroupHandler.createAndActivate()!!, "growx") val openBdvBtn = JButton("Open synced Mastodon BDV").apply { - addActionListener { bridge.openSyncedBDV() } + addActionListener { manvr3d.openSyncedBDV() } } windowPanel.add(openBdvBtn, "growx, wrap") // MIPMAP Level mipmapSpinner = addLabeledSpinner("Choose Mipmap Level", SpinnerNumberModel(0, 0, 6, 1)) { level -> - controlledBridge?.setMipmapLevel(level.toInt()) + this@Manvr3dUIMig.manvr3dContext?.setMipmapLevel(level.toInt()) } - controlledBridge?.let { + this@Manvr3dUIMig.manvr3dContext?.let { setMaxMipmapLevel(it.spimSource.numMipmapLevels - 1) } // Range Slider - intensityRangeSlider = AdjustableBoundsRangeSlider.createAndPlaceHere( + intensityRangeSlider = AdjustableBoundsRangeSlider.Companion.createAndPlaceHere( windowPanel, - bridge.intensity.rangeMin.toInt(), - bridge.intensity.rangeMax.toInt(), + manvr3d.intensity.rangeMin.toInt(), + manvr3d.intensity.rangeMax.toInt(), 0, 10000 ) @@ -65,33 +74,33 @@ class SciviewBridgeUIMig(controlledBridge: SciviewBridge, populateThisContainer: // Link range spinners linkRangeBackwards = addLabeledSpinner( "Link window range backwards", - SpinnerNumberModel(bridge.mastodon.maxTimepoint, 0, bridge.mastodon.maxTimepoint, 1) + SpinnerNumberModel(manvr3d.mastodon.maxTimepoint, 0, manvr3d.mastodon.maxTimepoint, 1) ) { value -> - bridge.sphereLinkNodes.linkBackwardRange = value.toInt() - bridge.sphereLinkNodes.updateLinkVisibility(bridge.lastUpdatedSciviewTP) + manvr3d.geometryHandler.linkBackwardRange = value.toInt() + manvr3d.geometryHandler.updateSegmentVisibility(manvr3d.currentTimepoint) } linkRangeForwards = addLabeledSpinner( "Link window range forwards", - SpinnerNumberModel(bridge.mastodon.maxTimepoint, 0, bridge.mastodon.maxTimepoint, 1) + SpinnerNumberModel(manvr3d.mastodon.maxTimepoint, 0, manvr3d.mastodon.maxTimepoint, 1) ) { value -> - bridge.sphereLinkNodes.linkForwardRange = value.toInt() - bridge.sphereLinkNodes.updateLinkVisibility(bridge.lastUpdatedSciviewTP) + manvr3d.geometryHandler.linkForwardRange = value.toInt() + manvr3d.geometryHandler.updateSegmentVisibility(manvr3d.currentTimepoint) } spotScaleFactor = addLabeledSpinner( "Spot scale factor", SpinnerNumberModel(1f, 0.1f, 10f, 0.1f) ) { value -> - bridge.sphereLinkNodes.sphereScaleFactor = value.toFloat() - bridge.redrawSciviewSpots() + manvr3d.geometryHandler.sphereScaleFactor = value.toFloat() + manvr3d.redrawSciviewSpots() } vrResolutionScale = addLabeledSpinner( "VR Resolution scale", SpinnerNumberModel(0.75f, 0.1f, 2f, 0.1f) ) { value -> - bridge.setVrResolutionScale(value.toFloat()) + manvr3d.setVrResolutionScale(value.toFloat()) } // Adding dropdowns for link LUTs and volume colors @@ -100,7 +109,7 @@ class SciviewBridgeUIMig(controlledBridge: SciviewBridge, populateThisContainer: // Link colors dropdown colorSelectorPanel.add(JLabel("Link colors:"), "gapright 10") val linkColorChoices = mutableListOf("By Spot") - val availableLUTs = bridge.sciviewWin.getAvailableLUTs() as MutableList + val availableLUTs = manvr3d.sciviewWin.getAvailableLUTs() as MutableList for (i in availableLUTs.indices) { availableLUTs[i] = availableLUTs[i].removeSuffix(".lut") } @@ -125,7 +134,7 @@ class SciviewBridgeUIMig(controlledBridge: SciviewBridge, populateThisContainer: visToggleSpots = JButton("Toggle spots").apply { addActionListener(toggleSpotsVisibility) } visToggleVols = JButton("Toggle volume").apply { addActionListener(toggleVolumeVisibility) } visToggleTracks = JButton("Toggle tracks").apply { addActionListener(toggleTrackVisibility) } - autoIntensityBtn = JToggleButton("Auto Intensity", bridge.isVolumeAutoAdjust).apply { + autoIntensityBtn = JToggleButton("Auto Intensity", manvr3d.isVolumeAutoAdjust).apply { addActionListener(autoAdjustIntensity) } @@ -140,13 +149,13 @@ class SciviewBridgeUIMig(controlledBridge: SciviewBridge, populateThisContainer: // Launch VR session toggleVR = JButton("Start VR").apply { addActionListener { - if (!bridge.isVRactive) { - val launched = bridge.launchVR(eyeTrackingToggle.isSelected) + if (!manvr3d.isVRactive) { + val launched = manvr3d.launchVR(eyeTrackingToggle.isSelected) if (launched) { toggleVR.text = "Stop VR" } } else { - bridge.stopVR() + manvr3d.stopVR() toggleVR.text = "Start VR" } } @@ -159,7 +168,7 @@ class SciviewBridgeUIMig(controlledBridge: SciviewBridge, populateThisContainer: }, "span, growx") // Close Button - val closeBtn = JButton("Close").apply { addActionListener { bridge.stopAndDetachUI() } } + val closeBtn = JButton("Close").apply { addActionListener { manvr3d.stopAndDetachUI() } } windowPanel.add(closeBtn, "span, right") } @@ -182,69 +191,72 @@ class SciviewBridgeUIMig(controlledBridge: SciviewBridge, populateThisContainer: } val rangeSliderListener = ChangeListener { - controlledBridge.intensity.rangeMin = intensityRangeSlider.value.toFloat() - controlledBridge.intensity.rangeMax = intensityRangeSlider.upperValue.toFloat() - controlledBridge.volumeNode.minDisplayRange = controlledBridge.intensity.rangeMin - controlledBridge.volumeNode.maxDisplayRange = controlledBridge.intensity.rangeMax + manvr3dContext.intensity.rangeMin = intensityRangeSlider.value.toFloat() + manvr3dContext.intensity.rangeMax = intensityRangeSlider.upperValue.toFloat() + manvr3dContext.volumeNode.minDisplayRange = manvr3dContext.intensity.rangeMin + manvr3dContext.volumeNode.maxDisplayRange = manvr3dContext.intensity.rangeMax } val chooseLinkColormap = ActionListener { _ -> when (linkColorSelector.selectedItem) { "By Spot" -> { - controlledBridge.sphereLinkNodes.currentColorMode = SphereLinkNodes.ColorMode.SPOT + manvr3dContext.geometryHandler.currentColorMode = GeometryHandler.ColorMode.SPOT logger.info("Coloring links by spot color") } + else -> { - controlledBridge.sphereLinkNodes.currentColorMode = SphereLinkNodes.ColorMode.LUT - controlledBridge.sphereLinkNodes.setLUT("${linkColorSelector.selectedItem}.lut") + manvr3dContext.geometryHandler.currentColorMode = GeometryHandler.ColorMode.LUT + manvr3dContext.geometryHandler.setLUT("${linkColorSelector.selectedItem}.lut") logger.info("Coloring links with LUT ${linkColorSelector.selectedItem}") } } - controlledBridge.sphereLinkNodes.updateLinkColors(controlledBridge.recentColorizer ?: controlledBridge.noTSColorizer) + manvr3dContext.geometryHandler.updateLinkColors( + manvr3dContext.currentColorizer + ) } val chooseVolumeColormap = ActionListener { - controlledBridge.sciviewWin.setColormap(controlledBridge.volumeNode, "${volumeColorSelector.selectedItem}.lut") + manvr3dContext.sciviewWin.setColormap(manvr3dContext.volumeNode, "${volumeColorSelector.selectedItem}.lut") logger.info("Coloring volume with LUT ${volumeColorSelector.selectedItem}") } val toggleSpotsVisibility = ActionListener { - val spots = controlledBridge.volumeNode.getChildrenByName("SpotInstance").first() + val spots = manvr3dContext.volumeNode.getChildrenByName("SpotInstance").first() val newState = !spots.visible spots.visible = newState } val toggleVolumeVisibility = ActionListener { - val newState = !controlledBridge.volumeNode.visible - controlledBridge.setVolumeOnlyVisibility(newState) + val newState = !manvr3dContext.volumeNode.visible + manvr3dContext.setVolumeOnlyVisibility(newState) } val toggleTrackVisibility = ActionListener { - val links = controlledBridge.volumeNode.getChildrenByName("LinkInstance").first() + val links = manvr3dContext.volumeNode.getChildrenByName("LinkInstance").first() val newState = !links.visible links.visible = newState } val autoAdjustIntensity = ActionListener { - controlledBridge.autoAdjustIntensity() + manvr3dContext.autoAdjustIntensity() } fun updatePaneValues() { - val bridge = this.controlledBridge ?: throw IllegalStateException("Bridge is null.") - val updVolAutoBackup = bridge.updateVolAutomatically + val manvr3d = this.manvr3dContext ?: throw IllegalStateException("Manvr3d context is null.") + val updVolAutoBackup = manvr3d.updateVolAutomatically //temporarily disable because setting the controls trigger their listeners //that trigger (not all of them) the expensive volume updating - bridge.updateVolAutomatically = false + manvr3d.updateVolAutomatically = false - spotScaleFactor.value = bridge.sphereLinkNodes.sphereScaleFactor - val upperValBackup = bridge.intensity.rangeMax + spotScaleFactor.value = manvr3d.geometryHandler.sphereScaleFactor + val upperValBackup = manvr3d.intensity.rangeMax - intensityRangeSlider.rangeSlider.value = bridge.intensity.rangeMin.toInt() + intensityRangeSlider.rangeSlider.value = manvr3d.intensity.rangeMin.toInt() //NB: this triggers a "value changed listener" which updates _both_ the value and upperValue, // which resets the value with the new one (so no change in the end) but clears upperValue // to the value the dialog was left with (forgets the new upperValue effectively) - bridge.intensity.rangeMax = upperValBackup - intensityRangeSlider.rangeSlider.upperValue = bridge.intensity.rangeMax.toInt() - autoIntensityBtn.isSelected = bridge.isVolumeAutoAdjust - bridge.updateVolAutomatically = updVolAutoBackup + manvr3d.intensity.rangeMax = upperValBackup + intensityRangeSlider.rangeSlider.upperValue = manvr3d.intensity.rangeMax.toInt() + autoIntensityBtn.isSelected = manvr3d.isVolumeAutoAdjust + manvr3d.updateVolAutomatically = updVolAutoBackup } fun deactivateAndForget() { @@ -258,13 +270,13 @@ class SciviewBridgeUIMig(controlledBridge: SciviewBridge, populateThisContainer: visToggleSpots.removeActionListener(toggleSpotsVisibility) visToggleVols.removeActionListener(toggleVolumeVisibility) autoIntensityBtn.removeActionListener(autoAdjustIntensity) - controlledBridge = null + this@Manvr3dUIMig.manvr3dContext = null } init { - this.controlledBridge = controlledBridge + this.manvr3dContext = manvr3dContext windowPanel = populateThisContainer populatePane() } -} +} \ No newline at end of file diff --git a/src/main/kotlin/util/CellTrackingButtonMapper.kt b/src/main/kotlin/util/CellTrackingButtonMapper.kt new file mode 100644 index 0000000..4bb2f1b --- /dev/null +++ b/src/main/kotlin/util/CellTrackingButtonMapper.kt @@ -0,0 +1,162 @@ +package util + +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.OpenVRHMD.OpenVRButton +import graphics.scenery.controls.OpenVRHMD.Manufacturer +import graphics.scenery.controls.TrackerRole +import graphics.scenery.utils.lazyLogger +import org.scijava.ui.behaviour.Behaviour + +/** This input mapping manager provides several preconfigured profiles for different VR controller layouts. + * The active profile is stored in [currentProfile]. + * To change profile, call [loadProfile] with the new [Manufacturer] type. + * Note that for Quest-like layouts, the lower button always equals [OpenVRButton.A] + * and the upper button is always [OpenVRButton.Menu]. */ +object CellTrackingButtonMapper { + + var eyeTracking: ButtonConfig? = null + var controllerTracking: ButtonConfig? = null + var grabObserver: ButtonConfig? = null + var grabSpot: ButtonConfig? = null + var playback: ButtonConfig? = null + var cycleMenu: ButtonConfig? = null + var faster: ButtonConfig? = null + var slower: ButtonConfig? = null + var stepFwd: ButtonConfig? = null + var stepBwd: ButtonConfig? = null + var addDeleteReset: ButtonConfig? = null + var select: ButtonConfig? = null + var move_forward_fast: ButtonConfig? = null + var move_back_fast: ButtonConfig? = null + var move_left_fast: ButtonConfig? = null + var move_right_fast: ButtonConfig? = null + var radiusIncrease: ButtonConfig? = null + var radiusDecrease: ButtonConfig? = null + + private var currentProfile: Manufacturer = Manufacturer.Oculus + + val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) + + private val profiles = mapOf( + Manufacturer.HTC to mapOf( + "eyeTracking" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Trigger), + "controllerTracking" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Trigger), + "grabObserver" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Side), + "grabSpot" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Side), + "playback" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Menu), + "cycleMenu" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Menu), + "faster" to null, + "slower" to null, + "radiusIncrease" to null, + "radiusDecrease" to null, + "stepFwd" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Left), + "stepBwd" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Right), + "addDeleteReset" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Up), + "select" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Down), + "move_forward_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Up), + "move_back_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Down), + "move_left_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Left), + "move_right_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Right), + ), + + Manufacturer.Oculus to mapOf( + "eyeTracking" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Trigger), + "controllerTracking" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Trigger), + "grabObserver" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Side), + "grabSpot" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Side), + "playback" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.A), + "cycleMenu" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Menu), +// "faster" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Up), +// "slower" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Down), + "stepFwd" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Left), + "stepBwd" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Right), + "addDeleteReset" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Menu), + "select" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.A), + "move_forward_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Up), + "move_back_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Down), + "move_left_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Left), + "move_right_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Right), + "radiusIncrease" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Up), + "radiusDecrease" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Down), + ) + ) + + init { + loadProfile(Manufacturer.Oculus) + } + + /** Load the current profile's button mapping */ + fun loadProfile(p: Manufacturer): Boolean { + currentProfile = p + val profile = profiles[currentProfile] ?: return false + eyeTracking = profile["eyeTracking"] + controllerTracking = profile["controllerTracking"] + grabObserver = profile["grabObserver"] + grabSpot = profile["grabSpot"] + playback = profile["playback"] + cycleMenu = profile["cycleMenu"] + faster = profile["faster"] + slower = profile["slower"] + stepFwd = profile["stepFwd"] + stepBwd = profile["stepBwd"] + addDeleteReset = profile["addDeleteReset"] + select = profile["select"] + move_forward_fast = profile["move_forward_fast"] + move_back_fast = profile["move_back_fast"] + move_left_fast = profile["move_left_fast"] + move_right_fast = profile["move_right_fast"] + radiusIncrease = profile["radiusIncrease"] + radiusDecrease = profile["radiusDecrease"] + return true + } + + fun getCurrentMapping(): Map?{ + return profiles[currentProfile] + } + + fun getMapFromName(name: String): ButtonConfig? { + return when (name) { + "eyeTracking" -> eyeTracking + "controllerTracking" -> controllerTracking + "grabObserver" -> grabObserver + "grabSpot" -> grabSpot + "playback" -> playback + "cycleMenu" -> cycleMenu + "faster" -> faster + "slower" -> slower + "stepFwd" -> stepFwd + "stepBwd" -> stepBwd + "addDeleteReset" -> addDeleteReset + "select" -> select + "move_forward_fast" -> move_forward_fast + "move_back_fast" -> move_back_fast + "move_left_fast" -> move_left_fast + "move_right_fast" -> move_right_fast + "radiusIncrease" -> radiusIncrease + "radiusDecrease" -> radiusDecrease + else -> null + } + } + + /** Sets a keybinding and behavior for an [hmd], using the [name] string, a [behavior] + * and the keybinding if found in the current profile. */ + fun setKeyBindAndBehavior(hmd: OpenVRHMD, name: String, behavior: Behaviour) { + val config = getMapFromName(name) + if (config != null) { + hmd.addKeyBinding(name, config.r, config.b) + hmd.addBehaviour(name, behavior) + logger.debug("Added behavior $behavior to ${config.r}, ${config.b}.") + } else { + logger.warn("No valid button mapping found for key '$name' in current profile!") + } + } +} + + +/** Combines the [TrackerRole] ([r]) and the [OpenVRHMD.OpenVRButton] ([b]) into a single configuration. */ +data class ButtonConfig ( + /** The [TrackerRole] of this button configuration. */ + var r: TrackerRole, + /** The [OpenVRButton] of this button configuration. */ + var b: OpenVRButton +) \ No newline at end of file diff --git a/src/main/kotlin/util/DataAxes.kt b/src/main/kotlin/util/DataAxes.kt new file mode 100644 index 0000000..160a420 --- /dev/null +++ b/src/main/kotlin/util/DataAxes.kt @@ -0,0 +1,41 @@ +package org.mastodon.mamut.util + +import graphics.scenery.DefaultNode +import graphics.scenery.Group +import graphics.scenery.Mesh +import graphics.scenery.Node +import graphics.scenery.attribute.renderable.HasRenderable +import graphics.scenery.attribute.spatial.HasSpatial +import graphics.scenery.primitives.Cylinder +import org.joml.Quaternionf +import org.joml.Vector3f + +class DataAxes: Mesh() { + + init { + //add the data axes + val AXES_LINE_WIDTHS = 0.01f + val AXES_LINE_LENGTHS = 0.1f + + this.name = "Data Axes" + + var c = Cylinder(AXES_LINE_WIDTHS / 2.0f, AXES_LINE_LENGTHS, 12) + c.name = "Data x axis" + c.material().diffuse = Vector3f(1f, 0f, 0f) + val halfPI = Math.PI.toFloat() / 2.0f + c.spatial().rotation = Quaternionf().rotateLocalZ(-halfPI) + this.addChild(c) + + c = Cylinder(AXES_LINE_WIDTHS / 2.0f, AXES_LINE_LENGTHS, 12) + c.name = "Data y axis" + c.material().diffuse = Vector3f(0f, 1f, 0f) + c.spatial().rotation = Quaternionf().rotateLocalZ(Math.PI.toFloat()) + this.addChild(c) + + c = Cylinder(AXES_LINE_WIDTHS / 2.0f, AXES_LINE_LENGTHS, 12) + c.name = "Data z axis" + c.material().diffuse = Vector3f(0f, 0f, 1f) + c.spatial().rotation = Quaternionf().rotateLocalX(-halfPI) + this.addChild(c) + } +} \ No newline at end of file diff --git a/src/main/kotlin/util/SphereLinkNodes.kt b/src/main/kotlin/util/GeometryHandler.kt similarity index 93% rename from src/main/kotlin/util/SphereLinkNodes.kt rename to src/main/kotlin/util/GeometryHandler.kt index 489c790..07a5198 100644 --- a/src/main/kotlin/util/SphereLinkNodes.kt +++ b/src/main/kotlin/util/GeometryHandler.kt @@ -24,7 +24,7 @@ import org.mastodon.collection.RefCollections import org.mastodon.collection.RefList import org.mastodon.collection.RefSet import org.mastodon.mamut.ProjectModel -import org.mastodon.mamut.SciviewBridge +import Manvr3dMain import org.mastodon.mamut.model.Link import org.mastodon.mamut.model.Spot import org.mastodon.spatial.SpatialIndex @@ -32,16 +32,15 @@ import org.mastodon.ui.coloring.GraphColorGenerator import org.mastodon.views.bdv.overlay.util.JamaEigenvalueDecomposition import org.scijava.event.EventService import sc.iview.SciView -import sc.iview.commands.analysis.HedgehogAnalysis.SpineGraphVertex +import analysis.HedgehogAnalysis.SpineGraphVertex import spim.fiji.spimdata.interestpoints.InterestPoint -import util.SphereLinkNodes.ColorMode.LUT -import util.SphereLinkNodes.ColorMode.SPOT +import util.GeometryHandler.ColorMode.LUT +import util.GeometryHandler.ColorMode.SPOT import java.awt.Color import java.lang.Math import java.util.* import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.atomic.AtomicInteger import kotlin.collections.plus import kotlin.collections.set import kotlin.math.pow @@ -52,14 +51,14 @@ import kotlin.time.TimeSource * in the form of instanced geometry. Spots are represented as spheres, and track segments (links) are represented as cylinders. * It handles initialization, updates and actions like addition, deletion and movement of spots. * @param sv The sciview instance to use. - * @param bridge And instance of the mastodon-sciview bridge. + * @param manvr3d An instance of manvr3d. * @param updateQueue Queue that executues scene updates sequentially in its own thread. * @param mastodonData Instance of the Mastodon ProjectModel * @param sphereParentNode Parent node for the instanced spheres * @param linkParentNode Parent node for the instanced links */ -class SphereLinkNodes( +class GeometryHandler( val sv: SciView, - val bridge: SciviewBridge, + val manvr3d: Manvr3dMain, val updateQueue: LinkedBlockingQueue<() -> Unit>, val mastodonData: ProjectModel, val sphereParentNode: Node, @@ -213,7 +212,7 @@ class SphereLinkNodes( var index = 0 logger.debug("we have ${visibleSpots.size()} spots in this Mastodon time point.") - bridge.bdvNotifier?.lockUpdates = true + manvr3d.bdvNotifier?.lockUpdates = true val vertexRef = mastodonData.model.graph.vertexRef() mastodonData.model.graph.lock.readLock().lock() for (spot in visibleSpots) { @@ -266,7 +265,7 @@ class SphereLinkNodes( index++ } - bridge.bdvNotifier?.lockUpdates = false + manvr3d.bdvNotifier?.lockUpdates = false mastodonData.model.graph.lock.readLock().unlock() // turn all leftover spots from the pool invisible var i = index @@ -392,19 +391,6 @@ class SphereLinkNodes( return quaternion } - // stretch color channels - private fun Vector3f.stretchColor(): Vector3f { - this.x.coerceIn(0f, 1f) - this.y.coerceIn(0f, 1f) - this.z.coerceIn(0f, 1f) - val max = this.max() - return this + Vector3f(1 - max) - } - - private fun Vector3f.toDoubleArray(): DoubleArray { - return this.toFloatArray().map { it.toDouble() }.toDoubleArray() - } - /** Extension function that takes a spot and colors the corresponding instance according to the [colorizer]. */ private fun InstancedNode.Instance.setColorFromSpot( s: Spot, @@ -418,7 +404,7 @@ class SphereLinkNodes( } if (!isPartyMode) { if (!randomColors) { - val col = unpackRGB(intColor) + val col = intColor.unpackRGB() this.instancedProperties["Color"] = { col } } else { val col = Random.random3DVectorFromRange(0f, 1f).stretchColor() @@ -547,7 +533,7 @@ class SphereLinkNodes( updateQueue.offer { val graph = mastodonData.model.graph mastodonData.model.setUndoPoint() - bridge.bdvNotifier?.lockUpdates = true + manvr3d.bdvNotifier?.lockUpdates = true val spatialIndex = mastodonData.model.spatioTemporalIndex.getSpatialIndex(tp) val queue = RefCollections.createRefDeque(graph.vertices()) queue.addAll(spatialIndex) @@ -574,7 +560,7 @@ class SphereLinkNodes( } mastodonData.model.graph.releaseRef(currentSpot) clearSelection() - bridge.bdvNotifier?.lockUpdates = false + manvr3d.bdvNotifier?.lockUpdates = false } } @@ -582,7 +568,10 @@ class SphereLinkNodes( * Original spots will be removed from the graph and a new merged spot is created. * Positions and radii are averaged. */ fun mergeSpots(spots: RefList) { - bridge.bdvNotifier?.lockUpdates = true + if (spots.isEmpty()) { + return + } + manvr3d.bdvNotifier?.lockUpdates = true val graph = mastodonData.model.graph val sourceRef = graph.vertexRef() val targetRef = graph.vertexRef() @@ -637,16 +626,17 @@ class SphereLinkNodes( } // Remove all old spots - spots.forEach { - graph.remove(it) + spots.forEach { spot -> + sourceRef.refTo(spot) + graph.remove(sourceRef) } logger.info("Newly merged spot now has incoming edges ${newSpot.incomingEdges().map { it.internalPoolIndex }}" + - "and outgoing edges ${newSpot.outgoingEdges().map { it.internalPoolIndex }}") + " and outgoing edges ${newSpot.outgoingEdges().map { it.internalPoolIndex }}") graph.lock.writeLock().unlock() graph.releaseRef(sourceRef) graph.releaseRef(targetRef) - bridge.bdvNotifier?.lockUpdates = false + manvr3d.bdvNotifier?.lockUpdates = false } /** Extension function that allows setting [Spot] covariances via passing a [radius]. */ @@ -666,8 +656,8 @@ class SphereLinkNodes( val selectClosestSpotsVR: ((pos: Vector3f, tp: Int, radius: Float, addOnly: Boolean) -> Pair) = { pos, tp, radius, addOnly -> val start = TimeSource.Monotonic.markNow() - val localPos = bridge.sciviewToMastodonCoords(pos) - val localRadius = bridge.sciviewToMastodonScale().max() * radius + val localPos = manvr3d.sciviewToMastodonCoords(pos) + val localRadius = manvr3d.sciviewToMastodonScale().max() * radius val spots = findSpotsInRange(tp, localPos, localRadius) // only proceed if we found at least one spot if (spots.isNotEmpty()) { @@ -695,7 +685,7 @@ class SphereLinkNodes( private fun selectSpot(spot: Spot) { findInstanceFromSpot(spot)?.let { - bridge.selectedSpotInstances.add(it) + manvr3d.selectedSpotInstances.add(it) it.instancedProperties["Color"] = { selectedColor } it.instancedParent.updateInstanceBuffers() mastodonData.selectionModel.setSelected(spot, true) @@ -704,7 +694,7 @@ class SphereLinkNodes( private fun deselectSpot(spot: Spot) { findInstanceFromSpot(spot)?.let { - bridge.selectedSpotInstances.remove(it) + manvr3d.selectedSpotInstances.remove(it) it.setColorFromSpot(spot, currentColorizer) mastodonData.selectionModel.setSelected(spot, false) } @@ -714,7 +704,7 @@ class SphereLinkNodes( /** Deletes the currently selected Spots from the graph. */ val deleteSpots: ((spots: RefSet) -> Unit) = { spots -> updateQueue.offer { - bridge.bdvNotifier?.lockUpdates = true + manvr3d.bdvNotifier?.lockUpdates = true mastodonData.model.setUndoPoint() mastodonData.model.graph.lock.writeLock().lock() spots.forEach { @@ -722,8 +712,8 @@ class SphereLinkNodes( logger.debug("Deleted spot {}", it) } mastodonData.model.graph.lock.writeLock().unlock() - bridge.selectedSpotInstances.clear() - bridge.bdvNotifier?.lockUpdates = false + this@GeometryHandler.manvr3d.selectedSpotInstances.clear() + manvr3d.bdvNotifier?.lockUpdates = false } } @@ -746,7 +736,7 @@ class SphereLinkNodes( logger.info("Trying to merge spot $nearestRef into $selectedRef") // Now that we found the nearest spot, let's merge it into the selected one nearestRef?.let { - bridge.bdvNotifier?.lockUpdates = true + manvr3d.bdvNotifier?.lockUpdates = true val sourceRef = graph.vertexRef() val targetRef = graph.vertexRef() @@ -768,14 +758,14 @@ class SphereLinkNodes( logger.debug("Merge event: added outgoing edge {}, deleted old edge {}", e, edge) } graph.remove(nearestRef) - bridge.bdvNotifier?.lockUpdates = false + manvr3d.bdvNotifier?.lockUpdates = false graph.notifyGraphChanged() } } } fun clearSelection() { - bridge.selectedSpotInstances.clear() + manvr3d.selectedSpotInstances.clear() mastodonData.focusModel.focusVertex(null) mastodonData.selectionModel.clearSelection() mastodonData.highlightModel.clearHighlight() @@ -884,13 +874,6 @@ class SphereLinkNodes( return sortedSpots } - /** Takes an integer-encoded RGB value and returns it as [Vector4f] where alpha is 1.0f. */ - private fun unpackRGB(intColor: Int): Vector4f { - val r = (intColor shr 16 and 0x000000FF) / 255f - val g = (intColor shr 8 and 0x000000FF) / 255f - val b = (intColor and 0x000000FF) / 255f - return Vector4f(r, g, b, 1.0f) - } fun updateSphereInstanceScales() { val tStart = TimeSource.Monotonic.markNow() @@ -905,13 +888,13 @@ class SphereLinkNodes( sphereScaleFactor -= 0.1f if (sphereScaleFactor < 0.1f) sphereScaleFactor = 0.1f updateSphereInstanceScales() - bridge.associatedUI?.updatePaneValues() + manvr3d.associatedUI?.updatePaneValues() } fun increaseSphereInstanceScale() { sphereScaleFactor += 0.1f updateSphereInstanceScales() - bridge.associatedUI?.updatePaneValues() + manvr3d.associatedUI?.updatePaneValues() } fun increaseLinkScale() { @@ -997,7 +980,7 @@ class SphereLinkNodes( inst.name = "${edge.internalPoolIndex}" inst.parent = linkParentNode // add a new key-value pair to the hash map - links[edge.internalPoolIndex] = LinkNode(inst, from, to, to.timepoint) + links[edge.target.hashCode()] = LinkNode(inst, from, to, to.timepoint) index++ } @@ -1063,14 +1046,14 @@ class SphereLinkNodes( * When set to [ColorMode.SPOT], it uses the [colorizer] to get the spot colors. */ fun updateLinkColors ( colorizer: GraphColorGenerator?, - cm: ColorMode = currentColorMode + cm: ColorMode = currentColorMode ) { val start = TimeSource.Monotonic.markNow() when (cm) { LUT -> { links.forEach {link -> val factor = link.value.tp / numTimePoints.toDouble() - val color = unpackRGB(lut.lookupARGB(0.0, 1.0, factor)) + val color = lut.lookupARGB(0.0, 1.0, factor).unpackRGB() link.value.instance.instancedProperties["Color"] = { color } } } @@ -1090,7 +1073,7 @@ class SphereLinkNodes( logger.debug("Updating link colors took ${end - start}.") } - fun updateLinkVisibility(currentTP: Int) { + fun updateSegmentVisibility(currentTP: Int) { links.forEach {link -> // turns the link on if it is within range, otherwise turns it off link.value.instance.visible = link.value.tp in currentTP - linkBackwardRange..currentTP + linkForwardRange @@ -1098,11 +1081,25 @@ class SphereLinkNodes( mainLinkInstance?.updateInstanceBuffers() } - /** Passed as callback to sciview to send a list of vertices from sciview to Mastodon. + fun setSpotVisibility(state: Boolean) { + mainSpotInstance?.let { + it.visible = state + it.updateInstanceBuffers() + } + } + + fun setTrackVisibility(state: Boolean) { + mainLinkInstance?.let { + it.visible = state + it.updateInstanceBuffers() + } + } + + /** Send a list of vertices from sciview to Mastodon. * If the boolean is true, the coordinates are in world space and will be converted to local Mastodon space first. * The first passed spot indicates that the user wants to start from an existing spot (aka clicked on it for starting the track). * The second spot is used to merge into existing spots. */ - val addTrackToMastodon = fun( + fun addTrackToMastodon( list: List>?, cursorRadius: Float, isWorldSpace: Boolean, @@ -1112,12 +1109,12 @@ class SphereLinkNodes( updateQueue.offer { val graph = mastodonData.model.graph var prevVertex = graph.vertexRef() - bridge.bdvNotifier?.lockUpdates = true + manvr3d.bdvNotifier?.lockUpdates = true // If the list isn't null, it was passed from eyetracking, and we have to treat it accordingly if (list != null) { list.forEachIndexed { index, (pos, vertex) -> val v = graph.addVertex() - val r = (bridge.sciviewToMastodonScale().max() * cursorRadius).toDouble() + val r = (manvr3d.sciviewToMastodonScale().max() * cursorRadius).toDouble() v.init(vertex.timepoint, pos.toDoubleArray(), r) logger.debug("added {}", v) // start adding edges once the first vertex was added @@ -1134,7 +1131,7 @@ class SphereLinkNodes( var localRadius: Float trackPointList.forEachIndexed { index, (pos, tp, pointRadius) -> // Calculate the equivalent radius in Mastodon from the raw radius from the cursor in sciview scale - localRadius = bridge.sciviewToMastodonScale().max() * pointRadius + localRadius = manvr3d.sciviewToMastodonScale().max() * pointRadius // If we reached the last spot in the list and a mergeSpot was passed, use it instead of the spot in the list if (index == trackPointList.size - 1 && mergeSpot != null) { graph.addEdge(mergeSpot, prevVertex) @@ -1144,7 +1141,7 @@ class SphereLinkNodes( v = startWithExisting } else { v = graph.addVertex() - // val localPos = if (isWorldSpace) bridge.sciviewToMastodonCoords(pos) else pos + // val localPos = if (isWorldSpace) manvr3d.sciviewToMastodonCoords(pos) else pos v.init(tp, pos.toDoubleArray(), localRadius.toDouble()) logger.debug("added {}", v) } @@ -1159,7 +1156,7 @@ class SphereLinkNodes( } } graph.releaseRef(prevVertex) - bridge.bdvNotifier?.lockUpdates = false + manvr3d.bdvNotifier?.lockUpdates = false // Once we send the new track to Mastodon, we can assume we no longer need the previews and can clear them mainLinkInstance?.instances?.removeAll(linkPreviewList.map { it.instance }.toSet()) linkPreviewList.clear() @@ -1174,7 +1171,7 @@ class SphereLinkNodes( val addOrRemoveSpots: (tp: Int, sciviewPos: Vector3f, radius: Float, deleteBranch: Boolean, isWorldSpace: Boolean) -> Unit = { tp, sciviewPos, radius, deleteBranch, isWorldSpace -> updateQueue.offer { - bridge.bdvNotifier?.lockUpdates = true + manvr3d.bdvNotifier?.lockUpdates = true // Check if a spot is selected, and perform deletion if true val selected = mastodonData.selectionModel.selectedVertices if (!selected.isEmpty()) { @@ -1199,27 +1196,27 @@ class SphereLinkNodes( } else { // If no spot is selected, add a new one val pos = if (isWorldSpace) { - bridge.sciviewToMastodonCoords(sciviewPos) + manvr3d.sciviewToMastodonCoords(sciviewPos) } else { sciviewPos } - val bb = bridge.volumeNode.boundingBox + val bb = manvr3d.volumeNode.boundingBox if (bb != null) { if (bb.isInside(pos)) { - val localRadius = bridge.sciviewToMastodonScale().max() * radius + val localRadius = manvr3d.sciviewToMastodonScale().max() * radius val v = mastodonData.model.graph.addVertex() v.init(tp, pos.toDoubleArray(), localRadius.toDouble()) logger.info("Added new spot at position $pos, radius is $localRadius") logger.debug("we now have ${mastodonData.model.graph.vertices().size} spots in total") } else { logger.warn("Not adding new spot, $pos is outside the volume!") - bridge.flashBoundingGrid() + manvr3d.flashVolumeGrid() } } else { logger.warn("Not adding new spot, volume has no bounding box!") } } - bridge.bdvNotifier?.lockUpdates = false + manvr3d.bdvNotifier?.lockUpdates = false } } @@ -1261,12 +1258,12 @@ class SphereLinkNodes( /** Adds a single link instance to the scene for visual feedback during controller based tracking. * No data are sent to Mastodon yet, but we keep track of the points in local space in a [trackPointList]. */ val addTrackedPoint: (pos: Vector3f, tp: Int, rawRadius: Float, preview: Boolean) -> Unit = { pos, tp, radius, preview -> - val localPos = bridge.sciviewToMastodonCoords(pos) + val localPos = manvr3d.sciviewToMastodonCoords(pos) // Once we tracked the first point, we can start adding link previews if (trackPointList.isNotEmpty() && mainLinkInstance != null) { val inst = mainLinkInstance!!.addInstance() val color = Vector4f(0.65f, 1f, 0.22f, 1f) -// val localFrom = bridge.sciviewToMastodonCoords(from) +// val localFrom = manvr3d.sciviewToMastodonCoords(from) inst.instancedProperties["Color"] = { color } inst.name = "${tp}_${localPos}" inst.parent = linkParentNode @@ -1301,14 +1298,8 @@ class SphereLinkNodes( val v = value / 100.0f val rgbInt = Color.HSBtoRGB(h, s, v) - return unpackRGB(rgbInt) + return rgbInt.unpackRGB() } } data class LinkNode (val instance: InstancedNode.Instance, val from: Spot, val to: Spot, val tp: Int) - -/** This extension function pushes updated instance buffers to the GPU. - * Without calling this function, instances will not update in the renderer. */ -fun InstancedNode.updateInstanceBuffers() { - this.metadata["MaxInstanceUpdateCount"] = AtomicInteger(1) -} diff --git a/src/main/kotlin/util/GroupLocksHandling.kt b/src/main/kotlin/util/GroupLocksHandling.kt index fbf29b8..7678181 100644 --- a/src/main/kotlin/util/GroupLocksHandling.kt +++ b/src/main/kotlin/util/GroupLocksHandling.kt @@ -4,7 +4,7 @@ import graphics.scenery.utils.lazyLogger import org.mastodon.app.ui.GroupLocksPanel import org.mastodon.grouping.GroupHandle import org.mastodon.mamut.ProjectModel -import org.mastodon.mamut.SciviewBridge +import Manvr3dMain import org.mastodon.mamut.model.Link import org.mastodon.mamut.model.Spot import org.mastodon.model.NavigationListener @@ -16,7 +16,7 @@ import org.scijava.event.SciJavaEvent import sc.iview.event.NodeActivatedEvent class GroupLocksHandling(//controls sciview via this bridge obj - private val bridge: SciviewBridge, mastodon: ProjectModel + private val manvr3d: Manvr3dMain, mastodon: ProjectModel ) { val logger by lazyLogger() //controls Mastodon @@ -31,7 +31,7 @@ class GroupLocksHandling(//controls sciview via this bridge obj fun createAndActivate(): GroupLocksPanel? { if (isActive) return null isActive = true - bridge.eventService?.subscribe(sciviewFocusHandler) + manvr3d.eventService?.subscribe(sciviewFocusHandler) myGroupHandle = projectModel.groupManager.createGroupHandle() myGroupHandle.getModel(projectModel.NAVIGATION).listeners().add(navigationRequestsHandler) myGroupHandle.getModel(projectModel.TIMEPOINT).listeners().add(navigationRequestsHandler) @@ -46,7 +46,7 @@ class GroupLocksHandling(//controls sciview via this bridge obj projectModel.groupManager.removeGroupHandle(myGroupHandle) //can fail, so we better do it as the last action here - val subs = bridge.eventService?.getSubscribers( + val subs = manvr3d.eventService?.getSubscribers( SciJavaEvent::class.java ) subs?.remove(sciviewFocusHandler) @@ -63,7 +63,7 @@ class GroupLocksHandling(//controls sciview via this bridge obj override fun timepointChanged() { logger.debug("timepoint changed to ${myGroupHandle.getModel(projectModel.TIMEPOINT).timepoint}") - bridge.showTimepoint(myGroupHandle.getModel(projectModel.TIMEPOINT).timepoint) + manvr3d.showTimepoint(myGroupHandle.getModel(projectModel.TIMEPOINT).timepoint) } } diff --git a/src/main/kotlin/util/SpineMetadata.kt b/src/main/kotlin/util/SpineMetadata.kt new file mode 100644 index 0000000..feab46a --- /dev/null +++ b/src/main/kotlin/util/SpineMetadata.kt @@ -0,0 +1,23 @@ +package util + +import org.joml.Quaternionf +import org.joml.Vector3f + +/** + * Data class to store metadata for spines of the hedgehog. + */ +data class SpineMetadata( + val timepoint: Int, + val origin: Vector3f, + val direction: Vector3f, + val distance: Float, + val localEntry: Vector3f, + val localExit: Vector3f, + val localDirection: Vector3f, + val headPosition: Vector3f, + val headOrientation: Quaternionf, + val position: Vector3f, + val confidence: Float, + val samples: List, + val samplePosList: List = ArrayList() +) \ No newline at end of file diff --git a/src/main/kotlin/vr/CellTrackingBase.kt b/src/main/kotlin/vr/CellTrackingBase.kt new file mode 100644 index 0000000..bf6be76 --- /dev/null +++ b/src/main/kotlin/vr/CellTrackingBase.kt @@ -0,0 +1,923 @@ +package vr + +import Manvr3dMain +import graphics.scenery.* +import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.* +import graphics.scenery.controls.behaviours.AnalogInputWrapper +import graphics.scenery.controls.behaviours.ConfirmableClickBehaviour +import graphics.scenery.controls.behaviours.VRTouch +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import graphics.scenery.ui.* +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.* +import graphics.scenery.utils.lazyLogger +import graphics.scenery.volumes.RAIVolume +import graphics.scenery.volumes.Volume +import org.joml.* +import org.mastodon.mamut.model.Spot +import org.scijava.ui.behaviour.ClickBehaviour +import org.scijava.ui.behaviour.DragBehaviour +import sc.iview.SciView +import analysis.HedgehogAnalysis.SpineGraphVertex +import graphics.scenery.controls.behaviours.MultiButtonManager +import graphics.scenery.controls.behaviours.VR2HandNodeTransform +import graphics.scenery.controls.behaviours.VRGrabTheWorld +import graphics.scenery.utils.TimepointObservable +import org.checkerframework.checker.units.qual.m +import util.CellTrackingButtonMapper +import util.GeometryHandler +import util.SpineMetadata +import java.io.BufferedWriter +import java.io.FileWriter +import java.nio.file.Path +import java.util.ArrayList +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread + +/** + * Base class for different VR cell tracking purposes. It includes functionality to add spines and edgehogs, + * as used by [EyeTracking], and registers controller bindings via [inputSetup]. It is possible to register observers + * that listen to timepoint changes with [registerObserver]. + * @param [sciview] The [SciView] instance to use + */ +open class CellTrackingBase( + open var sciview: SciView, + open var manvr3d: Manvr3dMain, + open var geometryHandler: GeometryHandler, + val resolutionScale: Float = 1f +): TimepointObservable() { + val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + + lateinit var hmd: OpenVRHMD + + val hedgehogs = Mesh() + val hedgehogIds = AtomicInteger(0) + lateinit var volume: Volume + + val referenceTarget = Icosphere(0.004f, 2) + + @Volatile var eyeTrackingActive = false + var playing = false + var direction = PlaybackDirection.Backward + var volumesPerSecond = 6f + var skipToNext = false + var skipToPrevious = false + + var volumeScaleFactor = 1.0f + + private lateinit var lightTetrahedron: List + + val volumeTimepointWidget = TextBoard() + + /** determines whether the volume and hedgehogs should keep listening for updates or not */ + var cellTrackingActive: Boolean = false + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + + enum class PlaybackDirection { Forward, Backward } + + enum class ElephantMode { StageSpots, TrainAll, PredictTP, PredictAll, NNLinking } + + var hedgehogVisibility = HedgehogVisibility.Hidden + var trackVisibility = true + var spotVisibility = true + + var leftVRController: TrackedDevice? = null + var rightVRController: TrackedDevice? = null + + val cursor = CursorTool() + val cursorSelectColor = Vector3f(1f, 0.25f, 0.25f) + val cursorTrackingColor = Vector3f(0.65f, 1f, 0.22f) + + lateinit var leftWristMenu: MultiWristMenu + + var enableTrackingPreview = true + + val grabButtonManager = MultiButtonManager() + val resetRotationBtnManager = MultiButtonManager() + + val buttonMapper = CellTrackingButtonMapper + + open fun run() { + sciview.toggleVRRendering(resolutionScale = resolutionScale) + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + + // Try to load the correct button mapping corresponding to the controller layout + val isProfileLoaded = buttonMapper.loadProfile(hmd.manufacturer) + if (!isProfileLoaded) { + throw IllegalStateException("Could not load profile, headset type unknown!") + } + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial { + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) + } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + shell.name = "Shell" + sciview.addNode(shell) + + lightTetrahedron = Light.createLightTetrahedron( + Vector3f(0.0f, 0.0f, 0.0f), + spread = 5.0f, + radius = 15.0f, + intensity = 5.0f + ) + lightTetrahedron.forEach { sciview.addNode(it) } + + val volumeNodes = sciview.findNodes { node -> Volume::class.java.isAssignableFrom(node.javaClass) } + + val v = (volumeNodes.firstOrNull() as? Volume) + if(v == null) { + logger.warn("No volume found, bailing") + return + } else { + logger.info("found ${volumeNodes.size} volume nodes. Using the first one: ${volumeNodes.first()}") + volume = v + } + + logger.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + logger.info("onDeviceConnect called, cam=${sciview.camera}") + if (device.type == TrackedDeviceType.Controller) { + logger.info("Got device ${device.name} at $timestamp") + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + when (device.role) { + TrackerRole.Invalid -> {} + TrackerRole.LeftHand -> leftVRController = device + TrackerRole.RightHand -> rightVRController = device + } + if (device.role == TrackerRole.RightHand) { + attachCursorAndTimepointWidget() + device.model?.name = "rightHand" + } else if (device.role == TrackerRole.LeftHand) { + device.model?.let { + it.name = "leftHand" + leftWristMenu = MultiWristMenu(it) + setupElephantMenu() + setupGeneralMenu() + leftWristMenu.hideAll() + } + + logger.info("Set up navigation and editing controls.") + } + } + } + inputSetup() + + cellTrackingActive = true + manvr3d.rebuildGeometry() + launchUpdaterThread() + } + + var controllerTrackingActive = false + + /** Intermediate storage for a single track created with the controllers. + * Once tracking is finished, this track is sent to Mastodon. */ + var controllerTrackList = mutableListOf() + var startWithExistingSpot: Spot? = null + + /** This lambda is called every time the user performs a click with controller-based tracking. */ + val trackCellsWithController = ClickBehaviour { _, _ -> + if (!controllerTrackingActive) { + controllerTrackingActive = true + cursor.setColor(cursorTrackingColor) + // we dont want animation, because we track step by step + playing = false + // Assume the user didn't click on an existing spot to start the track. + startWithExistingSpot = null + } + // play the volume backwards, step by step, so cell split events can simply be turned into a merge event + if (volume.currentTimepoint > 0) { + val p = cursor.getPosition() + // did the user click on an existing cell and wants to merge the track into it? + val (selected, isValidSelection) = + geometryHandler.selectClosestSpotsVR(p, volume.currentTimepoint, cursor.radius, false) ?: (null to false) + // If this is the first spot we track, and its a valid existing spot, mark it as such + if (isValidSelection && controllerTrackList.size == 0) { + startWithExistingSpot = selected + logger.debug("Set startWithExistingPost to $startWithExistingSpot") + } else { + controllerTrackList.add(p) + } + logger.debug("Tracked a new spot at position $p") + logger.debug("Do we want to merge? $isValidSelection. Selected spot is $selected") + // Create a placeholder link during tracking for immediate feedback + geometryHandler.addTrackedPoint(p, volume.currentTimepoint, cursor.radius, enableTrackingPreview) + + volume.goToTimepoint(volume.currentTimepoint - 1) + // If the user clicked a cell and its *not* the first in the track, we assume it is a merge event and end the tracking + if (isValidSelection && controllerTrackList.size > 1) { + endControllerTracking(selected) + } + // This will also redraw all geometry using Mastodon as source + notifyObservers(volume.currentTimepoint) + } else { + sciview.camera?.showMessage("Reached the first time point!", centered = true, distance = 2f, size = 0.2f) + // Let's head back to the last timepoint for starting a new track fast-like + volume.goToLastTimepoint() + endControllerTracking() + } + } + + /** Stops the current controller tracking process and sends the created track to Mastodon. */ + private fun endControllerTracking(mergeSpot: Spot? = null) { + if (controllerTrackingActive) { + logger.info("Ending controller tracking now and sending ${controllerTrackList.size} spots to Mastodon to chew on.") + controllerTrackingActive = false + // Radius can be 0 because the actual radii were already captured during tracking + geometryHandler.addTrackToMastodon(null, 0f, true, startWithExistingSpot, mergeSpot) + controllerTrackList.clear() + cursor.resetColor() + } + } + + fun setupElephantMenu() { + val unpressedColor = Vector3f(0.81f, 0.81f, 1f) + val touchingColor = Vector3f(0.7f, 0.65f, 1f) + val pressedColor = Vector3f(0.54f, 0.44f, 0.96f) + val colName = "Elephant Menu" + val delay = 500 + + leftWristMenu.addColumn(colName) + leftWristMenu.addButton(colName, "Stage all", + command = { updateElephantActions(ElephantMode.StageSpots) }, depressDelay = delay, + color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) + leftWristMenu.addButton(colName, "Train All TPs", + command = { updateElephantActions(ElephantMode.TrainAll) }, depressDelay = delay, + color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) + leftWristMenu.addButton(colName, "Predict All", + command = { updateElephantActions(ElephantMode.PredictAll) }, depressDelay = delay, + color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) + leftWristMenu.addButton(colName, "Predict TP", + command = { updateElephantActions(ElephantMode.PredictTP) }, depressDelay = delay, + color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) + leftWristMenu.addButton(colName, "NN linking", + command = { updateElephantActions(ElephantMode.NNLinking) }, depressDelay = delay, + color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) + } + + var lastButtonTime = System.currentTimeMillis() + + /** Ensure that only a single Elephant action is triggered at a time */ + private fun updateElephantActions(mode: ElephantMode) { + val buttonTime = System.currentTimeMillis() + if ((buttonTime - lastButtonTime) > 1000) { + thread { + when (mode) { + ElephantMode.StageSpots -> manvr3d.stageSpots() + ElephantMode.TrainAll -> manvr3d.trainSpots() + ElephantMode.PredictTP -> manvr3d.preditSpots(false) + ElephantMode.PredictAll -> manvr3d.preditSpots(true) + ElephantMode.NNLinking -> manvr3d.linkNearestNeighbors() + } + logger.info("We locked the buttons for ${(buttonTime-lastButtonTime)} ms ") + lastButtonTime = buttonTime + } + } else { + sciview.camera?.showMessage("Have some patience!", duration = 1500, distance = 2f, size = 0.2f, centered = true) + } + } + + fun setupGeneralMenu() { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + val color = Vector3f(0.8f) + val pressedColor = Vector3f(0.95f, 0.35f, 0.25f) + val touchingColor = Vector3f(0.7f, 0.55f, 0.55f) + + val undoButton = Button( + "Undo", + command = { manvr3d.undoRedo() }, byTouch = true, depressDelay = 250, + defaultColor = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + val redoButton = Button( + "Redo", + command = { manvr3d.undoRedo(redo = true) }, byTouch = true, depressDelay = 250, + defaultColor = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + val resetViewButton = Button( + "Recenter", command = { + manvr3d.resetView() + }, byTouch = true, depressDelay = 250, + defaultColor = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + + val togglePlaybackDirBtn = ToggleButton( + textFalse = "BW", textTrue = "FW", command = { + direction = if (direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + }, byTouch = true, + defaultColor = Vector3f(0.52f, 0.87f, 0.86f), + touchingColor = color, + pressedColor = Vector3f(0.84f, 0.87f, 0.52f) + ) + val playSlowerBtn = Button( + "<", command = { + volumesPerSecond = maxOf(volumesPerSecond - 1f, 1f) + cam.showMessage( + "Speed: ${"%.0f".format(volumesPerSecond)} vol/s", + distance = 1.2f, size = 0.2f, centered = true + ) + }, byTouch = true, depressDelay = 250, + defaultColor = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + val playFasterBtn = Button( + ">", command = { + volumesPerSecond = minOf(volumesPerSecond + 1f, 20f) + cam.showMessage( + "Speed: ${"%.0f".format(volumesPerSecond)} vol/s", + distance = 1.2f, size = 0.2f, centered = true + ) + }, byTouch = true, depressDelay = 250, + defaultColor = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + val goToLastBtn = Button( + ">|", command = { + playing = false + volume.goToLastTimepoint() + notifyObservers(volume.currentTimepoint) + cam.showMessage("Jumped to timepoint ${volume.currentTimepoint}.", + distance = 1.2f, size = 0.2f, centered = true) + }, byTouch = true, depressDelay = 250, + defaultColor = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + val goToFirstBtn = Button( + "|<", command = { + playing = false + volume.goToFirstTimepoint() + notifyObservers(volume.currentTimepoint) + cam.showMessage("Jumped to timepoint ${volume.currentTimepoint}.", + distance = 1.2f, size = 0.2f, centered = true) + }, byTouch = true, depressDelay = 250, + defaultColor = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + + + leftWristMenu.addColumn("General Menu") + leftWristMenu.addRow("General Menu", + goToFirstBtn, playSlowerBtn, togglePlaybackDirBtn, playFasterBtn, goToLastBtn, middleAlign = false) + leftWristMenu.addRow( + "General Menu", undoButton, redoButton, resetViewButton, middleAlign = false) + + + leftWristMenu.addColumn("Toggle Menu") + leftWristMenu.addToggleButton("Toggle Menu", "Volume off", "Volume on", command = { + val state = volume.visible + manvr3d.setVolumeOnlyVisibility(!state) + }, color = color, pressedColor = pressedColor, touchingColor = touchingColor, defaultState = true) + leftWristMenu.addToggleButton("Toggle Menu", "Tracks off", "Tracks on", + command = { + trackVisibility = !trackVisibility + geometryHandler.setTrackVisibility(trackVisibility) + }, color = color, pressedColor = pressedColor, touchingColor = touchingColor, defaultState = true ) + leftWristMenu.addToggleButton("Toggle Menu", "Spots off", "Spots on", + command = { + spotVisibility = !spotVisibility + geometryHandler.setSpotVisibility(spotVisibility) + }, color = color, pressedColor = pressedColor, touchingColor = touchingColor, defaultState = true ) + leftWristMenu.addToggleButton("Toggle Menu", "Preview Off", "Preview On", command = { + enableTrackingPreview = !enableTrackingPreview + geometryHandler.toggleLinkPreviews(enableTrackingPreview) + }, color = color, pressedColor = pressedColor, touchingColor = touchingColor, defaultState = true) + + + leftWristMenu.addColumn("Cleanup Menu") + leftWristMenu.addButton("Cleanup Menu", "Merge overlaps", command = { + manvr3d.mergeOverlapsAndUpdate(volume.currentTimepoint) + }, color = color, pressedColor = pressedColor, touchingColor = touchingColor) + leftWristMenu.addButton("Cleanup Menu", "Merge selected", command = { + manvr3d.mergeSelectionAndUpdate() + }, color = color, pressedColor = pressedColor, touchingColor = touchingColor) + leftWristMenu.addButton("Cleanup Menu", "Delete Graph", command = { + manvr3d.deleteGraphAndUpdate() + }, byTouch = true, color = color, pressedColor = pressedColor, touchingColor = touchingColor) + leftWristMenu.addButton("Cleanup Menu", "Delete TP", command = { + manvr3d.deleteTimepointAndUpdate(volume.currentTimepoint) + }, byTouch = true, color = color, pressedColor = pressedColor, touchingColor = touchingColor) + } + + fun addHedgehog() { + logger.info("added hedgehog") + val hedgehog = Cylinder(0.005f, 1.0f, 16) + hedgehog.visible = false + hedgehog.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.frag", "DeferredInstancedColor.vert")) + val hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.visible = false + hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} + hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } + hedgehogs.addChild(hedgehogInstanced) + } + + /** Attach a spherical cursor to the right controller. */ + private fun attachCursorAndTimepointWidget(debug: Boolean = false) { + // Only attach if not already attached + if (sciview.findNodes { it.name == "VR Cursor" }.isNotEmpty()) { + return + } + + volumeTimepointWidget.text = volume.currentTimepoint.toString() + volumeTimepointWidget.name = "Volume Timepoint Widget" + volumeTimepointWidget.fontColor = Vector4f(0.4f, 0.45f, 1f, 1f) + volumeTimepointWidget.spatial { + scale = Vector3f(0.07f) + position = Vector3f(-0.05f, -0.05f, 0.12f) + rotation = Quaternionf().rotationXYZ(-1.57f, -1.57f, 0f) + } + + rightVRController?.model?.let { + cursor.attachCursor(it) + sciview.addNode(volumeTimepointWidget, activePublish = false, parent = it) + } + } + + open fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + sciview.sceneryInputHandler?.let { handler -> + listOf( + "move_forward_fast", + "move_back_fast", + "move_left_fast", + "move_right_fast").forEach { name -> + handler.getBehaviour(name)?.let { behaviour -> + buttonMapper.setKeyBindAndBehavior(hmd, name, behaviour) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.entries.indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.entries.get((current + 1) % 3) + + when (hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogs.visible = false + hedgehogs.runRecursive { it.visible = false } + cam.showMessage("Hedgehogs hidden", distance = 2f, size = 0.2f, centered = true) + } + + HedgehogVisibility.PerTimePoint -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs shown per timepoint", distance = 2f, size = 0.2f, centered = true) + } + + HedgehogVisibility.Visible -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs visible", distance = 2f, size = 0.2f, centered = true) + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + class ScaleCursorOrSpotsBehavior(val factor: Float): DragBehaviour { + var selection = listOf() + override fun init(p0: Int, p1: Int) { + selection = manvr3d.selectedSpotInstances.toList() + } + + override fun drag(p0: Int, p1: Int) { + if (selection.isNotEmpty()) { + geometryHandler.changeSpotRadius(selection,factor, false) + } else { + // Make cursor movement a little faster than changing the spot radii + cursor.scaleByFactor(factor * factor) + } + } + + override fun end(p0: Int, p1: Int) { + geometryHandler.changeSpotRadius(selection, factor, true) + } + } + + val scaleCursorOrSpotsUp = AnalogInputWrapper(ScaleCursorOrSpotsBehavior(1.02f), sciview.currentScene) + + val scaleCursorOrSpotsDown = AnalogInputWrapper(ScaleCursorOrSpotsBehavior(0.98f), sciview.currentScene) + + val faster = ClickBehaviour { _, _ -> + volumesPerSecond = maxOf(minOf(volumesPerSecond+0.2f, 20f), 1f) + cam.showMessage("Speed: ${"%.1f".format(volumesPerSecond)} vol/s",distance = 1.2f, size = 0.2f, centered = true) + } + + val slower = ClickBehaviour { _, _ -> + volumesPerSecond = maxOf(minOf(volumesPerSecond-0.2f, 20f), 1f) + cam.showMessage("Speed: ${"%.1f".format(volumesPerSecond)} vol/s",distance = 2f, size = 0.2f, centered = true) + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if (playing) { + cam.showMessage("Playing", distance = 2f, size = 0.2f, centered = true) + } else { + cam.showMessage("Paused", distance = 2f, size = 0.2f, centered = true) + } + } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt(), + centered = true) + + }, + confirmAction = { + hedgehogs.children.removeLast() + volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> + volume.removeChild(lastTrack) + } + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000, + centered = true + ) + }) + + buttonMapper.setKeyBindAndBehavior(hmd, "stepFwd", nextTimepoint) + buttonMapper.setKeyBindAndBehavior(hmd, "stepBwd", prevTimepoint) + + buttonMapper.setKeyBindAndBehavior(hmd, "playback", playPause) + buttonMapper.setKeyBindAndBehavior(hmd, "radiusIncrease", scaleCursorOrSpotsUp) + buttonMapper.setKeyBindAndBehavior(hmd, "radiusDecrease", scaleCursorOrSpotsDown) + + /** Local class that handles double assignment of the left A key which is used to cycle menus as well as + * reset the rotation when pressed while the [VR2HandNodeTransform] is active. */ + class CycleMenuAndLockAxisBehavior(val button: OpenVRHMD.OpenVRButton, val role: TrackerRole) + : DragBehaviour { + fun registerConfig() { + logger.debug("Setting up keybinds for CycleMenuAndLockAxisBehavior") + resetRotationBtnManager.registerButtonConfig(button, role) + } + override fun init(x: Int, y: Int) { + resetRotationBtnManager.pressButton(button, role) + if (!resetRotationBtnManager.isTwoHandedActive()) { + leftWristMenu.cycleNext() + } + } + override fun drag(x: Int, y: Int) {} + override fun end(x: Int, y: Int) { + resetRotationBtnManager.releaseButton(button, role) + } + } + + val leftAButtonBehavior = CycleMenuAndLockAxisBehavior(OpenVRHMD.OpenVRButton.A, TrackerRole.LeftHand) + leftAButtonBehavior.let { + it.registerConfig() + buttonMapper.setKeyBindAndBehavior(hmd, "cycleMenu", it) + } + + buttonMapper.setKeyBindAndBehavior(hmd, "controllerTracking", trackCellsWithController) + + /** Several behaviors mapped per default to the right menu button. If controller tracking is active, + * end the tracking. If not, clicking will either create or delete a spot, depending on whether the user + * previously selected a spot. Holding the button for more than 0.5s deletes the whole connected branch. */ + class AddDeleteResetBehavior : DragBehaviour { + var start = System.currentTimeMillis() + var wasExecuted = false + override fun init(x: Int, y: Int) { + start = System.currentTimeMillis() + wasExecuted = false + } + override fun drag(x: Int, y: Int) { + if (System.currentTimeMillis() - start > 500 && !wasExecuted) { + val p = cursor.getPosition() + geometryHandler.addOrRemoveSpots( + volume.currentTimepoint, + p, + cursor.radius, + true, + true) + wasExecuted = true + } + } + override fun end(x: Int, y: Int) { + if (controllerTrackingActive) { + endControllerTracking() + } else { + val p = cursor.getPosition() + logger.debug("Got cursor position: $p") + if (!wasExecuted) { + geometryHandler.addOrRemoveSpots( + volume.currentTimepoint, + p, + cursor.radius, + false, + true) + } + } + } + } + + buttonMapper.setKeyBindAndBehavior(hmd, "addDeleteReset", AddDeleteResetBehavior()) + + class DragSelectBehavior: DragBehaviour { + var time = System.currentTimeMillis() + override fun init(x: Int, y: Int) { + time = System.currentTimeMillis() + val p = cursor.getPosition() + cursor.setColor(cursorSelectColor) + geometryHandler.selectClosestSpotsVR(p, volume.currentTimepoint, cursor.radius, false) + } + override fun drag(x: Int, y: Int) { + // Only perform the selection method ten times a second + if (System.currentTimeMillis() - time > 100) { + val p = cursor.getPosition() + geometryHandler.selectClosestSpotsVR(p, volume.currentTimepoint, cursor.radius, true) + time = System.currentTimeMillis() + } + } + override fun end(x: Int, y: Int) { + cursor.resetColor() + } + } + + buttonMapper.setKeyBindAndBehavior(hmd, "select", DragSelectBehavior()) + + // this behavior is needed for touching the menu buttons + VRTouch.createAndSet(sciview.currentScene, hmd, listOf(TrackerRole.RightHand), false, customTip = cursor.cursor) + + VRGrabTheWorld.createAndSet( + sciview.currentScene, + hmd, + listOf(OpenVRHMD.OpenVRButton.Side), + listOf(TrackerRole.LeftHand), + grabButtonManager, + 1.5f + ) + + VR2HandNodeTransform.createAndSet( + hmd, + OpenVRHMD.OpenVRButton.Side, + sciview.currentScene, + lockYaxis = false, + target = volume, + onStartCallback = { + geometryHandler.setSpotVisibility(false) + geometryHandler.setTrackVisibility(false) + }, + onEndCallback = { + manvr3d.rebuildGeometry() + // Only re-enable the spots or tracks if they were enabled in the first place + geometryHandler.setSpotVisibility(spotVisibility) + geometryHandler.setTrackVisibility(trackVisibility) + }, + resetRotationBtnManager = resetRotationBtnManager, + resetRotationButton = MultiButtonManager.ButtonConfig(leftAButtonBehavior.button, leftAButtonBehavior.role) + ) + + // drag behavior can stay enabled regardless of current tool mode + MoveInstanceVR.createAndSet(manvr3d, hmd, + listOf(OpenVRHMD.OpenVRButton.Side), listOf(TrackerRole.RightHand), + grabButtonManager, + { cursor.getPosition() } + ) + + hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand + logger.info("Registered VR controller bindings.") + + } + + /** + * Launches a thread that updates the volume time points, the hedgehog visibility and reference target color. + */ + fun launchUpdaterThread() { + thread { + while (!sciview.isInitialized) { + Thread.sleep(200) + } + + while (sciview.running && cellTrackingActive) { + if (playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + if (skipToNext || playing) { + skipToNext = false + if (direction == PlaybackDirection.Forward) { + notifyObservers(oldTimepoint + 1) + } else { + notifyObservers(oldTimepoint - 1) + } + } else { + skipToPrevious = false + if (direction == PlaybackDirection.Forward) { + notifyObservers(oldTimepoint - 1) + } else { + notifyObservers(oldTimepoint + 1) + } + } + + if (hedgehogs.visible) { + if (hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogs.children.forEach { hh -> + val hedgehog = hh as InstancedNode + hedgehog.instances.forEach { + if (it.metadata.isNotEmpty()) { + it.visible = + (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } + } else { + hedgehogs.children.forEach { hh -> + val hedgehog = hh as InstancedNode + hedgehog.instances.forEach { it.visible = true } + } + } + } + + updateLoopActions.forEach { it.invoke() } + } + + Thread.sleep((1000.0f / volumesPerSecond).toLong()) + } + logger.info("CellTracking updater thread has stopped.") + } + } + + private val updateLoopActions: ArrayList<() -> Unit> = ArrayList() + + /** Allows hooking lambdas into the main update loop. This is needed for eye tracking related actions. */ + protected fun attachToLoop(action: () -> Unit) { + updateLoopActions.add(action) + } + + /** Samples a given [volume] from an [origin] point along a [direction]. + * @return a pair of lists, containing the samples and sample positions, respectively. */ + protected fun sampleRayThroughVolume(origin: Vector3f, direction: Vector3f, volume: Volume): Pair?, List?> { + val intersection = volume.spatial().intersectAABB(origin, direction.normalize(), ignoreChildren = true) + + if (intersection is MaybeIntersects.Intersection) { + val localEntry = (intersection.relativeEntry) + val localExit = (intersection.relativeExit) + val (samples, samplePos) = volume.sampleRayGridTraversal(localEntry, localExit) ?: (null to null) + val volumeScale = (volume as RAIVolume).getVoxelScale() + return (samples?.map { it ?: 0.0f } to samplePos?.map { it?.mul(volumeScale) ?: Vector3f(0f) }) + } else { + logger.warn("Ray didn't intersect volume! Origin was $origin, direction was $direction.") + } + return (null to null) + } + + open fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = sphere.origin.minus(center) + val sphereDist = + Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = center + val temp = direction.mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + val spine = (hedgehogs.children.last() as InstancedNode).addInstance() + spine.spatial().orientBetweenPoints(p1, p2, true, true) + spine.visible = false + + val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize(), true) + + if (volume.boundingBox?.isInside(cam.spatial().position)!!) { + logger.info("Can't track inside the volume! Please move out of the volume and try again") + return + } + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) + val localExit = (intersection.relativeExit) + // TODO We dont need the local direction for grid traversal, but its still in the spine metadata for now + val localDirection = Vector3f(0f) + val (samples, samplePos) = volume.sampleRayGridTraversal(localEntry, localExit) ?: (null to null) + val volumeScale = (volume as RAIVolume).getVoxelScale() + + if (samples != null && samplePos != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.spatial().position, + confidence, + samples.map { it ?: 0.0f }, + samplePos.map { it?.mul(volumeScale) ?: Vector3f(0f) } + ) + val count = samples.filterNotNull().count { it > 0.2f } + + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } + // TODO: Show confidence as color for the spine + spine.instancedProperties["Metadata"] = + { Vector4f(confidence, timepoint.toFloat() / volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + + protected fun writeHedgehogToFile(hedgehog: InstancedNode, hedgehogId: Int) { + val hedgehogFile = + sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint;Origin;Direction;LocalEntry;LocalExit;LocalDirection;HeadPosition;HeadOrientation;Position;Confidence;Samples\n") + + val spines = hedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write( + "${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};" + + "${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};" + + "${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";") + }\n" + ) + } + hedgehogFileWriter.close() + logger.info("Wrote hedgehog to file ${hedgehogFile.name}") + } + + protected fun writeTrackToFile( + points: List>, + hedgehogId: Int + ) { + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + trackFileWriter.newLine() + trackFileWriter.newLine() + val parentId = 0 + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + val volumeDimensions = volume.getDimensions() + points.windowed(2, 1).forEach { pair -> + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)) // direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + + trackFileWriter.close() + } + + /** + * Stops the current tracking environment and restore the original state. + * This method should be overridden if functionality is extended, to make sure any extra objects are also deleted. + */ + open fun stop() { + logger.info("Objects in the scene: ${sciview.allSceneNodes.map { it.name }}") + cellTrackingActive = false + if (::lightTetrahedron.isInitialized) { + lightTetrahedron.forEach { sciview.deleteNode(it) } + } + // Try to find and delete possibly existing VR objects + listOf("Shell", "leftHand", "rightHand").forEach { + val n = sciview.find(it) + n?.let { sciview.deleteNode(n) } + } + rightVRController?.model?.let { + sciview.deleteNode(it) + } + leftVRController?.model?.let { + sciview.deleteNode(it) + } + + logger.info("Cleaned up basic VR objects. Objects left: ${sciview.allSceneNodes.map { it.name }}") + + sciview.toggleVRRendering() + logger.info("Shut down and disabled VR environment.") + manvr3d.rebuildGeometry() + } + +} diff --git a/src/main/kotlin/vr/EyeTracking.kt b/src/main/kotlin/vr/EyeTracking.kt new file mode 100644 index 0000000..4258b9d --- /dev/null +++ b/src/main/kotlin/vr/EyeTracking.kt @@ -0,0 +1,534 @@ +package vr + +import Manvr3dMain +import graphics.scenery.BoundingGrid +import graphics.scenery.Box +import graphics.scenery.BufferUtils +import graphics.scenery.DetachedHeadCamera +import graphics.scenery.Icosphere +import graphics.scenery.InstancedNode +import graphics.scenery.Mesh +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.eyetracking.PupilEyeTracker +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import graphics.scenery.textures.Texture +import graphics.scenery.ui.Button +import graphics.scenery.ui.Column +import graphics.scenery.ui.ToggleButton +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.plus +import graphics.scenery.utils.extensions.toDoubleArray +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.utils.gaussSmoothing +import graphics.scenery.utils.localMaxima +import graphics.scenery.utils.toVector3f +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.apache.commons.math3.ml.clustering.Clusterable +import org.apache.commons.math3.ml.clustering.DBSCANClusterer +import org.joml.Matrix4f +import org.joml.Vector2f +import org.joml.Vector3f +import org.joml.Vector3i +import org.joml.Vector4f +import org.scijava.ui.behaviour.ClickBehaviour +import sc.iview.SciView +import analysis.HedgehogAnalysis +import util.GeometryHandler +import util.SpineMetadata +import java.awt.image.DataBufferByte +import java.io.ByteArrayInputStream +import java.nio.file.Files +import java.nio.file.Paths +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException +import javax.imageio.ImageIO +import kotlin.concurrent.thread +import kotlin.math.PI +import kotlin.time.TimeSource + +/** + * Tracking class used for communicating with eye trackers, tracking cells with them in a sciview VR environment. + * It calls the Hedgehog analysis on the eye tracking results and communicates the results to Mastodon via + * [trackCreationCallback], which is called on every spine graph vertex that is extracted + */ +class EyeTracking( + sciview: SciView, + manvr3d: Manvr3dMain, + geometryHandler: GeometryHandler, + resolutionScale: Float = 1f, +): CellTrackingBase(sciview, manvr3d, geometryHandler, resolutionScale) { + + lateinit var pupilTracker: PupilEyeTracker + val calibrationTarget = Icosphere(0.02f, 2) + val laser = Cylinder(0.005f, 0.2f, 10) + + val confidenceThreshold = 0.60f + + private lateinit var debugBoard: TextBoard + + enum class TrackingType { Follow, Pick } + + private var currentTrackingType = TrackingType.Follow + + fun establishEyeTrackerConnection(): Boolean { + return try { + val future = CompletableFuture.supplyAsync { + PupilEyeTracker( + calibrationType = PupilEyeTracker.CalibrationType.WorldSpace, + port = System.getProperty("PupilPort", "50020").toInt() + ) + } + pupilTracker = future.get(4, TimeUnit.SECONDS) + true + } catch (e: TimeoutException) { + logger.warn("Eye tracker initialization timed out after 2 seconds. Resuming with default VR.") + false + } catch (e: Exception) { + logger.error("Eye tracker initialization failed with ${e.message}") + false + } + } + + override fun run() { + // Check whether we already initialized the pupilTracker or not + if (!::pupilTracker.isInitialized) { + if (!establishEyeTrackerConnection()) { + logger.error("Failed to initialize eye tracker") + // Handle the failure - maybe throw an exception or set a flag + return + } + } + + // Do all the things for general VR startup before setting up the eye tracking environment + super.run() + + sessionId = "BionicTracking-generated-${SystemHelpers.Companion.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + referenceTarget.name = "Reference Target" + sciview.camera?.addChild(referenceTarget) + + calibrationTarget.visible = false + calibrationTarget.material { + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(1.0f, 1.0f, 1.0f) + } + calibrationTarget.name = "Calibration Target" + sciview.camera?.addChild(calibrationTarget) + + laser.visible = false + laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f) } + laser.name = "Laser" + sciview.addNode(laser) + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + sciview.addNode(hedgehogs) + + val eyeFrames = Mesh("eyeFrames") + val left = Box(Vector3f(1.0f, 1.0f, 0.001f)) + val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) + left.spatial().position = Vector3f(-1.0f, 1.5f, 0.0f) + left.spatial().rotation = left.spatial().rotation.rotationZ(PI.toFloat()) + right.spatial().position = Vector3f(1.0f, 1.5f, 0.0f) + eyeFrames.addChild(left) + eyeFrames.addChild(right) + + sciview.addNode(eyeFrames) + + val pupilFrameLimit = 20 + var lastFrame = System.nanoTime() + + pupilTracker.subscribeFrames { eye, texture -> + if(System.nanoTime() - lastFrame < pupilFrameLimit*10e5) { + return@subscribeFrames + } + + val node = if(eye == 1) { + left + } else { + right + } + + val stream = ByteArrayInputStream(texture) + val image = ImageIO.read(stream) + val data = (image.raster.dataBuffer as DataBufferByte).data + + node.ifMaterial { + textures["diffuse"] = Texture( + Vector3i(image.width, image.height, 1), + 3, + UnsignedByteType(), + BufferUtils.Companion.allocateByteAndPut(data) + ) + } + + lastFrame = System.nanoTime() + } + + // TODO: Replace with cam.showMessage() + debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.spatial().scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.spatial().position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + if (device.type == TrackedDeviceType.Controller) { + setupEyeTracking() + setupEyeTrackingMenu() + } + } + + // Attach a behavior to the main loop that stops the eye tracking once we reached the first time point + // and analyzes the created track. + attachToLoop { + val newTimepoint = volume.viewerState.currentTimepoint + if (eyeTrackingActive && newTimepoint == 0) { + eyeTrackingActive = false + playing = false + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + logger.info("Deactivated eye tracking by reaching timepoint 0.") + sciview.camera!!.showMessage("Tracking deactivated.", distance = 2f, size = 0.2f, centered = true) + analyzeEyeTrack() + } + } + + } + + + private fun setupEyeTracking() { + val cam = sciview.camera as? DetachedHeadCamera ?: return + + val toggleTracking = ClickBehaviour { _, _ -> + if (!pupilTracker.isCalibrated) { + logger.warn("Can't do eye tracking because eye trackers are not calibrated yet.") + return@ClickBehaviour + } + if (eyeTrackingActive) { + logger.info("deactivated tracking through user input.") + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.", distance = 2f, size = 0.2f, centered = true) + if (currentTrackingType == TrackingType.Follow) { + analyzeEyeTrack() + } else { + analyzeGazeClusters() + } + playing = false + } else { + logger.info("activating tracking...") + playing = if (currentTrackingType == TrackingType.Follow) { + true + } else { + false + } + addHedgehog() + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.", distance = 2f, size = 0.2f, centered = true) + } + eyeTrackingActive = !eyeTrackingActive + } + + buttonMapper.setKeyBindAndBehavior(hmd, "eyeTracking", toggleTracking) + } + + private fun calibrateEyeTrackers(force: Boolean = false) { + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + pupilTracker.gazeConfidenceThreshold = confidenceThreshold + if (!pupilTracker.isCalibrated || force) { + logger.info("Calibrating pupil trackers...") + + volume.visible = false + + pupilTracker.onCalibrationInProgress = { + cam.showMessage( + "Crunching equations ...", + distance = 2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), + duration = 15000, centered = true + ) + } + + pupilTracker.onCalibrationFailed = { + cam.showMessage( + "Calibration failed.", + distance = 2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f), + centered = true + ) + } + + pupilTracker.onCalibrationSuccess = { + cam.showMessage( + "Calibration succeeded!", + distance = 2f, size = 0.2f, + messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f), + centered = true + ) + + for (i in 0 until 20) { + referenceTarget.ifMaterial { diffuse = Vector3f(0.0f, 1.0f, 0.0f) } + Thread.sleep(100) + referenceTarget.ifMaterial { diffuse = Vector3f(0.8f, 0.8f, 0.8f) } + Thread.sleep(30) + } + + if (!pupilTracker.isCalibrated) { + hmd.removeBehaviour("start_calibration") + hmd.removeKeyBinding("start_calibration") + } + + volume.visible = true + playing = false + } + + pupilTracker.unsubscribeFrames() + sciview.deleteNode(sciview.find("eyeFrames")) + + logger.info("Starting eye tracker calibration") + cam.showMessage( + "Follow the white rabbit.", + distance = 2f, + size = 0.2f, + duration = 1500, + centered = true + ) + + pupilTracker.calibrate( + cam, hmd, + generateReferenceData = true, + calibrationTarget = calibrationTarget + ) + + pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { + + PupilEyeTracker.CalibrationType.WorldSpace -> { gaze -> + if (gaze.confidence > confidenceThreshold) { + val p = gaze.gazePoint() + referenceTarget.visible = true + // Pupil has mm units, so we divide by 1000 here to get to scenery units + referenceTarget.spatial().position = p + (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = + "${String.format("%.2f", p.x())}, ${ + String.format( + "%.2f", + p.y() + ) + }, ${String.format("%.2f", p.z())}" + + val headCenter = cam.spatial().viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = Matrix4f(cam.spatial().world).transform(p.xyzw()).xyz() + val direction = (pointWorld - headCenter).normalize() + + if (eyeTrackingActive) { + addSpine( + headCenter, + direction, + volume, + gaze.confidence, + volume.viewerState.currentTimepoint + ) + } + } + } + } + logger.info("Calibration routine done.") + } + } + } + + private fun setupEyeTrackingMenu() { + + leftWristMenu.addColumn("Eye Tracking") + leftWristMenu.addButton( + "Eye Tracking", "Calibrate", + command = { calibrateEyeTrackers() }, depressDelay = 500 + ) + leftWristMenu.addToggleButton( + "Eye Tracking", "Hedgehogs Off", + "Hedgehogs On", + command = { + hedgehogVisibility = if (hedgehogVisibility == HedgehogVisibility.Hidden) { + HedgehogVisibility.PerTimePoint + } else { + HedgehogVisibility.Hidden + } + }) + leftWristMenu.addToggleButton( + "Eye Tracking", "Follow Cell", + "Count Cells", + command = { + currentTrackingType = if (currentTrackingType == TrackingType.Follow) { + TrackingType.Pick + } else { + TrackingType.Follow + } + }, color = Vector3f(0.65f, 1f, 0.22f), + pressedColor = Vector3f(0.15f, 0.2f, 1f) + ) + } + + /** Writes the accumulated gazes (hedgehog) to a file, analyzes it, + * sends the track to Mastodon and writes the track to a file. */ + private fun analyzeEyeTrack() { + val lastHedgehog = hedgehogs.children.last() as InstancedNode + val hedgehogId = hedgehogIds.incrementAndGet() + + writeHedgehogToFile(lastHedgehog, hedgehogId) + + val spines = getSpinesFromHedgehog(lastHedgehog) + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) + h.run() + } + + if(track == null) { + logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + geometryHandler.addTrackToMastodon(track.points, cursor.radius,false, null, null) + manvr3d.rebuildGeometry() + + writeTrackToFile(track.points, hedgehogId) + + } + + private fun getSpinesFromHedgehog(hedgehog: InstancedNode): List { + return hedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + } + + /** Performs an analysis of collected gazes (hedgehogs) by first calculating the rotational distance between + * subsequent gazes, then discards all gazes larger than 0.3x median distance, clusters the remaining directions + * and samples the volume using the cluster centers as directions. It then extracts the first local minima and sends + * them as spots to Mastodon. */ + private fun analyzeGazeClusters() { + logger.info("Starting analysis of gaze clusters...") + val lastHedgehog = hedgehogs.children.last() as InstancedNode + val hedgehogId = hedgehogIds.incrementAndGet() + + writeHedgehogToFile(lastHedgehog, hedgehogId) + // Get spines from the most recent hedgehog + val spines = getSpinesFromHedgehog(lastHedgehog) + logger.info("Starting with ${spines.size} spines") + + // Calculate the distance from each direction to its neighbor + val speeds = spines.zipWithNext { a, b -> a.direction.distance(b.direction) } + logger.info("Min speed: ${speeds.min()}, max speed: ${speeds.max()}") + val medianSpeed = speeds.sorted()[speeds.size/2] + logger.info("Median speed: $medianSpeed") + + // Clean the list of spines by removing the ones that are too far from their neighbors + val cleanedSpines = spines.filterIndexed { index, _ -> speeds[index] < 0.3 * medianSpeed } + logger.info("After cleaning: ${cleanedSpines.size} spines remain") + + var start = TimeSource.Monotonic.markNow() + // Assuming ten times the median distance is a good clustering value... + val clustering = DBSCANClusterer((10 * medianSpeed).toDouble(), 3) + + // Create a map to efficiently find spine metadata by direction + val spineByDirection = cleanedSpines.associateBy { it.direction.toDoubleArray().contentHashCode() } + + val clusters = clustering.cluster(cleanedSpines.map { + Clusterable { + // On the fly conversion of a Vector3f to a double array + it.direction.toDoubleArray() + } + }) + logger.info("Clustering took ${TimeSource.Monotonic.markNow() - start}") + logger.info("We got ${clusters.size} clusters") + + // Extract the mean direction for each cluster, + // and find the corresponding start positions and average them too + val clusterCenters = clusters.map { cluster -> + var meanDir = Vector3f() + var meanPos = Vector3f() + + // Each "point" in the cluster is actually the ray direction + cluster.points.forEach { point -> + // Accumulate the directions + meanDir + point.point.toVector3f() + // Now grab the spine itself so we can also access its origin + val spine = spineByDirection[point.point.contentHashCode()] + if (spine != null) { + meanPos += spine.origin + } else { + logger.warn("Could not find spine for direction: ${point.point.contentToString()}") + } + } + // Calculate means by dividing by cluster size + meanDir /= cluster.points.size.toFloat() + meanPos /= cluster.points.size.toFloat() + + logger.debug("MeanDir for cluster is $meanDir") + logger.debug("MeanPos for cluster is $meanPos") + + (meanPos to meanDir) + } + + // We only need the analyzer to access the smoothing and maxima search functions + val analyzer = HedgehogAnalysis(cleanedSpines, Matrix4f(volume.spatial().world)) + + start = TimeSource.Monotonic.markNow() + val spots = clusterCenters.map { (origin, direction) -> + val (samples, samplePos) = sampleRayThroughVolume(origin, direction, volume) + var spotPos: Vector3f? = null + if (samples != null && samplePos != null) { + val smoothed = gaussSmoothing(samples, 4) + val rayMax = smoothed.max() + // take the first local maximum that is at least 20% of the global maximum to prevent spot creation in noisy areas + localMaxima(smoothed).firstOrNull {it.second > 0.2 * rayMax}?.let { (index, sample) -> + spotPos = samplePos[index] + } + } + spotPos + } + logger.info("Sampling volume and spot extraction took ${TimeSource.Monotonic.markNow() - start}") + spots.filterNotNull().forEach { spot -> + geometryHandler.addOrRemoveSpots(volume.currentTimepoint, spot, cursor.radius, false, false) + } + } + + /** Toggles the VR rendering off, cleans up eyetracking-related scene objects and removes the light tetrahedron + * that was created for the calibration routine. */ + override fun stop() { + pupilTracker.unsubscribeFrames() + logger.info("Stopped volume and hedgehog updater thread.") + val n = sciview.find("eyeFrames") + n?.let { sciview.deleteNode(it) } + // Delete definitely existing objects + listOf(referenceTarget, calibrationTarget, laser, debugBoard, hedgehogs).forEach { + try { + sciview.deleteNode(it) + } catch (e: Exception) { + logger.warn("Failed to delete $it") + } + } + logger.info("Successfully cleaned up eye tracking environment.") + super.stop() + } + +} \ No newline at end of file diff --git a/src/main/kotlin/vr/MoveInstanceVR.kt b/src/main/kotlin/vr/MoveInstanceVR.kt new file mode 100644 index 0000000..c890552 --- /dev/null +++ b/src/main/kotlin/vr/MoveInstanceVR.kt @@ -0,0 +1,125 @@ +package vr + +import Manvr3dMain +import graphics.scenery.Scene +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.MultiButtonManager +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.plus +import graphics.scenery.utils.lazyLogger +import org.joml.Vector3f +import org.scijava.ui.behaviour.DragBehaviour + +class MoveInstanceVR( + val manvr3d: Manvr3dMain, + val buttonmanager: MultiButtonManager, + val button: OpenVRHMD.OpenVRButton, + val trackerRole: TrackerRole, + val getTipPosition: () -> Vector3f +): DragBehaviour { + + val logger by lazyLogger() + + val adjacentEdges = mutableListOf() + + var currentControllerPos = Vector3f() + + override fun init(x: Int, y: Int) { + buttonmanager.pressButton(button, trackerRole) + if (!buttonmanager.isTwoHandedActive()) { + val pos = getTipPosition() + if (manvr3d.mastodon.selectionModel.selectedVertices == null) { + manvr3d.selectedSpotInstances.clear() + return + } else { + manvr3d.bdvNotifier?.lockUpdates = true + manvr3d.selectedSpotInstances.forEach { inst -> + logger.debug("selected spot instance is $inst") + val spot = manvr3d.geometryHandler.findSpotFromInstance(inst) + val selectedTP = spot?.timepoint ?: -1 + if (selectedTP != manvr3d.volumeNode.currentTimepoint) { + manvr3d.selectedSpotInstances.clear() + logger.warn("Tried to move a spot that was outside the current timepoint. Aborting.") + return + } else { + manvr3d.bdvNotifier?.lockUpdates = true + currentControllerPos = manvr3d.sciviewToMastodonCoords(pos) + spot?.let { s -> + adjacentEdges.addAll(s.edges().map { it.internalPoolIndex }) + logger.debug("Moving edges $manvr3d.adjacentEdges for spot ${spot.internalPoolIndex}.") + } + } + } + } + } + } + + override fun drag(x: Int, y: Int) { + // Only perform the single hand behavior when no other grab button is currently active + // to prevent simultaneous execution of behaviors + if (!buttonmanager.isTwoHandedActive()) { + val pos = getTipPosition() + val newPos = manvr3d.sciviewToMastodonCoords(pos) + val movement = newPos - currentControllerPos + manvr3d.selectedSpotInstances.forEach { + it.spatial { + position += movement + } + manvr3d.geometryHandler.moveSpotInBDV(it, movement) + } + manvr3d.geometryHandler.mainSpotInstance?.updateInstanceBuffers() + manvr3d.geometryHandler.updateLinkTransforms(adjacentEdges) + currentControllerPos = newPos + } + } + + override fun end(x: Int, y: Int) { + if (!buttonmanager.isTwoHandedActive()) { + manvr3d.bdvNotifier?.lockUpdates = false + manvr3d.geometryHandler.showInstancedSpots(manvr3d.currentTimepoint, + manvr3d.currentColorizer) + adjacentEdges.clear() + manvr3d.bdvNotifier?.lockUpdates = false + } + buttonmanager.releaseButton(button, trackerRole) + } + + companion object { + + /** + * Convenience method for adding grab behaviour + */ + fun createAndSet( + manvr3d: Manvr3dMain, + hmd: OpenVRHMD, + buttons: List, + controllerSide: List, + buttonmanager: MultiButtonManager, + getTipPosition: () -> Vector3f + ) { + hmd.events.onDeviceConnect.add { _, device, _ -> + if (device.type == TrackedDeviceType.Controller) { + device.model?.let { controller -> + if (controllerSide.contains(device.role)) { + buttons.forEach { button -> + val name = "VRDrag:${hmd.trackingSystemName}:${device.role}:$button" + val grabBehaviour = MoveInstanceVR( + manvr3d, + buttonmanager, + button, + device.role, + getTipPosition + ) + buttonmanager.registerButtonConfig(button, device.role) + hmd.addBehaviour(name, grabBehaviour) + hmd.addKeyBinding(name, device.role, button) + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/src/test/kotlin/org/mastodon/mamut/StartSciviewBridgeDirectly.kt b/src/test/kotlin/org/mastodon/mamut/StartManvr3dDirectly.kt similarity index 87% rename from src/test/kotlin/org/mastodon/mamut/StartSciviewBridgeDirectly.kt rename to src/test/kotlin/org/mastodon/mamut/StartManvr3dDirectly.kt index df49489..5935286 100644 --- a/src/test/kotlin/org/mastodon/mamut/StartSciviewBridgeDirectly.kt +++ b/src/test/kotlin/org/mastodon/mamut/StartManvr3dDirectly.kt @@ -1,5 +1,6 @@ package org.mastodon.mamut +import Manvr3dMain import graphics.scenery.utils.lazyLogger import mpicbg.spim.data.SpimDataException import org.mastodon.mamut.io.ProjectLoader @@ -9,7 +10,7 @@ import sc.iview.SciView.Companion.create import java.io.IOException import javax.swing.WindowConstants -object StartSciviewBridgeDirectly { +object StartManvr3dDirectly { private val logger by lazyLogger() @Throws(IOException::class, SpimDataException::class) @@ -43,12 +44,11 @@ object StartSciviewBridgeDirectly { // --------------->> <<--------------- val sv = createSciview() val mastodon = giveMeMastodonOfThisProject(sv.scijavaContext, projectPath) - val bridge = SciviewBridge(mastodon, targetSciviewWindow = sv) - bridge.createAndShowControllingUI() -// bridge.openSyncedBDV(); + val manvr3dContext = Manvr3dMain(mastodon, targetSciviewWindow = sv) + manvr3dContext.createAndShowControllingUI() mastodon.projectClosedListeners().add(CloseListener { logger.debug("Mastodon project was closed, cleaning up in sciview:") - bridge.close() //calls also bridge.detachControllingUI(); + manvr3dContext.close() //calls also manvr3dMain.detachControllingUI(); }) } catch (e: Exception) { throw e diff --git a/src/test/kotlin/org/mastodon/mamut/TestingAdjustableRange.kt b/src/test/kotlin/org/mastodon/mamut/TestingAdjustableRange.kt index 7845dd4..77c95fb 100644 --- a/src/test/kotlin/org/mastodon/mamut/TestingAdjustableRange.kt +++ b/src/test/kotlin/org/mastodon/mamut/TestingAdjustableRange.kt @@ -1,7 +1,7 @@ import bdv.ui.rangeslider.RangeSlider -import util.AbstractAdjustableSliderBasedControl -import util.AdjustableBoundsRangeSlider +import ui.AbstractAdjustableSliderBasedControl +import ui.AdjustableBoundsRangeSlider import java.awt.GridBagConstraints import java.awt.GridBagLayout import java.awt.Insets diff --git a/src/test/kotlin/org/mastodon/mamut/TestingAdjustableSlider.kt b/src/test/kotlin/org/mastodon/mamut/TestingAdjustableSlider.kt index c9eb9b6..5d32341 100644 --- a/src/test/kotlin/org/mastodon/mamut/TestingAdjustableSlider.kt +++ b/src/test/kotlin/org/mastodon/mamut/TestingAdjustableSlider.kt @@ -1,6 +1,6 @@ package org.mastodon.mamut -import util.AdjustableBoundsSlider +import ui.AdjustableBoundsSlider import java.awt.GridBagConstraints import java.awt.GridBagLayout import javax.swing.JButton