Skip to content

Commit 71b37a6

Browse files
authored
Merge pull request #479 from EarthyScience/jp/modular-zarr
Modular Zarr/Refactor
2 parents 3fd3ded + 25e8a95 commit 71b37a6

File tree

11 files changed

+403
-577
lines changed

11 files changed

+403
-577
lines changed

src/components/LandingHome.tsx

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import * as THREE from 'three'
33
THREE.Cache.enabled = true;
44
import { GetZarrMetadata, GetVariableNames, GetTitleDescription } from '@/components/zarr/GetMetadata';
5-
import { ZarrDataset, GetStore } from '@/components/zarr/ZarrLoaderLRU';
5+
import { GetStore } from '@/components/zarr/ZarrLoaderLRU';
66
import { useEffect, useMemo } from 'react';
77
import { PlotArea, Plot, LandingShapes } from '@/components/plots';
88
import { MainPanel } from '@/components/ui';
@@ -62,8 +62,6 @@ export function LandingHome() {
6262
setCurrentStore(newStore)
6363
}, [initStore, setCurrentStore])
6464

65-
const ZarrDS = useMemo(() => new ZarrDataset(currentStore), [currentStore])
66-
6765
useEffect(() => {
6866
let isMounted = true;
6967

@@ -80,9 +78,9 @@ export function LandingHome() {
8078
return () => { isMounted = false; };
8179
}, [currentStore, setZMeta, setVariables, setTitleDescription])
8280

83-
useEffect(()=>{
84-
sendPing()
85-
},[])
81+
// useEffect(()=>{
82+
// sendPing()
83+
// },[])
8684

8785
return (
8886
<>
@@ -93,7 +91,7 @@ export function LandingHome() {
9391
<Loading />
9492

9593
{/* {variable === "Default" && <ScrollableLinksTable />} */}
96-
{variable != "Default" && <Plot ZarrDS={ZarrDS} />}
94+
{variable != "Default" && <Plot />}
9795
{Object.keys(timeSeries).length >= 1 && <PlotArea />}
9896
</>
9997
);

src/components/plots/AnalysisWG.tsx

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import React, { useEffect } from 'react';
55
import { DataReduction, Convolve, Multivariate2D, Multivariate3D, CUMSUM3D, Convolve2D } from '../computation/webGPU';
66
import { useGlobalStore, useAnalysisStore, usePlotStore } from '@/utils/GlobalStates';
77
import { useShallow } from 'zustand/shallow';
8-
import { ZarrDataset } from '../zarr/ZarrLoaderLRU';
8+
import { GetArray } from '../zarr/ZarrLoaderLRU';
99
import { CreateTexture } from '../textures';
1010

1111
// The new centralized map for all operations
@@ -40,7 +40,7 @@ const ShaderMap = {
4040
// Define a type for our operations based on the ShaderMap keys
4141
type Operation = keyof typeof ShaderMap;
4242

43-
const AnalysisWG = ({ setTexture, ZarrDS }: { setTexture: React.Dispatch<React.SetStateAction<THREE.Data3DTexture[] | THREE.DataTexture[] | null>>, ZarrDS: ZarrDataset }) => {
43+
const AnalysisWG = ({ setTexture, }: { setTexture: React.Dispatch<React.SetStateAction<THREE.Data3DTexture[] | THREE.DataTexture[] | null>> }) => {
4444

4545
// Global state hooks remain the same
4646
const { strides, dataShape, valueScales, isFlat, setIsFlat, setStatus, setValueScales } = useGlobalStore(useShallow(state => ({
@@ -82,7 +82,7 @@ const AnalysisWG = ({ setTexture, ZarrDS }: { setTexture: React.Dispatch<React.S
8282
let var2Data: ArrayBufferView | null = null;
8383
if (useTwo) {
8484
setStatus("Fetching second variable...")
85-
const var2Array = await ZarrDS.GetArray(variable2, {zSlice, ySlice, xSlice});
85+
const var2Array = await GetArray();
8686
var2Data = var2Array?.data;
8787
setStatus("Computing...");
8888
if (!var2Data) {

src/components/plots/FlatMap.tsx

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import { flatFrag3D, fragmentFlat } from '../textures/shaders';
88
import { useShallow } from 'zustand/shallow'
99
import { ThreeEvent } from '@react-three/fiber';
1010
import { GetCurrentArray, GetTimeSeries, parseUVCoords } from '@/utils/HelperFuncs';
11-
import { ZarrDataset } from '../zarr/ZarrLoaderLRU';
1211
import { evaluate_cmap } from 'js-colormaps-es';
1312

1413
interface InfoSettersProps{
@@ -23,7 +22,7 @@ function Rescale(value: number, scales: {minVal: number, maxVal: number}){
2322
return value * range + scales.minVal
2423
}
2524

26-
const FlatMap = ({textures, infoSetters, ZarrDS} : {textures : THREE.DataTexture | THREE.Data3DTexture[], infoSetters : InfoSettersProps, ZarrDS: ZarrDataset}) => {
25+
const FlatMap = ({textures, infoSetters} : {textures : THREE.DataTexture | THREE.Data3DTexture[], infoSetters : InfoSettersProps}) => {
2726
const {setLoc, setShowInfo, val, coords} = infoSetters;
2827
const {flipY, colormap, valueScales, dimArrays, dimNames, dimUnits,
2928
isFlat, dataShape, textureArrayDepths, strides, timeSeries,
@@ -134,7 +133,7 @@ const FlatMap = ({textures, infoSetters, ZarrDS} : {textures : THREE.DataTexture
134133
function HandleTimeSeries(event: THREE.Intersection){
135134
const uv = event.uv;
136135
const normal = new THREE.Vector3(0,0,1)
137-
if(ZarrDS && uv){
136+
if(uv){
138137
const tsUV = flipY ? new THREE.Vector2(uv.x, 1-uv.y) : uv
139138
const tempTS = GetTimeSeries({data:analysisMode ? analysisArray : GetCurrentArray(), shape:dataShape, stride:strides},{uv:tsUV,normal})
140139
setPlotDim(0) //I think this 2 is only if there are 3-dims. Need to rework the logic

src/components/plots/Plot.tsx

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@ import * as THREE from 'three';
44
import { PointCloud, UVCube, DataCube, FlatMap, Sphere, CountryBorders, AxisLines, SphereBlocks, FlatBlocks, KeyFramePreviewer } from '@/components/plots';
55
import { Canvas, invalidate, useThree } from '@react-three/fiber';
66
import { ArrayToTexture, CreateTexture } from '@/components/textures';
7-
import { ZarrDataset } from '../zarr/ZarrLoaderLRU';
7+
import { GetArray, GetAttributes } from '../zarr/ZarrLoaderLRU';
88
import { useAnalysisStore, useGlobalStore, useImageExportStore, usePlotStore, useZarrStore } from '@/utils/GlobalStates';
99
import { useShallow } from 'zustand/shallow';
1010
import { Navbar, Colorbar, ExportExtent } from '../ui';
1111
import AnalysisInfo from './AnalysisInfo';
1212
import { OrbitControls as OrbitControlsImpl } from 'three-stdlib';
1313
import AnalysisWG from './AnalysisWG';
14-
import { ParseExtent } from '@/utils/HelperFuncs';
14+
import { ParseExtent, GetDimInfo } from '@/utils/HelperFuncs';
1515
import ExportCanvas from '@/utils/ExportCanvas';
1616
import KeyFrames from '../ui/KeyFrames';
1717

@@ -130,7 +130,7 @@ const Orbiter = ({isFlat} : {isFlat : boolean}) =>{
130130
);
131131
}
132132

133-
const Plot = ({ZarrDS}:{ZarrDS: ZarrDataset}) => {
133+
const Plot = () => {
134134
const {
135135
setShape, setDataShape, setFlipY, setValueScales, setMetadata, setDimArrays,
136136
setDimNames, setDimUnits, setPlotOn, setStatus} = useGlobalStore(
@@ -197,7 +197,7 @@ const Plot = ({ZarrDS}:{ZarrDS: ZarrDataset}) => {
197197
setZSlice(zSlice);
198198
setYSlice(ySlice);
199199
setXSlice(xSlice);
200-
ZarrDS.GetArray(variable, {xSlice, ySlice, zSlice}).then((result) => {
200+
GetArray().then((result) => {
201201
const [tempTexture, scaling] = ArrayToTexture({
202202
data: result.data,
203203
shape: result.shape
@@ -232,37 +232,40 @@ const Plot = ({ZarrDS}:{ZarrDS: ZarrDataset}) => {
232232
return;
233233
}
234234
//Get Metadata
235-
ZarrDS.GetAttributes(variable).then((result)=>{
235+
GetAttributes().then((result)=>{
236236
setMetadata(result);
237237
setStableMetadata(result);
238-
let [dimArrs, dimUnits, dimNames] = ZarrDS.GetDimArrays()
238+
})
239+
GetDimInfo(variable).then((arrays)=>{
240+
let {dimArrays, dimUnits, dimNames}= arrays;
239241
if (is4D){
240-
dimArrs = dimArrs.slice(1);
242+
dimArrays = dimArrays.slice(1);
241243
dimUnits = dimUnits.slice(1);
242244
dimNames = dimNames.slice(1);
243245
}
244-
setDimArrays(dimArrs)
245246
setDimNames(dimNames)
246-
if (dimArrs.length > 2){
247-
if (dimArrs[1][1] < dimArrs[1][0])
247+
setDimArrays(dimArrays)
248+
if (dimArrays.length > 2){
249+
if (dimArrays[1][1] < dimArrays[1][0])
248250
{setFlipY(true)}
249251
else
250252
{setFlipY(false)}
251253
}
252254
else{
253-
if (dimArrs[0][1] < dimArrs[0][0])
255+
if (dimArrays[0][1] < dimArrays[0][0])
254256
{setFlipY(true)}
255257
else
256258
{setFlipY(false)}
257259
}
258260
setDimUnits(dimUnits)
259-
ParseExtent(dimUnits, dimArrs)
260-
})
261+
ParseExtent(dimUnits, dimArrays)
262+
})
261263
}else{
262264
setMetadata(null)
263265
}
264266
}, [reFetch])
265267

268+
266269
useEffect(()=>{ // Reset after analysis mode
267270
if(!analysisMode && show){
268271
const {dataShape} = useGlobalStore.getState();
@@ -322,7 +325,7 @@ const Plot = ({ZarrDS}:{ZarrDS: ZarrDataset}) => {
322325
<ExportExtent />
323326
{keyFrameEditor && <KeyFrames />}
324327
<TransectNotice />
325-
<AnalysisWG setTexture={setTextures} ZarrDS={ZarrDS}/>
328+
<AnalysisWG setTexture={setTextures} />
326329
{show && <Colorbar units={stableMetadata?.units} metadata={stableMetadata} valueScales={valueScales}/>}
327330
<Nav />
328331
{(isFlat || plotType == "flat") && <AnalysisInfo loc={loc} show={showInfo} info={[...coords.current,val.current]}/> }
@@ -338,22 +341,22 @@ const Plot = ({ZarrDS}:{ZarrDS: ZarrDataset}) => {
338341
{plotType == "volume" && show &&
339342
<>
340343
<DataCube volTexture={textures}/>
341-
<UVCube ZarrDS={ZarrDS} />
344+
<UVCube />
342345
</>
343346
}
344347
{plotType == "point-cloud" && show &&
345348
<>
346-
<PointCloud textures={{texture: textures as THREE.Data3DTexture[],colormap}} ZarrDS={ZarrDS}/>
349+
<PointCloud textures={{texture: textures as THREE.Data3DTexture[],colormap}}/>
347350
</>
348351
}
349352
{plotType == "sphere" && show &&
350353
<>
351-
{displaceSurface ? <Sphere textures={textures} ZarrDS={ZarrDS} /> : <SphereBlocks textures={textures} />}
354+
{displaceSurface ? <Sphere textures={textures} /> : <SphereBlocks textures={textures} />}
352355
</>
353356
}
354357
<Orbiter isFlat={plotType == "flat"} />
355358
{plotType == "flat" && show && <>
356-
{displaceSurface && <FlatMap textures={textures as THREE.DataTexture | THREE.Data3DTexture[]} infoSetters={infoSetters} ZarrDS={ZarrDS}/> }
359+
{displaceSurface && <FlatMap textures={textures as THREE.DataTexture | THREE.Data3DTexture[]} infoSetters={infoSetters} /> }
357360
{!displaceSurface && <FlatBlocks textures={textures} />}
358361
</>
359362
}

src/components/plots/PointCloud.tsx

Lines changed: 50 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import { useEffect, useMemo, useState, useRef } from 'react'
44
import { pointFrag, pointVert } from '@/components/textures/shaders'
55
import { useAnalysisStore, useGlobalStore, usePlotStore } from '@/utils/GlobalStates';
66
import { useShallow } from 'zustand/shallow';
7-
import { ZarrDataset } from '../zarr/ZarrLoaderLRU';
87
import { parseUVCoords, getUnitAxis, GetTimeSeries, GetCurrentArray } from '@/utils/HelperFuncs';
98
import { evaluate_cmap } from 'js-colormaps-es';
109

@@ -25,7 +24,7 @@ interface pointSetters{
2524
setDimWidth: React.Dispatch<React.SetStateAction<number>>;
2625
}
2726

28-
const MappingCube = ({dimensions, ZarrDS, setters} : {dimensions: dimensionsProps, ZarrDS: ZarrDataset, setters:pointSetters}) =>{
27+
const MappingCube = ({dimensions, setters} : {dimensions: dimensionsProps, setters:pointSetters}) =>{
2928
const {width, height, depth} = dimensions;
3029
const {setPoints, setStride, setDimWidth} = setters;
3130
const selectTS = usePlotStore(state => state.selectTS)
@@ -80,60 +79,56 @@ const MappingCube = ({dimensions, ZarrDS, setters} : {dimensions: dimensionsProp
8079
setPoints({})
8180
}
8281
lastNormal.current = dimAxis;
83-
84-
if(ZarrDS){
85-
const tempTS = GetTimeSeries({data: analysisMode? analysisArray : GetCurrentArray(), shape: dataShape, stride: strides},{uv,normal})
86-
const plotDim = (normal.toArray()).map((val, idx) => {
87-
if (Math.abs(val) > 0) {
88-
return idx;
89-
}
90-
return null;}).filter(idx => idx !== null);
91-
setPlotDim(2-plotDim[0]) //I think this 2 is only if there are 3-dims. Need to rework the logic
92-
const coordUV = parseUVCoords({normal:normal,uv:uv})
93-
let dimCoords = coordUV.map((val,idx)=>val ? dimSlices[idx][Math.round(val*dimSlices[idx].length-.5)] : null)
94-
const thisDimNames = dimNames.filter((_,idx)=> dimCoords[idx] !== null)
95-
const thisDimUnits = dimUnits.filter((_,idx)=> dimCoords[idx] !== null)
96-
dimCoords = dimCoords.filter(val => val !== null)
97-
const tsID = `${dimCoords[0]}_${dimCoords[1]}`
98-
const tsObj = {
99-
color:evaluate_cmap(getColorIdx()/10,"Paired"),
100-
data:tempTS
82+
const tempTS = GetTimeSeries({data: analysisMode? analysisArray : GetCurrentArray(), shape: dataShape, stride: strides},{uv,normal})
83+
const plotDim = (normal.toArray()).map((val, idx) => {
84+
if (Math.abs(val) > 0) {
85+
return idx;
10186
}
102-
incrementColorIdx();
103-
updateTimeSeries({ [tsID] : tsObj})
104-
const dimObj = {
105-
first:{
106-
name:thisDimNames[0],
107-
loc:dimCoords[0] ?? 0,
108-
units:thisDimUnits[0]
109-
},
110-
second:{
111-
name:thisDimNames[1],
112-
loc:dimCoords[1] ?? 0,
113-
units:thisDimUnits[1]
114-
},
115-
plot:{
116-
units:dimUnits[2-plotDim[0]]
117-
}
87+
return null;}).filter(idx => idx !== null);
88+
setPlotDim(2-plotDim[0]) //I think this 2 is only if there are 3-dims. Need to rework the logic
89+
const coordUV = parseUVCoords({normal:normal,uv:uv})
90+
let dimCoords = coordUV.map((val,idx)=>val ? dimSlices[idx][Math.round(val*dimSlices[idx].length-.5)] : null)
91+
const thisDimNames = dimNames.filter((_,idx)=> dimCoords[idx] !== null)
92+
const thisDimUnits = dimUnits.filter((_,idx)=> dimCoords[idx] !== null)
93+
dimCoords = dimCoords.filter(val => val !== null)
94+
const tsID = `${dimCoords[0]}_${dimCoords[1]}`
95+
const tsObj = {
96+
color:evaluate_cmap(getColorIdx()/10,"Paired"),
97+
data:tempTS
98+
}
99+
incrementColorIdx();
100+
updateTimeSeries({ [tsID] : tsObj})
101+
const dimObj = {
102+
first:{
103+
name:thisDimNames[0],
104+
loc:dimCoords[0] ?? 0,
105+
units:thisDimUnits[0]
106+
},
107+
second:{
108+
name:thisDimNames[1],
109+
loc:dimCoords[1] ?? 0,
110+
units:thisDimUnits[1]
111+
},
112+
plot:{
113+
units:dimUnits[2-plotDim[0]]
118114
}
119-
updateDimCoords({[tsID] : dimObj})
120-
const dims = [depth, height, width].filter((_,idx)=> coordUV[idx] != null)
121-
const dimWidth = [depth, height, width].filter((_,idx)=> coordUV[idx] == null)
122-
const newUV = coordUV.filter((v)=> v != null)
123-
const thisStride = strides.filter((_,idx)=> coordUV[idx] != null)
124-
const uIdx = Math.round((newUV[0])*dims[0]-.5)
125-
const vIdx = Math.round(newUV[1]*dims[1]-.5)
126-
const newIdx = uIdx * thisStride[0] + vIdx * thisStride[1]
127-
const dimStride = strides.filter((_,idx)=> coordUV[idx] == null)
128-
setDimWidth(dimWidth[0])
129-
setPoints(prevItems => {
130-
const newEntry = {[tsID] : newIdx}
131-
const updated = {...newEntry, ...prevItems};
132-
return updated; // keep only first 10 items
133-
})
134-
setStride(dimStride[0])
135115
}
136-
116+
updateDimCoords({[tsID] : dimObj})
117+
const dims = [depth, height, width].filter((_,idx)=> coordUV[idx] != null)
118+
const dimWidth = [depth, height, width].filter((_,idx)=> coordUV[idx] == null)
119+
const newUV = coordUV.filter((v)=> v != null)
120+
const thisStride = strides.filter((_,idx)=> coordUV[idx] != null)
121+
const uIdx = Math.round((newUV[0])*dims[0]-.5)
122+
const vIdx = Math.round(newUV[1]*dims[1]-.5)
123+
const newIdx = uIdx * thisStride[0] + vIdx * thisStride[1]
124+
const dimStride = strides.filter((_,idx)=> coordUV[idx] == null)
125+
setDimWidth(dimWidth[0])
126+
setPoints(prevItems => {
127+
const newEntry = {[tsID] : newIdx}
128+
const updated = {...newEntry, ...prevItems};
129+
return updated; // keep only first 10 items
130+
})
131+
setStride(dimStride[0])
137132
}
138133
return(
139134
<mesh scale={[2*globalScale, 2*shapeRatio*globalScale, 2*depthRatio*globalScale]} position={[-offset, -offset, offset]} onClick={HandleTimeSeries}>
@@ -143,7 +138,7 @@ const MappingCube = ({dimensions, ZarrDS, setters} : {dimensions: dimensionsProp
143138
)
144139
}
145140

146-
export const PointCloud = ({textures, ZarrDS} : {textures:PCProps, ZarrDS: ZarrDataset} )=>{
141+
export const PointCloud = ({textures} : {textures:PCProps} )=>{
147142
const { colormap } = textures;
148143
const {timeSeries, flipY, dataShape, textureData} = useGlobalStore(useShallow(state=>({
149144
timeSeries: state.timeSeries,
@@ -265,7 +260,7 @@ export const PointCloud = ({textures, ZarrDS} : {textures:PCProps, ZarrDS: ZarrD
265260
<mesh scale={[1,flipY ? -1:1, 1]} >
266261
<points geometry={geometry} material={shaderMaterial} frustumCulled={false}/>
267262
</mesh>
268-
<MappingCube dimensions={{width,height,depth}} ZarrDS={ZarrDS} setters={{setPoints:setPointsObj, setStride, setDimWidth}}/>
263+
<MappingCube dimensions={{width,height,depth}} setters={{setPoints:setPointsObj, setStride, setDimWidth}}/>
269264
</>
270265
);
271266
}

0 commit comments

Comments
 (0)