to zarr store

Author: lazarusA

src/GlobalStates/GlobalStore.ts

@@ -1,7 +1,6 @@
 import { create } from "zustand";
 import * as THREE from 'three';
 import { GetColorMapTexture } from "@/components/textures";
-import { IcechunkStoreOptions, FetchStoreOptions } from "@/components/zarr/Interfaces";
 
 const ESDC = 'https://s3.bgc-jena.mpg.de:9000/esdl-esdc-v3.0.2/esdc-16d-2.5deg-46x72x1440-3.0.2.zarr'
 
@@ -32,7 +31,6 @@ type StoreState = {
   plotDim: number;
   flipY:boolean;
   initStore:string;
-  fetchKey: number;
   variable: string;
   variables: string[];
   openVariables: boolean;
@@ -48,9 +46,6 @@ type StoreState = {
   textureArrayDepths: number[];
   textureData: Uint8Array;
   clampExtremes: boolean;
-  icechunkOptions: IcechunkStoreOptions | null;
-  fetchOptions: FetchStoreOptions | null;
-  abortController: AbortController | null;
 
   // setters
   setDataShape: (dataShape: number[]) => void;
@@ -70,7 +65,6 @@ type StoreState = {
   setPlotDim: (plotDim: number) => void;
   setFlipY: (flipY:boolean) => void;
   setInitStore: (initStore:string) => void;
-  bumpFetchKey: () => void;
   setVariable: (variable: string) => void;
   setVariables: (variables: string[]) => void;
   setOpenVariables: (openVariables: boolean) => void;
@@ -86,9 +80,6 @@ type StoreState = {
   setDPR: (DPR: number) => void;
   setScalingFactor: (scalingFactor: number | null) => void;
   setClampExtremes: (clampExtremes: boolean) => void;
-  setIcechunkOptions: (options: IcechunkStoreOptions | null) => void;
-  setFetchOptions: (options: FetchStoreOptions | null) => void;
-  setAbortController: (controller: AbortController | null) => void;
 };
 
 export const useGlobalStore = create<StoreState>((set, get) => ({
@@ -107,7 +98,6 @@ export const useGlobalStore = create<StoreState>((set, get) => ({
   plotDim: 0,
   flipY: false,
   initStore: ESDC,
-  fetchKey: 0,
   variable: 'Default',
   variables: [],
   openVariables: false,
@@ -123,9 +113,6 @@ export const useGlobalStore = create<StoreState>((set, get) => ({
   DPR: 1,
   scalingFactor: null,
   clampExtremes: false,
-  icechunkOptions: null,
-  fetchOptions: null,
-  abortController: null,
   // setters
 
   setDataShape: (dataShape) => set({ dataShape }),
@@ -165,7 +152,6 @@ export const useGlobalStore = create<StoreState>((set, get) => ({
   setPlotDim: (plotDim) => set({ plotDim }),
   setFlipY: (flipY) => set({ flipY }),
   setInitStore: (initStore) => set({ initStore }),
-  bumpFetchKey: () => set(s => ({ fetchKey: s.fetchKey + 1 })),
   setVariable: (variable) => set({ variable }),
   setVariables: (variables) => set({ variables }),
   setOpenVariables: (openVariables) => set({ openVariables }),
@@ -180,7 +166,4 @@ export const useGlobalStore = create<StoreState>((set, get) => ({
   setDPR: (DPR) => set({ DPR }),
   setScalingFactor: (scalingFactor) => set({ scalingFactor }),
   setClampExtremes: (clampExtremes) => set({ clampExtremes }),
-  setIcechunkOptions: (options) => set({ icechunkOptions: options }),
-  setFetchOptions: (options) => set({ fetchOptions: options }),
-  setAbortController: (controller) => set({ abortController: controller }),
 }));

src/GlobalStates/ZarrStore.ts

@@ -1,5 +1,6 @@
 import { create } from "zustand";
 import { GetStore } from "@/components/zarr/ZarrLoaderLRU";
+import { FetchStoreOptions, IcechunkStoreOptions } from "@/components/zarr/Interfaces";
 
 const ESDC = 'https://s3.bgc-jena.mpg.de:9000/esdl-esdc-v3.0.2/esdc-16d-2.5deg-46x72x1440-3.0.2.zarr'
 
@@ -18,7 +19,11 @@ type ZarrState = {
   coarsen: boolean,
   kernelSize: number,
   kernelDepth: number,
-
+  icechunkOptions: IcechunkStoreOptions | null;
+  fetchOptions: FetchStoreOptions | null;
+  abortController: AbortController | null;
+  fetchKey: number;
+  
   setZSlice: (zSlice: [number , number | null]) => void;
   setYSlice: (ySlice: [number , number | null]) => void;
   setXSlice: (xSlice: [number , number | null]) => void;
@@ -33,6 +38,10 @@ type ZarrState = {
   setCoarsen: (coarsen: boolean) => void;
   setKernelSize: (kernelSize: number) => void;
   setKernelDepth: (kernelDepth: number) => void;
+  setIcechunkOptions: (options: IcechunkStoreOptions | null) => void;
+  setFetchOptions: (options: FetchStoreOptions | null) => void;
+  setAbortController: (controller: AbortController | null) => void;
+  bumpFetchKey: () => void;
 }
 
 export const useZarrStore = create<ZarrState>((set, get) => ({
@@ -50,6 +59,10 @@ export const useZarrStore = create<ZarrState>((set, get) => ({
   coarsen: false,
   kernelSize: 2,
   kernelDepth: 2,
+  icechunkOptions: null,
+  fetchOptions: null,
+  abortController: null,
+  fetchKey: 0,
 
   setZSlice: (zSlice) => set({ zSlice }),
   setYSlice: (ySlice) => set({ ySlice }),
@@ -65,4 +78,8 @@ export const useZarrStore = create<ZarrState>((set, get) => ({
   setCoarsen: (coarsen) => set({ coarsen }),
   setKernelSize: (kernelSize) => set({ kernelSize }),
   setKernelDepth: (kernelDepth) => set({ kernelDepth }),
+  setIcechunkOptions: (options) => set({ icechunkOptions: options }),
+  setFetchOptions: (options) => set({ fetchOptions: options }),
+  setAbortController: (controller) => set({ abortController: controller }),
+  bumpFetchKey: () => set(s => ({ fetchKey: s.fetchKey + 1 })),
 }))

src/components/LandingHome.tsx

@@ -25,11 +25,10 @@ async function sendPing() {
 
 export function LandingHome() {
   const {
-    initStore, fetchKey, timeSeries, variable, plotOn,
+    initStore, timeSeries, variable, plotOn,
     setZMeta, setVariables, setTitleDescription,
   } = useGlobalStore(useShallow(state => ({
     initStore: state.initStore,
-    fetchKey: state.fetchKey,
     timeSeries: state.timeSeries,
     variable: state.variable,
     plotOn: state.plotOn,
@@ -38,8 +37,11 @@ export function LandingHome() {
     setTitleDescription: state.setTitleDescription,
   })))
 
-  const { currentStore, setCurrentStore, setZSlice, setYSlice, setXSlice, setUseNC } = useZarrStore(useShallow(state => ({
+  const { currentStore, fetchKey,
+    setCurrentStore, setZSlice, setYSlice, setXSlice, setUseNC 
+  } = useZarrStore(useShallow(state => ({
     currentStore: state.currentStore,
+    fetchKey: state.fetchKey,
     setCurrentStore: state.setCurrentStore,
     setZSlice: state.setZSlice,
     setYSlice: state.setYSlice,
@@ -74,16 +76,16 @@ export function LandingHome() {
     }
     if (initStore.startsWith('local')) return; // local_ set by LocalNetCDF/LocalZarr after load
     setUseNC(false)
-    const { icechunkOptions, fetchOptions } = useGlobalStore.getState();
+    const { icechunkOptions, fetchOptions } = useZarrStore.getState();
     const newStore = GetStore(
       initStore,
       fetchOptions   ?? undefined,
       icechunkOptions ?? undefined
     );
     setCurrentStore(newStore);
     // Clear after use
-    useGlobalStore.getState().setIcechunkOptions(null);
-    useGlobalStore.getState().setFetchOptions(null);
+    useZarrStore.getState().setIcechunkOptions(null);
+    useZarrStore.getState().setFetchOptions(null);
   }, [initStore, fetchKey, setCurrentStore])
 
   useEffect(() => {

src/components/ui/MainPanel/RemoteIcechunk.tsx

@@ -12,6 +12,7 @@ import {
 } from "@/components/ui/select";
 import { ChevronDown, ChevronUp, Plus, Trash2 } from 'lucide-react';
 import { useGlobalStore } from '@/GlobalStates/GlobalStore';
+import { useZarrStore } from '@/GlobalStates/ZarrStore';
 
 type RefType     = 'branch' | 'tag' | 'snapshot';
 type HeaderRow   = { key: string; value: string };
@@ -120,8 +121,8 @@ const RemoteIcechunk = ({ setInitStore, onOpenDescription }: Props) => {
     const builtStorageHeaders     = buildHeaders(storageHeaders);
     const builtFetchClientHeaders = buildHeaders(fetchClientHeaders);
 
-    useGlobalStore.getState().setFetchOptions(null);
-    useGlobalStore.getState().setIcechunkOptions({
+    useZarrStore.getState().setFetchOptions(null);
+    useZarrStore.getState().setIcechunkOptions({
       [refType]: refValue,
       ...(Object.keys(builtStorageHeaders).length > 0     && { headers: builtStorageHeaders }),
       ...(credentials                                     && { credentials }),
@@ -142,7 +143,7 @@ const RemoteIcechunk = ({ setInitStore, onOpenDescription }: Props) => {
       retryDelay,
     });
     useGlobalStore.getState().setStatus('Fetching...');
-    useGlobalStore.getState().bumpFetchKey();
+    useZarrStore.getState().bumpFetchKey();
     setInitStore(url);
     onOpenDescription();
   };

src/components/ui/MainPanel/RemoteZarr.tsx

@@ -5,6 +5,7 @@ import { Input } from '@/components/ui/';
 import { Button } from '@/components/ui/button-enhanced';
 
 import { ChevronDown, ChevronUp, Plus, Trash2 } from 'lucide-react';
+import { useZarrStore } from '@/GlobalStates/ZarrStore';
 
 type HeaderRow  = { key: string; value: string };
 type AuthPreset = 'none' | 'bearer' | 'basic' | 'apikey';
@@ -82,13 +83,13 @@ const RemoteZarr = ({ initStore, setInitStore, onOpenDescription }: Props) => {
           ...(overrides && { overrides }),
         };
 
-        useGlobalStore.getState().setIcechunkOptions(null);
-        useGlobalStore.getState().setFetchOptions(
+        useZarrStore.getState().setIcechunkOptions(null);
+        useZarrStore.getState().setFetchOptions(
           Object.keys(fetchOptions).length > 0 ? fetchOptions : null
         );
         useGlobalStore.getState().setStatus('Fetching...');
 
-        useGlobalStore.getState().bumpFetchKey();
+        useZarrStore.getState().bumpFetchKey();
         setInitStore(url);
         onOpenDescription();
       }}

src/components/zarr/icechunk-store.ts

@@ -4,7 +4,7 @@ import { useGlobalStore } from "@/GlobalStates/GlobalStore";
 import { useErrorStore, ZarrError } from "@/GlobalStates/ErrorStore";
 import { IcechunkStore } from "icechunk-js";
 import type { NodeSnapshot } from "icechunk-js";
-import { Repository, HttpStorage } from "icechunk-js";
+// import { Repository, HttpStorage } from "icechunk-js";
 import { ZarrMetadata, ZarrTitleDescription } from "./Interfaces";
 import { useCacheStore } from "@/GlobalStates/CacheStore";
 import { getDtypeSize, calculateTotalElements, calculateChunkCount, formatBytes } from "./utils";
@@ -16,8 +16,6 @@ export async function getIcechunkNodes(store: IcechunkStore): Promise<NodeSnapsh
   }
   return await (store as any)._cachedNodes;
 }
-// For testing purposes only.
-// https://carbonplan-share.s3.us-west-2.amazonaws.com/zarr-layer-examples/pipeline/multi_level_virtual_hybrid_icechunk.icechunk
 
 export async function getIcechunkStore(
   storePath: string,
@@ -36,11 +34,11 @@ export async function getIcechunkStore(
         ...(options?.snapshot && { snapshot: options.snapshot }),
         ...(fetchClient       && { fetchClient }),
       });
-      console.log(icechunkStore);
-      const storage = new HttpStorage(storePath);
-      console.log('Storage initialized:', storage);
-      const repo = await Repository.open({ storage });
-      console.log('Repository opened:', repo);
+      // console.log(icechunkStore);
+      // const storage = new HttpStorage(storePath);
+      // console.log('Storage initialized:', storage);
+      // const repo = await Repository.open({ storage });
+      // console.log('Repository opened:', repo);
       // const branches = await repo.listBranches();
       // const tags = await repo.listTags(); 
       // console.log('Branches:', branches);
@@ -73,11 +71,9 @@ export async function getIcechunkMetadata(
   const variables: ZarrMetadata[] = [];
 
   const allNodes   = await getIcechunkNodes(icechunkStore);
-  console.log('All icechunk nodes:', allNodes);
   const arrayNodes = allNodes.filter(node =>
     (node.nodeData as { type: string }).type === 'array'
   );
-  console.log('Array nodes:', arrayNodes);
 
   for (const node of arrayNodes) {
     const nodeData = node.nodeData as {
@@ -122,7 +118,6 @@ export async function getIcechunkMetadata(
       groupPath,
     });
   }
-  console.log('Icechunk variables:', variables);
   return variables;
 }
 
@@ -175,7 +170,6 @@ export async function getIcechunkDims(
 ) {
   const { cache } = useCacheStore.getState();
   const allNodes = await getIcechunkNodes(group.store as IcechunkStore);
-  console.log('All icechunk nodes:', allNodes);
   const normalizedVariable = variable.startsWith('/') ? variable : `/${variable}`;
   const node = allNodes.find(n => n.path === normalizedVariable);
   const dimArrays: unknown[] = [];