Known limitations on data loading

[rachelstephlee]

Data loading remains a problem.

For code ocean, the known limit is around 250 sessions (demonstrated in 11/27/2024 by @ZhixiaoSu ).

@alexpiet has this script for downloading massive amounts of data from CO as well:

import os
import pandas as pd
from cloudpathlib import CloudPath

RAW_DATA_BUCKET = 'aind-private-data-prod-o5171v'
RAW_DATA_DIR = '/Users/alex.piet/data/raw_data/'
NWB_DATA_BUCKET = 'codeocean-s3resultsbucket-1182nktl2bh9f'
NWB_DATA_BUCKET = 'codeocean-s3publicbucket-g8yj6utko6qv'
NWB_DATA_DIR = '/Users/alex.piet/data/nwb/'

def download_raw_data(asset_name,skip=['behavior-videos','fib']):
    '''
        Downloads the asset_name, which is the same as the session_id
        Skips all files or directories in skip
    '''
    cp = CloudPath("s3://{}/{}".format(RAW_DATA_BUCKET,asset_name))
    download_path = os.path.join(RAW_DATA_DIR,asset_name)
    os.makedirs(download_path, exist_ok = True)

    for i in cp.iterdir():
        file = str(i).split('/')[-1]
        if file not in skip:
            filepath = os.path.join(download_path, file)
            print('Downloading: {}'.format(file))
            this_cp = CloudPath("s3://{}/{}/{}".format(RAW_DATA_BUCKET,asset_name,file))
            this_cp.download_to(filepath)
        else:
            print('Skipping: {}'.format(file))

def download_nwb_data(asset_name):
    # First, determine the location
    # It might be better to query the docdb directly
    # https://github.com/AllenNeuralDynamics/foraging-behavior-browser/blob/main/code/util/fetch_data_docDB.py
    df = pd.read_csv('/Users/alex.piet/code/dynamic_foraging_analysis/data_download/session_table.csv')
    matches = df.query('session_name == @asset_name')
    if len(matches) > 1:
        print(matches[['session_name','results']])
        return
    elif len(matches) == 0:
        print('No sessions found')
        return
    print('Found the file location')
    results_location = os.path.join(matches.iloc[0]['results_location'],'nwb')

    # Now download the NWB file
    print('Downloading ...')
    cp = CloudPath(results_location)
    cp.download_to(NWB_DATA_DIR)