diff --git a/.gitignore b/.gitignore index 2a410b2..67e671a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Ignore MacOS-specific files +.DS_Store + # Ignore PyCharm IDE files .idea diff --git a/README.md b/README.md index 8ab8728..467e3f9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,22 @@ # IFCB Classifier -This repo host an image classifying program designed to be trained on plankton images from an IFCB datasource. +This repository is a fork of the original ```WHOIGit/ibcb_classifier repo```, an image classifying program designed to be trained on plankton images from an IFCB datasource. The ```legacy``` branch is identical to ```WHOIGit/ifcb_classifier v0.3.1```, with the exception that the dependencies have been updated to rely on conda-forge channels only. The ```main``` branch contains the code that has been migrated to the modern PyTorch packages (```pytorch>=2.5.0```, etc.). In addition, he SLURM functionality has been removed. + +For details on usage, please see [the WHOIGit repository wiki](https://github.com/WHOIGit/ifcb_classifier/wiki) + +## Changes to original code + +- "ptl.callbacks.base.Callback" replaced with "ptl.callbacks.Callback". +- Loaders are now used as parameters for the NeustonModel object, as they don't work as parameters when the trainer fits the model anymore. +- 'input_classes', 'output_classes', 'input_srcs' and 'outputs' are now placed into a separate dictionary object ('unloggable_dict') instead of being logged, as logging lists doesn't work anymore. +- 'training_epoch_end' changed to 'on_train_epoch_end' as the lingo was out of date (ditto for validation and testing). +- 'steps' now stored in separate lists and cleared after their respective 'on_epoch_end' function is called, as said function no longer accepts 'steps' as a parameter. +- 'gpus' and 'checkpoint_callback' removed as parameters from the Trainers, as they are no longer valid for whatever reason. +- Added parameters to Trainers: accelerator='gpu', devices=1 +- Added parameter to dataloaders: persistent_workers=True + +## Comparison of v2025.07a2 with v0.3.1 + +To confirm successful migration to PyTorch 2.x, comparison between a model trained with the updated code and the ```legacy_pytorch_1.7.1``` was made. (TBD) -For details on usage and installation, please see [this repository's wiki](https://github.com/WHOIGit/ifcb_classifier/wiki) diff --git a/batches/templates/example.RUN.sbatch b/batches/templates/example.RUN.sbatch deleted file mode 100644 index 46e3fc1..0000000 --- a/batches/templates/example.RUN.sbatch +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -#SBATCH --job-name=RUN_ExampleDataset_ExampleModel -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem-per-cpu=10240 -#SBATCH --time=24:00:00 -#SBATCH --mail-type=ALL -#SBATCH --mail-user=username@whoi.edu -#SBATCH --partition=gpu -#SBATCH --gres=gpu:1 -#SBATCH --output=slurm-logs/%j.%x.out - -# SETTING OPERATIVE DIRECTORY # -cd /vortexfs1/scratch/username/ifcbnn - -# LOGGING JOB DETAILS # -echo "Job ID: $SLURM_JOB_ID, JobName: $SLURM_JOB_NAME" -hostname; pwd; date - -# SETTING UP ENVIRONMENT # -module load cuda10.1/toolkit cuda10.1/blas cuda10.1/cudnn/8.0.2 cuda91/fft -module load anaconda -source activate ifcbnn -echo "Environment... Loaded" - -# PARAMS # -RUN_ID=ExampleRunID -MODEL=training-output/TrainedExample/TrainedExample.ptl -DATASET=run-data/ExampleDataset - -# RUN SCRIPT # -python ./neuston_net.py RUN "$DATASET" "$MODEL" "$RUN_ID" diff --git a/batches/templates/example.TRAINING.sbatch b/batches/templates/example.TRAINING.sbatch deleted file mode 100644 index bc7ba13..0000000 --- a/batches/templates/example.TRAINING.sbatch +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -#SBATCH --job-name=TRAIN_Example -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem-per-cpu=10240 -#SBATCH --time=24:00:00 -#SBATCH --mail-type=ALL -#SBATCH --mail-user=username@whoi.edu -#SBATCH --partition=gpu -#SBATCH --gres=gpu:1 -#SBATCH --output=slurm-logs/%j.%x.out - -# SETTING OPERATIVE DIRECTORY # -cd /vortexfs1/scratch/username/ifcbnn - -# LOGGING JOB DETAILS # -echo "Job ID: $SLURM_JOB_ID, JobName: $SLURM_JOB_NAME" -hostname; pwd; date - -# SETTING UP ENVIRONMENT # -module load cuda10.1/toolkit cuda10.1/blas cuda10.1/cudnn/8.0.2 cuda91/fft -module load anaconda -source activate ifcbnn -echo "Environment... Loaded" - -## PARAMS ## -TRAIN_ID=ExampleTrainingID -MODEL=inception_v3 -DATASET=training-data/ExampleTrainingData - -# RUN SCRIPT # -python ./neuston_net.py TRAIN "$DATASET" "$MODEL" "$TRAIN_ID" --flip xy - diff --git a/environment-linux-64.yml b/environment-linux-64.yml new file mode 100644 index 0000000..bbf670b --- /dev/null +++ b/environment-linux-64.yml @@ -0,0 +1,22 @@ +name: ifcbnn +channels: + - conda-forge +dependencies: + - python=3.12 + - pytorch=2.8.0=cuda129_mkl_py312_hda324a2_301 + - torchvision=0.24.0=cuda129_py312_h9f56bbf_2 + - pytorch-lightning=2.5.5 + - scikit-learn + - scipy=1.13.1 + - pandas=2.2.3 + - h5py=3.12.1 + - requests=2.32.4 + - Pillow=11.1.0 + - rectpack=0.2.2 + - scikit-image=0.24.0 + - pysmb=1.2.10 + - smbprotocol=1.15.0 + - pyyaml=6.0.2 + - pip + - pip: + - git+https://github.com/joefutrelle/pyifcb@v1.2.1 diff --git a/environment-win-64.yml b/environment-win-64.yml new file mode 100644 index 0000000..8261305 --- /dev/null +++ b/environment-win-64.yml @@ -0,0 +1,23 @@ +name: ifcbnn +channels: + - conda-forge +dependencies: + - python=3.11 + - pytorch + - torchvision + - pytorch-lightning + - Pillow + - scikit-learn + - scikit-image + - pandas + - h5py + - scipy + - numpy + - pyyaml + - requests + - rectpack + - pysmb + - smbprotocol + - pip + - pip: + - git+https://github.com/joefutrelle/pyifcb@v1.2.1 diff --git a/neuston_callbacks.py b/neuston_callbacks.py index 2640321..a1e94c9 100644 --- a/neuston_callbacks.py +++ b/neuston_callbacks.py @@ -17,7 +17,8 @@ ## Training ## -class SaveValidationResults(ptl.callbacks.base.Callback): +#class SaveValidationResults(ptl.callbacks.base.Callback): # deprecated in PyTorch 2.x +class SaveValidationResults(ptl.callbacks.Callback): def __init__(self, outdir, outfile, series, best_only=True): self.outdir = outdir @@ -27,6 +28,7 @@ def __init__(self, outdir, outfile, series, best_only=True): def on_validation_end(self, trainer, pl_module): log = trainer.callback_metrics # flattened dict + unlog = pl_module.unloggable_dict #log: val_loss input_classes output_classes input_srcs outputs epoch best train_loss f1_macro f1_weighted if not(log['best'] or not self.best_only): @@ -45,11 +47,14 @@ def on_validation_end(self, trainer, pl_module): training_image_basenames = [os.path.splitext(os.path.basename(img))[0] for img in training_image_fullpaths] training_classes = train_dataset.targets - output_scores = log['outputs'] + #output_scores = log['outputs'] + output_scores = unlog['outputs'] output_winscores = np.max(output_scores, axis=1) output_classes = np.argmax(output_scores, axis=1) - input_classes = log['input_classes'] - image_fullpaths = log['input_srcs'] + #input_classes = log['input_classes'] + input_classes = unlog['input_classes'] + #image_fullpaths = log['input_srcs'] + image_fullpaths = unlog['input_srcs'] image_basenames = [os.path.splitext(os.path.basename(img))[0] for img in image_fullpaths] assert output_scores.shape[0] == len(input_classes), 'wrong number inputs-to-outputs' @@ -272,7 +277,8 @@ def _save_run_results_hdf(outfile, results): if outfile.endswith('.h5'): _save_run_results_hdf(outfile, results) -class SaveTestResults(ptl.callbacks.base.Callback): +#class SaveTestResults(ptl.callbacks.base.Callback): # deprecated in PyTorch 2.x +class SaveTestResults(ptl.callbacks.Callback): def __init__(self, outdir, outfile, timestamp): self.outdir = outdir @@ -281,7 +287,8 @@ def __init__(self, outdir, outfile, timestamp): def on_test_end(self, trainer, pl_module): - RRs = trainer.callback_metrics['RunResults'] + #RRs = trainer.callback_metrics['RunResults'] + RRs = pl_module.unloggable_dict['RunResults'] # RunResult rr: inputs, outputs, bin_id if not isinstance(RRs,list): RRs = [RRs] diff --git a/neuston_models.py b/neuston_models.py index a2dbdcb..f9a0f01 100644 --- a/neuston_models.py +++ b/neuston_models.py @@ -46,7 +46,8 @@ def get_namebrand_model(model_name, num_o_classes, pretrained=False): class NeustonModel(ptl.LightningModule): - def __init__(self, hparams): + #def __init__(self, hparams): + def __init__(self, hparams, training_loader=None, validation_loader=None, testing_loader=None): super().__init__() if isinstance(hparams,dict): @@ -59,6 +60,15 @@ def __init__(self, hparams): self.best_val_loss = np.inf self.best_epoch = 0 self.agg_train_loss = 0.0 + + # Holly's additions + self.train_steps = [] + self.validation_steps = [] + self.test_steps = [] + self.training_loader = training_loader + self.validation_loader = validation_loader + self.testing_loader = testing_loader + self.unloggable_dict = {} def configure_optimizers(self): return Adam(self.parameters(), lr=0.001) @@ -83,10 +93,14 @@ def training_step(self, batch, batch_nb): outputs = self.forward(input_data) batch_loss = self.loss(input_classes, outputs) self.agg_train_loss += batch_loss.item() + self.train_steps.append(dict(loss=batch_loss)) return dict(loss=batch_loss) - def training_epoch_end(self, steps): - train_loss = torch.stack([batch['loss'] for batch in steps]).sum().item() + #def training_epoch_end(self, steps): + def on_train_epoch_end(self): + #train_loss = torch.stack([batch['loss'] for batch in steps]).sum().item() + train_loss = torch.stack([batch['loss'] for batch in self.train_steps]).sum().item() + self.train_steps = [] #print('training_epoch_end: self.agg_train_loss={:.5f}, train_loss={:.5f}, DIFF={:.9f}'.format(self.agg_train_loss, train_loss, self.agg_train_loss-train_loss), end='\n\n') #return dict(train_loss=train_loss) @@ -97,16 +111,24 @@ def validation_step(self, batch, batch_idx): val_batch_loss = self.loss(input_classes, outputs) outputs = outputs.logits if isinstance(outputs,InceptionOutputs) else outputs outputs = softmax(outputs,dim=1) - return dict(val_batch_loss=val_batch_loss, + #return dict(val_batch_loss=val_batch_loss, + # val_outputs=outputs, + # val_input_classes=input_classes, + # val_input_srcs=input_src) + outp = dict(val_batch_loss=val_batch_loss, val_outputs=outputs, val_input_classes=input_classes, val_input_srcs=input_src) + self.validation_steps.append(outp) + return outp - def validation_epoch_end(self, steps): + #def validation_epoch_end(self, steps): + def on_validation_epoch_end(self): print(end='\n\n') # give space for progress bar if self.current_epoch==0: self.best_val_loss = np.inf # takes care of any lingering val_loss from sanity checks - validation_loss = torch.stack([batch['val_batch_loss'] for batch in steps]).sum() + #validation_loss = torch.stack([batch['val_batch_loss'] for batch in steps]).sum() + validation_loss = torch.stack([batch['val_batch_loss'] for batch in self.validation_steps]).sum() #eoe0 = 'validation_epoch_end: best_val_loss={}, curr_val_loss={}, curr')) - sbatch_ofile_dict['PID'] = 'xxxxxx' - else: - sbatch_ofile_dict['PID'] = 'xxxxxx' - - # record sbatch file to outdir directory - sbatch_ofile = args.ofile.format(**sbatch_ofile_dict) - print('SBATCH script: ' + sbatch_ofile) - with open(sbatch_ofile,'w') as f: - f.write(sbatch_content) - - -def argparse_sbatch(): - parser = argparse.ArgumentParser(description='SLURM SBATCH auto-submitter for neuston_net.py') - - slurm = parser.add_argument_group(title='SLURM Args', description=None) - slurm.add_argument('--job-name', metavar='STR', - help='Job Name that will appear in slurm jobs list. Defaults is "{}"'.format(SBATCH_DDICT['JOB_NAME'])) - slurm.add_argument('--email', - help='Email address to send slurm notifications to. Your default is "{}"'.format(SBATCH_DDICT['EMAIL'])) - slurm.add_argument('--walltime', metavar='HH:MM:SS', - help='Set Slurm Task max runtime. Default is "{}"'.format(SBATCH_DDICT['WALLTIME'])) - slurm.add_argument('--gpu-num', metavar='INT', type=int, - help='Number of GPUs to allocate per task. Default is {}'.format(SBATCH_DDICT['GPU_NUM'])) - slurm.add_argument('--cpu-num', metavar='INT', type=int, - help='Number of CPUs to allocate per task. Default is {}'.format(SBATCH_DDICT['CPU_NUM'])) - slurm.add_argument('--mem-per-cpu', metavar='MB', type=int, - help='Memory to allocate per cpu in MB. Default is {}MB'.format(SBATCH_DDICT['MEM_PER_CPU'])) - slurm.add_argument('--slurm-log-dir', metavar='DIR', - help='Directory to save slurm log file to. Defaults to OUTDIR (as defined by TRAIN or RUN subcommand)') - slurm.add_argument('--ofile', default="{OUTDIR}/{PID}.{JOB_NAME}.sbatch", - help='Save location for generated sbatch file. Defaults to "{OUTDIR}/{PID}.{JOB_NAME}.sbatch"') - slurm.add_argument('--conda-env', default='ifcbnn', help='The conda environment to activate for neuston_net.py. Default is "ifcbnn"') - slurm.add_argument('--dry-run', default=False, action='store_true', help='Create the sbatch script but do not run it') - - return parser - -if __name__ == '__main__': - parser = argparse_sbatch() - - parser = nn.argparse_nn(parser) - - main(parser) - - -#TODO ok this test_tube thing is stupid. diff --git a/neuston_util.py b/neuston_util.py index 5896cfb..6484df9 100644 --- a/neuston_util.py +++ b/neuston_util.py @@ -63,18 +63,26 @@ def write_csv(outfile, rows): print(','.join(row)) +# Slightly modified by Holly def make_dataset_config(args): # parsing datasets datasets = [] priorities = [] for src in args.dataset: - src = src.split(':',1) - if len(src)==2: - datasets.append(src[1]) - priorities.append(int(src[0])) + split = src.split(':',1) + startsWithInt = False + try: + int(split[0]) + startsWithInt = True + except: + pass + if len(split)==2 and startsWithInt: + datasets.append(split[1]) + priorities.append(int(split[0])) else: - datasets.append(src[0]) + #datasets.append(split[0]) + datasets.append(src) priorities.append(0) priorities = [p if p>0 else max(priorities)+1 for p in priorities] diff --git a/requirements/env.dev.yml b/requirements/env.dev.yml deleted file mode 100644 index 2a99a21..0000000 --- a/requirements/env.dev.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: ifcbnn -channels: - - conda-forge - - pytorch -dependencies: - - python>=3.8 - - cudatoolkit=11.0.221=h6bb024c_0 - - pytorch=1.7.1=py3.8_cuda11.0.221_cudnn8.0.5_0 - - torchvision=0.8.2=py38_cu110 - - pytorch-lightning=1.3.8=pyhd8ed1ab_0 - - onnxruntime=1.7.2=py38h14f2e07_4 - - scikit-learn - - pandas - - numpy - - h5py - - pip -- pip: - - git+https://github.com/joefutrelle/pyifcb.git diff --git a/requirements/env.hpc.yml b/requirements/env.hpc.yml deleted file mode 100644 index 6d443ae..0000000 --- a/requirements/env.hpc.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: ifcbnn -channels: - - conda-forge - - pytorch -dependencies: - - python>=3.8 - - cudatoolkit=10.1.243=h6bb024c_0 - - pytorch=1.7.1=py3.8_cuda10.1.243_cudnn7.6.3_0 - - torchvision=0.8.2=py38_cu101 - - pytorch-lightning=1.3.8=pyhd8ed1ab_0 - - onnxruntime=1.7.2=py38h14f2e07_4 - - scikit-learn - - pandas - - numpy - - h5py - - pip -- pip: - - git+https://github.com/joefutrelle/pyifcb.git diff --git a/requirements/pkgs.dev.txt b/requirements/pkgs.dev.txt deleted file mode 100644 index f88a32c..0000000 --- a/requirements/pkgs.dev.txt +++ /dev/null @@ -1,120 +0,0 @@ -# This file may be used to create an environment using: -# $ conda create --name --file -# platform: linux-64 -@EXPLICIT -https://repo.anaconda.com/pkgs/main/linux-64/_libgcc_mutex-0.1-main.conda -https://repo.anaconda.com/pkgs/main/linux-64/blas-1.0-mkl.conda -https://repo.anaconda.com/pkgs/main/linux-64/ca-certificates-2021.10.26-h06a4308_2.conda -https://repo.anaconda.com/pkgs/main/linux-64/intel-openmp-2021.4.0-h06a4308_3561.conda -https://repo.anaconda.com/pkgs/main/linux-64/ld_impl_linux-64-2.35.1-h7274673_9.conda -https://repo.anaconda.com/pkgs/main/linux-64/libgfortran4-7.5.0-ha8ba4b0_17.conda -https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-9.3.0-hd4cf53a_17.conda -https://repo.anaconda.com/pkgs/main/linux-64/libgfortran-ng-7.5.0-ha8ba4b0_17.conda -https://repo.anaconda.com/pkgs/main/linux-64/libgomp-9.3.0-h5101ec6_17.conda -https://repo.anaconda.com/pkgs/main/linux-64/mkl-2021.4.0-h06a4308_640.conda -https://repo.anaconda.com/pkgs/main/linux-64/_openmp_mutex-4.5-1_gnu.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-9.3.0-h5101ec6_17.conda -https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.17.1-h7f98852_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/cudatoolkit-11.0.221-h6bb024c_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/giflib-5.2.1-h7b6447c_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/jpeg-9b-h024ee3a_2.conda -https://repo.anaconda.com/pkgs/main/linux-64/libffi-3.3-he6710b0_2.conda -https://repo.anaconda.com/pkgs/main/linux-64/libuv-1.40.0-h7b6447c_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/libwebp-base-1.2.0-h27cfd23_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/lz4-c-1.9.3-h295c915_1.conda -https://repo.anaconda.com/pkgs/main/linux-64/ncurses-6.3-h7f8727e_2.conda -https://repo.anaconda.com/pkgs/main/linux-64/openssl-1.1.1l-h7f8727e_0.conda -https://conda.anaconda.org/conda-forge/linux-64/re2-2021.08.01-h9c3ff4c_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/xz-5.2.5-h7b6447c_0.conda -https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h516909a_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/zlib-1.2.11-h7b6447c_3.conda -https://repo.anaconda.com/pkgs/main/linux-64/hdf5-1.10.6-hb1b8bf9_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/libpng-1.6.37-hbc83047_0.conda -https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-3.16.0-h780b84a_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/readline-8.1-h27cfd23_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/tk-8.6.11-h1ccaba5_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/zstd-1.4.9-haebb681_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/freetype-2.11.0-h70c0345_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/libtiff-4.2.0-h85742a9_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/sqlite-3.36.0-hc218d9a_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/lcms2-2.12-h3be6417_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/libwebp-1.2.0-h89dd481_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/python-3.8.12-h12debd9_0.conda -https://conda.anaconda.org/conda-forge/noarch/async-timeout-3.0.1-py_1000.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/attrs-21.2.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/blinker-1.4-py_1.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/cachetools-4.2.4-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/certifi-2021.10.8-py38h06a4308_0.conda -https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.4-pyh9f0ad1d_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/dataclasses-0.8-pyhc8e2a94_3.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/fsspec-2021.11.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/idna-2.10-pyh9f0ad1d_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/joblib-1.1.0-pyhd3eb1b0_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/ninja-1.10.2-py38hd09550d_3.conda -https://repo.anaconda.com/pkgs/main/noarch/olefile-0.46-pyhd3eb1b0_0.conda -https://conda.anaconda.org/conda-forge/noarch/pyasn1-0.4.8-py_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pycparser-2.21-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pydeprecate-0.3.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pyjwt-2.3.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.0.6-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.8-2_cp38.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/pytz-2021.3-pyhd3eb1b0_0.conda -https://repo.anaconda.com/pkgs/main/noarch/six-1.16.0-pyhd3eb1b0_0.conda -https://conda.anaconda.org/conda-forge/noarch/tensorboard-plugin-wit-1.8.0-pyh44b312d_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/threadpoolctl-2.2.0-pyh0d69192_0.conda -https://repo.anaconda.com/pkgs/main/noarch/typing_extensions-3.10.0.2-pyh06a4308_0.conda -https://repo.anaconda.com/pkgs/main/noarch/wheel-0.37.0-pyhd3eb1b0_1.conda -https://conda.anaconda.org/conda-forge/noarch/zipp-3.6.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/absl-py-1.0.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/cffi-1.14.6-py38ha65f79e_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/chardet-4.0.0-py38h578d9bd_2.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/click-8.0.3-py38h578d9bd_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/future-0.18.2-py38h578d9bd_4.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/importlib-metadata-4.8.2-py38h578d9bd_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/mkl-service-2.4.0-py38h7f8727e_0.conda -https://conda.anaconda.org/conda-forge/linux-64/multidict-5.1.0-py38h497a2fe_1.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/packaging-21.0-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/pillow-8.4.0-py38h5aabda8_0.conda -https://conda.anaconda.org/conda-forge/noarch/pyasn1-modules-0.2.7-py_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/pysocks-1.7.1-py38h578d9bd_4.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/python-dateutil-2.8.2-pyhd3eb1b0_0.conda -https://conda.anaconda.org/conda-forge/noarch/pyu2f-0.1.5-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/pyyaml-5.4.1-py38h497a2fe_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/rsa-4.7.2-pyh44b312d_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/setuptools-58.0.4-py38h06a4308_0.conda -https://conda.anaconda.org/conda-forge/linux-64/tensorboard-data-server-0.6.0-py38h2b97feb_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/tqdm-4.62.3-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/typing-extensions-3.10.0.2-hd3eb1b0_0.conda -https://conda.anaconda.org/conda-forge/noarch/werkzeug-2.0.1-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/brotlipy-0.7.0-py38h497a2fe_1001.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/cryptography-35.0.0-py38ha5dfef3_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/grpcio-1.38.1-py38hdd6454d_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/markdown-3.3.5-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/numpy-base-1.21.2-py38h79a1101_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/pip-21.2.4-py38h06a4308_0.conda -https://conda.anaconda.org/conda-forge/linux-64/protobuf-3.16.0-py38h709712a_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/yarl-1.6.3-py38h497a2fe_2.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/aiohttp-3.7.4.post0-py38h497a2fe_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/oauthlib-3.1.1-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pyopenssl-21.0.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.7-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/requests-2.25.1-pyhd3deb0d_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/google-auth-2.3.3-pyh6c4a22f_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/requests-oauthlib-1.3.0-pyh9f0ad1d_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/google-auth-oauthlib-0.4.6-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/onnx-1.10.1-py38h6f20ad4_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/onnxruntime-1.7.2-py38h14f2e07_4.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/tensorboard-2.7.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/torchmetrics-0.6.0-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/bottleneck-1.3.2-py38heb32a55_1.conda -https://repo.anaconda.com/pkgs/main/linux-64/h5py-3.6.0-py38ha0f2276_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/mkl_fft-1.3.1-py38hd3c417c_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/mkl_random-1.2.2-py38h51133e4_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/numpy-1.21.2-py38h20f2e39_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/numexpr-2.7.3-py38h22e1b3c_1.conda -https://conda.anaconda.org/pytorch/linux-64/pytorch-1.7.1-py3.8_cuda11.0.221_cudnn8.0.5_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/scipy-1.7.1-py38h292c36d_2.conda -https://repo.anaconda.com/pkgs/main/linux-64/pandas-1.3.4-py38h8c16a72_0.conda -https://conda.anaconda.org/conda-forge/noarch/pytorch-lightning-1.3.8-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/scikit-learn-1.0.1-py38h51133e4_0.conda -https://conda.anaconda.org/pytorch/linux-64/torchvision-0.8.2-py38_cu110.tar.bz2 diff --git a/requirements/pkgs.hpc.txt b/requirements/pkgs.hpc.txt deleted file mode 100644 index 5057451..0000000 --- a/requirements/pkgs.hpc.txt +++ /dev/null @@ -1,118 +0,0 @@ -# This file may be used to create an environment using: -# $ conda create --name --file -# platform: linux-64 -@EXPLICIT -https://repo.anaconda.com/pkgs/main/linux-64/_libgcc_mutex-0.1-main.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/blas-1.0-mkl.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2021.10.8-ha878542_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/cudatoolkit-10.1.243-h6bb024c_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/intel-openmp-2021.4.0-h06a4308_3561.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/ld_impl_linux-64-2.35.1-h7274673_9.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/libgfortran4-7.5.0-ha8ba4b0_17.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-9.3.0-hd4cf53a_17.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/libgfortran-ng-7.5.0-ha8ba4b0_17.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/libgomp-9.3.0-h5101ec6_17.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/mkl-2021.4.0-h06a4308_640.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/_openmp_mutex-4.5-1_gnu.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-9.3.0-h5101ec6_17.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.17.1-h7f98852_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/giflib-5.2.1-h7b6447c_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/jpeg-9b-h024ee3a_2.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/libffi-3.3-he6710b0_2.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/libuv-1.40.0-h7b6447c_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/libwebp-base-1.2.0-h27cfd23_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/lz4-c-1.9.3-h295c915_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/ncurses-6.3-h7f8727e_2.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/openssl-1.1.1l-h7f8727e_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/re2-2021.08.01-h9c3ff4c_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/xz-5.2.5-h7b6447c_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h516909a_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/zlib-1.2.11-h7b6447c_3.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/hdf5-1.10.6-hb1b8bf9_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/libpng-1.6.37-hbc83047_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-3.16.0-h780b84a_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/readline-8.1-h27cfd23_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/tk-8.6.11-h1ccaba5_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/zstd-1.4.9-haebb681_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/freetype-2.11.0-h70c0345_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/libtiff-4.2.0-h85742a9_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/sqlite-3.36.0-hc218d9a_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/lcms2-2.12-h3be6417_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/libwebp-1.2.0-h89dd481_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/python-3.8.12-h12debd9_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/async-timeout-3.0.1-py_1000.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/attrs-21.2.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/blinker-1.4-py_1.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/cachetools-4.2.4-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.4-pyh9f0ad1d_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/dataclasses-0.8-pyhc8e2a94_3.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/fsspec-2021.11.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/idna-2.10-pyh9f0ad1d_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/joblib-1.1.0-pyhd3eb1b0_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/ninja-1.10.2-py38hd09550d_3.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/olefile-0.46-pyhd3eb1b0_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pyasn1-0.4.8-py_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pycparser-2.21-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pydeprecate-0.3.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pyjwt-2.3.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.0.6-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.8-2_cp38.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/pytz-2021.3-pyhd3eb1b0_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/six-1.16.0-pyhd3eb1b0_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/tensorboard-plugin-wit-1.8.0-pyh44b312d_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/threadpoolctl-2.2.0-pyh0d69192_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/typing_extensions-3.10.0.2-pyh06a4308_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/wheel-0.37.0-pyhd3eb1b0_1.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/zipp-3.6.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/absl-py-1.0.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/certifi-2021.10.8-py38h578d9bd_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/cffi-1.14.6-py38ha65f79e_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/chardet-3.0.4-py38h924ce5b_1008.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/click-8.0.3-py38h578d9bd_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/future-0.18.2-py38h578d9bd_4.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/importlib-metadata-4.8.2-py38h578d9bd_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/mkl-service-2.4.0-py38h7f8727e_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/multidict-5.1.0-py38h497a2fe_1.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/packaging-21.0-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/pillow-8.4.0-py38h5aabda8_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pyasn1-modules-0.2.7-py_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/pysocks-1.7.1-py38h578d9bd_4.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/python-dateutil-2.8.2-pyhd3eb1b0_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pyu2f-0.1.5-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/pyyaml-5.4.1-py38h497a2fe_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/rsa-4.7.2-pyh44b312d_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/tensorboard-data-server-0.6.0-py38h2b97feb_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/tqdm-4.62.3-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/werkzeug-2.0.1-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/brotlipy-0.7.0-py38h497a2fe_1001.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/cryptography-35.0.0-py38ha5dfef3_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/markdown-3.3.4-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/numpy-base-1.21.2-py38h79a1101_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/setuptools-58.0.4-py38h06a4308_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/yarl-1.6.3-py38h497a2fe_2.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/aiohttp-3.7.0-py38h1e0a361_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/grpcio-1.38.1-py38hdd6454d_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/oauthlib-3.1.1-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/pip-21.2.4-py38h06a4308_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/protobuf-3.16.0-py38h709712a_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pyopenssl-21.0.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.7-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/requests-2.25.1-pyhd3deb0d_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/google-auth-2.3.3-pyh6c4a22f_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/requests-oauthlib-1.3.0-pyh9f0ad1d_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/google-auth-oauthlib-0.4.6-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/onnxruntime-1.7.2-py38h14f2e07_4.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/tensorboard-2.7.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/torchmetrics-0.6.0-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/bottleneck-1.3.2-py38heb32a55_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/h5py-3.5.0-py38hd430a98_100.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/mkl_fft-1.3.1-py38hd3c417c_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/mkl_random-1.2.2-py38h51133e4_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/numpy-1.21.2-py38h20f2e39_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/numexpr-2.7.3-py38h22e1b3c_1.tar.bz2 -https://conda.anaconda.org/pytorch/linux-64/pytorch-1.7.1-py3.8_cuda10.1.243_cudnn7.6.3_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/scipy-1.7.1-py38h292c36d_2.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/pandas-1.3.4-py38h8c16a72_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pytorch-lightning-1.3.8-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/scikit-learn-1.0.1-py38h51133e4_0.tar.bz2 -https://conda.anaconda.org/pytorch/linux-64/torchvision-0.8.2-py38_cu101.tar.bz2 diff --git a/slurm-logs/.gitignore b/slurm-logs/.gitignore deleted file mode 100644 index df3359d..0000000 --- a/slurm-logs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*/* \ No newline at end of file diff --git a/version b/version index 937cd78..c1f9f78 100644 --- a/version +++ b/version @@ -1 +1 @@ -v0.3.1 +v2025.11a1 \ No newline at end of file