🎨 🔥 ✨ Revisions

- Add IV_Revisions.ipynb: new features, improved perturbation scheme and performance evaluation.
- Add prom-CHiC preprocessing scripts
- Update Readme.md

Author: mpielies

.gitignore

@@ -5,16 +5,29 @@ configurations/conf_microc_rotated_pure_conv_tutorial
 figures
 GEO_files
 inputs/DNA_sequences
+inputs/landscape_arrays/K562/
 inputs/landscape_arrays/training/
+inputs/landscape_arrays/training_20kbp_context/
+inputs/landscape_arrays/training_no_H3K27ac/
+inputs/landscape_arrays/test_20kbp_context/
+inputs/landscape_arrays/test_chr19/
+inputs/landscape_arrays/test_no_H3K27ac/
 inputs/microC
+inputs/Promoter-CHiC/
 inputs/microC_rotated/training/
 inputs/perturbed_landscape_arrays/
+results_CRISPR/
 runs
 runs_tutorial
 scripts/*.py
 scripts/*.sh
-targets/perturbed_targets.csv
-targets/test_targets_Enformer.csv
+tables/
+targets/K562/
+targets/Enhancer_centered_targets.csv
+targets/*perturbed_targets.csv
+targets/test_targets_*.csv
+targets/*1kbp*
 targets/training_targets_Enformer.csv
 training_targets.csv
+HiC-Pro-3.1.0
 *.log

Readme.md

@@ -26,7 +26,7 @@ The folder contains the test set inputs for both data modalities, i.e. samples e
 ### `scripts`
 
 - [`0_Tutorial.ipynb`](https://github.com/RasmussenLab/CLASTER/blob/master/scripts/0_Tutorial.ipynb): The notebook provides a rapid overview of the most important steps in CLASTER's pipeline, including training and validating the network using the EIR framework. 
-- `1_Data_obtention.ipynb`: This notebook guides the user through the data obtention process, including:
+- `I_Data_obtention.ipynb`: This notebook guides the user through the data obtention process, including:
     - Data download from publicly available repositories:
         - Inputs: Chromatin landscape (ATAC-seq, H3K4me3, H3K27ac and H3K27me3 in mESCs) and structure (Micro-C maps in mESCs)
         - Outputs: Nascent transcription profiles (EU-seq).
@@ -35,15 +35,16 @@ The folder contains the test set inputs for both data modalities, i.e. samples e
     - Data filtering and preprocessing:
         - Obtain numpy arrays for the inputs.
         - Obtain csv files for the targets.
-- `2_Run_CLASTER.ipynb`: This notebook creates the configuration files required to train and test CLASTER using the EIR framework.
-- `2b_Run_HyenaDNA_and_Enformer.ipynb`: The notebook contains our adaptations of the code building
+- `II_Run_CLASTER.ipynb`: This notebook creates the configuration files required to train and test CLASTER using the EIR framework.
+- `IIb_Run_HyenaDNA_and_Enformer.ipynb`: The notebook contains our adaptations of the code building
     - Hyena-DNA (https://github.com/HazyResearch/hyena-dna) in its public colab version.
     - Enformer (https://github.com/lucidrains/enformer-pytorch) in its python implementation. 
 These were used to benchmark CLASTER. It includes:
     - The obtention of sequence embeddings from both model's backbones when loading the pretrained weights. 
     - The addition of a model head on top of the embeddings to match our regression outputs.
     - Code to fine-tune Hyena-DNA's backbone and the added head together.
-- `3_Data_analysis.ipynb`: The notebook contains the functions used to perform the data analysis and create the figures included in the manuscript.
+- `III_Data_analysis.ipynb`: The notebook contains the functions used to perform the data analysis and create the figures included in the manuscript.
+- `IV_Revisions.ipynb`: Code and analyses during the revisions.
 
 ### `targets`
 

index.rst

@@ -17,7 +17,8 @@ Modeling nascent RNA transcription from chromatin landscape and structure
     :maxdepth: 2
     :caption: Scripts:
 
-    scripts/1_Data_obtention.ipynb
-    scripts/2_Run_CLASTER.ipynb
-    scripts/2b_Run_HyenaDNA_and_Enformer.ipynb
-    scripts/3_Data_analysis.ipynb
+    scripts/I_Data_obtention.ipynb
+    scripts/II_Run_CLASTER.ipynb
+    scripts/IIb_Run_HyenaDNA_and_Enformer.ipynb
+    scripts/III_Data_analysis.ipynb
+    scripts/IV_Revisions.ipynb

scripts/0_Tutorial.ipynb

@@ -1658,7 +1658,7 @@
    "formats": "ipynb,py:percent"
   },
   "kernelspec": {
-   "display_name": "Python 3.11.9 ('claster_env_tutorial')",
+   "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
@@ -1672,12 +1672,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.undefined.undefined"
-  },
-  "vscode": {
-   "interpreter": {
-    "hash": "12187645b0fe86aeeeb7c7a27e94503590520c2ee4730b6cfd81a67885c82988"
-   }
+   "version": "3.11.5"
   }
  },
  "nbformat": 4,

scripts/3_Data_analysis.ipynb scripts/III_Data_analysis.ipynbscripts/3_Data_analysis.ipynb renamed to scripts/III_Data_analysis.ipynb

@@ -21,7 +21,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Packages"
+    "### Packages and Functions"
    ]
   },
   {
@@ -34,6 +34,7 @@
    },
    "outputs": [],
    "source": [
+    "# %%writefile claster_utils.py\n",
     "import os\n",
     "import numpy as np\n",
     "import matplotlib as mpl\n",
@@ -54,26 +55,9 @@
     "import matplotlib.cm as cm\n",
     "from scipy.ndimage import rotate\n",
     "\n",
-    "plt.rcParams['font.family'] = 'Nimbus Roman'"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Functions"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 69,
-   "metadata": {
-    "tags": [
-     "hide-input"
-    ]
-   },
-   "outputs": [],
-   "source": [
+    "plt.rcParams['font.family'] = 'Nimbus Roman'\n",
+    "\n",
+    "################################# Functions #####################################\n",
     "def read_gene_positions(csv_path, resolution=1):\n",
     "    \"\"\" \n",
     "    This function reads a csv containing genes and enhancers (entities), and creates a dictionary with the relative coordinates of\n",
@@ -2557,7 +2541,7 @@
    "formats": "ipynb,py:percent"
   },
   "kernelspec": {
-   "display_name": "Python 3.11.8",
+   "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
@@ -2572,11 +2556,6 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.11.5"
-  },
-  "vscode": {
-   "interpreter": {
-    "hash": "c305a8ed7bd3fb5b79b1e9049e998ba6b84af3f1b497d7bcc87b4717f669b9d3"
-   }
   }
  },
  "nbformat": 4,

scripts/2_Run_CLASTER.ipynb scripts/II_Run_CLASTER.ipynbscripts/2_Run_CLASTER.ipynb renamed to scripts/II_Run_CLASTER.ipynb

@@ -5843,7 +5843,7 @@
    "formats": "ipynb,py:percent"
   },
   "kernelspec": {
-   "display_name": "Python 3.11.5",
+   "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
@@ -5858,11 +5858,6 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.11.5"
-  },
-  "vscode": {
-   "interpreter": {
-    "hash": "c305a8ed7bd3fb5b79b1e9049e998ba6b84af3f1b497d7bcc87b4717f669b9d3"
-   }
   }
  },
  "nbformat": 4,