diff --git a/common b/common index 876036f71..f01ff2170 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 876036f71713cbd79285b108ab0a9a8238f2b5e1 +Subproject commit f01ff2170161295e89014ee5453c61b29b4e4e77 diff --git a/docs/build/doctrees/dataset.doctree b/docs/build/doctrees/dataset.doctree index 3ea7f7bad..b39003cbc 100644 Binary files a/docs/build/doctrees/dataset.doctree and b/docs/build/doctrees/dataset.doctree differ diff --git a/docs/build/doctrees/environment.pickle b/docs/build/doctrees/environment.pickle index 68395e875..02d83a60b 100644 Binary files a/docs/build/doctrees/environment.pickle and b/docs/build/doctrees/environment.pickle differ diff --git a/docs/build/doctrees/evaluation.doctree b/docs/build/doctrees/evaluation.doctree index 43a7d2af9..44a3e89d8 100644 Binary files a/docs/build/doctrees/evaluation.doctree and b/docs/build/doctrees/evaluation.doctree differ diff --git a/docs/build/doctrees/extending.doctree b/docs/build/doctrees/extending.doctree index 1a657e459..5b3375b48 100644 Binary files a/docs/build/doctrees/extending.doctree and b/docs/build/doctrees/extending.doctree differ diff --git a/docs/build/doctrees/index.doctree b/docs/build/doctrees/index.doctree index 283a92c78..21de8b4e6 100644 Binary files a/docs/build/doctrees/index.doctree and b/docs/build/doctrees/index.doctree differ diff --git a/docs/build/doctrees/inference.doctree b/docs/build/doctrees/inference.doctree index 2fe1018d1..cfd503de4 100644 Binary files a/docs/build/doctrees/inference.doctree and b/docs/build/doctrees/inference.doctree differ diff --git a/docs/build/doctrees/leaderboard.doctree b/docs/build/doctrees/leaderboard.doctree index b8230257f..3dca66a90 100644 Binary files a/docs/build/doctrees/leaderboard.doctree and b/docs/build/doctrees/leaderboard.doctree differ diff --git a/docs/build/doctrees/repo/objects.doctree b/docs/build/doctrees/repo/objects.doctree index b0bc255df..e18920ea4 100644 Binary files a/docs/build/doctrees/repo/objects.doctree and b/docs/build/doctrees/repo/objects.doctree differ diff --git a/docs/build/html/_images/datasets.png b/docs/build/html/_images/datasets.png index b08a554c9..9ac5c6238 100644 Binary files a/docs/build/html/_images/datasets.png and b/docs/build/html/_images/datasets.png differ diff --git a/docs/build/html/_images/datasets_metrics.png b/docs/build/html/_images/datasets_metrics.png deleted file mode 100644 index cdec25940..000000000 Binary files a/docs/build/html/_images/datasets_metrics.png and /dev/null differ diff --git a/docs/build/html/_images/grn_models.png b/docs/build/html/_images/grn_models.png deleted file mode 100644 index a7d9c7d69..000000000 Binary files a/docs/build/html/_images/grn_models.png and /dev/null differ diff --git a/docs/build/html/_images/metric_quality_evaluation.png b/docs/build/html/_images/metric_quality_evaluation.png new file mode 100644 index 000000000..d321fc49e Binary files /dev/null and b/docs/build/html/_images/metric_quality_evaluation.png differ diff --git a/docs/build/html/_images/metrics.png b/docs/build/html/_images/metrics.png index f5c2fd02c..2f9e1bd36 100644 Binary files a/docs/build/html/_images/metrics.png and b/docs/build/html/_images/metrics.png differ diff --git a/docs/build/html/_images/raw_scores_300BCG.png b/docs/build/html/_images/raw_scores_300BCG.png index f4b06b0cf..cec454d14 100644 Binary files a/docs/build/html/_images/raw_scores_300BCG.png and b/docs/build/html/_images/raw_scores_300BCG.png differ diff --git a/docs/build/html/_images/raw_scores_adamson.png b/docs/build/html/_images/raw_scores_adamson.png deleted file mode 100644 index 53a2614f1..000000000 Binary files a/docs/build/html/_images/raw_scores_adamson.png and /dev/null differ diff --git a/docs/build/html/_images/raw_scores_ibd.png b/docs/build/html/_images/raw_scores_ibd.png deleted file mode 100644 index cc4e8bf19..000000000 Binary files a/docs/build/html/_images/raw_scores_ibd.png and /dev/null differ diff --git a/docs/build/html/_images/raw_scores_ibd_cd.png b/docs/build/html/_images/raw_scores_ibd_cd.png new file mode 100644 index 000000000..968629360 Binary files /dev/null and b/docs/build/html/_images/raw_scores_ibd_cd.png differ diff --git a/docs/build/html/_images/raw_scores_ibd_uc.png b/docs/build/html/_images/raw_scores_ibd_uc.png new file mode 100644 index 000000000..1d402ed77 Binary files /dev/null and b/docs/build/html/_images/raw_scores_ibd_uc.png differ diff --git a/docs/build/html/_images/raw_scores_nakatake.png b/docs/build/html/_images/raw_scores_nakatake.png index 0da93a104..4fed0c14f 100644 Binary files a/docs/build/html/_images/raw_scores_nakatake.png and b/docs/build/html/_images/raw_scores_nakatake.png differ diff --git a/docs/build/html/_images/raw_scores_norman.png b/docs/build/html/_images/raw_scores_norman.png index 538446d00..b3611239b 100644 Binary files a/docs/build/html/_images/raw_scores_norman.png and b/docs/build/html/_images/raw_scores_norman.png differ diff --git a/docs/build/html/_images/raw_scores_op.png b/docs/build/html/_images/raw_scores_op.png index b55325147..7228b96c7 100644 Binary files a/docs/build/html/_images/raw_scores_op.png and b/docs/build/html/_images/raw_scores_op.png differ diff --git a/docs/build/html/_images/raw_scores_parsebioscience.png b/docs/build/html/_images/raw_scores_parsebioscience.png index 723d508b6..3c382b4f6 100644 Binary files a/docs/build/html/_images/raw_scores_parsebioscience.png and b/docs/build/html/_images/raw_scores_parsebioscience.png differ diff --git a/docs/build/html/_images/raw_scores_replogle.png b/docs/build/html/_images/raw_scores_replogle.png index ab0d3a5ef..76915237e 100644 Binary files a/docs/build/html/_images/raw_scores_replogle.png and b/docs/build/html/_images/raw_scores_replogle.png differ diff --git a/docs/build/html/_images/raw_scores_xaira_HCT116.png b/docs/build/html/_images/raw_scores_xaira_HCT116.png index 946595ffd..307473dd8 100644 Binary files a/docs/build/html/_images/raw_scores_xaira_HCT116.png and b/docs/build/html/_images/raw_scores_xaira_HCT116.png differ diff --git a/docs/build/html/_images/raw_scores_xaira_HEK293T.png b/docs/build/html/_images/raw_scores_xaira_HEK293T.png index c451309a2..ec5856c85 100644 Binary files a/docs/build/html/_images/raw_scores_xaira_HEK293T.png and b/docs/build/html/_images/raw_scores_xaira_HEK293T.png differ diff --git a/docs/build/html/_images/summary_figure.pdf b/docs/build/html/_images/summary_figure.pdf new file mode 100644 index 000000000..a151e4a0d Binary files /dev/null and b/docs/build/html/_images/summary_figure.pdf differ diff --git a/docs/build/html/_images/summary_figure.png b/docs/build/html/_images/summary_figure.png new file mode 100644 index 000000000..5c13b7046 Binary files /dev/null and b/docs/build/html/_images/summary_figure.png differ diff --git a/docs/build/html/_images/table_datasets_summary.pdf b/docs/build/html/_images/table_datasets_summary.pdf new file mode 100644 index 000000000..edbe45376 Binary files /dev/null and b/docs/build/html/_images/table_datasets_summary.pdf differ diff --git a/docs/build/html/_images/table_datasets_summary.png b/docs/build/html/_images/table_datasets_summary.png new file mode 100644 index 000000000..27fcd22b9 Binary files /dev/null and b/docs/build/html/_images/table_datasets_summary.png differ diff --git a/docs/build/html/_sources/dataset.rst.txt b/docs/build/html/_sources/dataset.rst.txt index 8d1b096c0..9bfcb6ad5 100644 --- a/docs/build/html/_sources/dataset.rst.txt +++ b/docs/build/html/_sources/dataset.rst.txt @@ -1,13 +1,16 @@ Datasets ======== -The list of datasets integrated into geneRNIB is provided below with their perturbation signatures as well as the type of perturbation used in each dataset. +The list of datasets integrated into geneRNIB is provided below: .. image:: images/datasets.png - :width: 80% + :width: 50% :align: center ---- -All datasets provide RNA data, while the `OPSCA` and `IBD` datasets also includes scATAC data. +.. image:: images/table_datasets_summary.png + :width: 100% + :align: center +---- You need `awscli` to download the datasets. diff --git a/docs/build/html/_sources/evaluation.rst.txt b/docs/build/html/_sources/evaluation.rst.txt index cccc945ba..6c420b546 100644 --- a/docs/build/html/_sources/evaluation.rst.txt +++ b/docs/build/html/_sources/evaluation.rst.txt @@ -9,13 +9,14 @@ The evaluation metrics used in geneRNIB are summarized below. :align: center ---- -.. image:: images/datasets_metrics.png - :width: 90% +.. image:: images/metric_quality_evaluation.png + :width: 100% :align: center ---- -For a detailed description of each metric, refer to the geneRNIB paper. +For a detailed description of each metric, refer to the geneRNIB paper. Not all the metrics were applicable to all datasets, as shown in the table. In addition, only those datasets with * passed the applicability criteria for a given metric, which includes minimal variability and performance threshold set for each metric. +In addition, not all metrics passed the additional criteria for inclusion in the final score calculation, as explained in the paper, and marked with ** in the table. This includes context specificity and robustness in stability analysis. The evaluation metrics expect the inferred network to be in the form of an AnnData object with specific format as explained here. It should be noted that the metric currently evaluate only the **top TF-gene pairs**, currently limited to **50,000 edges**, ranked by their assigned weight. @@ -28,32 +29,42 @@ The inferred network should have a tabular format with the following columns: See `resources/grn_benchmark/prior/collectri.h5ad` for an example of the expected format. -Running GRN evaluation using standard pipeline ----------------------------------------- -To run the evalution for a given GRN and dataset, use the following command: + +Running GRN evaluation without docker +---------------------------------------- +Considering that Docker is not supported by certtain systems, you can run the evaluation without Docker by following these steps: ```bash -bash scripts/run_grn_evaluation.sh --prediction= --save_dir= --dataset= --build_images= +bash src/metrics/all_metrics/run_local.sh --dataset --prediction= --score --num_workers ``` example command: ```bash -bash scripts/run_grn_evaluation.sh --prediction=resources/grn_models/op/collectri.h5ad --save_dir=output/ --dataset=op --build_images=true +bash src/metrics/all_metrics/run_local.sh --dataset op --prediction=resources/grn_models/op/collectri.h5ad --score=output_score_file.h5ad --num_workers=20 ``` +If you are evaluating a new GRN model, which is not part of geneRNIB, make you to generate the consensus prior file for the dataset you are evaluating on. -Running GRN evaluation without docker +```bash +bash scripts/prior/run_consensus.sh --dataset op --new_model {new_grn_model_file.h5ad} +``` + +This will add your model to the previous ones and create the new consensus prior file needed for evaluation. + +Running GRN evaluation using standard pipeline ---------------------------------------- -Considering that Docker is not supported by certtain systems, you can run the evaluation without Docker by following these steps: + +To run the evalution for a given GRN and dataset, use the following command: ```bash -bash src/metrics/all_metrics/run_local.sh --dataset --prediction= --score --num_workers +bash scripts/run_grn_evaluation.sh --prediction= --save_dir= --dataset= --build_images= ``` example command: ```bash -bash src/metrics/all_metrics/run_local.sh --dataset op --prediction=resources/grn_models/op/collectri.h5ad --score=output_score_file.h5ad --num_workers=20 -``` \ No newline at end of file +bash scripts/run_grn_evaluation.sh --prediction=resources/grn_models/op/collectri.h5ad --save_dir=output/ --dataset=op --build_images=true +``` + diff --git a/docs/build/html/_sources/installation.rst.txt b/docs/build/html/_sources/installation.rst.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/build/html/_sources/leaderboard.rst.txt b/docs/build/html/_sources/leaderboard.rst.txt index 2130ba971..0ee707263 100644 --- a/docs/build/html/_sources/leaderboard.rst.txt +++ b/docs/build/html/_sources/leaderboard.rst.txt @@ -1,14 +1,14 @@ Leaderboard ================= -.. The overal comparitive performance of the integrated GRN inference methods is summarized in the leaderboard below. +The overal comparitive performance of the integrated GRN inference methods is summarized in the leaderboard below. It should be noted that not all metrics count towards the final score, as some metrics did not pass the applicability criteria. See `here `_ for more details on the metrics applicability. -.. .. image:: images/leaderboard.png -.. :width: 90% -.. :align: center -.. ---- +.. image:: images/summary_figure.png + :width: 100% + :align: center +---- -The individual performance of the methods on each dataset is summarized below. +The individual performance of the methods on each dataset is summarized below. .. image:: images/raw_scores_op.png :width: 70% @@ -16,7 +16,7 @@ The individual performance of the methods on each dataset is summarized below. ---- .. image:: images/raw_scores_nakatake.png - :width: 60% + :width: 70% :align: center ---- @@ -25,10 +25,6 @@ The individual performance of the methods on each dataset is summarized below. :align: center ---- -.. image:: images/raw_scores_adamson.png - :width: 70% - :align: center ----- .. image:: images/raw_scores_replogle.png :width: 70% @@ -45,8 +41,13 @@ The individual performance of the methods on each dataset is summarized below. :align: center ---- -.. image:: images/raw_scores_ibd.png - :width: 90% +.. image:: images/raw_scores_ibd_uc.png + :width: 70% + :align: center +---- + +.. image:: images/raw_scores_ibd_cd.png + :width: 70% :align: center ---- diff --git a/docs/build/html/_sources/overview.rst.txt b/docs/build/html/_sources/overview.rst.txt deleted file mode 100644 index eadc7f5b6..000000000 --- a/docs/build/html/_sources/overview.rst.txt +++ /dev/null @@ -1,102 +0,0 @@ -Overview -======== - -Overview of geneRNIB - - -geneRNIB is a cloud-hosted platform designed to evaluate gene regulatory network (GRN) inference methods in a standardized and reproducible way. It brings together datasets, GRN models, evaluation metrics, and a dynamic leaderboard to track the latest advancements in GRN benchmarks. - -To ensure fair comparisons, geneRNIB provides five benchmark datasets, each tailored to assess different aspects of GRN inference. These datasets originate from the same cell types and experiments, allowing context-specific evaluations. The platform supports both transcriptomics-based GRN inference, which relies solely on gene expression data, and multi-omics approaches that integrate chromatin accessibility and gene expression to uncover regulatory interactions. - -Evaluating GRN performance is challenging due to the lack of a definitive “ground truth” network. To address this, geneRNIB employs eight standardized evaluation metrics that use perturbation data to assess inferred interactions. The Wasserstein (WS) distance measures shifts in gene expression after perturbations, helping to determine how well a model captures true regulatory effects. Additionally, regression-based metrics (R₁ and R₂) assess predictive accuracy, ensuring that inferred regulatory links contribute to meaningful predictions. - -To put GRN models into context, geneRNIB also includes three control models. A simple baseline computes Pearson correlations between genes, serving as a quick reference for benchmarking. A positive control model sets an upper bound by incorporating all available variation, while a negative control model generates random networks to ensure meaningful performance comparisons. - -Built with modern computational tools like Docker and Viash, geneRNIB prioritizes scalability and reproducibility. It provides a structured framework for integrating new datasets, inference methods, and evaluation metrics, making it a powerful resource for advancing GRN research. - - -#TODO: Fig of the datasts and the table that shows inference and evaluation datassts -#TODO: Fig of the evaluation metrics and explain - - -Installation ------------- - -For installation, follow the `task_gen_benchmark `_. - -Once the repository is cloned and the required software installed, proceed to the next steps. - -Download resources for GRN inference and evalation: ------------------- - -.. code-block:: bash - - cd task_grn_benchmark - - # download resources - scripts/download_resources.sh - -The full resources is acceesible - -Infer a GRN ------------ - -.. code-block:: bash - - viash run src/methods/dummy/config.vsh.yaml -- --multiomics_rna resources/grn-benchmark/multiomics_rna.h5ad --multiomics_atac resources/grn-benchmark/multiomics_atac.h5ad --prediction output/dummy.csv - - -Similarly, run the command for other methods. - -Evaluate a GRN --------------- - -.. code-block:: bash - - scripts/run_evaluation.sh --grn resources/grn-benchmark/grn_models/collectri.csv - -Similarly, run the command for other GRN models. - - -See examples of interacting with the framework can be found in section :doc:`examples`. - - ----- TODO: improve this with info given above - -The pipeline can evaluate algorithms that leverage only one of the multi-omic data types (RNA-Seq or ATAC-Seq) or both. -It also evaluates the performance of two controls: - -#. As a *negative control*, the pipeline evaluates the performance of a random network. -#. As a *positive control*, the pipeline evaluates the performance of a network derived from correlation of genes in the perturbation dataset used for evaluation. - -The two types of regression models are: - -#. Regression from GRN regulations to target expression -#. Regression from TF expression of predicted regulators to target expression - -The evaluation is done with the help of pertubation data, using two different approaches: - -#. Regression from GRN regulations to target expression -#. Regression from TF expression of predicted regulators to target expression - -| - -.. image:: images/regressions.png - :width: 100 % - :alt: overview of the two regression evaluation approaches - :align: center - -| -| - - -Evaluation 1: Regression from GRN regulations to target expression ------------------------------------------------------------------- -The first approach we used is similar to GRaNPA and the multivariate decision tree in Decoupler, where regulatory weights from the GRN form the feature space to predict perturbation data. In this method, we train one model per sample. The feature space matrix has dimensions of genes by transcription factors (TFs), with values being the regulatory weights from the GRN or 0 if the link is absent. The target space matrix represents the perturbation data for each sample. We evaluate the model's predictive performance using a 5-fold cross-validation scheme and the coefficient of determination (R²) as the metric. LightGBM is used for computational efficiency. - - -Evaluation 2: Regression from TF expression of predicted regulators to target expression ----------------------------------------------------------------------------------------- -In the second approach, instead of using regulatory weights, we utilized the expression of putative regulators (TFs) from the perturbation data to construct the feature space. We fit one model per gene, selecting regulators based on the regulatory weights suggested by the GRNs. This method is similar to many modern GRN inference techniques. - - diff --git a/docs/build/html/dataset.html b/docs/build/html/dataset.html index 49f90320e..76e640288 100644 --- a/docs/build/html/dataset.html +++ b/docs/build/html/dataset.html @@ -3,7 +3,7 @@ - + Datasets — OpenProblems GRN benchmarking 0.1 documentation @@ -82,10 +82,13 @@

Datasets

-

The list of datasets integrated into geneRNIB is provided below with their perturbation signatures as well as the type of perturbation used in each dataset.

-_images/datasets.png +

The list of datasets integrated into geneRNIB is provided below:

+_images/datasets.png + +
+_images/table_datasets_summary.png +
-

All datasets provide RNA data, while the OPSCA and IBD datasets also includes scATAC data.

You need awscli to download the datasets.

Downloading the main datasets

diff --git a/docs/build/html/evaluation.html b/docs/build/html/evaluation.html index 0a6ed1712..37dae3ead 100644 --- a/docs/build/html/evaluation.html +++ b/docs/build/html/evaluation.html @@ -3,7 +3,7 @@ - + GRN evaluation — OpenProblems GRN benchmarking 0.1 documentation @@ -46,8 +46,8 @@
  • Datasets
  • GRN Inference
  • GRN evaluation
  • Extending
    • @@ -81,11 +81,14 @@

    GRN evaluation

    The evaluation metrics used in geneRNIB are summarized below.

    -_images/metrics.png +_images/metrics.png +
    -_images/datasets_metrics.png +_images/metric_quality_evaluation.png +
    -

    For a detailed description of each metric, refer to the geneRNIB paper.

    +

    For a detailed description of each metric, refer to the geneRNIB paper. Not all the metrics were applicable to all datasets, as shown in the table. In addition, only those datasets with * passed the applicability criteria for a given metric, which includes minimal variability and performance threshold set for each metric. +In addition, not all metrics passed the additional criteria for inclusion in the final score calculation, as explained in the paper, and marked with ** in the table. This includes context specificity and robustness in stability analysis.

    The evaluation metrics expect the inferred network to be in the form of an AnnData object with specific format as explained here. It should be noted that the metric currently evaluate only the top TF-gene pairs, currently limited to 50,000 edges, ranked by their assigned weight.

    The inferred network should have a tabular format with the following columns:

    @@ -97,26 +100,31 @@

    GRN evaluation -

    Running GRN evaluation using standard pipeline

    -

    To run the evalution for a given GRN and dataset, use the following command:

    +
    +

    Running GRN evaluation without docker

    +

    Considering that Docker is not supported by certtain systems, you can run the evaluation without Docker by following these steps:

    `bash -bash scripts/run_grn_evaluation.sh --prediction=<inferred GRN (e.g.collectri.h5ad)> --save_dir=<e.g.output/> --dataset=<e.g. replogle> --build_images=<true or false. true for the first time running> +bash src/metrics/all_metrics/run_local.sh --dataset <dataset_name> --prediction=<inferred GRN (e.g.collectri.h5ad)> --score <output_score_file.h5ad> --num_workers <number_of_workers> `

    example command:

    `bash -bash scripts/run_grn_evaluation.sh --prediction=resources/grn_models/op/collectri.h5ad --save_dir=output/ --dataset=op --build_images=true +bash src/metrics/all_metrics/run_local.sh --dataset op --prediction=resources/grn_models/op/collectri.h5ad --score=output_score_file.h5ad --num_workers=20 `

    +

    If you are evaluating a new GRN model, which is not part of geneRNIB, make you to generate the consensus prior file for the dataset you are evaluating on.

    +

    `bash +bash scripts/prior/run_consensus.sh --dataset op --new_model {new_grn_model_file.h5ad} +`

    +

    This will add your model to the previous ones and create the new consensus prior file needed for evaluation.

    -
    -

    Running GRN evaluation without docker

    -

    Considering that Docker is not supported by certtain systems, you can run the evaluation without Docker by following these steps:

    +
    +

    Running GRN evaluation using standard pipeline

    +

    To run the evalution for a given GRN and dataset, use the following command:

    `bash -bash src/metrics/all_metrics/run_local.sh --dataset <dataset_name> --prediction=<inferred GRN (e.g.collectri.h5ad)> --score <output_score_file.h5ad> --num_workers <number_of_workers> +bash scripts/run_grn_evaluation.sh --prediction=<inferred GRN (e.g.collectri.h5ad)> --save_dir=<e.g.output/> --dataset=<e.g. replogle> --build_images=<true or false. true for the first time running> `

    example command:

    `bash -bash src/metrics/all_metrics/run_local.sh --dataset op --prediction=resources/grn_models/op/collectri.h5ad --score=output_score_file.h5ad --num_workers=20 +bash scripts/run_grn_evaluation.sh --prediction=resources/grn_models/op/collectri.h5ad --save_dir=output/ --dataset=op --build_images=true `

    diff --git a/docs/build/html/extending.html b/docs/build/html/extending.html index 44729c935..6cad5d6ec 100644 --- a/docs/build/html/extending.html +++ b/docs/build/html/extending.html @@ -3,7 +3,7 @@ - + Extending — OpenProblems GRN benchmarking 0.1 documentation diff --git a/docs/build/html/index.html b/docs/build/html/index.html index 8b77f1e8a..478251dfb 100644 --- a/docs/build/html/index.html +++ b/docs/build/html/index.html @@ -3,7 +3,7 @@ - + geneRNIB: A living benchmark for gene regulatory network inference — OpenProblems GRN benchmarking 0.1 documentation @@ -79,7 +79,8 @@

    geneRNIB: A living benchmark for gene regulatory network inference_images/overview.png +_images/overview.png +

    This documentation is supplementary to:

      @@ -114,8 +115,8 @@

      Contents

  • GRN evaluation
  • Extending
      diff --git a/docs/build/html/inference.html b/docs/build/html/inference.html index 689bfe218..e5035f54b 100644 --- a/docs/build/html/inference.html +++ b/docs/build/html/inference.html @@ -3,7 +3,7 @@ - + GRN Inference — OpenProblems GRN benchmarking 0.1 documentation diff --git a/docs/build/html/installation.html b/docs/build/html/installation.html deleted file mode 100644 index e7f000560..000000000 --- a/docs/build/html/installation.html +++ /dev/null @@ -1,102 +0,0 @@ - - - - - - - - - <no title> — OpenProblems GRN benchmarking 0.1 documentation - - - - - - - - - - - - - - - -
      - - -
      - -
      -
      -
      - -
      -
      -
      -
      - - - -
      -
      - -
      -
      -
      -
      - - - - \ No newline at end of file diff --git a/docs/build/html/leaderboard.html b/docs/build/html/leaderboard.html index 0f547f774..599c02e72 100644 --- a/docs/build/html/leaderboard.html +++ b/docs/build/html/leaderboard.html @@ -3,7 +3,7 @@ - + Leaderboard — OpenProblems GRN benchmarking 0.1 documentation @@ -75,26 +75,40 @@

      Leaderboard

      +

      The overal comparitive performance of the integrated GRN inference methods is summarized in the leaderboard below. It should be noted that not all metrics count towards the final score, as some metrics did not pass the applicability criteria. See here for more details on the metrics applicability.

      +_images/summary_figure.png + +

      The individual performance of the methods on each dataset is summarized below.

      -_images/raw_scores_op.png +_images/raw_scores_op.png +
      -_images/raw_scores_nakatake.png +_images/raw_scores_nakatake.png +
      -_images/raw_scores_norman.png +_images/raw_scores_norman.png +
      -_images/raw_scores_adamson.png +_images/raw_scores_replogle.png +
      -_images/raw_scores_replogle.png +_images/raw_scores_300BCG.png +
      -_images/raw_scores_300BCG.png +_images/raw_scores_parsebioscience.png +
      -_images/raw_scores_parsebioscience.png +_images/raw_scores_ibd_uc.png +
      -_images/raw_scores_ibd.png +_images/raw_scores_ibd_cd.png +
      -_images/raw_scores_xaira_HEK293T.png +_images/raw_scores_xaira_HEK293T.png +
      -_images/raw_scores_xaira_HCT116.png +_images/raw_scores_xaira_HCT116.png +
      diff --git a/docs/build/html/overview.html b/docs/build/html/overview.html deleted file mode 100644 index eb24f9bf2..000000000 --- a/docs/build/html/overview.html +++ /dev/null @@ -1,177 +0,0 @@ - - - - - - - - - Overview — OpenProblems GRN benchmarking 0.1 documentation - - - - - - - - - - - - - - - -
      - - -
      - -
      -
      -
      - -
      -
      -
      -
      - -
      -

      Overview

      -

      Overview of geneRNIB

      -

      geneRNIB is a cloud-hosted platform designed to evaluate gene regulatory network (GRN) inference methods in a standardized and reproducible way. It brings together datasets, GRN models, evaluation metrics, and a dynamic leaderboard to track the latest advancements in GRN benchmarks.

      -

      To ensure fair comparisons, geneRNIB provides five benchmark datasets, each tailored to assess different aspects of GRN inference. These datasets originate from the same cell types and experiments, allowing context-specific evaluations. The platform supports both transcriptomics-based GRN inference, which relies solely on gene expression data, and multi-omics approaches that integrate chromatin accessibility and gene expression to uncover regulatory interactions.

      -

      Evaluating GRN performance is challenging due to the lack of a definitive “ground truth” network. To address this, geneRNIB employs eight standardized evaluation metrics that use perturbation data to assess inferred interactions. The Wasserstein (WS) distance measures shifts in gene expression after perturbations, helping to determine how well a model captures true regulatory effects. Additionally, regression-based metrics (R₁ and R₂) assess predictive accuracy, ensuring that inferred regulatory links contribute to meaningful predictions.

      -

      To put GRN models into context, geneRNIB also includes three control models. A simple baseline computes Pearson correlations between genes, serving as a quick reference for benchmarking. A positive control model sets an upper bound by incorporating all available variation, while a negative control model generates random networks to ensure meaningful performance comparisons.

      -

      Built with modern computational tools like Docker and Viash, geneRNIB prioritizes scalability and reproducibility. It provides a structured framework for integrating new datasets, inference methods, and evaluation metrics, making it a powerful resource for advancing GRN research.

      -

      #TODO: Fig of the datasts and the table that shows inference and evaluation datassts -#TODO: Fig of the evaluation metrics and explain

      -
      -

      Installation

      -

      For installation, follow the task_gen_benchmark.

      -

      Once the repository is cloned and the required software installed, proceed to the next steps.

      -
      -
      -

      Download resources for GRN inference and evalation:

      -
      cd task_grn_benchmark
-
-# download resources
-scripts/download_resources.sh
-
      -
      -

      The full resources is acceesible

      -
      -
      -

      Infer a GRN

      -
      viash run src/methods/dummy/config.vsh.yaml -- --multiomics_rna resources/grn-benchmark/multiomics_rna.h5ad --multiomics_atac resources/grn-benchmark/multiomics_atac.h5ad --prediction output/dummy.csv
-
      -
      -

      Similarly, run the command for other methods.

      -
      -
      -

      Evaluate a GRN

      -
      scripts/run_evaluation.sh --grn resources/grn-benchmark/grn_models/collectri.csv
-
      -
      -

      Similarly, run the command for other GRN models.

      -

      See examples of interacting with the framework can be found in section examples.

      -

      —- TODO: improve this with info given above

      -

      The pipeline can evaluate algorithms that leverage only one of the multi-omic data types (RNA-Seq or ATAC-Seq) or both. -It also evaluates the performance of two controls:

      -
        -

      1. As a negative control, the pipeline evaluates the performance of a random network.

        2. -

      2. As a positive control, the pipeline evaluates the performance of a network derived from correlation of genes in the perturbation dataset used for evaluation.

        3. -
      -

      The two types of regression models are:

      -
        -

      1. Regression from GRN regulations to target expression

        2. -

      2. Regression from TF expression of predicted regulators to target expression

        3. -
      -

      The evaluation is done with the help of pertubation data, using two different approaches:

      -
        -

      1. Regression from GRN regulations to target expression

        2. -

      2. Regression from TF expression of predicted regulators to target expression

        3. -
      -
      -

      -
      -overview of the two regression evaluation approaches - -
      -

      -

      -
      -
      -
      -

      Evaluation 1: Regression from GRN regulations to target expression

      -

      The first approach we used is similar to GRaNPA and the multivariate decision tree in Decoupler, where regulatory weights from the GRN form the feature space to predict perturbation data. In this method, we train one model per sample. The feature space matrix has dimensions of genes by transcription factors (TFs), with values being the regulatory weights from the GRN or 0 if the link is absent. The target space matrix represents the perturbation data for each sample. We evaluate the model’s predictive performance using a 5-fold cross-validation scheme and the coefficient of determination (R²) as the metric. LightGBM is used for computational efficiency.

      -
      -
      -

      Evaluation 2: Regression from TF expression of predicted regulators to target expression

      -

      In the second approach, instead of using regulatory weights, we utilized the expression of putative regulators (TFs) from the perturbation data to construct the feature space. We fit one model per gene, selecting regulators based on the regulatory weights suggested by the GRNs. This method is similar to many modern GRN inference techniques.

      -
      -
      - - -
      -
      - -
      -
      -
      -
      - - - - \ No newline at end of file diff --git a/docs/build/html/repo/objects.html b/docs/build/html/repo/objects.html index 27b963c00..7bb339eec 100644 --- a/docs/build/html/repo/objects.html +++ b/docs/build/html/repo/objects.html @@ -3,7 +3,7 @@ - + Objects — OpenProblems GRN benchmarking 0.1 documentation diff --git a/docs/build/html/searchindex.js b/docs/build/html/searchindex.js index a24f5b86d..a2c88acdb 100644 --- a/docs/build/html/searchindex.js +++ b/docs/build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {"Add a GRN evaluation metric": [[2, "add-a-grn-evaluation-metric"]], "Add a GRN inference and evalaution dataset": [[2, "add-a-grn-inference-and-evalaution-dataset"]], "Add a GRN inference method": [[2, "add-a-grn-inference-method"]], "AnnData object": [[6, "id1"], [6, "id4"], [6, "id5"], [6, "id6"]], "Contents": [[3, "contents"]], "Control Method": [[6, "control-method"]], "Datasets": [[0, null]], "Downloading the extended datasets": [[0, "downloading-the-extended-datasets"]], "Downloading the main datasets": [[0, "downloading-the-main-datasets"]], "Downloading the raw/unprocessed data": [[0, "downloading-the-raw-unprocessed-data"]], "Downloading the results": [[0, "downloading-the-results"]], "Extending": [[2, null]], "GRN": [[6, "grn"]], "GRN Inference": [[4, null]], "GRN evaluation": [[1, null]], "GRN inference using integrated methods": [[4, "grn-inference-using-integrated-methods"]], "GRN inference without method integration": [[4, "grn-inference-without-method-integration"]], "Getting started": [[3, "getting-started"]], "Label": [[6, "label"]], "Leaderboard": [[5, null]], "Method": [[6, "method"]], "Multiomics ATAC": [[6, "multiomics-atac"]], "Multiomics RNA": [[6, "multiomics-rna"]], "Objects": [[6, null]], "Perturbation": [[6, "perturbation"]], "Prior data": [[6, "prior-data"]], "Running GRN evaluation using standard pipeline": [[1, "running-grn-evaluation-using-standard-pipeline"]], "Running GRN evaluation without docker": [[1, "running-grn-evaluation-without-docker"]], "Score": [[6, "score"]], "Tabular data": [[6, "id2"], [6, "id3"]], "geneRNIB: A living benchmark for gene regulatory network inference": [[3, null]]}, "docnames": ["dataset", "evaluation", "extending", "index", "inference", "leaderboard", "repo/objects"], "envversion": {"sphinx": 64, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.intersphinx": 1}, "filenames": ["dataset.rst", "evaluation.rst", "extending.rst", "index.rst", "inference.rst", "leaderboard.rst", "repo/objects.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"0": [2, 4], "000": [1, 4], "1": [2, 6], "2": 2, "20": 1, "3": [], "4": 2, "5": [], "50": [1, 4], "50k": 2, "6": 6, "80": [], "A": [2, 6], "For": [1, 2, 4], "If": 4, "In": 2, "It": [1, 2, 3], "The": [0, 1, 2, 3, 4, 5, 6], "There": 2, "These": 0, "To": [0, 1, 3, 4], "__merge__": 2, "_viash": [], "about": 2, "access": 0, "accordingli": 3, "accuraci": 6, "activ": [], "ad": [2, 3, 4], "adamson": [], "add": 3, "addit": 2, "addition": [0, 2], "aertslab": 6, "after": 4, "aim": [], "algorithm": 4, "align": [], "all": [0, 2], "all_metr": 1, "alreadi": [], "also": [0, 2], "amazon": [], "among": [], "an": [1, 4, 6], "anndata": [1, 2, 4], "annot": 6, "api": 2, "applic": [], "approach": [4, 6], "ar": [0, 1, 2, 3], "argument": 6, "art": 3, "assess": 3, "assign": [1, 4], "astyp": [2, 4], "atac": [], "attribut": 6, "avail": 0, "aw": [0, 2], "awscli": 0, "base": [2, 6], "base_python": 2, "base_r": [], "base_requir": 2, "baselin": [], "bash": 1, "below": [0, 1, 5], "benchmark": 6, "better": 2, "between": [], "beyond": 0, "bio": 2, "biorxiv": 3, "bioscienc": 0, "block": [], "both": [2, 3], "build_imag": 1, "bulk": 2, "c": 4, "call": 2, "can": [1, 2, 4], "categori": [], "causal": [], "cell": [0, 2, 6], "cell_count": 6, "cell_typ": 6, "celloracl": 2, "center": [], "certtain": 1, "chang": 2, "charact": 4, "check": 2, "choic": 2, "chosen": 6, "cistarget": 6, "cli": [], "cloud": 3, "code": 2, "collectri": [1, 4, 6], "column": [1, 4, 6], "com": [], "combin": 0, "command": [0, 1, 2, 4], "common": 2, "comp_method": 2, "comparison": [0, 3], "comparit": [], "competit": 3, "complet": 6, "complic": 2, "compon": [], "compound": 6, "compress": 4, "comput": [0, 3], "config": [2, 4], "connect": 6, "consensu": 0, "consid": [1, 4], "consist": [], "construct": [], "contact": 0, "contain": 0, "content": [], "control": 0, "control_method": [], "core": 0, "correct": [], "correl": 4, "could": 2, "csv": [2, 4, 6], "curat": 3, "current": [1, 2, 4], "data": [2, 3], "datafram": [2, 4], "dataset": [1, 3, 4, 5, 6], "dataset_id": [2, 4], "dataset_nam": [1, 2, 4], "datasets_raw": 0, "de": 2, "default": [0, 6], "defin": [], "definit": 2, "depend": 2, "descript": [1, 2, 6], "design": 3, "detail": 1, "develop": 3, "did": [], "differ": 2, "differenti": 2, "direct": 2, "directori": 0, "distanc": [], "distribut": [], "do": 2, "docker": [2, 3], "document": [2, 3], "documentation_url": 2, "don": [], "done": 2, "donor": 6, "donor_id": 6, "dont": 2, "doubl": 6, "download": [2, 3, 4], "draft": [], "drawn": 6, "dtype": 2, "due": 0, "dummpi": [], "dure": [], "dynam": 3, "e": [1, 2, 4], "each": [0, 1, 2, 4, 5, 6], "edg": [1, 2, 4], "effect": [], "effici": 0, "eight": [], "end": 2, "engin": 2, "enhanc": [], "ensur": [2, 4], "essenti": 2, "etc": 1, "evalaut": 3, "evalu": [0, 3, 4, 6], "evaluation_data": 2, "evalut": 1, "exampl": [1, 2, 4, 6], "except": [], "execut": 2, "expect": [1, 2, 4], "experi": 6, "explain": [1, 2], "explan": 6, "express": [2, 6], "extend": 3, "extended_data": 0, "extra": 2, "factor": 4, "fals": 1, "far": [], "featur": [2, 3], "few": [0, 2], "file": [0, 2, 6], "filter": 2, "find": [0, 6], "first": 1, "float": 6, "folder": [0, 2], "follow": [1, 2, 4], "form": 1, "format": [1, 2, 4, 6], "format_save_scor": 2, "found": 2, "from": [2, 4, 6], "full": 0, "further": 0, "futur": 3, "g": [1, 2, 4], "gene": [1, 2, 4, 6], "gene1": 2, "gene2": 2, "genernbi": 4, "genernib": [0, 1, 2, 4], "get": 2, "ghcr": 2, "github": 3, "given": [1, 4], "grn": [0, 3], "grn_benchmark": [0, 1, 2, 4], "grn_method": 2, "grn_model": 1, "grnboost2": [2, 4], "group": [], "guidelin": [], "gzip": 4, "h5ad": [1, 2, 4, 6], "ha": 2, "have": [0, 1, 2, 4], "hct116": [], "hek293t": [], "helper": 2, "here": [1, 2], "how": 2, "howev": 2, "http": 6, "i": [0, 1, 2, 3, 4, 5, 6], "ibd": 0, "id": [2, 6], "identifi": 2, "imag": 2, "implement": [], "import": [1, 2, 4], "improv": [], "includ": [0, 2], "incorpor": 0, "indic": 6, "individu": 5, "infer": [0, 1, 6], "inference_data": [2, 4], "info": 2, "inform": 2, "infrastructur": 3, "initi": [], "instal": [2, 3, 4], "instruct": 2, "integ": 6, "integr": [0, 2, 3], "io": 2, "know": [], "known": [0, 6], "label": 2, "labels_tw": 2, "larg": 0, "latest": [], "layer": [2, 6], "leaderboard": [0, 3], "let": [], "level": 6, "librari": [2, 4], "likelihood": [1, 4], "limit": [1, 4], "linc": 6, "link": 2, "list": [0, 2, 4, 6], "live": [], "load": 2, "loadtxt": 2, "locat": 2, "logarithm": 6, "lognorm": [2, 6], "main": [2, 3], "make": [], "manuscript": [], "map": 6, "mask": [], "matrix": 4, "me": [], "mean": 6, "measur": [], "mention": 0, "merg": 2, "method": [0, 3, 5], "method_id": [2, 4], "method_nam": 2, "metric": [0, 1, 3, 6], "metric_key_1": 2, "metric_key_2": 2, "metric_value_1": 2, "metric_value_2": 2, "midcpu": 2, "midmem": 2, "midtim": 2, "modal": 3, "model": [], "more": [2, 4], "most": [], "multi": 3, "multiom": [], "multiomics_atac": 6, "multiomics_rna": 6, "n_count": 6, "nakatak": [], "name": [1, 2, 6], "namespac": 2, "nativ": [], "ncol": 4, "necessari": 2, "necessarili": [], "need": [0, 2], "neg": [], "net": [2, 4], "network": [1, 2, 4], "new": [2, 3], "next": 4, "nextflow": 2, "none": [2, 4], "normal": [2, 6], "norman": [], "note": 1, "novel": 3, "np": 2, "nrow": 4, "num_work": 1, "number": 6, "number_of_work": 1, "numpi": 2, "ob": 6, "object": 1, "observ": [], "obtain": 6, "offer": 3, "omic": 3, "omit": 2, "onc": [2, 4], "one": 4, "onli": [1, 2, 4], "op": [1, 4], "op_rna": 4, "openproblem": [0, 2, 3], "opsca": 0, "option": [2, 6], "org": 6, "organ": 2, "origin": [], "other": 0, "our": [3, 4], "out": [], "outperform": [], "output": [1, 2, 4, 6], "output_score_fil": 1, "over": 3, "overlook": 2, "overview": 6, "own": [], "packag": 2, "page": [3, 4], "pair": [1, 2, 4], "panda": [2, 4], "paper": [0, 1, 3], "par": 2, "param": 2, "paramet": 2, "parent": 6, "pars": 0, "parsebiosci": [], "pass": 2, "path": [2, 6], "pd": 2, "peak": 6, "pearson": [4, 6], "pearson_corr": 4, "per": [], "perform": [3, 4, 5, 6], "permit": [], "perturb": 0, "perturbation_data": 6, "pip": [], "pipelin": [2, 3], "pl": 0, "place": 2, "plate": 6, "plate_nam": 6, "platform": [2, 3], "pleas": [], "png": [], "posit": 0, "possibl": [], "power": [], "predict": [1, 2, 4, 6], "prerequisit": 2, "pretti": 2, "previou": 0, "previous": [0, 3], "primari": 6, "prior": [0, 1, 2, 4], "prior_data": 6, "proce": 4, "process": 0, "project": [], "protocol": 3, "prove": [], "provid": [0, 2, 3, 6], "pseudo": 2, "pseudobulk": [0, 6], "pull": 2, "py": 2, "python": 2, "python_script": 2, "r": [2, 4], "r_script": 2, "randomli": 6, "rank": [1, 4], "raw": 3, "re": 3, "reach": [], "read": 2, "read_h5ad": 2, "read_predict": 2, "readabl": [], "recent": 0, "reduc": [], "refer": [1, 2], "refin": [], "reflect": 3, "reg_typ": 6, "regress": 6, "regul": 6, "regulatori": [1, 4], "relationship": 6, "remov": [], "replac": 2, "replogl": [0, 1, 4], "repositori": 2, "repres": 0, "represent": 6, "request": [0, 2], "requir": 2, "residu": 6, "resourc": [0, 1, 2, 4, 6], "resources_test": [2, 4], "result": [2, 3], "ridg": 6, "rna": [0, 2, 4], "rna_op": 2, "row": 6, "run": [0, 2, 3, 4], "run_grn_evalu": 1, "run_loc": 1, "run_test": [], "runner": 2, "s3": [0, 2], "same": 2, "sampl": 6, "save": [2, 4], "save_dir": 1, "save_to_fil": 4, "scatac": 0, "scenic": 2, "schema": 2, "scheme": 2, "score": [1, 2, 3, 4], "score_fil": 2, "script": [1, 2], "section": [2, 4], "see": [1, 2, 3, 4], "set": [], "setup": 2, "sh": 1, "shift": 6, "should": [1, 2, 4], "sign": [0, 2], "signatur": 0, "similar": 2, "sinc": 4, "singl": [0, 2, 3, 6], "size": 0, "slot": 6, "sm_name": 6, "so": 2, "some": [2, 6], "sourc": [1, 2, 4, 6], "space": [], "specif": 1, "src": [1, 2, 4, 6], "standard": [0, 3, 6], "start": 2, "state": 3, "step": 1, "still": [], "store": [2, 3, 4], "str": [2, 4], "string": [2, 4, 6], "structur": 2, "submetr": 2, "submit": 2, "subsampl": 6, "substanti": [], "success": 2, "summar": [1, 5], "summari": 2, "supplementari": [0, 3], "support": [1, 3], "sy": 2, "sync": [0, 2], "system": 1, "t": [], "tabular": 1, "tag": 0, "target": [1, 2, 4, 6], "task": 6, "terminologi": [], "test": 2, "test_run": [], "tf": [0, 1, 2, 4, 6], "tf1": 2, "tf2": 2, "tf_all": [2, 4], "tf_list": 6, "thei": [], "them": 3, "thi": [0, 2, 3, 4, 6], "those": [], "three": [2, 4], "thu": [], "time": [1, 3], "tip": 2, "togeth": 2, "top": [1, 2, 4], "track": 3, "transcript": 4, "transcriptom": [], "true": 1, "type": [0, 2, 6], "typo": [], "u": 0, "un": [2, 4, 6], "under": [], "uniqu": 2, "unprocess": 3, "updat": [2, 3], "us": [0, 2, 3, 6], "util": 2, "valu": [2, 6], "variat": 0, "veri": 0, "version": 0, "viash": [2, 4], "view": 3, "vsh": [2, 4], "wa": [], "wasserstein": [], "we": [0, 2, 4], "weight": [1, 2, 4, 6], "well": [0, 6], "were": 0, "when": 3, "where": 2, "which": [0, 2, 6], "while": [0, 2], "width": [], "within": 0, "without": 3, "work": [2, 4], "write": [2, 4], "write_h5ad": 4, "x": [2, 4], "xaira": 0, "xaira_hct116": [], "xaira_hek293t": [], "yaml": [2, 4], "yet": [], "you": [0, 1, 2, 4], "your": [2, 4], "your_method": 2, "your_method_nam": 4, "zellkonvert": 4}, "titles": ["Datasets", "GRN evaluation", "Extending", "geneRNIB: A living benchmark for gene regulatory network inference", "GRN Inference", "Leaderboard", "Objects"], "titleterms": {"A": 3, "ad": [], "add": 2, "anndata": 6, "atac": 6, "benchmark": 3, "content": 3, "control": 6, "data": [0, 6], "dataset": [0, 2], "docker": 1, "document": [], "download": 0, "evalaut": 2, "evalu": [1, 2], "extend": [0, 2], "gene": 3, "genernib": 3, "get": 3, "grn": [1, 2, 4, 6], "infer": [2, 3, 4], "integr": 4, "label": 6, "leaderboard": 5, "live": 3, "main": 0, "method": [2, 4, 6], "metric": 2, "model": [], "multiom": 6, "network": 3, "object": 6, "perturb": 6, "pipelin": 1, "prior": 6, "raw": 0, "regulatori": 3, "result": 0, "rna": 6, "run": 1, "score": 6, "standard": 1, "start": 3, "tabular": 6, "test": [], "unprocess": 0, "us": [1, 4], "without": [1, 4]}}) \ No newline at end of file +Search.setIndex({"alltitles": {"Add a GRN evaluation metric": [[2, "add-a-grn-evaluation-metric"]], "Add a GRN inference and evalaution dataset": [[2, "add-a-grn-inference-and-evalaution-dataset"]], "Add a GRN inference method": [[2, "add-a-grn-inference-method"]], "AnnData object": [[6, "id1"], [6, "id4"], [6, "id5"], [6, "id6"]], "Contents": [[3, "contents"]], "Control Method": [[6, "control-method"]], "Datasets": [[0, null]], "Downloading the extended datasets": [[0, "downloading-the-extended-datasets"]], "Downloading the main datasets": [[0, "downloading-the-main-datasets"]], "Downloading the raw/unprocessed data": [[0, "downloading-the-raw-unprocessed-data"]], "Downloading the results": [[0, "downloading-the-results"]], "Extending": [[2, null]], "GRN": [[6, "grn"]], "GRN Inference": [[4, null]], "GRN evaluation": [[1, null]], "GRN inference using integrated methods": [[4, "grn-inference-using-integrated-methods"]], "GRN inference without method integration": [[4, "grn-inference-without-method-integration"]], "Getting started": [[3, "getting-started"]], "Label": [[6, "label"]], "Leaderboard": [[5, null]], "Method": [[6, "method"]], "Multiomics ATAC": [[6, "multiomics-atac"]], "Multiomics RNA": [[6, "multiomics-rna"]], "Objects": [[6, null]], "Perturbation": [[6, "perturbation"]], "Prior data": [[6, "prior-data"]], "Running GRN evaluation using standard pipeline": [[1, "running-grn-evaluation-using-standard-pipeline"]], "Running GRN evaluation without docker": [[1, "running-grn-evaluation-without-docker"]], "Score": [[6, "score"]], "Tabular data": [[6, "id2"], [6, "id3"]], "geneRNIB: A living benchmark for gene regulatory network inference": [[3, null]]}, "docnames": ["dataset", "evaluation", "extending", "index", "inference", "leaderboard", "repo/objects"], "envversion": {"sphinx": 64, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.intersphinx": 1}, "filenames": ["dataset.rst", "evaluation.rst", "extending.rst", "index.rst", "inference.rst", "leaderboard.rst", "repo/objects.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"0": [2, 4], "000": [1, 4], "1": [2, 6], "2": 2, "20": 1, "4": 2, "50": [1, 4], "50k": 2, "6": 6, "A": [2, 6], "For": [1, 2, 4], "If": [1, 4], "In": [1, 2], "It": [1, 2, 3, 5], "Not": 1, "The": [0, 1, 2, 3, 4, 5, 6], "There": 2, "These": 0, "To": [0, 1, 3, 4], "__merge__": 2, "about": 2, "access": 0, "accordingli": 3, "accuraci": 6, "ad": [2, 3, 4], "add": [1, 3], "addit": [1, 2], "addition": [0, 2], "aertslab": 6, "after": 4, "algorithm": 4, "all": [0, 1, 2, 5], "all_metr": 1, "also": 2, "an": [1, 4, 6], "analysi": 1, "anndata": [1, 2, 4], "annot": 6, "api": 2, "applic": [1, 5], "approach": [4, 6], "ar": [0, 1, 2, 3], "argument": 6, "art": 3, "assess": 3, "assign": [1, 4], "astyp": [2, 4], "attribut": 6, "avail": 0, "aw": [0, 2], "awscli": 0, "base": [2, 6], "base_python": 2, "base_requir": 2, "bash": 1, "below": [0, 1, 5], "benchmark": 6, "better": 2, "beyond": 0, "bio": 2, "biorxiv": 3, "bioscienc": 0, "both": [2, 3], "build_imag": 1, "bulk": 2, "c": 4, "calcul": 1, "call": 2, "can": [1, 2, 4], "cell": [0, 2, 6], "cell_count": 6, "cell_typ": 6, "celloracl": 2, "certtain": 1, "chang": 2, "charact": 4, "check": 2, "choic": 2, "chosen": 6, "cistarget": 6, "cloud": 3, "code": 2, "collectri": [1, 4, 6], "column": [1, 4, 6], "combin": 0, "command": [0, 1, 2, 4], "common": 2, "comp_method": 2, "comparison": [0, 3], "comparit": 5, "competit": 3, "complet": 6, "complic": 2, "compound": 6, "compress": 4, "comput": [0, 3], "config": [2, 4], "connect": 6, "consensu": [0, 1], "consid": [1, 4], "contact": 0, "contain": 0, "context": 1, "control": 0, "core": 0, "correl": 4, "could": 2, "count": 5, "creat": 1, "criteria": [1, 5], "csv": [2, 4, 6], "curat": 3, "current": [1, 2, 4], "data": [2, 3], "datafram": [2, 4], "dataset": [1, 3, 4, 5, 6], "dataset_id": [2, 4], "dataset_nam": [1, 2, 4], "datasets_raw": 0, "de": 2, "default": [0, 6], "definit": 2, "depend": 2, "descript": [1, 2, 6], "design": 3, "detail": [1, 5], "develop": 3, "did": 5, "differ": 2, "differenti": 2, "direct": 2, "directori": 0, "do": 2, "docker": [2, 3], "document": [2, 3], "documentation_url": 2, "done": 2, "donor": 6, "donor_id": 6, "dont": 2, "doubl": 6, "download": [2, 3, 4], "drawn": 6, "dtype": 2, "due": 0, "dynam": 3, "e": [1, 2, 4], "each": [1, 2, 4, 5, 6], "edg": [1, 2, 4], "effici": 0, "end": 2, "engin": 2, "ensur": [2, 4], "essenti": 2, "etc": 1, "evalaut": 3, "evalu": [0, 3, 4, 6], "evaluation_data": 2, "evalut": 1, "exampl": [1, 2, 4, 6], "execut": 2, "expect": [1, 2, 4], "experi": 6, "explain": [1, 2], "explan": 6, "express": [2, 6], "extend": 3, "extended_data": 0, "extra": 2, "factor": 4, "fals": 1, "featur": [2, 3], "few": [0, 2], "file": [0, 1, 2, 6], "filter": 2, "final": [1, 5], "find": [0, 6], "first": 1, "float": 6, "folder": [0, 2], "follow": [1, 2, 4], "form": 1, "format": [1, 2, 4, 6], "format_save_scor": 2, "found": 2, "from": [2, 4, 6], "full": 0, "further": 0, "futur": 3, "g": [1, 2, 4], "gene": [1, 2, 4, 6], "gene1": 2, "gene2": 2, "gener": 1, "genernbi": 4, "genernib": [0, 1, 2, 4], "get": 2, "ghcr": 2, "github": 3, "given": [1, 4], "grn": [0, 3, 5], "grn_benchmark": [0, 1, 2, 4], "grn_method": 2, "grn_model": 1, "grnboost2": [2, 4], "gzip": 4, "h5ad": [1, 2, 4, 6], "ha": 2, "have": [0, 1, 2, 4], "helper": 2, "here": [1, 2, 5], "how": 2, "howev": 2, "http": 6, "i": [0, 1, 2, 3, 4, 5, 6], "id": [2, 6], "identifi": 2, "imag": 2, "import": [1, 2, 4], "includ": [0, 1, 2], "inclus": 1, "incorpor": 0, "indic": 6, "individu": 5, "infer": [0, 1, 5, 6], "inference_data": [2, 4], "info": 2, "inform": 2, "infrastructur": 3, "instal": [2, 3, 4], "instruct": 2, "integ": 6, "integr": [0, 2, 3, 5], "io": 2, "known": [0, 6], "label": 2, "labels_tw": 2, "larg": 0, "layer": [2, 6], "leaderboard": [0, 3], "level": 6, "librari": [2, 4], "likelihood": [1, 4], "limit": [1, 4], "linc": 6, "link": 2, "list": [0, 2, 4, 6], "load": 2, "loadtxt": 2, "locat": 2, "logarithm": 6, "lognorm": [2, 6], "main": [2, 3], "make": 1, "map": 6, "mark": 1, "matrix": 4, "mean": 6, "mention": 0, "merg": 2, "method": [0, 3, 5], "method_id": [2, 4], "method_nam": 2, "metric": [0, 1, 3, 5, 6], "metric_key_1": 2, "metric_key_2": 2, "metric_value_1": 2, "metric_value_2": 2, "midcpu": 2, "midmem": 2, "midtim": 2, "minim": 1, "modal": 3, "model": 1, "more": [2, 4, 5], "multi": 3, "multiomics_atac": 6, "multiomics_rna": 6, "n_count": 6, "name": [1, 2, 6], "namespac": 2, "ncol": 4, "necessari": 2, "need": [0, 1, 2], "net": [2, 4], "network": [1, 2, 4], "new": [1, 2, 3], "new_grn_model_fil": 1, "new_model": 1, "next": 4, "nextflow": 2, "none": [2, 4], "normal": [2, 6], "note": [1, 5], "novel": 3, "np": 2, "nrow": 4, "num_work": 1, "number": 6, "number_of_work": 1, "numpi": 2, "ob": 6, "object": 1, "obtain": 6, "offer": 3, "omic": 3, "omit": 2, "onc": [2, 4], "one": 4, "ones": 1, "onli": [1, 2, 4], "op": [1, 4], "op_rna": 4, "openproblem": [0, 2, 3], "option": [2, 6], "org": 6, "organ": 2, "other": 0, "our": [3, 4], "output": [1, 2, 4, 6], "output_score_fil": 1, "over": [3, 5], "overlook": 2, "overview": 6, "packag": 2, "page": [3, 4], "pair": [1, 2, 4], "panda": [2, 4], "paper": [0, 1, 3], "par": 2, "param": 2, "paramet": 2, "parent": 6, "pars": 0, "part": 1, "pass": [1, 2, 5], "path": [2, 6], "pd": 2, "peak": 6, "pearson": [4, 6], "pearson_corr": 4, "perform": [1, 3, 4, 5, 6], "perturb": 0, "perturbation_data": 6, "pipelin": [2, 3], "pl": 0, "place": 2, "plate": 6, "plate_nam": 6, "platform": [2, 3], "posit": 0, "predict": [1, 2, 4, 6], "prerequisit": 2, "pretti": 2, "previou": [0, 1], "previous": [0, 3], "primari": 6, "prior": [0, 1, 2, 4], "prior_data": 6, "proce": 4, "process": 0, "protocol": 3, "provid": [0, 2, 3, 6], "pseudo": 2, "pseudobulk": [0, 6], "pull": 2, "py": 2, "python": 2, "python_script": 2, "r": [2, 4], "r_script": 2, "randomli": 6, "rank": [1, 4], "raw": 3, "re": 3, "read": 2, "read_h5ad": 2, "read_predict": 2, "recent": 0, "refer": [1, 2], "reflect": 3, "reg_typ": 6, "regress": 6, "regul": 6, "regulatori": [1, 4], "relationship": 6, "replac": 2, "replogl": [0, 1, 4], "repositori": 2, "repres": 0, "represent": 6, "request": [0, 2], "requir": 2, "residu": 6, "resourc": [0, 1, 2, 4, 6], "resources_test": [2, 4], "result": [2, 3], "ridg": 6, "rna": [2, 4], "rna_op": 2, "robust": 1, "row": 6, "run": [0, 2, 3, 4], "run_consensu": 1, "run_grn_evalu": 1, "run_loc": 1, "runner": 2, "s3": [0, 2], "same": 2, "sampl": 6, "save": [2, 4], "save_dir": 1, "save_to_fil": 4, "scenic": 2, "schema": 2, "scheme": 2, "score": [1, 2, 3, 4, 5], "score_fil": 2, "script": [1, 2], "section": [2, 4], "see": [1, 2, 3, 4, 5], "set": 1, "setup": 2, "sh": 1, "shift": 6, "should": [1, 2, 4, 5], "shown": 1, "sign": [0, 2], "similar": 2, "sinc": 4, "singl": [0, 2, 3, 6], "size": 0, "slot": 6, "sm_name": 6, "so": 2, "some": [2, 5, 6], "sourc": [1, 2, 4, 6], "specif": 1, "src": [1, 2, 4, 6], "stabil": 1, "standard": [0, 3, 6], "start": 2, "state": 3, "step": 1, "store": [2, 3, 4], "str": [2, 4], "string": [2, 4, 6], "structur": 2, "submetr": 2, "submit": 2, "subsampl": 6, "success": 2, "summar": [1, 5], "summari": 2, "supplementari": [0, 3], "support": [1, 3], "sy": 2, "sync": [0, 2], "system": 1, "tabl": 1, "tabular": 1, "tag": 0, "target": [1, 2, 4, 6], "task": 6, "test": 2, "tf": [0, 1, 2, 4, 6], "tf1": 2, "tf2": 2, "tf_all": [2, 4], "tf_list": 6, "them": 3, "thi": [0, 1, 2, 3, 4, 6], "those": 1, "three": [2, 4], "threshold": 1, "time": [1, 3], "tip": 2, "togeth": 2, "top": [1, 2, 4], "toward": 5, "track": 3, "transcript": 4, "true": 1, "type": [2, 6], "u": 0, "un": [2, 4, 6], "uniqu": 2, "unprocess": 3, "updat": [2, 3], "us": [0, 2, 3, 6], "util": 2, "valu": [2, 6], "variabl": 1, "variat": 0, "veri": 0, "version": 0, "viash": [2, 4], "view": 3, "vsh": [2, 4], "we": [0, 2, 4], "weight": [1, 2, 4, 6], "well": 6, "were": [0, 1], "when": 3, "where": 2, "which": [0, 1, 2, 6], "while": 2, "within": 0, "without": 3, "work": [2, 4], "write": [2, 4], "write_h5ad": 4, "x": [2, 4], "xaira": 0, "yaml": [2, 4], "you": [0, 1, 2, 4], "your": [1, 2, 4], "your_method": 2, "your_method_nam": 4, "zellkonvert": 4}, "titles": ["Datasets", "GRN evaluation", "Extending", "geneRNIB: A living benchmark for gene regulatory network inference", "GRN Inference", "Leaderboard", "Objects"], "titleterms": {"A": 3, "add": 2, "anndata": 6, "atac": 6, "benchmark": 3, "content": 3, "control": 6, "data": [0, 6], "dataset": [0, 2], "docker": 1, "download": 0, "evalaut": 2, "evalu": [1, 2], "extend": [0, 2], "gene": 3, "genernib": 3, "get": 3, "grn": [1, 2, 4, 6], "infer": [2, 3, 4], "integr": 4, "label": 6, "leaderboard": 5, "live": 3, "main": 0, "method": [2, 4, 6], "metric": 2, "multiom": 6, "network": 3, "object": 6, "perturb": 6, "pipelin": 1, "prior": 6, "raw": 0, "regulatori": 3, "result": 0, "rna": 6, "run": 1, "score": 6, "standard": 1, "start": 3, "tabular": 6, "unprocess": 0, "us": [1, 4], "without": [1, 4]}}) \ No newline at end of file diff --git a/docs/source/dataset.rst b/docs/source/dataset.rst index 8d1b096c0..9bfcb6ad5 100644 --- a/docs/source/dataset.rst +++ b/docs/source/dataset.rst @@ -1,13 +1,16 @@ Datasets ======== -The list of datasets integrated into geneRNIB is provided below with their perturbation signatures as well as the type of perturbation used in each dataset. +The list of datasets integrated into geneRNIB is provided below: .. image:: images/datasets.png - :width: 80% + :width: 50% :align: center ---- -All datasets provide RNA data, while the `OPSCA` and `IBD` datasets also includes scATAC data. +.. image:: images/table_datasets_summary.png + :width: 100% + :align: center +---- You need `awscli` to download the datasets. diff --git a/docs/source/evaluation.rst b/docs/source/evaluation.rst index cccc945ba..6c420b546 100644 --- a/docs/source/evaluation.rst +++ b/docs/source/evaluation.rst @@ -9,13 +9,14 @@ The evaluation metrics used in geneRNIB are summarized below. :align: center ---- -.. image:: images/datasets_metrics.png - :width: 90% +.. image:: images/metric_quality_evaluation.png + :width: 100% :align: center ---- -For a detailed description of each metric, refer to the geneRNIB paper. +For a detailed description of each metric, refer to the geneRNIB paper. Not all the metrics were applicable to all datasets, as shown in the table. In addition, only those datasets with * passed the applicability criteria for a given metric, which includes minimal variability and performance threshold set for each metric. +In addition, not all metrics passed the additional criteria for inclusion in the final score calculation, as explained in the paper, and marked with ** in the table. This includes context specificity and robustness in stability analysis. The evaluation metrics expect the inferred network to be in the form of an AnnData object with specific format as explained here. It should be noted that the metric currently evaluate only the **top TF-gene pairs**, currently limited to **50,000 edges**, ranked by their assigned weight. @@ -28,32 +29,42 @@ The inferred network should have a tabular format with the following columns: See `resources/grn_benchmark/prior/collectri.h5ad` for an example of the expected format. -Running GRN evaluation using standard pipeline ----------------------------------------- -To run the evalution for a given GRN and dataset, use the following command: + +Running GRN evaluation without docker +---------------------------------------- +Considering that Docker is not supported by certtain systems, you can run the evaluation without Docker by following these steps: ```bash -bash scripts/run_grn_evaluation.sh --prediction= --save_dir= --dataset= --build_images= +bash src/metrics/all_metrics/run_local.sh --dataset --prediction= --score --num_workers ``` example command: ```bash -bash scripts/run_grn_evaluation.sh --prediction=resources/grn_models/op/collectri.h5ad --save_dir=output/ --dataset=op --build_images=true +bash src/metrics/all_metrics/run_local.sh --dataset op --prediction=resources/grn_models/op/collectri.h5ad --score=output_score_file.h5ad --num_workers=20 ``` +If you are evaluating a new GRN model, which is not part of geneRNIB, make you to generate the consensus prior file for the dataset you are evaluating on. -Running GRN evaluation without docker +```bash +bash scripts/prior/run_consensus.sh --dataset op --new_model {new_grn_model_file.h5ad} +``` + +This will add your model to the previous ones and create the new consensus prior file needed for evaluation. + +Running GRN evaluation using standard pipeline ---------------------------------------- -Considering that Docker is not supported by certtain systems, you can run the evaluation without Docker by following these steps: + +To run the evalution for a given GRN and dataset, use the following command: ```bash -bash src/metrics/all_metrics/run_local.sh --dataset --prediction= --score --num_workers +bash scripts/run_grn_evaluation.sh --prediction= --save_dir= --dataset= --build_images= ``` example command: ```bash -bash src/metrics/all_metrics/run_local.sh --dataset op --prediction=resources/grn_models/op/collectri.h5ad --score=output_score_file.h5ad --num_workers=20 -``` \ No newline at end of file +bash scripts/run_grn_evaluation.sh --prediction=resources/grn_models/op/collectri.h5ad --save_dir=output/ --dataset=op --build_images=true +``` + diff --git a/docs/source/images/datasets.png b/docs/source/images/datasets.png index b08a554c9..9ac5c6238 100644 Binary files a/docs/source/images/datasets.png and b/docs/source/images/datasets.png differ diff --git a/docs/source/images/datasets_metrics.png b/docs/source/images/datasets_metrics.png deleted file mode 100644 index cdec25940..000000000 Binary files a/docs/source/images/datasets_metrics.png and /dev/null differ diff --git a/docs/source/images/metric_quality_evaluation.png b/docs/source/images/metric_quality_evaluation.png new file mode 100644 index 000000000..d321fc49e Binary files /dev/null and b/docs/source/images/metric_quality_evaluation.png differ diff --git a/docs/source/images/metrics.png b/docs/source/images/metrics.png index f5c2fd02c..2f9e1bd36 100644 Binary files a/docs/source/images/metrics.png and b/docs/source/images/metrics.png differ diff --git a/docs/source/images/raw_scores_300BCG.png b/docs/source/images/raw_scores_300BCG.png index b6ef7f7ed..cec454d14 100644 Binary files a/docs/source/images/raw_scores_300BCG.png and b/docs/source/images/raw_scores_300BCG.png differ diff --git a/docs/source/images/raw_scores_adamson.png b/docs/source/images/raw_scores_adamson.png deleted file mode 100644 index 2fdfd1794..000000000 Binary files a/docs/source/images/raw_scores_adamson.png and /dev/null differ diff --git a/docs/source/images/raw_scores_ibd.png b/docs/source/images/raw_scores_ibd.png deleted file mode 100644 index 6774168b0..000000000 Binary files a/docs/source/images/raw_scores_ibd.png and /dev/null differ diff --git a/docs/source/images/raw_scores_ibd_cd.png b/docs/source/images/raw_scores_ibd_cd.png new file mode 100644 index 000000000..968629360 Binary files /dev/null and b/docs/source/images/raw_scores_ibd_cd.png differ diff --git a/docs/source/images/raw_scores_ibd_uc.png b/docs/source/images/raw_scores_ibd_uc.png new file mode 100644 index 000000000..1d402ed77 Binary files /dev/null and b/docs/source/images/raw_scores_ibd_uc.png differ diff --git a/docs/source/images/raw_scores_nakatake.png b/docs/source/images/raw_scores_nakatake.png index be958d245..4fed0c14f 100644 Binary files a/docs/source/images/raw_scores_nakatake.png and b/docs/source/images/raw_scores_nakatake.png differ diff --git a/docs/source/images/raw_scores_norman.png b/docs/source/images/raw_scores_norman.png index a5f14c27d..b3611239b 100644 Binary files a/docs/source/images/raw_scores_norman.png and b/docs/source/images/raw_scores_norman.png differ diff --git a/docs/source/images/raw_scores_op.png b/docs/source/images/raw_scores_op.png index feb142808..7228b96c7 100644 Binary files a/docs/source/images/raw_scores_op.png and b/docs/source/images/raw_scores_op.png differ diff --git a/docs/source/images/raw_scores_parsebioscience.png b/docs/source/images/raw_scores_parsebioscience.png index 0a0571644..3c382b4f6 100644 Binary files a/docs/source/images/raw_scores_parsebioscience.png and b/docs/source/images/raw_scores_parsebioscience.png differ diff --git a/docs/source/images/raw_scores_replogle.png b/docs/source/images/raw_scores_replogle.png index ec79345c7..76915237e 100644 Binary files a/docs/source/images/raw_scores_replogle.png and b/docs/source/images/raw_scores_replogle.png differ diff --git a/docs/source/images/raw_scores_xaira_HCT116.png b/docs/source/images/raw_scores_xaira_HCT116.png index 8f02270f4..307473dd8 100644 Binary files a/docs/source/images/raw_scores_xaira_HCT116.png and b/docs/source/images/raw_scores_xaira_HCT116.png differ diff --git a/docs/source/images/raw_scores_xaira_HEK293T.png b/docs/source/images/raw_scores_xaira_HEK293T.png index 3d181dce7..ec5856c85 100644 Binary files a/docs/source/images/raw_scores_xaira_HEK293T.png and b/docs/source/images/raw_scores_xaira_HEK293T.png differ diff --git a/docs/source/images/summary_figure.png b/docs/source/images/summary_figure.png new file mode 100644 index 000000000..5c13b7046 Binary files /dev/null and b/docs/source/images/summary_figure.png differ diff --git a/docs/source/images/table_datasets_summary.png b/docs/source/images/table_datasets_summary.png new file mode 100644 index 000000000..27fcd22b9 Binary files /dev/null and b/docs/source/images/table_datasets_summary.png differ diff --git a/docs/source/leaderboard.rst b/docs/source/leaderboard.rst index 2130ba971..0ee707263 100644 --- a/docs/source/leaderboard.rst +++ b/docs/source/leaderboard.rst @@ -1,14 +1,14 @@ Leaderboard ================= -.. The overal comparitive performance of the integrated GRN inference methods is summarized in the leaderboard below. +The overal comparitive performance of the integrated GRN inference methods is summarized in the leaderboard below. It should be noted that not all metrics count towards the final score, as some metrics did not pass the applicability criteria. See `here `_ for more details on the metrics applicability. -.. .. image:: images/leaderboard.png -.. :width: 90% -.. :align: center -.. ---- +.. image:: images/summary_figure.png + :width: 100% + :align: center +---- -The individual performance of the methods on each dataset is summarized below. +The individual performance of the methods on each dataset is summarized below. .. image:: images/raw_scores_op.png :width: 70% @@ -16,7 +16,7 @@ The individual performance of the methods on each dataset is summarized below. ---- .. image:: images/raw_scores_nakatake.png - :width: 60% + :width: 70% :align: center ---- @@ -25,10 +25,6 @@ The individual performance of the methods on each dataset is summarized below. :align: center ---- -.. image:: images/raw_scores_adamson.png - :width: 70% - :align: center ----- .. image:: images/raw_scores_replogle.png :width: 70% @@ -45,8 +41,13 @@ The individual performance of the methods on each dataset is summarized below. :align: center ---- -.. image:: images/raw_scores_ibd.png - :width: 90% +.. image:: images/raw_scores_ibd_uc.png + :width: 70% + :align: center +---- + +.. image:: images/raw_scores_ibd_cd.png + :width: 70% :align: center ---- diff --git a/jobs.sh b/jobs.sh deleted file mode 100644 index 06f785dee..000000000 --- a/jobs.sh +++ /dev/null @@ -1,5 +0,0 @@ -sbatch src/metrics/regression_r/run_global.sh -sbatch src/metrics/regression_r/run_local.sh - -sbatch src/metrics/experimental/annotation/run_global.sh -sbatch src/metrics/experimental/annotation/run_local.sh \ No newline at end of file diff --git a/scripts/local_workflows/run_grn_evaluation.sh b/scripts/local_workflows/run_grn_evaluation.sh new file mode 100755 index 000000000..a2e8966b9 --- /dev/null +++ b/scripts/local_workflows/run_grn_evaluation.sh @@ -0,0 +1,288 @@ +#!/bin/bash +# Local GRN Evaluation Script +# This script evaluates all GRN predictions for all datasets using local SLURM jobs +# +# set -e +# Default parameters +RUN_CONSENSUS=false +RUN_METRICS=false +PROCESS_RESULTS=false +TEMP_DIR="output/evaluation" +RESULTS_FILE="resources/results/all_scores.csv" + + +LAYER="lognorm" +NUM_WORKERS=20 + +# get the arguments +while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + --run_consensus) + RUN_CONSENSUS=true + shift + ;; + --run_consensus) + RUN_CONSENSUS=true + shift + ;; + --run_metrics) + RUN_METRICS=true + shift + ;; + --no_run_metrics) + RUN_METRICS=false + shift + ;; + --process_results) + PROCESS_RESULTS=true + shift + ;; + --no_process_results) + PROCESS_RESULTS=false + shift + ;; + --temp_dir) + TEMP_DIR="$2" + shift + shift + ;; + --results_file) + RESULTS_FILE="$2" + shift + shift + ;; + --num_workers) + NUM_WORKERS="$2" + shift + shift + ;; + *) + echo "Unknown option: $key" + exit 1 + ;; + esac +done + + +echo "==========================================" +echo "GRN Evaluation Configuration" +echo "==========================================" +echo "Run consensus: $RUN_CONSENSUS" +echo "Run metrics: $RUN_METRICS" +echo "Output directory: $TEMP_DIR" +echo "Results file: $RESULTS_FILE" +echo "Process results: $PROCESS_RESULTS" +echo "Number of workers: $NUM_WORKERS" +echo "==========================================" + +# Create output directory +mkdir -p "$TEMP_DIR" + +# Generate and source dataset configuration +echo "Generating dataset configuration..." +python src/utils/config.py +source src/utils/config.env + +# Get list of datasets from config +DATASETS=(${DATASETS//,/ }) +METHODS=(${METHODS//,/ }) + +# Function to submit a metric evaluation job +submit_metric_job() { + local dataset=$1 + local method=$2 + local prediction_file=$3 + + local job_name="${dataset}_${method}" + local score_file="${TEMP_DIR}/${dataset}_${method}_score.h5ad" + + # Skip if score file already exists + if [[ -f "$score_file" ]]; then + echo " Skipping ${job_name} - score file already exists" + return + fi + + echo " Submitting job: ${job_name}" + + + sbatch \ + --job-name="${job_name}" \ + src/metrics/all_metrics/run_local.sh \ + --dataset "${dataset}" \ + --prediction "${prediction_file}" \ + --score "${score_file}" \ + --num_workers "${NUM_WORKERS}" || echo " [ERROR] Failed to submit job for ${job_name}" +} + +# Function to process all results into a single CSV +process_all_results() { + echo "" + echo "==========================================" + echo "Processing Results" + echo "==========================================" + + local results_file="${RESULTS_FILE}" + + # Get list of datasets for parsing + local datasets_list=$(python -c "from src.utils.config import DATASET_GROUPS; print(','.join(DATASET_GROUPS.keys()))") + + # Create Python script to aggregate results + python - "$TEMP_DIR" "$RESULTS_FILE" "$datasets_list" << 'EOF' +import sys +import os +import pandas as pd +import anndata as ad +from pathlib import Path + +temp_dir = sys.argv[1] +results_file = sys.argv[2] +datasets_str = sys.argv[3] +scores_dir = Path(temp_dir) + +# Parse datasets list +known_datasets = datasets_str.split(',') + +print(f"Looking for score files in: {scores_dir}") +print(f"Will save results to: {results_file}") +print(f"Known datasets: {known_datasets[:5]}...") + +all_results = [] + +# Iterate through all score files +for score_file in scores_dir.glob("*_score.h5ad"): + try: + # Parse filename: {dataset}_{method}_score.h5ad + filename = score_file.stem # Remove .h5ad + + # Remove _score suffix + if filename.endswith('_score'): + filename = filename[:-6] # Remove last 6 characters '_score' + + # Try to match against known datasets - find which dataset this file belongs to + dataset = None + method = None + + for ds in known_datasets: + if filename.startswith(ds + '_'): + dataset = ds + method = filename[len(ds) + 1:] # Everything after dataset_ + break + + if not dataset or not method: + print(f"Warning: Could not parse filename {score_file.name}") + continue + + # Load score file + adata = ad.read_h5ad(score_file) + + # Extract metric scores from uns + if 'metric_ids' in adata.uns and 'metric_values' in adata.uns: + metric_ids = adata.uns['metric_ids'] + metric_values = adata.uns['metric_values'] + + # Create row for each metric + for metric_id, metric_value in zip(metric_ids, metric_values): + # Convert metric_value to float if it's a string + try: + score_value = float(metric_value) if isinstance(metric_value, str) else metric_value + except (ValueError, TypeError): + score_value = metric_value + + all_results.append({ + 'dataset': dataset, + 'method': method, + 'metric': metric_id, + 'score': score_value + }) + + print(f"Processed: {dataset} - {method} ({len(metric_ids)} metrics)") + else: + print(f"Warning: No metric data in {score_file.name}") + + except Exception as e: + print(f"Error processing {score_file.name}: {e}") + +# Create DataFrame +if all_results: + df = pd.DataFrame(all_results) + + # Pivot to wide format: rows=dataset+method, columns=metrics + df_wide = df.pivot_table( + index=['dataset', 'method'], + columns='metric', + values='score' + ).reset_index() + + # Save results (create directory if needed) + results_path = Path(results_file) + results_path.parent.mkdir(parents=True, exist_ok=True) + df_wide.to_csv(results_file, index=False) + print(f"\n{'='*50}") + print(f"Results saved to: {results_file}") + print(f"Total evaluations: {len(df_wide)}") + print(f"Datasets: {df_wide['dataset'].nunique()}") + print(f"Methods: {df_wide['method'].nunique()}") + print(f"Metrics: {len([c for c in df_wide.columns if c not in ['dataset', 'method']])}") + print(f"{'='*50}") +else: + print("No results found to process!") +EOF +} + +# Function to run consensus for a dataset +run_consensus() { + local dataset=$1 + bash scripts/prior/run_consensus.sh --dataset "$dataset" +} + +# Main execution +if [[ "$RUN_CONSENSUS" == "true" ]]; then + echo "" + echo "==========================================" + echo "Running Consensus for All Datasets" + echo "==========================================" + + for dataset in "${DATASETS[@]}"; do + run_consensus "$dataset" + done +fi + +if [[ "$RUN_METRICS" == "true" ]]; then + echo "" + echo "==========================================" + echo "Submitting Metric Evaluation Jobs" + echo "==========================================" + + job_count=0 + + for dataset in "${DATASETS[@]}"; do + echo "" + echo "Dataset: $dataset" + echo "----------------------------------------" + + models_folder="resources/results/${dataset}/" + # echo "Looking in: $models_folder" + + # Check each method for this dataset + for method in "${METHODS[@]}"; do + prediction_file="${models_folder}/${dataset}.${method}.${method}.prediction.h5ad" + + if [[ -f "$prediction_file" ]]; then + submit_metric_job "$dataset" "$method" "$prediction_file" + ((job_count++)) + else + echo " [NOT FOUND] ${prediction_file}" + fi + done + done + +fi + +if [[ "$PROCESS_RESULTS" == "true" ]]; then + process_all_results +fi + +echo "" +echo "Done!" diff --git a/src/local_workflows/run.sh b/scripts/local_workflows/run_grn_inference.sh similarity index 64% rename from src/local_workflows/run.sh rename to scripts/local_workflows/run_grn_inference.sh index 35775bba9..0baab01a8 100644 --- a/src/local_workflows/run.sh +++ b/scripts/local_workflows/run_grn_inference.sh @@ -3,11 +3,14 @@ set -e run_prefix='sbatch' #bash -DATASETS=('op' 'adamson' 'replogle' 'norman' 'nakatake' 'parsebioscience' '300BCG' 'xaira_HCT116' 'xaira_HEK293T') #'op' 'adamson' 'replogle' 'norman' 'nakatake' 'parsebioscience' '300BCG' 'xaira_HCT116' 'xaira_HEK293T' 'ibd_uc' 'ibd_cd' -DATASETS=('ibd_uc' 'ibd_cd') +python src/utils/config.py +source src/utils/config.env -# METHODS=('negative_control' 'positive_control' 'pearson_corr' 'portia' 'ppcor' 'grnboost' 'scenic' 'scenicplus' 'scglue' 'figr' 'granie') -METHODS=( 'ppcor' ) #'negative_control' 'positive_control' 'pearson_corr' 'portia' 'ppcor' 'grnboost' 'scenic' 'scenicplus' 'scglue' 'figr' 'granie' +DATASETS=(${DATASETS//,/ }) +# DATASETS=('parsebioscience') #'op' 'adamson' 'replogle' 'norman' 'nakatake' 'parsebioscience' '300BCG' 'xaira_HCT116' 'xaira_HEK293T' 'ibd_uc' 'ibd_cd' + +METHODS=(${METHODS//,/ }) +# METHODS=( 'scenic' 'grnboost') #'negative_control' 'positive_control' 'pearson_corr' 'portia' 'ppcor' 'grnboost' 'scenic' 'scenicplus' 'scglue' 'figr' 'granie' methods_dir='src/methods/' ctr_methods_dir='src/methods/' @@ -33,10 +36,8 @@ run_func() { if [[ "$run_prefix" == "bash" ]]; then bash "$script" $arguments elif [[ "$run_prefix" == "sbatch" ]]; then - # submit the job and capture the job ID output=$(sbatch "$script" $arguments) echo "$output" - # sbatch usually returns: "Submitted batch job 12345678" jobid=$(echo "$output" | awk '{print $4}') echo "Job ID: $jobid" else diff --git a/scripts/prior/run_consensus.sh b/scripts/prior/run_consensus.sh index 2c8622bf0..6f891ace2 100644 --- a/scripts/prior/run_consensus.sh +++ b/scripts/prior/run_consensus.sh @@ -13,21 +13,55 @@ set -e -DATASET=$1 +DATASET="" +NEW_MODEL_PATH="" + +while [[ $# -gt 0 ]]; do + case $1 in + --dataset) + DATASET="$2" + shift 2 + ;; + --new_model) + NEW_MODEL_PATH="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + echo "Usage: sbatch run_consensus.sh --dataset [--new_model ]" + exit 1 + ;; + esac +done + if [ -z "$DATASET" ]; then - echo "Usage: sbatch run_consensus.sh " + echo "Usage: sbatch run_consensus.sh --dataset [--new_model ]" exit 1 fi models_dir="resources/results/$DATASET" models=("pearson_corr" "positive_control" "portia" "ppcor" "scenic" "scprint" "grnboost" "scenicplus" "scglue" "granie" "figr" "celloracle" "scgpt" "geneformer" "spearman_corr") +python src/utils/config.py +source src/utils/config.env +METHODS=(${METHODS//,/ }) + predictions=() -for model in "${models[@]}"; do +for model in "${METHODS[@]}"; do file="${models_dir}/${DATASET}.${model}.${model}.prediction.h5ad" if [ -e "$file" ]; then predictions+=("$file") fi done + +if [ -n "$NEW_MODEL_PATH" ]; then + if [ -e "$NEW_MODEL_PATH" ]; then + echo "Adding new model: $NEW_MODEL_PATH" + predictions+=("$NEW_MODEL_PATH") + else + echo "Warning: New model path does not exist: $NEW_MODEL_PATH" + fi +fi + printf '%s\n' "${predictions[@]}" echo "Running consensus for Regression" diff --git a/scripts/process_data/rest.sh b/scripts/process_data/rest.sh index 7935f22e4..9d4fb4230 100755 --- a/scripts/process_data/rest.sh +++ b/scripts/process_data/rest.sh @@ -15,7 +15,7 @@ set -e # python src/process_data/main/adamson/script.py # python src/process_data/main/nakatake/script.py -# python src/process_data/main/norman/script.py +python src/process_data/main/norman/script.py # echo "Processing opsca" # python src/process_data/main/opsca/script.py @@ -27,5 +27,5 @@ set -e # echo "Processing 300BCG" # python src/process_data/main/300BCG/script.py -echo "Processing IBD" -python src/process_data/main/ibd/script.py +# echo "Processing IBD" +# python src/process_data/main/ibd/script.py diff --git a/scripts/run_all.sh b/scripts/run_all.sh index 32b3ac925..cf6fbcb61 100644 --- a/scripts/run_all.sh +++ b/scripts/run_all.sh @@ -1,17 +1,19 @@ set -e -# datasets=( 'replogle' 'op' 'nakatake' 'adamson' 'norman' 'xaira_HEK293T' 'xaira_HCT116' 'parsebioscience' 'ibd_uc' 'ibd_cd' '300BCG' ) #'replogle' 'op' 'nakatake' 'adamson' 'norman' 'xaira_HEK293T' 'xaira_HCT116' 'parsebioscience' 'ibd_uc' 'ibd_cd' '300BCG') # -datasets=( 'ibd_uc' 'ibd_cd' ) #'replogle' 'op' 'nakatake' 'adamson' 'norman' 'xaira_HEK293T' 'xaira_HCT116' 'parsebioscience' 'ibd_uc' 'ibd_cd' '300BCG') # -run_local=false # set to true to run locally, false to run on AWS +python src/utils/config.py +source src/utils/config.env +DATASETS=(${DATASETS//,/ }) -run_grn_inference=true -run_grn_evaluation=false -run_download=false +run_local=false +run_grn_inference=false #arg +run_consensus=true +run_grn_evaluation=true #arg +run_sync=false num_workers=20 -for dataset in "${datasets[@]}"; do +for dataset in "${DATASETS[@]}"; do trace_file="resources/results/$dataset/trace.txt" if [ "$run_grn_inference" = true ]; then @@ -48,12 +50,14 @@ for dataset in "${datasets[@]}"; do # fi - if [ "$run_local" = false ]; then - echo "Downloading inference results from AWS" - aws s3 sync s3://openproblems-data/resources/grn/results/$dataset resources/results/$dataset + # if [ "$run_local" = false ]; then + # echo "Downloading inference results from AWS" + # aws s3 sync s3://openproblems-data/resources/grn/results/$dataset resources/results/$dataset + # fi + if [ "$run_consensus" = true ]; then + echo "Running consensus for dataset: $dataset" + bash scripts/prior/run_consensus.sh --dataset $dataset # run consensus for Regression and ws distance -> needs to be run after adding each method and dataset fi - echo "Running consensus for dataset: $dataset" - bash scripts/prior/run_consensus.sh $dataset # run consensus for Regression and ws distance -> needs to be run after adding each method and dataset if [ "$run_local" = false ]; then echo "Syncing prior results to AWS" @@ -64,7 +68,7 @@ for dataset in "${datasets[@]}"; do bash scripts/run_grn_evaluation.sh --dataset=$dataset --run_local=$run_local --build_images=false --num_workers=$num_workers fi - if [ "$run_download" = true ]; then + if [ "$run_sync" = true ]; then if [ "$run_local" = false ]; then echo "Downloading evaluation results from AWS" aws s3 sync s3://openproblems-data/resources/grn/results/$dataset resources/results/$dataset diff --git a/scripts/run_consensus.sh b/scripts/run_consensus.sh new file mode 100644 index 000000000..b1a8d6b2b --- /dev/null +++ b/scripts/run_consensus.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# Consensus Calculation Script +# This script runs consensus calculations for both Regression and WS distance metrics +# Usage: bash scripts/run_consensus.sh [run_mode] +# dataset: name of the dataset (e.g., replogle, op, norman) +# run_mode: 'local' (default) or 'aws' + +set -e + +DATASET=$1 + +if [ -z "$DATASET" ]; then + echo "Usage: bash scripts/run_consensus.sh [run_mode]" + echo " dataset: name of the dataset (required)" + echo " run_mode: 'local' (default) or 'aws'" + exit 1 +fi + +echo "==========================================" +echo "Running Consensus Calculation" +echo "Dataset: $DATASET" +echo "Run mode: $RUN_MODE" +echo "==========================================" + +# Set paths based on run mode +resources_dir="./resources" +models_dir="${resources_dir}/results/$DATASET" + +# Get available methods from config +echo "Checking available methods..." +available_methods=$(python -c " +from src.utils.config import METHODS +import os +methods = [] +for method in METHODS: + file = f'resources/results/$DATASET/$DATASET.{method}.{method}.prediction.h5ad' + if os.path.exists(file): + methods.append(method) +print(' '.join(methods)) +") + +if [ -z "$available_methods" ]; then + echo "No prediction files found for dataset: $DATASET" + exit 1 +fi + +echo "Available methods: $available_methods" + +# Convert space-separated list to array +methods_array=($available_methods) + +# Build predictions list +predictions=() +for method in "${methods_array[@]}"; do + file="resources/results/${DATASET}/${DATASET}.${method}.${method}.prediction.h5ad" + if [ -e "$file" ]; then + predictions+=("$file") + fi +done + +if [ ${#predictions[@]} -eq 0 ]; then + echo "No prediction files found for consensus calculation" + exit 1 +fi + +echo "Found ${#predictions[@]} prediction files for consensus calculation" +printf '%s\n' "${predictions[@]}" + +# Run Regression consensus +echo "" +echo "Running Regression consensus..." +python src/metrics/regression/consensus/script.py \ + --dataset "$DATASET" \ + --regulators_consensus "resources/grn_benchmark/prior/regulators_consensus_${DATASET}.json" \ + --evaluation_data "resources/grn_benchmark/evaluation_data/${DATASET}_bulk.h5ad" \ + --predictions "${predictions[@]}" + +echo "Regression consensus completed successfully" + +# Run WS distance consensus (only for applicable datasets) +applicable_datasets=("norman" "adamson" "replogle" "xaira_HEK293T" "xaira_HCT116") +skip_ws=true +for d in "${applicable_datasets[@]}"; do + if [[ "$DATASET" == "$d" ]]; then + skip_ws=false + break + fi +done + +if [ "$skip_ws" = true ]; then + echo "" + echo "Skipping WS distance consensus (not applicable for dataset: $DATASET)" +else + echo "" + echo "Running WS distance consensus..." + python src/metrics/ws_distance/consensus/script.py \ + --dataset "$DATASET" \ + --models_dir "resources/results/$DATASET" \ + --ws_consensus "resources/grn_benchmark/prior/ws_consensus_${DATASET}.csv" \ + --tf_all "resources/grn_benchmark/prior/tf_all.csv" \ + --evaluation_data_sc "resources/processed_data/${DATASET}_evaluation_sc.h5ad" \ + --models "${methods_array[@]}" + + echo "WS distance consensus completed successfully" +fi + +# Sync results to AWS if needed +if [ "$RUN_MODE" = "aws" ]; then + echo "" + echo "Syncing consensus results to AWS..." + aws s3 sync resources/grn_benchmark/prior s3://openproblems-data/resources/grn/grn_benchmark/prior + echo "Sync completed" +fi + +echo "" +echo "==========================================" +echo "Consensus calculation completed for $DATASET" +echo "==========================================" diff --git a/scripts/run_grn_evaluation.sh b/scripts/run_grn_evaluation.sh index 79b7f7241..c4dc0bd6a 100755 --- a/scripts/run_grn_evaluation.sh +++ b/scripts/run_grn_evaluation.sh @@ -106,8 +106,8 @@ param_aws="s3://openproblems-data/resources/grn/results/params/${RUN_ID}_param_l # Generate and source config file echo "Generating dataset configuration..." -python src/utils/config.py --output src/utils/dataset_config.env -source src/utils/dataset_config.env +python src/utils/config.py +source src/utils/config.env if [ "$RUN_LOCAL" = true ]; then @@ -181,29 +181,13 @@ HERE if [ "$PREDICTION" != "none" ]; then append_entry "single_run" $PREDICTION "$DATASET" else - grn_names=( - "positive_control" - "pearson_corr" - "negative_control" - "spearman_corr" - "scglue" - "scenicplus" - "celloracle" - "granie" - "figr" - "grnboost" - "portia" - "scenic" - "scprint" - "geneformer" - "scgpt" - ) + METHODS =(${METHODS//,/ }) grn_models_folder="${resources_dir}/results/${models_folder}/" grn_models_folder_local="./resources/results/${models_folder}/" # just to control the hetergenity of the models for different datasets # Iterate over GRN models available_methods=() - for grn_name in "${grn_names[@]}"; do + for grn_name in "${METHODS[@]}"; do prediction_file="${grn_models_folder_local}/${DATASET}.${grn_name}.${grn_name}.prediction.h5ad" if [[ -f "${prediction_file}" ]]; then prediction_file=${grn_models_folder}/${DATASET}.${grn_name}.${grn_name}.prediction.h5ad diff --git a/scripts/run_grn_inference.sh b/scripts/run_grn_inference.sh index d95691305..2772f4397 100755 --- a/scripts/run_grn_inference.sh +++ b/scripts/run_grn_inference.sh @@ -87,13 +87,12 @@ append_entry() { else layer_="$layer" fi - + group_id="${dataset}" if [ "$use_train_sc" = true ]; then rna_file="${resources_dir}/extended_data/${dataset}_train_sc.h5ad" - group_id="${dataset}_sc" else rna_file="${resources_dir}/grn_benchmark/inference_data/${dataset}_rna.h5ad" - group_id="${dataset}" + fi cat >> "$param_local" << HERE diff --git a/scripts/sync_resources.sh b/scripts/sync_resources.sh index 4f27282a8..75d1b3c24 100755 --- a/scripts/sync_resources.sh +++ b/scripts/sync_resources.sh @@ -22,14 +22,14 @@ set -e # aws s3 sync s3://openproblems-data/resources/grn/grn_benchmark resources/grn_benchmark/ --no-sign-request # aws s3 sync resources/grn_benchmark/prior s3://openproblems-data/resources/grn/grn_benchmark/prior --delete -aws s3 sync resources/extended_data/ s3://openproblems-data/resources/grn/extended_data --delete +# aws s3 sync resources/extended_data/ s3://openproblems-data/resources/grn/extended_data --delete # aws s3 sync resources/results/experiment s3://openproblems-data/resources/grn/results/experiment --delete # aws s3 sync resources_test s3://openproblems-data/resources_test/grn/ --delete # aws s3 sync s3://openproblems-data/resources_test/grn/ resources_test --delete # aws s3 sync s3://openproblems-data/resources/grn/grn_benchmark/ground_truth resources_test/grn_benchmark/ground_truth # aws s3 sync resources/grn_benchmark/ground_truth s3://openproblems-data/resources/grn/grn_benchmark/ground_truth -aws s3 sync resources/grn_benchmark/ s3://openproblems-data/resources/grn/grn_benchmark/ --delete -# aws s3 sync resources/results/ibd_uc s3://openproblems-data/resources/grn/results/ibd_uc +# aws s3 sync resources/grn_benchmark/ s3://openproblems-data/resources/grn/grn_benchmark/ --delete +aws s3 sync s3://openproblems-data/resources/grn/results/ resources/results/ --delete # aws s3 sync resources/results/ibd_cd s3://openproblems-data/resources/grn/results/ibd_cd # aws s3 sync s3://openproblems-data/resources/grn/grn_benchmark/ground_truth resources/grn_benchmark/ground_truth --no-sign-request diff --git a/src/metrics/all_metrics/helper.py b/src/metrics/all_metrics/helper.py index ec5756750..659f9ac1d 100644 --- a/src/metrics/all_metrics/helper.py +++ b/src/metrics/all_metrics/helper.py @@ -10,7 +10,7 @@ from regression.helper import main as regression from ws_distance.helper import main as ws_distance from sem.helper import main as sem -from anchor_regression.helper import main as anchor_regression +# from anchor_regression.helper import main as anchor_regression from tf_recovery.helper import main as tf_recovery from tf_binding.helper import main as tf_binding from rc_tf_act.helper import main as rc_tf_act @@ -23,7 +23,7 @@ 'regression': regression, 'ws_distance': ws_distance, 'sem': sem, - 'anchor_regression': anchor_regression, + # 'anchor_regression': anchor_regression, 'tf_recovery': tf_recovery, 'tf_binding': tf_binding, 'rc_tf_act': rc_tf_act, diff --git a/src/metrics/all_metrics/run_all.sh b/src/metrics/all_metrics/run_all.sh deleted file mode 100644 index a40711e08..000000000 --- a/src/metrics/all_metrics/run_all.sh +++ /dev/null @@ -1,20 +0,0 @@ -run_type='sbatch' - -metrics=( - rc_tf_act - regression - tf_recovery - ar - rc_tf_act - tf_binding - vc - ws_distance - sem - gs_recovery -) -for metric in "${metrics[@]}"; do - echo "Running metric: ${metric}" - ${run_type} src/metrics/${metric}/run_local.sh - echo "----------------------------------------" -done - diff --git a/src/metrics/all_metrics/run_local.sh b/src/metrics/all_metrics/run_local.sh index 00a69707d..2f3cd28c4 100644 --- a/src/metrics/all_metrics/run_local.sh +++ b/src/metrics/all_metrics/run_local.sh @@ -5,7 +5,7 @@ #SBATCH --ntasks=1 #SBATCH --cpus-per-task=20 #SBATCH --time=2:00:00 -#SBATCH --mem=120GB +#SBATCH --mem=60GB #SBATCH --partition=gpu #SBATCH --gres=gpu:1 #SBATCH --mail-type=END,FAIL @@ -77,7 +77,7 @@ fi # exit 1 # fi -source src/utils/dataset_config.env +source src/utils/config.env cell_type_var="CELLTYPE_${dataset}" cell_type="${!cell_type_var}" diff --git a/src/metrics/anchor_regression/config.vsh.yaml b/src/metrics/experimental/anchor_regression/config.novsh.yaml similarity index 100% rename from src/metrics/anchor_regression/config.vsh.yaml rename to src/metrics/experimental/anchor_regression/config.novsh.yaml diff --git a/src/metrics/anchor_regression/helper.py b/src/metrics/experimental/anchor_regression/helper.py similarity index 100% rename from src/metrics/anchor_regression/helper.py rename to src/metrics/experimental/anchor_regression/helper.py diff --git a/src/metrics/anchor_regression/run_global.sh b/src/metrics/experimental/anchor_regression/run_global.sh similarity index 100% rename from src/metrics/anchor_regression/run_global.sh rename to src/metrics/experimental/anchor_regression/run_global.sh diff --git a/src/metrics/anchor_regression/run_local.sh b/src/metrics/experimental/anchor_regression/run_local.sh similarity index 100% rename from src/metrics/anchor_regression/run_local.sh rename to src/metrics/experimental/anchor_regression/run_local.sh diff --git a/src/metrics/anchor_regression/script.py b/src/metrics/experimental/anchor_regression/script.py similarity index 100% rename from src/metrics/anchor_regression/script.py rename to src/metrics/experimental/anchor_regression/script.py diff --git a/src/metrics/rc_tf_act/run_local.sh b/src/metrics/rc_tf_act/run_local.sh index 6e961cf41..a1240c000 100755 --- a/src/metrics/rc_tf_act/run_local.sh +++ b/src/metrics/rc_tf_act/run_local.sh @@ -14,9 +14,9 @@ set -euo pipefail save_dir="output/rc_tf_act" mkdir -p "$save_dir" - + # Datasets to process (only those with rc_tf_ac grouping defined) -datasets=('300BCG' 'parsebioscience' 'op') +datasets=('ibd_uc') # datasets=('op' ) # Methods to process methods=("grnboost" "pearson_corr" "negative_control" "positive_control" "ppcor" "portia" "scenic" "scprint" "scenicplus" "celloracle" "scglue" "figr" "granie") diff --git a/src/metrics/regression/helper.py b/src/metrics/regression/helper.py index 7a07d3186..4aad5eafd 100644 --- a/src/metrics/regression/helper.py +++ b/src/metrics/regression/helper.py @@ -336,10 +336,23 @@ def main(par: Dict[str, Any]) -> Tuple[pd.DataFrame, pd.DataFrame]: # Load data prturb_adata = ad.read_h5ad(par['evaluation_data']) layer = manage_layer(prturb_adata, par) - gene_names = prturb_adata.var.index.to_numpy() + all_gene_names = prturb_adata.var.index.to_numpy() with open(par['regulators_consensus'], 'r') as f: data = json.load(f) - print(len(data), len(gene_names)) + + # Filter genes to only include those in consensus + gene_names = np.array([gene for gene in all_gene_names if gene in data]) + missing_genes = set(all_gene_names) - set(gene_names) + + if len(missing_genes) > 0: + print(f"Warning: {len(missing_genes)} genes not in consensus data and will be omitted from evaluation") + print(f"Missing genes: {sorted(list(missing_genes))[:10]}..." if len(missing_genes) > 10 else f"Missing genes: {sorted(list(missing_genes))}") + + # Filter the adata to only include genes in consensus + gene_mask = prturb_adata.var.index.isin(gene_names) + prturb_adata = prturb_adata[:, gene_mask] + + print(f"Evaluating {len(gene_names)} genes (consensus data available for all)") n_features_theta_025 = np.asarray([data[gene_name]['0.25'] for gene_name in gene_names], dtype=int) n_features_theta_075 = np.asarray([data[gene_name]['0.75'] for gene_name in gene_names], dtype=int) diff --git a/src/metrics/regression/run_local.sh b/src/metrics/regression/run_local.sh index 3e3c1d7fc..c008e80f3 100644 --- a/src/metrics/regression/run_local.sh +++ b/src/metrics/regression/run_local.sh @@ -17,7 +17,7 @@ mkdir -p "$save_dir" # datasets to process datasets=('ibd_uc' 'ibd_cd' 'op' 'parsebioscience' "300BCG" "adamson" "replogle" "xaira_HEK293T" "xaira_HCT116" "nakatake" "norman" ) #"300BCG" "ibd" 'parsebioscience', 'xaira_HEK293T' -# datasets=('xaira_HEK293T' ) #"300BCG" "ibd" 'parsebioscience', 'xaira_HEK293T' +datasets=('norman' ) #"300BCG" "ibd" 'parsebioscience', 'xaira_HEK293T' # methods to process methods=( "pearson_corr" "positive_control" "negative_control" "ppcor" "portia" "scenic" "grnboost" "scprint" "scenicplus" "celloracle" "scglue" "figr" "granie") diff --git a/src/metrics/sem/helper.py b/src/metrics/sem/helper.py index c74022b06..840b2c4da 100644 --- a/src/metrics/sem/helper.py +++ b/src/metrics/sem/helper.py @@ -510,8 +510,8 @@ def main(par): print(f"SEM HVG score (max): {np.max(valid_scores_hvg):.4f}") results = { - 'sem_grn': [float(sem_grn_score)], - 'sem_hvg': [float(sem_hvg_score)], + # 'sem_grn': [float(sem_grn_score)], + # 'sem_hvg': [float(sem_hvg_score)], 'sem': [float((sem_grn_score + sem_hvg_score) / 2)] } diff --git a/src/metrics/vc/analyze_permutation.py b/src/metrics/vc/analyze_permutation.py deleted file mode 100644 index 520ea135c..000000000 --- a/src/metrics/vc/analyze_permutation.py +++ /dev/null @@ -1,153 +0,0 @@ -""" -Analyze how vc metric scores change when GRN structure is permuted. -Tests robustness by shuffling TF-gene pairs in the network. -""" - -import numpy as np -import pandas as pd -import anndata as ad -from helper import main as main_vc -import sys - -def permute_grn_edges(net_df, permutation_degree=1.0, seed=42): - """ - Permute the GRN by shuffling TF-gene pairs. - - Args: - net_df: DataFrame with 'source', 'target', 'weight' columns - permutation_degree: Fraction of edges to permute (0.0 to 1.0) - seed: Random seed - - Returns: - Permuted network DataFrame - """ - np.random.seed(seed) - net_permuted = net_df.copy() - - n_edges = len(net_df) - n_to_permute = int(n_edges * permutation_degree) - - if n_to_permute == 0: - return net_permuted - - # Select random edges to permute - indices = np.random.choice(n_edges, n_to_permute, replace=False) - - # Shuffle the targets for selected edges - targets = net_permuted.loc[indices, 'target'].values - np.random.shuffle(targets) - net_permuted.loc[indices, 'target'] = targets - - return net_permuted - - -def run_permutation_analysis(par, permutation_degrees=[0.0, 0.2, 0.5, 0.8, 1.0]): - """ - Run vc metric with different levels of GRN permutation. - - Args: - par: Parameters dict with paths to data and prediction - permutation_degrees: List of permutation fractions to test - - Returns: - DataFrame with results for each permutation level - """ - results = [] - - # Load original prediction - read as DataFrame from var - pred_adata = ad.read_h5ad(par['prediction']) - original_net = pred_adata.var.copy() - - if 'source' not in original_net.columns or 'target' not in original_net.columns: - # Try to get from varm - if 'skeleton' in pred_adata.varm: - original_net = pd.DataFrame(pred_adata.varm['skeleton'], columns=['source', 'target', 'weight']) - else: - raise ValueError("Cannot find GRN data in prediction file") - - for degree in permutation_degrees: - print(f"\n{'='*60}") - print(f"Testing permutation degree: {degree:.0%}") - print(f"{'='*60}") - - # Permute network - if degree > 0: - permuted_net = permute_grn_edges(original_net, degree, seed=42) - - # Create permuted anndata with same structure - permuted_adata = ad.AnnData(uns=pred_adata.uns.copy()) - permuted_adata.var = permuted_net - - permuted_path = par['prediction'].replace('.h5ad', f'_perm{int(degree*100)}.h5ad') - permuted_adata.write_h5ad(permuted_path) - - # Update parameter - par_temp = par.copy() - par_temp['prediction'] = permuted_path - else: - par_temp = par.copy() - - # Run evaluation - try: - df_result = main_vc(par_temp) - r2_score = df_result['vc'].values[0] - - results.append({ - 'permutation_degree': degree, - 'r2_score': r2_score - }) - - print(f"R² score: {r2_score:.4f}") - - except Exception as e: - print(f"Error at permutation degree {degree}: {e}") - import traceback - traceback.print_exc() - results.append({ - 'permutation_degree': degree, - 'r2_score': np.nan - }) - - return pd.DataFrame(results) - - -if __name__ == '__main__': - # Configuration - dataset = 'op' # Changed from 'replogle' to 'op' - method = 'grnboost' - - par = { - 'prediction': f'resources/results/{dataset}/{dataset}.{method}.{method}.prediction.h5ad', - 'evaluation_data': f'resources/grn_benchmark/evaluation_data/{dataset}_bulk.h5ad', - 'score': f'output/vc/vc_{dataset}_{method}_permutation.h5ad' - } - - print(f"Dataset: {dataset}") - print(f"Method: {method}") - - # Run permutation analysis - results_df = run_permutation_analysis( - par, - permutation_degrees=[0.0, 0.2, 0.5, 0.8, 1.0] - ) - - # Save results - output_path = f'output/vc/permutation_analysis_{dataset}_{method}.csv' - results_df.to_csv(output_path, index=False) - - print(f"\n{'='*60}") - print("Permutation Analysis Results") - print(f"{'='*60}") - print(results_df.to_string(index=False)) - print(f"\nResults saved to: {output_path}") - - # Calculate score degradation - if len(results_df) > 0 and not results_df['r2_score'].isna().all(): - original_score = results_df.loc[results_df['permutation_degree'] == 0.0, 'r2_score'].values[0] - final_score = results_df.loc[results_df['permutation_degree'] == 1.0, 'r2_score'].values[0] - - if not np.isnan(original_score) and not np.isnan(final_score): - degradation = (original_score - final_score) / original_score * 100 - print(f"\nScore degradation with 100% permutation: {degradation:.1f}%") - print(f"Original R²: {original_score:.4f}") - print(f"Permuted R²: {final_score:.4f}") diff --git a/src/metrics/vc/analyze_robustness.py b/src/metrics/vc/analyze_robustness.py index be8021b45..04e266544 100644 --- a/src/metrics/vc/analyze_robustness.py +++ b/src/metrics/vc/analyze_robustness.py @@ -21,7 +21,7 @@ sys.path.append('src/utils') from helper import main as main_vc -from util import process_links +from util import process_links, parse_args def permute_grn(prediction: pd.DataFrame, degree: float, noise_type: str) -> pd.DataFrame: @@ -316,6 +316,7 @@ def create_robustness_plot(df_results, output_dir): help='Path to evaluation data h5ad file') parser.add_argument('--output_dir', type=str, required=True, help='Output directory for results') + parser.add_argument('--n_top_genes', type=int, default=3000) parser.add_argument('--degrees', type=float, nargs='+', default=[0.0, 0.2, 0.5, 1.0], help='Permutation degrees (0.0 to 1.0)') @@ -332,7 +333,8 @@ def create_robustness_plot(df_results, output_dir): 'output_dir': args.output_dir, 'degrees': args.degrees, 'noise_types': args.noise_types, - 'score': f"{args.output_dir}/vc_score.h5ad" + 'score': f"{args.output_dir}/vc_score.h5ad", + 'n_top_genes': args.n_top_genes } # Run analysis diff --git a/src/metrics/vc/helper.py b/src/metrics/vc/helper.py index f0c5cdd1d..7a933b51d 100644 --- a/src/metrics/vc/helper.py +++ b/src/metrics/vc/helper.py @@ -499,7 +499,8 @@ def main(par): gene_mask_grn = np.logical_or(np.any(A_full, axis=1), np.any(A_full, axis=0)) in_degrees = np.sum(A_full != 0, axis=0) out_degrees = np.sum(A_full != 0, axis=1) - n_genes_grn = par['n_top_genes'] + # n_genes_grn = par['n_top_genes'] + n_genes_grn = 1000 # Select top n_genes_grn by connectivity gene_connectivity = in_degrees + out_degrees @@ -621,9 +622,10 @@ def main(par): print(f"VC HVG: {vc_hvg_score:.4f}") print(f"VC (average): {(vc_grn_score + vc_hvg_score) / 2:.4f}") + # add baseline grn that has same target genes -> assumption: cetrain genes are easy to predict results = { - 'vc_grn': [vc_grn_score], - 'vc_hvg': [vc_hvg_score], + # 'vc_grn': [vc_grn_score], + # 'vc_hvg': [vc_hvg_score], 'vc': [float((vc_grn_score + vc_hvg_score) / 2)] } diff --git a/src/metrics/vc/run_local.sh b/src/metrics/vc/run_local.sh index 65c0cb377..93fc032f6 100644 --- a/src/metrics/vc/run_local.sh +++ b/src/metrics/vc/run_local.sh @@ -17,7 +17,7 @@ mkdir -p "$save_dir" # datasets to process datasets=( "replogle" "xaira_HEK293T" "xaira_HCT116" "nakatake" "norman" "adamson" 'parsebioscience' 'op' "300BCG" 'ibd_uc' 'ibd_cd') #"300BCG" "ibd" 'parsebioscience', 'xaira_HEK293T' -datasets=( "op" "300BCG" "parsebioscience" "ibd" ) +datasets=( "norman" ) # methods to process methods=( "pearson_corr" "positive_control" "negative_control" "ppcor" "portia" "scenic" "grnboost" "scprint" "scenicplus" "celloracle" "scglue" "figr" "granie") # methods=( "grnboost") diff --git a/src/metrics/vc/run_robustness.sh b/src/metrics/vc/run_robustness.sh index ba482e4a9..8ac3c3baf 100644 --- a/src/metrics/vc/run_robustness.sh +++ b/src/metrics/vc/run_robustness.sh @@ -12,7 +12,7 @@ set -euo pipefail # Configuration DATASET="op" -METHOD="grnboost" +METHOD="pearson_corr" OUTPUT_DIR="output/vc/robustness" # Input files @@ -36,7 +36,7 @@ python src/metrics/vc/analyze_robustness.py \ --prediction "${PREDICTION}" \ --evaluation_data "${EVALUATION_DATA}" \ --output_dir "${OUTPUT_DIR}" \ - --degrees 0.0 0.2 0.5 1.0 \ + --degrees 0.0 0.5 1.0 \ --noise_types net echo "" diff --git a/src/process_data/helper_data.py b/src/process_data/helper_data.py index 1e726fcad..43518bcdd 100644 --- a/src/process_data/helper_data.py +++ b/src/process_data/helper_data.py @@ -66,7 +66,7 @@ def normalize_func(adata, log_norm=True, pearson_residual=False, target_sum=1e4) import scipy.sparse as sp counts = adata.layers['counts'].copy() if 'counts' in adata.layers else adata.X.copy() adata.layers['counts'] = sp.csr_matrix(counts) if not sp.issparse(counts) else counts - assert sp.issparse(counts), "Counts matrix must be sparse." + assert sp.issparse(adata.layers['counts']), "Counts matrix must be sparse." print("min:", counts.min(), "max:", counts.max(), "mean:", counts.mean()) if pearson_residual: diff --git a/src/process_data/main/ibd/script.py b/src/process_data/main/ibd/script.py index 0259ba581..d34167b1f 100644 --- a/src/process_data/main/ibd/script.py +++ b/src/process_data/main/ibd/script.py @@ -191,5 +191,7 @@ def harmonize(adata_rna, adata_atac): adata_rna_bulk_d = adata_rna_bulk[adata_rna_bulk.obs['disease']==disease] adata_rna_bulk_d.uns['dataset_id'] = f'ibd_{disease.lower()}' adata_rna_bulk_d.write(f'resources/extended_data/ibd_{disease}_bulk.h5ad') +adata_rna_bulk.uns['dataset_id'] = f'ibd' +adata_rna_bulk.write(f'resources/extended_data/ibd_bulk.h5ad') print('Done') \ No newline at end of file diff --git a/src/process_data/main/norman/script.py b/src/process_data/main/norman/script.py index 359f3f474..d88cba64c 100644 --- a/src/process_data/main/norman/script.py +++ b/src/process_data/main/norman/script.py @@ -29,6 +29,7 @@ sys.path.append(meta["resources_dir"]) from helper_data import sum_by +from helper_data import wrapper_large_perturbation_data, split_data_gene_perturbation, split_control_groups def add_metadata(adata): @@ -39,7 +40,7 @@ def add_metadata(adata): adata.uns['dataset_id'] = 'norman' adata.uns['dataset_name'] = 'Norman' adata.uns['dataset_organism'] = 'human' - adata.uns['normalization_id'] = 'X_norm' + adata.uns['normalization_id'] = 'lognorm' return adata if __name__ == '__main__': @@ -62,48 +63,80 @@ def add_metadata(adata): adata.var.index = adata.var.index.astype(str) adata.obs = adata.obs[['perturbation', 'is_control', 'perturbation_type']] - # preprocess - sc.pp.filter_cells(adata, min_genes=100) - sc.pp.filter_genes(adata, min_cells=10) - - # - - adata.layers['X_norm'] = adata.X.copy() - - # - split to inference and evaluation datasets - ctr_pertb = adata[adata.obs['is_control']].obs['perturbation'].unique() - non_ctr_pertubs =adata[~adata.obs['is_control']].obs['perturbation'].unique() - train_perturbs, test_perturbs = train_test_split(non_ctr_pertubs, test_size=.5, random_state=32) - train_perturbs = np.concatenate([train_perturbs, ctr_pertb]) # add control perturbations to test set for ws_distance - test_perturbs = np.concatenate([test_perturbs, ctr_pertb]) - - adata_train_sc = adata[adata.obs['perturbation'].isin(train_perturbs)] - adata_test_sc = adata[adata.obs['perturbation'].isin(test_perturbs)] - - - # - filter genes and cells - sc.pp.filter_cells(adata_train_sc, min_genes=100) - sc.pp.filter_genes(adata_train_sc, min_cells=10) - - sc.pp.filter_cells(adata_test_sc, min_genes=100) - sc.pp.filter_genes(adata_test_sc, min_cells=10) - - # - pseudo bulk - adata_bulk = sum_by(adata, unique_mapping=True, col='perturbation') - norman_test_bulk = sum_by(adata_test_sc, unique_mapping=True, col='perturbation') # summing over X_norm - - # - normalize evaluation data - norman_test_bulk.layers['X_norm'] = norman_test_bulk.X.copy() - adata_train_sc.layers['X_norm'] = adata_train_sc.X.copy() - adata_bulk.layers['X_norm'] = adata_bulk.X.copy() - - # - add metadata - adata_bulk = add_metadata(adata_bulk) - norman_test_bulk = add_metadata(norman_test_bulk) - adata_test_sc = add_metadata(adata_test_sc) - adata_train_sc = add_metadata(adata_train_sc) - # - save - print('saving...') - adata_bulk.write(par['norman_bulk']) - adata_test_sc.write(par['norman_test_sc']) - norman_test_bulk.write(par['norman_test_bulk']) - adata_train_sc.write(par['norman_train_sc']) \ No newline at end of file + # preprocess + if False: + sc.pp.filter_cells(adata, min_genes=100) + sc.pp.filter_genes(adata, min_cells=10) + + # - + adata.layers['X_norm'] = adata.X.copy() + + # - split to inference and evaluation datasets + ctr_pertb = adata[adata.obs['is_control']].obs['perturbation'].unique() + non_ctr_pertubs =adata[~adata.obs['is_control']].obs['perturbation'].unique() + train_perturbs, test_perturbs = train_test_split(non_ctr_pertubs, test_size=.5, random_state=32) + train_perturbs = np.concatenate([train_perturbs, ctr_pertb]) # add control perturbations to test set for ws_distance + test_perturbs = np.concatenate([test_perturbs, ctr_pertb]) + + adata_train_sc = adata[adata.obs['perturbation'].isin(train_perturbs)] + adata_test_sc = adata[adata.obs['perturbation'].isin(test_perturbs)] + + + # - filter genes and cells + sc.pp.filter_cells(adata_train_sc, min_genes=100) + sc.pp.filter_genes(adata_train_sc, min_cells=10) + + sc.pp.filter_cells(adata_test_sc, min_genes=100) + sc.pp.filter_genes(adata_test_sc, min_cells=10) + + # - pseudo bulk + adata_bulk = sum_by(adata, unique_mapping=True, col='perturbation') + norman_test_bulk = sum_by(adata_test_sc, unique_mapping=True, col='perturbation') # summing over X_norm + + # - normalize evaluation data (proper log normalization to avoid overflow) + # For pseudobulk data, normalize and log transform + sc.pp.normalize_total(norman_test_bulk, target_sum=1e4) + sc.pp.log1p(norman_test_bulk) + norman_test_bulk.layers['X_norm'] = norman_test_bulk.X.copy() + + sc.pp.normalize_total(adata_bulk, target_sum=1e4) + sc.pp.log1p(adata_bulk) + adata_bulk.layers['X_norm'] = adata_bulk.X.copy() + + # For single-cell data, just copy (already filtered and should be reasonable) + adata_train_sc.layers['X_norm'] = adata_train_sc.X.copy() + adata_test_sc.layers['X_norm'] = adata_test_sc.X.copy() + + # - clean infinity values from all layers + for adata_obj, name in [(norman_test_bulk, 'norman_test_bulk'), + (adata_train_sc, 'adata_train_sc'), + (adata_bulk, 'adata_bulk'), + (adata_test_sc, 'adata_test_sc')]: + for layer_name in adata_obj.layers.keys(): + layer_data = adata_obj.layers[layer_name] + if csr_matrix is not None and isinstance(layer_data, csr_matrix): + layer_data = layer_data.toarray() + if np.any(np.isinf(layer_data)): + print(f"Warning: Found {np.sum(np.isinf(layer_data))} infinity values in {name}.layers['{layer_name}']. Replacing with 0.") + layer_data[np.isinf(layer_data)] = 0 + adata_obj.layers[layer_name] = csr_matrix(layer_data) if isinstance(adata_obj.layers[layer_name], csr_matrix) else layer_data + + # - add metadata + adata_bulk = add_metadata(adata_bulk) + norman_test_bulk = add_metadata(norman_test_bulk) + adata_test_sc = add_metadata(adata_test_sc) + adata_train_sc = add_metadata(adata_train_sc) + # - save + print('saving...') + adata_bulk.write(par['norman_bulk']) + adata_test_sc.write(par['norman_test_sc']) + norman_test_bulk.write(par['norman_test_bulk']) + adata_train_sc.write(par['norman_train_sc']) + else: + adata = split_control_groups(adata, perturbation_col='perturbation', control_flag_col='is_control', new_col='control_split') + + wrapper_large_perturbation_data(adata, split_func=split_data_gene_perturbation, + covariates=['perturbation', 'control_split'], + qc_perturbation_effect=False, + add_metadata=add_metadata, + save_name='norman') \ No newline at end of file diff --git a/src/utils/config.env b/src/utils/config.env new file mode 100644 index 000000000..7283c74fb --- /dev/null +++ b/src/utils/config.env @@ -0,0 +1,32 @@ +# Auto-generated dataset configuration +# Format: DATASET_VARIABLE=value + +# Global lists (comma-separated) +METHODS="positive_control,pearson_corr,grnboost,ppcor,portia,scenic,geneformer,scgpt,ppcor,scenicplus,celloracle,figr,granie,scglue,scprint,negative_control" +METRICS="r_precision,r_recall,ws_precision,ws_recall,vc,sem,t_rec_precision,t_rec_recall,rc_tf_act,tfb_f1,gs_f1" +DATASETS="op,parsebioscience,300BCG,ibd_uc,ibd_cd,replogle,xaira_HEK293T,xaira_HCT116,nakatake,norman" +FINAL_METRICS="r_precision,r_recall,vc,sem,ws_precision,ws_recall" + +# Cell types +CELLTYPE_replogle="K562" +CELLTYPE_norman="K562" +CELLTYPE_xaira_HEK293T="HEK293T" +CELLTYPE_xaira_HCT116="HCT116" +CELLTYPE_op="PBMC" +CELLTYPE_parsebioscience="PBMC" +CELLTYPE_300BCG="PBMC" +CELLTYPE_ibd_uc="PBMC" +CELLTYPE_ibd_cd="PBMC" +CELLTYPE_nakatake="" + +# Metrics (comma-separated) +METRICS_replogle="regression,ws_distance,tf_recovery,tf_binding,sem,gs_recovery,vc" +METRICS_norman="regression,ws_distance,tf_binding,gs_recovery,vc" +METRICS_nakatake="regression,sem,gs_recovery,vc" +METRICS_op="regression,vc,rc_tf_act,tf_binding,sem,gs_recovery" +METRICS_300BCG="regression,vc,rc_tf_act,tf_binding,sem,gs_recovery" +METRICS_ibd_uc="regression,tf_binding,gs_recovery,rc_tf_act" +METRICS_ibd_cd="regression,tf_binding,gs_recovery,rc_tf_act" +METRICS_parsebioscience="regression,vc,rc_tf_act,tf_binding,sem,gs_recovery" +METRICS_xaira_HEK293T="regression,ws_distance,tf_recovery,tf_binding,sem,gs_recovery,vc" +METRICS_xaira_HCT116="regression,ws_distance,tf_recovery,tf_binding,sem,gs_recovery,vc" diff --git a/src/utils/config.py b/src/utils/config.py index 230a51bee..1ad57379b 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -4,6 +4,88 @@ METHODS = ['positive_control', 'pearson_corr', 'grnboost', 'ppcor', 'portia', 'scenic', 'geneformer', 'scgpt', 'ppcor', 'scenicplus', 'celloracle', 'figr', 'granie', 'scglue', 'scprint', 'negative_control'] + + +DATASET_INFO = { + "op": { + "cell_type": "PBMC", + "perturbation_type": "Drugs", + "Inference data": " sc", + 'Measurement time': "24 hours", + "Modality": 'Multiomics' + }, + "ibd_uc": { + "cell_type": "PBMC", + "perturbation_type": "Chemicals/ bacteria", + "Inference data": "sc", + 'Measurement time': "24 hours", + "Modality": 'Multiomics' + }, + "ibd_cd": { + "cell_type": "PBMC", + "perturbation_type": "Chemicals/ bacteria", + "Inference data": "sc", + 'Measurement time': "24 hours", + "Modality": 'Multiomics' + }, + "300BCG": { + "cell_type": "PBMC", + "perturbation_type": "Chemicals", + "Inference data": "sc", + 'Measurement time': 'T0 and 3 months', + "Modality": 'Transcriptmoics' + }, + "parsebioscience": { + "cell_type": "PBMC", + "perturbation_type": "Cytokines", + "Inference data": " sc/bulk", + 'Measurement time': "24 hours", + "Modality": 'Transcriptmoics' + }, + "xaira_HEK293T": { + "cell_type": "HEK293T", + "perturbation_type": "Knockout", + "Inference data": " sc/bulk", + 'Measurement time': "7 days", + "Modality": 'Transcriptmoics' + }, + "xaira_HCT116": { + "cell_type": "HCT116", + "perturbation_type": "Knockout", + "Inference data": " sc/bulk", + 'Measurement time': "7 days", + "Modality": 'Transcriptmoics' + }, + "replogle": { + "cell_type": "K562", + "perturbation_type": "Knockout", + "Inference data": " sc/bulk", + 'Measurement time': "7 days", + "Modality": 'Transcriptmoics' + }, + "nakatake": { + "cell_type": "SEES3 (PSC)", + "perturbation_type": "Overexpression", + "Inference data": "bulk", + 'Measurement time': "2 days", + "Modality": 'Transcriptmoics' + }, + "norman": { + "cell_type": "K562", + "perturbation_type": "Activation", + "Inference data": "sc", + 'Measurement time': "7 days", + "Modality": 'Transcriptmoics' + }, + "adamson": { + "cell_type": "K562", + "perturbation_type": "Knockout", + "Inference data": "sc", + 'Measurement time': "7 days", + "Modality": 'Transcriptmoics' + }, + } + DATASET_GROUPS = { "op": { "match": ["plate_name", "donor_id", "cell_type", "well"], @@ -31,12 +113,14 @@ "match": ["donor_id", "cell_type"], "loose_match": ["donor_id", "cell_type"], "cv": ["perturbation", "cell_type"], + "rc_tf_ac": ["perturbation", "cell_type"] }, "ibd_cd": { 'anchors': ['donor_id'], "match": ["donor_id", "cell_type"], "loose_match": ["donor_id", "cell_type"], "cv": ["perturbation", "cell_type"], + "rc_tf_ac": ["perturbation", "cell_type"] }, "replogle": { "match": ["perturbation"], @@ -88,17 +172,17 @@ } DATASETS_METRICS = { - 'replogle': ['regression', 'ws_distance', 'tf_recovery', 'tf_binding', 'sem', 'gs_recovery'], + 'replogle': ['regression', 'ws_distance', 'tf_recovery', 'tf_binding', 'sem', 'gs_recovery', 'vc'], # 'adamson': ['regression', 'tf_binding', 'sem', 'gs_recovery'], - 'norman': ['regression', 'ws_distance', 'tf_binding', 'sem', 'gs_recovery'], - 'nakatake': ['regression', 'sem', 'gs_recovery'], + 'norman': ['regression', 'ws_distance', 'tf_binding', 'gs_recovery', 'vc'], + 'nakatake': ['regression', 'sem', 'gs_recovery', 'vc'], 'op': ['regression', 'vc', 'rc_tf_act', 'tf_binding', 'sem', 'gs_recovery'], '300BCG': ['regression', 'vc', 'rc_tf_act', 'tf_binding', 'sem', 'gs_recovery'], - 'ibd_uc': ['regression', 'tf_binding', 'gs_recovery'], - 'ibd_cd': ['regression', 'tf_binding', 'gs_recovery'], + 'ibd_uc': ['regression', 'tf_binding', 'gs_recovery', 'rc_tf_act'], + 'ibd_cd': ['regression', 'tf_binding', 'gs_recovery', 'rc_tf_act'], 'parsebioscience': ['regression', 'vc', 'rc_tf_act', 'tf_binding', 'sem', 'gs_recovery'], - 'xaira_HEK293T': ['regression', 'ws_distance', 'tf_recovery', 'tf_binding', 'sem', 'gs_recovery'], - 'xaira_HCT116': ['regression', 'ws_distance', 'tf_recovery', 'tf_binding', 'sem', 'gs_recovery'], + 'xaira_HEK293T': ['regression', 'ws_distance', 'tf_recovery', 'tf_binding', 'sem', 'gs_recovery', 'vc'], + 'xaira_HCT116': ['regression', 'ws_distance', 'tf_recovery', 'tf_binding', 'sem', 'gs_recovery', 'vc'], } @@ -108,27 +192,61 @@ METRICS_DATASETS.setdefault(metric, []).append(dataset) METRICS = [ - 'r_precision', 'r_recall', 'r_f1', - 'ws_precision', 'ws_recall', 'ws_f1', - 'vc', 'vc_raw', 'vc_precision', - 'sem', 'sem_precision', 'sem_raw', - 't_rec_precision', 't_rec_recall', 't_rec_f1', + 'r_precision', 'r_recall', + 'ws_precision', 'ws_recall', + 'vc', + 'sem', + # 'sem_raw', + 't_rec_precision', 't_rec_recall', + # 't_rec_f1', 'rc_tf_act', - 'tfb_precision', 'tfb_recall', 'tfb_f1', - 'gs_precision', 'gs_recall', 'gs_f1', + # 'tfb_precision', 'tfb_recall', + 'tfb_f1', + # 'gs_precision', 'gs_recall', + 'gs_f1', ] FINAL_METRICS = [ - 'r_precision', 'r_recall', + 'r_precision', + 'r_recall', 'vc', 'sem', - 'ws_precision', 'ws_recall', - 't_rec_precision', 't_rec_recall', - 'rc_tf_act', - 'tfb_f1', - 'gs_f1', + 'ws_precision', + 'ws_recall', + # 'tfb_f1', + # 'gs_f1', ] - +METRIC_THRESHOLDS = { + # Regression metrics: R2-based, meaningful if > 0.1 + 'r_precision': 0.1, + 'r_recall': 0.1, + + # Wasserstein distance metrics: precision/recall based, meaningful if > 0.05 + 'ws_precision': 0.5, + 'ws_recall': 0.5, + + # Virtual cell: r2 scores + 'vc': 0.1, + + # SEM (Structural Equation Modeling): goodness of fit (0-1), meaningful if > 0.4 + 'sem': 0.1, + + # TF recovery metrics: t-statistics from paired t-test, meaningful if > 2.0 (p<0.05) + 't_rec_precision': 2.0, + 't_rec_recall': 2.0, + + # Replica consistency (RC) for TF activity: consistency score (0-1) based on MAD, meaningful if > 0.3 + # Measures consistency of TF activity across biological replicates (1=perfect, 0=no consistency) + 'rc_tf_act': 0.3, + + # TF binding F1: F1 for TF-target binding, meaningful if > 0.05 + # Based on ChIP-seq or other binding data + 'tfb_f1': 0.05, + + # Gene set recovery F1: F1 for gene set enrichment, meaningful if > 0.1 + # Tests if predicted regulators recover known gene sets + 'gs_f1': 0.1, +} surrogate_names = { 'scprint': 'scPRINT', 'collectri': 'CollectRI', @@ -149,15 +267,17 @@ 'negative_control':'Negative Ctrl', 'scgpt': 'scGPT', 'spearman_corr': 'Spearman Corr.', + 'geneformer': 'GeneFormer', 'regression': 'Regression', 'tf_recovery': 'TF Recovery', - 'r_precision': "R (precision)", - 'r_recall': "R (recall)", - 'r_f1': "R (F1)", - 'r_raw': "R (raw)", + 'r_precision': "Regression (precision)", + 'r_recall': "Regression (recall)", + 'r_f1': "Regression (F1)", + 'r_raw': "Regression (raw)", 'ws_precision': "WS (precision)", 'ws_recall': "WS (recall)", + 'ws_distance': "WS distance", 'ws_f1': "WS (F1)", 'ws_raw': "WS (raw)", 'sem': 'SEM', @@ -170,9 +290,11 @@ 'tfb_precision': 'TF binding (precision)', 'tfb_recall': 'TF binding (recall)', 'tfb_f1': 'TF binding', + 'tf_binding': 'TF binding', 'gs_precision': 'GS (precision)', 'gs_recall': 'GS (recall)', - 'gs_f1': 'Gene sets', + 'gs_f1': 'Gene sets recovery', + 'gs_recovery': 'Gene sets recovery', 'op':'OPSCA', 'nakatake': 'Nakatake', @@ -187,15 +309,22 @@ '300BCG': '300BCG' } -def generate_config_env(output_path='src/utils/dataset_config.env'): +def generate_config_env(output_path='src/utils/config.env'): """Generate a simple env-style config file with dataset-specific configurations.""" with open(output_path, 'w') as f: f.write("# Auto-generated dataset configuration\n") f.write("# Format: DATASET_VARIABLE=value\n\n") + # Global lists + f.write("# Global lists (comma-separated)\n") + f.write(f'METHODS="{",".join(METHODS)}"\n') + f.write(f'METRICS="{",".join(METRICS)}"\n') + f.write(f'DATASETS="{",".join(DATASETS)}"\n') + f.write(f'FINAL_METRICS="{",".join(FINAL_METRICS)}"\n') + # Cell types - f.write("# Cell types\n") + f.write("\n# Cell types\n") for dataset, cell_type in DATASETS_CELLTYPES.items(): var_name = f"CELLTYPE_{dataset}" f.write(f'{var_name}="{cell_type}"\n') @@ -214,7 +343,7 @@ def generate_config_env(output_path='src/utils/dataset_config.env'): if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description='Generate dataset configuration') - parser.add_argument('--output', type=str, default='src/utils/dataset_config.env', + parser.add_argument('--output', type=str, default='src/utils/config.env', help='Output path for the config file') args = parser.parse_args() diff --git a/src/utils/dataset_config.env b/src/utils/dataset_config.env index 861b35c55..c6b08ff4e 100644 --- a/src/utils/dataset_config.env +++ b/src/utils/dataset_config.env @@ -4,7 +4,6 @@ # Cell types CELLTYPE_replogle="K562" CELLTYPE_norman="K562" -CELLTYPE_adamson="K562" CELLTYPE_xaira_HEK293T="HEK293T" CELLTYPE_xaira_HCT116="HCT116" CELLTYPE_op="PBMC" @@ -15,14 +14,13 @@ CELLTYPE_ibd_cd="PBMC" CELLTYPE_nakatake="" # Metrics (comma-separated) -METRICS_replogle="regression,ws_distance,tf_recovery,tf_binding,sem,gs_recovery" -METRICS_adamson="regression,tf_binding,sem,gs_recovery" -METRICS_norman="regression,ws_distance,tf_binding,sem,gs_recovery" -METRICS_nakatake="regression,sem,gs_recovery" +METRICS_replogle="regression,ws_distance,tf_recovery,tf_binding,sem,gs_recovery,vc" +METRICS_norman="regression,ws_distance,tf_binding,gs_recovery,vc" +METRICS_nakatake="regression,sem,gs_recovery,vc" METRICS_op="regression,vc,rc_tf_act,tf_binding,sem,gs_recovery" METRICS_300BCG="regression,vc,rc_tf_act,tf_binding,sem,gs_recovery" -METRICS_ibd_uc="regression,vc,tf_binding,sem,gs_recovery" -METRICS_ibd_cd="regression,vc,tf_binding,sem,gs_recovery" +METRICS_ibd_uc="regression,tf_binding,gs_recovery,rc_tf_act" +METRICS_ibd_cd="regression,tf_binding,gs_recovery,rc_tf_act" METRICS_parsebioscience="regression,vc,rc_tf_act,tf_binding,sem,gs_recovery" -METRICS_xaira_HEK293T="regression,ws_distance,tf_recovery,tf_binding,sem,gs_recovery" -METRICS_xaira_HCT116="regression,ws_distance,tf_recovery,tf_binding,sem,gs_recovery" +METRICS_xaira_HEK293T="regression,ws_distance,tf_recovery,tf_binding,sem,gs_recovery,vc" +METRICS_xaira_HCT116="regression,ws_distance,tf_recovery,tf_binding,sem,gs_recovery,vc" diff --git a/src/workflows/run_grn_evaluation/config.vsh.yaml b/src/workflows/run_grn_evaluation/config.vsh.yaml index bf3833843..e16609966 100644 --- a/src/workflows/run_grn_evaluation/config.vsh.yaml +++ b/src/workflows/run_grn_evaluation/config.vsh.yaml @@ -145,7 +145,7 @@ dependencies: - name: metrics/vc - name: metrics/rc_tf_act - name: metrics/sem - - name: metrics/anchor_regression + # - name: metrics/anchor_regression - name: metrics/tf_binding - name: metrics/gs_recovery - name: utils/extract_uns_metadata diff --git a/src/workflows/run_grn_evaluation/main.nf b/src/workflows/run_grn_evaluation/main.nf index 5b93c9bf9..0027271d3 100644 --- a/src/workflows/run_grn_evaluation/main.nf +++ b/src/workflows/run_grn_evaluation/main.nf @@ -17,7 +17,7 @@ workflow run_wf { regression, ws_distance, tf_recovery, - anchor_regression, + // anchor_regression, rc_tf_act, sem, vc, diff --git a/test.ipynb b/test.ipynb index 2719ea804..9aaea7f49 100644 --- a/test.ipynb +++ b/test.ipynb @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -85,9 +85,9 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAACsCAYAAADxNoA4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAuH0lEQVR4nO3dd1QUV/sH8O8usDRpKgooiggqFuwNNaggalSMPWA0xo6FYKLGXolGE6xE8bVgS3yjRlAjUVFRBLEgYBdsFBEpIihFyu7z+4OX/bECCbuiC8vzOcdz3Dszd565Z/fhzsydOwIiIjDGmAoRKjsAxhirbJzYGGMqhxMbY0zlcGJjjKkcTmyMMZXDiY0xpnI4sTHGVA4nNsaYyuHExhhTOZzY2EchEAgwa9asf11v7969EAgEiI2N/fhBlaN3797o3bu30vbPKh8nNsaYyhHws6LsYxAIBJg5cya8vb3/cT2xWIyCggJoampCIBB8ouhk5efnAwBEIpFS9s8qn7qyA2DKRUR49+4dtLW1lbJ/NTU1qKmpKWXfxTihqR4+Fa3iVqxYAYFAgIcPH2L06NHQ19dHnTp18O233+Ldu3fS9Xx9fdG3b1/Uq1cPmpqaaNmyJbZv316qPgsLCwwePBhnzpxBp06doK2tjR07dgAAAgMD0bNnTxgaGqJWrVpo3rw5Fi1aJLN9Xl4eli9fDisrK2hqasLc3Bzz589HXl5emfH/9ttvaN68ObS0tNCxY0cEBwfLLC/vGtu2bdvQqlUraGpqwszMDDNnzkRGRobMOo8ePcKIESNgYmICLS0tNGzYEF9++SUyMzNl1jt48CC6dOkCHR0dGBkZ4bPPPsPZs2ely8u6xlbR4yy+lujv74/WrVtDU1MTrVq1wunTp0u1RWJiIiZNmgQzMzNoamqiSZMmcHNzk/YYASAjIwMeHh4wNzeHpqYmrKyssG7dOkgkkjLbl5WNe2zVxOjRo2FhYYG1a9fi6tWr2LJlC16/fo39+/cDALZv345WrVrB2dkZ6urqOHnyJGbMmAGJRIKZM2fK1BUdHQ0XFxdMmzYNU6ZMQfPmzXHv3j0MHjwYtra2WLVqFTQ1NfH48WOEhoZKt5NIJHB2dkZISAimTp0KGxsb3LlzBxs3bkRMTAz8/f1l9nPp0iX88ccfcHd3h6amJrZt24YBAwbg+vXraN26dbnHumLFCqxcuRKOjo5wc3NDdHQ0tm/fjhs3biA0NBQaGhrIz89H//79kZeXh9mzZ8PExASJiYn466+/kJGRAQMDAwDAypUrsWLFCtjZ2WHVqlUQiUS4du0aLly4ACcnpzL3L+9xhoSE4NixY5gxYwb09PSwZcsWjBgxAvHx8ahTpw4A4MWLF+jSpQsyMjIwdepUtGjRAomJiTh69ChycnIgEomQk5MDe3t7JCYmYtq0aWjUqBGuXLmChQsXIikpCZs2barIV4UBALEqbfny5QSAnJ2dZcpnzJhBAOjWrVtERJSTk1Nq2/79+5OlpaVMWePGjQkAnT59WqZ848aNBIBSU1PLjeXAgQMkFArp8uXLMuU+Pj4EgEJDQ6VlAAgAhYeHS8vi4uJIS0uLhg0bJi3z9fUlAPTs2TMiIkpJSSGRSEROTk4kFoul63l7exMA2rNnDxERRUZGEgA6cuRIufE+evSIhEIhDRs2TKYuIiKJRCL9v729Pdnb2yt8nCKRiB4/fiwtu3XrFgGgrVu3SsvGjx9PQqGQbty4USrO4lhWr15Nurq6FBMTI7N8wYIFpKamRvHx8eUeK5PFp6LVxPu9rtmzZwMAAgICAEDmGllmZibS0tJgb2+Pp0+fljo1a9KkCfr37y9TZmhoCAA4fvx4uac9R44cgY2NDVq0aIG0tDTpv759+wIAgoKCZNbv3r07OnbsKP3cqFEjDB06FGfOnIFYLC5zH+fOnUN+fj48PDwgFP7/13PKlCnQ19fHqVOnAEDaIztz5gxycnLKrMvf3x8SiQTLli2TqQvAP96okPc4HR0d0bRpU+lnW1tb6Ovr4+nTpwCKeoD+/v4YMmQIOnXqVGp/xbEcOXIEvXr1gpGRkcx+HR0dIRaLS53Gs/LxqWg1YW1tLfO5adOmEAqF0mtToaGhWL58OcLCwkr90DMzM6WJAChKbO8bM2YMdu3ahcmTJ2PBggVwcHDA8OHDMXLkSGlSePToER48eABjY+MyY0xJSfnHmAGgWbNmyMnJQWpqKkxMTEotj4uLAwA0b95cplwkEsHS0lK6vEmTJvjuu++wYcMG/Pbbb+jVqxecnZ3x1VdfSY/1yZMnEAqFaNmyZZnxlkfe42zUqFGpdYyMjPD69WsAQGpqKt68efOPp9/F+719+3aF98vKx4mtmirZ43jy5AkcHBzQokULbNiwAebm5hCJRAgICMDGjRtL9cDKugOqra2N4OBgBAUF4dSpUzh9+jT++OMP9O3bF2fPnoWamhokEgnatGmDDRs2lBmTubl55R7kv/Dy8sKECRNw/PhxnD17Fu7u7tJrkA0bNlS4XnmPs7y7uiTnSCqJRIJ+/fph/vz5ZS5v1qyZXPXVZJzYqolHjx7J9LQeP34MiUQCCwsLnDx5Enl5eThx4oRM7+H9U6Z/IxQK4eDgAAcHB2zYsAFr1qzB4sWLERQUJD3dunXrFhwcHCo05uzRo0elymJiYqCjo1Nur6Rx48YAim5wWFpaSsvz8/Px7NkzODo6yqzfpk0btGnTBkuWLMGVK1fQo0cP+Pj4wNPTE02bNoVEIsH9+/fRrl27CreDvMf5b4yNjaGvr4+7d+/+636zsrJKHSOTH19jqyZ+/fVXmc9bt24FAAwcOFDaYyjZQ8jMzISvr2+F609PTy9VVpwMioc4jB49GomJidi5c2epdXNzc5GdnS1TFhYWhoiICOnnhIQEHD9+HE5OTuX2chwdHSESibBlyxaZ49m9ezcyMzMxaNAgAMCbN29QWFgos22bNm0gFAql8X7xxRcQCoVYtWpVqV7rP/Wm5D3OfyMUCvHFF1/g5MmTCA8PL7W8OJbRo0cjLCwMZ86cKbVORkZGqeNl5eMeWzXx7NkzODs7Y8CAAQgLC8PBgwfh6uqKtm3bQktLCyKRCEOGDMG0adOQlZWFnTt3ol69ekhKSqpQ/atWrUJwcDAGDRqExo0bIyUlBdu2bUPDhg3Rs2dPAMC4ceNw+PBhTJ8+HUFBQejRowfEYjEePnyIw4cPS8fGFWvdujX69+8vM9wDKBqCUR5jY2MsXLgQK1euxIABA+Ds7Izo6Ghs27YNnTt3xldffQUAuHDhAmbNmoVRo0ahWbNmKCwsxIEDB6CmpoYRI0YAAKysrLB48WKsXr0avXr1wvDhw6GpqYkbN27AzMwMa9euLTMGeY+zItasWYOzZ8/C3t5eOoQkKSkJR44cQUhICAwNDTFv3jycOHECgwcPxoQJE9CxY0dkZ2fjzp07OHr0KGJjY1G3bl259ltjKfWeLPtXxcM97t+/TyNHjiQ9PT0yMjKiWbNmUW5urnS9EydOkK2tLWlpaZGFhQWtW7eO9uzZIzOUgqhouMegQYNK7ef8+fM0dOhQMjMzI5FIRGZmZuTi4lJq6EF+fj6tW7eOWrVqRZqammRkZEQdO3aklStXUmZmpnQ9ADRz5kw6ePAgWVtbk6amJrVv356CgoJk6nt/uEcxb29vatGiBWloaFD9+vXJzc2NXr9+LV3+9OlTmjhxIjVt2pS0tLSodu3a1KdPHzp37lypY9uzZw+1b99eGq+9vT0FBgZKl78/3EOR43xf48aN6euvv5Ypi4uLo/Hjx5OxsTFpamqSpaUlzZw5k/Ly8qTrvH37lhYuXEhWVlYkEomobt26ZGdnR7/88gvl5+eX2g8rGz8rWsUVD1ZNTU3lv9aMVRBfY2OMqRxObIwxlcOJjTGmcvgaG2NM5XCPjTGmcjixMcZUDic2xpjK4cTGVMqKFSvkei6UqSa+ecCqvN69e6Ndu3YVmkE2KysLeXl50plrWc3Ez4oylUBEEIvFqFWrFmrVqqXscJiS8akoq1S9e/fG7Nmz4eHhASMjI9SvXx87d+5EdnY2vvnmG+jp6cHKygp///23dJu7d+9i4MCBqFWrFurXr49x48YhLS0NADBhwgRcunQJmzdvhkAgkL745eLFixAIBPj777/RsWNHaGpqIiQkpMxT0T179khfDGNqalqhFzmz6o0TG6t0+/btQ926dXH9+nXMnj0bbm5uGDVqFOzs7BAREQEnJyeMGzcOOTk5yMjIQN++fdG+fXuEh4fj9OnTSE5OxujRowEAmzdvRvfu3TFlyhQkJSUhKSlJZqLHBQsW4KeffsKDBw9ga2tbKpbt27dj5syZmDp1Ku7cuYMTJ07Aysrqk7UFUxIlPoDPVJC9vT317NlT+rmwsJB0dXVp3Lhx0rKkpCQCQGFhYbR69WpycnKSqSMhIYEAUHR0tLTOb7/9VmadoKAgAkD+/v4y5cuXL6e2bdtKP5uZmdHixYsr6ehYdcHX2FilK9lzUlNTQ506ddCmTRtpWf369QEUzeF/69YtBAUFlXld7MmTJ/86HfY/zYuWkpKCFy9ewMHBQd5DYNUcJzZW6TQ0NGQ+CwQCmbLi6bYlEgmysrIwZMgQrFu3rlQ9pqam/7ovXV3dcpcp6+32TPk4sTGl6tChA/78809YWFhAXb3sr6NIJCr3dX3/RE9PDxYWFjh//jz69OnzoaGyaoRvHjClmjlzJtLT0+Hi4oIbN27gyZMnOHPmDL755htpMrOwsMC1a9cQGxuLtLS0ct97WpYVK1bAy8sLW7ZswaNHjxARESF9XwRTXZzYmFKZmZkhNDQUYrEYTk5OaNOmDTw8PGBoaCh9n+ncuXOhpqaGli1bwtjYGPHx8RWu/+uvv8amTZuwbds2tGrVCoMHDy7z7VlMtfCTB4wxlcM9NsaYyuHExhhTOZzYGGMqhxMbY0zlcGJjjKkcTmyMMZXDiY0xpnI4sTHGVA4nNsaYUn2MZwQ4sTHGlIaIpLO9/PHHHwgMDKyUenl2D8aYUkgkEunzwNevX8fmzZshEolgYGCALl26fFDd3GNjrBIVz0gizwwkNRERSZPaypUrsWXLFmRnZyMsLAw//PADgoODP6h+TmyMVQIvLy+4urrCxcUFN2/elP5oWdmKTz+3bt0KLy8vTJkyBQEBAfD19YVEIoGnpydCQkIUrp9bn7EPtHLlSqxbtw7a2tpISUlB165d8eeffyo7rCpNIpFAIpEgNDQUI0eOhL29PRo0aABXV1fMnTsXiYmJWLJkCcLCwhSqnxMbYx8gMTERRAQ/Pz/s3r0bAQEB8PDwgIuLC44ePars8KqUknc/8/PzIRQKYWhoiLS0NOTl5UmXDRkyBC4uLrh69SpWr16NK1euyL0vTmyMKSgwMBDm5uY4ePAgRCIRAEBHRwdr167Ft99+i7Fjx3LPrYTi08/du3dj//79AIDWrVsjODgYFy9elEl85ubmcHBwgEAgwG+//Yb8/Hy59sWJjTEFeHt7g4gwa9YsxMbGIjk5GUDRKZaGhgbWrFkDDw8PjBo1CkFBQUqOtmr5888/sXv3bgDArFmz8Pnnn8PV1RV+fn54+vQp3r59i2PHjmHw4MEYOHAgfH19pe1bYUp78R9j1dSvv/5KpqamdPXqVUpKSqKxY8dSrVq16OrVq0REJJFIiIgoLy+PvL29qaCgQJnhVhmFhYVERPTkyROytLQkHx8f6bKJEyeSqakpmZmZkZWVFTVr1owkEgmFh4eTtbU1xcfHy7UvnhqcMTlcv34du3btgqOjo/Rt9SkpKfDw8MDJkydx7tw5dO3aVWbgKQAUFhaW+xYuVfV+GxSXvXnzBrNmzYJQKMS+ffukyy5evIjU1FQUFBRgzJgxUFNTw+zZs3HlyhWcP38ehoaGcu2cMVYBJ0+epObNm5OZmRkFBgbKLEtOTiZXV1cyNDSk4OBgJUVYNe3evZuWLl1KhYWF0t7r2bNnSSgU0t9//13mNpGRkTRp0iSqU6cORUVFyb1PvsbGWAX169cPffv2xdu3b3HkyBFkZ2dLl9WrVw+bNm1Ct27dsGrVKiVGWXUQEVJTU3H//n14e3vD3t4eq1evRnJyMvr16wc3Nzf4+vri1atXMtvl5eUhPT0dycnJCAoKQtu2beXeN5+KMvYvDh8+jNq1a8PR0REFBQXw8PDAlStXMHbsWMyYMQM6OjrSdTMyMqCvr19jB+iWd8qdmZmJFStWIDIyEnfu3IGnpyfu3buHqKgo7Nq1Cy1atCi1TW5uLrS1tRWKgxMbY//g+vXr+Pbbb6Gjo4OffvoJnTt3Rn5+PmbNmoWoqCiMGTMGbm5uMskNkH0OsiaIjY2FhYWF9LOvry9u3boFGxsbdO/eHba2tpBIJHj79i02bdqEkJAQvHr1ClFRUfjhhx+wdu3ayg1I0fNmxlTdqlWr6KuvvqJWrVqRhoYG9evXjy5fvkxERXc8p06dSt26daPly5fTu3fvlByt8ixfvpx69OhBt27dIiKihQsXUu3atal3795kbW1NTk5OdO7cOZlt7t+/T/7+/jRy5EjKz8+v9Jhqzp8UxuSwefNm/Pzzz5g4cSJOnToFHx8f5OTk4Mcff8SVK1cgEomwdetWmJubIzExUTpAtyZq0qQJtLW1sXTpUpw+fRovXrzA33//jaCgIPz666/Q1dXFsmXLZMbztWjRAkOHDsWRI0egoaGBgoKCSo2JT0UZK4GIQEQYPXo06tSpgx07dkiX/fnnn1iyZAkaNmwIT09PdO3aFQUFBRAKhVBTUytzeENNcfToUezatQsSiQR5eXk4ceIEDAwMAABBQUHYunUrUlJS4Onpid69e3/0eLjHxlgJAoEAQqEQBgYG0jFVxUaMGIERI0bg8uXL8PT0xLVr16ChoQE1NTVIJJIamdSK+0UjR47EN998g5ycHNy6dQuxsbHSdfr06QN3d3eYmppi6tSpiIiI+OhxcWJjrAxt2rRBcHAwQkNDZcotLCzQt29fFBYW4tChQygsLASAGnWjoCSBQCBNbmPGjMG8efNgY2ODRYsWISoqSrpe7969MXnyZIwcOVKh4Rtyx8WnooyVbcSIEQgNDcWuXbtga2uLOnXqwNXVFYMGDcKbN2+watUqREdHw9TUVNmhKl3J0/AjR45gx44d0NXVxapVq8pMZGKxGGpqah8tHk5sjL2n5I/O1dUVly5dglAohI6ODogIMTExuHr1Kr7++msEBQXBzMxMyRFXDSWT2+HDh7Fz507o6elh0aJF6NSp0yeNpWY9vMZYBZS8EfD7778jMDAQKSkpEIvFGDt2LADgwIEDMDAwgK6urpKjrTqKT0sFAgFGjx4NoVAIT09PHD169JMnNu6xMVaOsk6Xbt26BW9vbxw7dgxBQUGwtbVVUnRVV8me24ULF9C7d+9Pfg2yZl7xVGH8d0p+5b14pTipFS/Py8tDcnIyEhISalRSK699yisXCATSZX379oVQKPzk30vusVVzkZGRSEtLg66uLuzs7ADUvMd55HXkyBHk5eXB0NAQgwcPBlD+xeySvY+nT5+iQYMGEIvFpR6hUlXv3xRQpN2ePHmCBg0aQEtL65MGzqqpY8eOkYGBATVo0ICaNWtG06dPly4Ti8VKjKzq+uGHH8jIyIgsLS3J2tqaJk2aJF1WPBFiseIJI4mINm3aRJaWlpSQkPDJYq1KFG23zZs3K6XdOLFVQxKJhHJycmjgwIF04MABevjwIe3YsYMaN25MX375pXQ9Tm7/TyKRUFpaGg0YMIDu3LlD8fHxtHfvXmrUqJFMmxX/SEv+OH18fMjIyIgOHTr0yeNWluLvTnVtN05s1dDbt2/p9evX9OWXX1JsbCwREWVnZ9OhQ4fI3Nyck1sZXr58Sbdv36ahQ4dSWloaEcm2mYuLi3Tdkg9l+/j4kL6+Ph09evSTx6wsJb8z1bXdOLFVM/7+/tSpUycaPHgwNW7cWGYu+OIvXJMmTWjQoEFKjLJqWbRoEVlaWlLXrl3J2tqaXr9+LV2WnZ1N//3vf8nCwoL69+8vs91//vMfMjAwqFFJraSS7WZlZUXp6enSZdnZ2fTHH3+QhYUFOTk5yWxXFdqNE1s1cu3aNTI0NCQPDw9yc3MjY2NjGjJkiMw6OTk5tHfvXmrVqhU9f/5cSZFWHQcOHCBzc3Py8fGhZcuWkbGxMTk7O8usk5OTQ3v27KFhw4ZJeyuHDx8mgUBAx44dU0bYSlGyp3bw4EEyNzenHTt20PLly6levXplftd8fX1l2u3IkSNVot34rmg1cfv2bcTExODhw4dYsmQJ3r17h7CwMLi4uKBbt27w9/eXrpubm4vCwkLo6ekpL+AqwN/fHwkJCdDV1cXEiRNRUFCAkJCQMtssLy8Pmpqa0s9ZWVm4evUqHB0dlRC5cqlEuyk1rbIKycrKIhMTExIIBKXufAYFBVG9evVoxIgRSoyw6klNTSVdXV0SCAS0atUqaXlxm5mYmNCwYcPK3LYmvy5PVdqNE1s1cefOHWrdujV17NiREhMTpeVisZguXrxIampqNHbsWCVGWPXcu3ePWrRoQXZ2dpSUlCQtl0gkdPHiRRIIBPTDDz8oMULlK3kXs5gqtBsntiosJyeHCgsLpdNO37lzh0xNTWnAgAGUmpoqXU8sFtPly5cpOjpaWaFWGWlpaZSRkUGZmZlEVNRmJiYmZbZZREREqTFYNUnJa2qq1m6c2KqoU6dOkYuLC3Xq1IlmzJhBf/31FxEVfeHMzMxKfeEY0erVq6lfv35kaWlJrq6u5OfnR0REd+/eJTMzMxo4cGCZbVbVf6QfQ8memiq2Gye2Kuj48eOkpaVFnp6etH37dhozZgwJhUK6f/8+ERV94Ro1akTdu3eXji2q6ZYsWUJ16tShY8eO0V9//UV9+vQhIyMj6Yj3u3fvkrm5OXXu3JkyMjKUHG3Voartxomtinn9+jU5OjrSxo0biYgoJSWFzMzMaObMmTLrRUVFkY2NDcXFxSkhyqolLi6OunXrRufPnyciotOnT5O+vj795z//IaL/HzgaGRlJzs7OPGj5f1S53TixVTEpKSlkZWVFN2/epMTERGrQoAFNmTJFuvzo0aMUExNDREWvgGNE0dHRZGpqSi9fvqQTJ05QrVq1aPv27URUdJ3Sx8eHnj59KrNNdfqRVpb3j1mV242ngKgioqKikJCQAAMDA9jY2CAiIgI9evTA559/ju3btwMAnj9/jlOnTuH+/fsgohr9yjcA8PPzQ1xcHGrXrg0bGxv4+Phg3Lhx+PnnnzF9+nQAQExMDAIDA5GQkCCzbU2b/aTkjC81ot2UnVkZkZ+fH5mZmdGSJUtILBbTzJkzSSAQyIzoJiJasGABtWzZUuYxqppq4cKF1KBBA9qyZQsREU2dOpUEAgHNmzdPuk5WVhZ9/vnnNGDAgGrT0/gYSt4oqCntxlODK9mpU6fg6uqKLVu2YMCAARAKhfD29kZWVhYCAgKwbt06CIVCPH36FIcOHcLly5dhbm6u7LCVavXq1di5cycCAgLQrFkzAMCOHTuQnZ2N/fv3Izc3F+rq6oiKikJaWhoiIiIgFApr7Dx1xfOi1aR248SmRO/evcO+ffswZ84cTJ48GTk5OYiJicGJEycwfPhwZGRkIDg4GMnJyWjdujWuXLmC1q1bKztspUpPT0dwcDA2bdqEzp07IzExERERETh06BAGDBgAgUCAV69eITc3F3Z2dli5ciXU1dVRWFgIdfWa+3Wvae1W/SJWIUSEZ8+ewcTEBOnp6Vi+fDlu376Nx48fQ0NDA+7u7pg6dSqEQiHU1dVr/DU1oKj3cf/+fTx48ADBwcHYtm0bnj17BolEgoCAACxZsgTTp0+XmcFVLBZXyx9nZapx7abkU+Eab9++faStrU36+vo0bNgw2rdvHxERubu7U58+farU83dVxa5du8jIyIj09fVp/vz5FBgYSEREY8eOpfHjxys5uqqrJrVbNU3HqmP8+PHo1KkTEhMT0a9fP+lLMMRiMczNzav3X82PZNKkSejXrx/y8vJgbW0NoOiu38uXL9GtWzclR1d11aR242mLqpiHDx/iwIED+PXXXxESElLjr6n9m6ysLERFRWHdunWIi4tDREQE/yGoAFVvN9U5EhVw8+ZNeHl5ISoqCpcuXeKk9i+ICOHh4fDy8kJBQQFu3rwJdXX1ct+cxIrUhHbjHlsVkpubi/DwcFhYWNT4IR0VlZeXh/v376Nt27YQCoXV9i7ep6bq7caJjamM6jjeqipQxXbjxMYYUzlKT9PBwcEoLCwsVV5YWIjg4GAlRMQYq+6U3mNTU1NDUlIS6tWrJ1P+6tUr1KtXD2KxWEmRMcaqK6X32KjESOeSXr16BV1dXSVExBir7pR2G2T48OEAih71mDBhgswrvMRiMW7fvg07OztlhccYq8aUltgMDAwAFPXY9PT0oK2tLV0mEonQrVs3TJkyRVnhMcaqMaVfY1u5ciXmzp3Lp52MsUpT4cR24sSJClfq7Oxc4XVzc3NBRNDR0QEAxMXFwc/PDy1btoSTk1OF62GMsWIVTmzvD+ATCAQouWnJGwDy3Ml0cnLC8OHDMX36dGRkZKB58+YQiURIS0vDhg0b4ObmVuG6GGMMkOOuqEQikf47e/Ys2rVrh7///hsZGRnIyMhAQEAAOnTogNOnT8sVQEREBHr16gUAOHr0KExMTBAXF4f9+/djy5Yt8h0NY4xBwZsHHh4e8PHxQc+ePaVl/fv3h46ODqZOnYoHDx5UuK6cnBzo6ekBAM6ePYvhw4dDKBSiW7duiIuLUyQ8WG7xUmg7xtin8dT9+49av0Lj2J48eQJDQ8NS5QYGBoiNjZWrLisrK/j7+yMhIQFnzpyRXldLSUmBvr6+IuExxmo4hRJb586d8d133yE5OVlalpycjHnz5qFLly5y1bVs2TLMnTsXFhYW6NKlC7p37w6gqPfWvn17RcJjjNVwCg33ePz4MYYNG4aYmBjp9DoJCQmwtraGv78/rKys5Krv5cuXSEpKkk6hAgDXr1+Hvr4+WrRoIW94fCrKWBX3sU9FFR7HRkQIDAzEw4cPAQA2NjZwdHQs8/Goinr+/DkAoGHDhgrXAXBiY6yq+9iJTeEnDwQCAZycnD54rJlEIoGnpye8vLyQlZUFANDT08P333+PxYsXq9w8UYyxj0/hrHHp0iUMGTIEVlZWsLKygrOzMy5fvix3PYsXL4a3tzd++uknREZGIjIyEmvWrMHWrVuxdOlSRcNjjNVgCiW2gwcPwtHRETo6OnB3d4e7uzu0tLTg4OCA33//Xa669u3bh127dsHNzQ22trawtbXFjBkzsHPnTuzdu1eR8BhjNZxC19hsbGwwdepUzJkzR6Z8w4YN2Llzp1zj2LS0tHD79m00a9ZMpjw6Ohrt2rVDbm6uvOHxNTbGqrgqOY7t6dOnGDJkSKlyZ2dnPHv2TK662rZtC29v71Ll3t7eaNu2rSLhMcZqOIVuHpibm+P8+fOlhnWcO3dO7rcrrV+/HoMGDcK5c+ekY9jCwsKQkJCAgIAARcJjjNVwCiW277//Hu7u7oiKipJOBhkaGoq9e/di8+bNctVlb2+PmJgY/Prrr9KhI8OHD8eMGTNgZmamSHiMsRpO4XFsfn5+8PLykl5Ps7Gxwbx58zB06NBKDVARfI2Nsaqtyo5jGzZsGIYNG1YpQWRkZGD37t3SJNmqVStMnDhROssuY4zJ44Nm0M3Pz0dKSgokEolMeaNGjSpcR3h4OPr37w9tbW3pc6Y3btxAbm4uzp49iw4dOsgd16fssY2zbYcpHTrBWEcXD9JSseLSBdxOflnmuta162BONzu0rlcfDfUNsDo4CL5RETLrfNu1O77tKvuuhyfp6eh30PejHcOnJk+bjWnVBsNbtESzOnUBAHdTkvFzWIjM+v2bWsG1TVu0Nq4PI21tDPp9Px6kpX6SY/mU5Gk3ABho1QzfdeuBhvr6iM14jXWhl3Ex7v9v7uloaGC+XS/0a2oFIy0tJLx5g31REfj97u2PfixVssf26NEjTJw4EVeuXJEpL37jlDwTTc6ZMwfOzs7YuXMn1NWLwiksLMTkyZPh4eFRpd8tOsi6ORb1ssfSC+cQlZyEb9p1xL6hI+B4YA9elTFMRVtdHfGZmQh4FIMln/Uut97oV2kY53dE+lksUZ13WsvbZt0amONkzEPcTHqBPLEY0zt2xv4vRqD/wX1Izi56UkVbQwPhLxJx6lEMfnJQzVmX5W23DiZm2DxgEH6+chkXnj2Fc/MW8Bk8FM6HDiAm/RUAYHGv3uje0BzfnQnA8zdv0KtRY6zq44jk7Gycf/bkUx9ipVIosU2YMAHq6ur466+/YGpq+kHPh4aHh8skNQBQV1fH/Pnz0alTJ4Xr/RQmte+IP+7ewdEH9wAASy4Eoo9FE4xq2QY+N6+XWv92SjJupxTNiDK/R69y6xVLJEjLyfk4QSuZvG0256zsnfEF58+iv5U17Mwbwe/hfQCA/8OiSxgN9FR3mit5221Cuw4IjnuGnRHhAICNV6+gZ6PGGN+2PZYEnQMAdDA1w7EH93EtsegZ7f/euwOXNm3Rtr5JzUxsUVFRuHnzpkIzb7xPX18f8fHxpepKSEiQTkBZFWkIhWhdrz62h///l4oAhCbEo72p6QfVbWFohLCJ05AnLkRkUhJ+vnIZL7LefmDEylcZbaatrg4NoRCZ7959pCirHkXarYOpKXZH3pQpuxwXh36WTaWfI5JewNGyKY7cv4vk7Cx0a2iOJoZG8IwP+ijH8SkplNhatmyJtLS0SglgzJgxmDRpEn755ReZoSPz5s2Di4tLpezjYzDS1oa6UIi0nGyZ8rScHDQ1qq1wvVEvkzAv8DSevU6Hsa4u3Lva4Y+RX2LAb3uRXVDwoWErVWW02Q89PkNydjZCEhSbXbk6UqTd6urolur1p+Vkw7jE2+BWXrqAH/v2Q9ikaSgQiyEBYdH5QNx4kVj5B/GJVTixvXnzRvr/devWYf78+VizZg3atGkDDQ0NmXXlmfn2l19+gUAgwPjx41FYWAgA0NDQgJubG3766acK16MqLsXFSv//8FUaol6+RMg3UzDIujkO37+rvMCqgOkdu2Bws+Zw/fMw8uW4jsvKNt62PdqbmGLyST+8ePMGnRs0xMreDkjJzkJoQryyw/sgFU5shoaGMtfSiAgODg4y6yhy80AkEmHz5s1Yu3YtnjwpOq9v2rSp9HV8VdXr3FwUSiSoqyP7PtS6OjpIfe8v64d4m5+HZxmv0biMqdirmw9ps8ntO2F6p84Y53cUD19VztlCdaFIu6XlZKPue7+hujq6SM0uWl9TTR1z7XrC7dRxBMUW3Sl9+CoNLY3rYXKHTjUnsQUFfdzzbh0dHel7FKp6UgOAAokEd1OSYWfeCIFPHwMABADszBvhwK2oStuPjoYGGhkYwO9h5SVLZVG0zaZ26IyZnbvi6+N/4k5KcrnrqSpF2i0iKQl25o1khhP1aNQYkS+TAAAaakKI1NQgeW+0l1gigfADbgZWFRVObPb29tL/x8fHw9zcvNTdUCJCQkKCXAEUFhZi5cqV2LJli3SiyVq1amH27NlYvnx5qdPcqmR35E380m8A7iS/xK3kl/imXQfoqGvg6P9OGX/pNwDJ2Vn4+UoIgKKLwFa16/zv/2qor1sLNnWNkVNQgLjMDADAwp72OP/sCRLfvEF93Vrw6GYHMRFOxjxUyjFWNnnbbFrHzvDoZoc5pwPw/E2mtBeSU1CAnP9dczTQ1IKZnh7q69YCAFj+77pTak62ytxdlrfd9kZF4NCI0ZjUviOCYp9hSLPmaFOvPhafPwsAyMrPx9XnCVjQ0x7vCguR+PYNujYwx3Cblvjx8iWlHWdlUejmQZMmTZCUlIR69erJlKenp6NJkyZynYrOnj0bx44dw/r162Uegl+xYgVevXqF7du3KxLiJ3HqUTRqa2tjTrceqKurgwepqZhw/E+k5Rb9mMz09GX+ItbTrYVTruOln6d27IypHTvj6vMEuB47DAAwqVULm/sPgqG2FtJzcxH+IhEjDv+OdAWmb6qK5G2zsW3aQlNNHdsGOcvUs/naFWy+FgYAcLRsip/7DZAu2zpwcKl1qjt52y3i5Qt4nAnA9917YK5dT8RmZGD6X8elY9gAwP30X5hv1wsb+38OQy0tJL55C6+wUPx259YnP77KptCTB0KhEMnJyTA2NpYpj4uLQ8uWLZGdXfHTJgMDA/z3v//FwIEDZcoDAgLg4uKCzMxMecPjZ0UZq+Kq1JMH3333HYCi9x0sXbpU5lqYWCzGtWvX0K5dO7kC0NTUhIWFRanyJk2aQCQSyVUXY4wBcia2yMhIAEXX0u7cuSOTeEQiEdq2bYu5c+fKFcCsWbOwevVq+Pr6QlNTEwCQl5eHH3/8EbNmzZKrLsYYA+RMbMV3Rr/55hts3rxZ4Te1Dx8+XObzuXPn0LBhQ+mMubdu3UJ+fn6p4SSMMVYRCt088PUtmmni8ePHePLkCT777DNoa2tLx7H9m/enIxoxYoTMZ3ln4WWMsZIUSmzp6ekYNWoUgoKCIBAI8OjRI1haWmLSpEkwMjKCl9c/X7wvToyMMfYxKPQyFw8PD2hoaCA+Pl7mBsKYMWNw+vTpSguOMcYUoVBiO3v2LNatW4eGDRvKlFtbWyMuTr6Hk5OTkzFu3DiYmZlBXV0dampqMv8YY0xeCp2KZmdnl/nYU3p6uvTOZkVNmDAB8fHxWLp06QfP7cYYY4CCia1Xr17Yv38/Vq9eDaBoXJtEIsH69evRp08fueoKCQnB5cuX5R7/xhhj5VEosa1fvx4ODg4IDw9Hfn4+5s+fj3v37iE9PR2hoaFy1WVubo4PeO0CY4yVotA1ttatWyM6Oho9e/bE0KFDkZ2djeHDhyMyMhJNmzb99wpK2LRpExYsWIDY2FhFQmGMsVIUfkvVu3fvcPv27TLfUuXs7FzOVqUZGRkhJycHhYWF0NHRKTWbR3p6utyx8bOijFVtVepZ0WKnT5/GuHHjkJ6eXuo0Ut6JJjdt2qRICIwxVi6FemzW1tZwcnLCsmXLUL9+fYV3XlBQgGnTpmHp0qVo0qSJwvUwxlhJCiU2fX19ha6nlcXAwABRUVGc2BhjlUahmwcjR47ExYsXKyWAL774Av7+/pVSF2OMAQr22HJycjBq1CgYGxuX+ZYqd3f3Ctfl6ekJLy8vODg4oGPHjtDVlX1hhTx1McYYoGBi2717N6ZPnw4tLS3UqVNH5mkBgUCAp0+fVriufzoFlbcuxhgDFExsJiYmcHd3x4IFCyAUKnQ2yxhjH41Cwz3y8/MxZsyYSklqxdONv08gEEBLSwtWVlYYOnQoatdW/O3qjLGaRaEe25w5c2BsbIxFixZ9cAB9+vRBREQExGIxmjdvDgCIiYmBmpoaWrRogejoaAgEAoSEhKBly5YfvD/GmOpTqMcmFouxfv16nDlzBra2tqVuHmzYsKHCdRX3xnx9faVTjWdmZmLy5Mno2bMnpkyZAldXV8yZMwdnzpxRJFzGWA2jUI/tn2bwEAgEuHDhQoXratCgAQIDA0v1xu7duwcnJyckJiYiIiICTk5OSEtLkzdUxlgNpFCPrfilLpUhMzMTKSkppRJbamoq3rx5AwAwNDREfn5+pe2TMabalH5Lc+jQoZg4cSL8/Pzw/PlzPH/+HH5+fpg0aRK++OILAMD169fRrFkz5QbKGKs2FJ7do7JkZWVhzpw52L9/PwoLCwEA6urq+Prrr7Fx40bo6uoiKioKAHgySsZYhSg9sRXLysqSDsa1tLRErVq1lBwRY6y6qjKJjTHGKovSr7Exxlhl48TGGFM5nNgYYyqHExtjTOVwYmOMqRxObIwxlcOJjTGmcv4PBcJr//YINNYAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZwAAAGHCAYAAACEUORhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB0L0lEQVR4nO3deXxMVxvA8d9M9j2RIEEWZBEk9r1Fxb621uKttdROLaVVtbaqte9a+1K1Fq29aqnYRQhCghAiBFmIkG3u+0dqGImWSCaL5/v5zOc1555773POm85zz7ln7qgURVEQQgghspk6pwMQQgjxbpCEI4QQQi8k4QghhNALSThCCCH0QhKOEEIIvZCEI4QQQi8k4QghhNALSThCCCH0QhKOEEIIvZCEI8Q7RKVSMWDAgP+st3z5clQqFdevX8/+oF6hbt261K1bN8fOL7KeJBwhhBB6oZJnqQnx7lCpVPTv35+5c+f+a73U1FSSk5MxMTFBpVLpKTpdSUlJABgbG+fI+UXWM8zpAIQQ6SmKwtOnTzEzM8uR8xsYGGBgYJAj535GEk3+I1NqQmTCuHHjUKlUXLp0ifbt22NtbY29vT2DBw/m6dOn2nrLli2jXr16FCpUCBMTE0qXLs2CBQvSHc/NzY3mzZuze/duKleujJmZGYsWLQJg7969vPfee9ja2mJpaYmXlxdfffWVzv6JiYmMHTsWd3d3TExMcHZ25osvviAxMTHD+NesWYOXlxempqZUqlSJQ4cO6Wx/1T2c+fPnU6ZMGUxMTChSpAj9+/cnNjZWp05oaCht2rTB0dERU1NTihUrxscff0xcXJxOvdWrV1O1alXMzc2xs7Ojdu3a7NmzR7s9o3s4r9vOZ/eqtmzZQtmyZTExMaFMmTLs2rUrXV9ERETQs2dPihQpgomJCcWLF6dv377aERZAbGwsQ4YMwdnZGRMTE9zd3ZkyZQoajSbD/hUZkxGOEG+hffv2uLm5MXnyZI4dO8bs2bOJiYlh5cqVACxYsIAyZcrQsmVLDA0N+f333+nXrx8ajYb+/fvrHOvy5ct07NiRzz77jF69euHl5cWFCxdo3rw5vr6+TJgwARMTE65cuYK/v792P41GQ8uWLTl8+DC9e/fG29uboKAgZsyYQUhICFu2bNE5z8GDB1m3bh2DBg3CxMSE+fPn07hxY06cOEHZsmVf2dZx48Yxfvx46tevT9++fbl8+TILFizg5MmT+Pv7Y2RkRFJSEo0aNSIxMZGBAwfi6OhIREQEf/zxB7GxsdjY2AAwfvx4xo0bR82aNZkwYQLGxsYcP36cv/76i4YNG2Z4/jdt5+HDh9m8eTP9+vXDysqK2bNn06ZNG8LDw7G3twfg9u3bVK1aldjYWHr37k2pUqWIiIhg48aNJCQkYGxsTEJCAnXq1CEiIoLPPvsMFxcXjhw5wpdffklkZCQzZ858nT8VAaAIId7Y2LFjFUBp2bKlTnm/fv0UQDl79qyiKIqSkJCQbt9GjRopJUqU0ClzdXVVAGXXrl065TNmzFAA5d69e6+MZdWqVYparVb+/vtvnfKFCxcqgOLv768tAxRAOXXqlLbsxo0biqmpqfLRRx9py5YtW6YASlhYmKIoihIVFaUYGxsrDRs2VFJTU7X15s6dqwDK0qVLFUVRlDNnziiAsmHDhlfGGxoaqqjVauWjjz7SOZaiKIpGo9H+u06dOkqdOnUy3U5jY2PlypUr2rKzZ88qgDJnzhxtWZcuXRS1Wq2cPHkyXZzPYpk4caJiYWGhhISE6GwfNWqUYmBgoISHh7+yrUKXTKkJ8RZeHqUMHDgQgB07dgDo3IOJi4vj/v371KlTh2vXrqWbYipevDiNGjXSKbO1tQVg69atr5y+2bBhA97e3pQqVYr79+9rX/Xq1QNg//79OvVr1KhBpUqVtO9dXFxo1aoVu3fvJjU1NcNz/PnnnyQlJTFkyBDU6ucfG7169cLa2prt27cDaEcwu3fvJiEhIcNjbdmyBY1GwzfffKNzLOBfFyi8aTvr169PyZIlte99fX2xtrbm2rVrQNqIacuWLbRo0YLKlSunO9+zWDZs2MD777+PnZ2dznnr169PampquulI8WoypSbEW/Dw8NB5X7JkSdRqtfbeh7+/P2PHjuXo0aPpPoDj4uK0H9CQlnBe1qFDBxYvXsynn37KqFGj8PPzo3Xr1rRt21b7YR0aGkpwcDAFCxbMMMaoqKh/jRnA09OThIQE7t27h6OjY7rtN27cAMDLy0un3NjYmBIlSmi3Fy9enKFDhzJ9+nTWrFnD+++/T8uWLfnf//6nbevVq1dRq9WULl06w3hf5U3b6eLikq6OnZ0dMTExANy7d4+HDx/+6zTis/OeO3futc8rXk0SjhBZ6MUr9KtXr+Ln50epUqWYPn06zs7OGBsbs2PHDmbMmJFuxJLRijQzMzMOHTrE/v372b59O7t27WLdunXUq1ePPXv2YGBggEajwcfHh+nTp2cYk7Ozc9Y28j9MmzaNbt26sXXrVvbs2cOgQYO097iKFSuW6eO+aTtftcpOecNvgmg0Gho0aMAXX3yR4XZPT883Ot67TBKOEG8hNDRUZ2Ry5coVNBoNbm5u/P777yQmJrJt2zadq+2Xp37+i1qtxs/PDz8/P6ZPn853333H6NGj2b9/v3ba6OzZs/j5+b3Wd2ZCQ0PTlYWEhGBubv7Kq3hXV1cgbWFDiRIltOVJSUmEhYVRv359nfo+Pj74+Pjw9ddfc+TIEWrVqsXChQuZNGkSJUuWRKPRcPHiRcqXL//a/fCm7fwvBQsWxNramvPnz//neePj49O1Ubw5uYcjxFuYN2+ezvs5c+YA0KRJE+0V9otX1HFxcSxbtuy1jx8dHZ2u7NmH9LOlwO3btyciIoKff/45Xd0nT57w+PFjnbKjR48SEBCgfX/z5k22bt1Kw4YNXzkqqF+/PsbGxsyePVunPUuWLCEuLo5mzZoB8PDhQ1JSUnT29fHxQa1Wa+P98MMPUavVTJgwId0o799GH2/azv+iVqv58MMP+f333zl16lS67c9iad++PUePHmX37t3p6sTGxqZrr3g1GeEI8RbCwsJo2bIljRs35ujRo6xevZpOnTpRrlw5TE1NMTY2pkWLFnz22WfEx8fz888/U6hQISIjI1/r+BMmTODQoUM0a9YMV1dXoqKimD9/PsWKFeO9994D4JNPPmH9+vX06dOH/fv3U6tWLVJTU7l06RLr16/XfrfnmbJly9KoUSOdZdGQtlT5VQoWLMiXX37J+PHjady4MS1btuTy5cvMnz+fKlWq8L///Q+Av/76iwEDBtCuXTs8PT1JSUlh1apVGBgY0KZNGwDc3d0ZPXo0EydO5P3336d169aYmJhw8uRJihQpwuTJkzOM4U3b+Tq+++479uzZQ506dbRLrSMjI9mwYQOHDx/G1taWESNGsG3bNpo3b063bt2oVKkSjx8/JigoiI0bN3L9+nUcHBze6LzvrBxdIydEHvVsWfTFixeVtm3bKlZWVoqdnZ0yYMAA5cmTJ9p627ZtU3x9fRVTU1PFzc1NmTJlirJ06VKdJceKkrYsulmzZunOs2/fPqVVq1ZKkSJFFGNjY6VIkSJKx44d0y3RTUpKUqZMmaKUKVNGMTExUezs7JRKlSop48ePV+Li4rT1AKV///7K6tWrFQ8PD8XExESpUKGCsn//fp3jvbws+pm5c+cqpUqVUoyMjJTChQsrffv2VWJiYrTbr127pvTo0UMpWbKkYmpqqhQoUED54IMPlD///DNd25YuXapUqFBBG2+dOnWUvXv3are/vCw6M+18maurq9K1a1edshs3bihdunRRChYsqJiYmCglSpRQ+vfvryQmJmrrPHr0SPnyyy8Vd3d3xdjYWHFwcFBq1qypTJ06VUlKSkp3HpExeZaaEJnw7EuQ9+7dk6tbIV6T3MMRQgihF5JwhBBC6IUkHCGEEHoh93CEEELohYxwhBBC6IUkHCGEEHohCUcIIYReSMIRQujFuHHj3ujZaSL/kUUDQohMq1u3LuXLl3+tX72Mj48nMTFR+2ub4t0jz1ITQmQrRVFITU3F0tISS0vLnA5H5CCZUhPiHVG3bl0GDhzIkCFDsLOzo3Dhwvz88888fvyY7t27Y2Vlhbu7Ozt37tTuc/78eZo0aYKlpSWFCxfmk08+4f79+wB069aNgwcPMmvWLFQqFSqViuvXr3PgwAFUKhU7d+6kUqVKmJiYcPjw4Qyn1JYuXUqZMmUwMTHBycmJAQMG6LNLhJ5JwhHiHbJixQocHBw4ceIEAwcOpG/fvrRr146aNWsSEBBAw4YN+eSTT0hISCA2NpZ69epRoUIFTp06xa5du7h79y7t27cHYNasWdSoUYNevXoRGRlJZGSkzo+gjRo1iu+//57g4GB8fX3TxbJgwQL69+9P7969CQoKYtu2bbi7u+utL0QOyMEHhwoh9KhOnTrKe++9p32fkpKiWFhYKJ988om2LDIyUgGUo0ePKhMnTlQaNmyoc4ybN28qgHL58mXtMQcPHqxTZ//+/QqgbNmyRad87NixSrly5bTvixQpoowePTqLWifyArmHI8Q75MWRhoGBAfb29vj4+GjLChcuDEBUVBRnz55l//79Gd53uXr16n/+tPK//TZNVFQUt2/fxs/P702bIPIwSThCvEOMjIx03qtUKp2yZz/drNFoiI+Pp0WLFkyZMiXdcZycnP7zXBYWFq/cZmZm9rohi3xEEo4QIkMVK1Zk06ZNuLm5YWiY8UeFsbExqampb3xsKysr3Nzc2LdvHx988MHbhiryCFk0IITIUP/+/YmOjqZjx46cPHmSq1evsnv3brp3765NMm5ubhw/fpzr169z//59NBrNax9/3LhxTJs2jdmzZxMaGkpAQABz5szJruaIXEASjhAiQ0WKFMHf35/U1FQaNmyIj48PQ4YMwdbWFrU67aNj+PDhGBgYULp0aQoWLEh4ePhrH79r167MnDmT+fPnU6ZMGZo3b05oaGh2NUfkAvKkASGEEHohIxwhhBB6IQlHCCGEXkjCEUIIoReScIQQQuiFJBwhhBB6IQlHCCGEXkjCEUIIoReScIQQQuiFJBwhhBDpZMczASThCCGE0KEoivbJ4evWrWPv3r1Zclx5WrQQQggtjUajfVbeiRMnmDVrFsbGxtjY2FC1atW3OraMcIQQ+d6zp1u/ydOs30WKomiTzfjx45k9ezaPHz/m6NGjjBw5kkOHDr3V8SXhCCHyrWnTptGpUyc6duzI6dOntR+mImPPptHmzJnDtGnT6NWrFzt27GDZsmVoNBomTZrE4cOHM3186X0hRL40fvx4pkyZgpmZGVFRUVSrVo1NmzbldFi5mkajQaPR4O/vT9u2balTpw5FixalU6dODB8+nIiICL7++muOHj2aqeNLwhFC5DsREREoisJvv/3GkiVL2LFjB0OGDKFjx45s3Lgxp8PLVV5cjZaUlIRarcbW1pb79++TmJio3daiRQs6duzIsWPHmDhxIkeOHHnjc0nCEULkK3v37sXZ2ZnVq1djbGwMgLm5OZMnT2bw4MF07txZRjoveDaNtmTJElauXAlA2bJlOXToEAcOHNBJSM7Ozvj5+aFSqVizZg1JSUlvdC5JOEKIfGPu3LkoisKAAQO4fv06d+/eBdKmioyMjPjuu+8YMmQI7dq1Y//+/Tkcbe6yadMmlixZAsCAAQNo2rQpnTp14rfffuPatWs8evSIzZs307x5c5o0acKyZcu0/fvaFCGEyAfmzZunODk5KceOHVMiIyOVzp07K5aWlsqxY8cURVEUjUajKIqiJCYmKnPnzlWSk5NzMtxcIyUlRVEURbl69apSokQJZeHChdptPXr0UJycnJQiRYoo7u7uiqenp6LRaJRTp04pHh4eSnh4+BudS35iWgiR5504cYLFixdTv3592rdvD0BUVBRDhgzh999/588//6RatWo6X2gESElJwdDw3fo64st98Kzs4cOHDBgwALVazYoVK7TbDhw4wL1790hOTqZDhw4YGBgwcOBAjhw5wr59+7C1tX2jkwshRJ71+++/K15eXkqRIkWUvXv36my7e/eu0qlTJ8XW1lY5dOhQDkWYOy1ZskQZM2aMkpKSoh3t7dmzR1Gr1crOnTsz3OfMmTNKz549FXt7eyUwMPCNzyn3cIQQeVqDBg2oV68ejx49YsOGDTx+/Fi7rVChQsycOZPq1aszYcKEHIwy91AUhXv37nHx4kXmzp1LnTp1mDhxInfv3qVBgwb07duXZcuW8eDBA539EhMTiY6O5u7du+zfv59y5cq98bllSk0IkSetX7+eAgUKUL9+fZKTkxkyZAhHjhyhc+fO9OvXD3Nzc23d2NhYrK2t39kvfr5q6jAuLo5x48Zx5swZgoKCmDRpEhcuXCAwMJDFixdTqlSpdPs8efIEMzOzTMUhCUcIkeecOHGCwYMHY25uzvfff0+VKlVISkpiwIABBAYG0qFDB/r27auTdED3OWHvguvXr+Pm5qZ9v2zZMs6ePYu3tzc1atTA19cXjUbDo0ePmDlzJocPH+bBgwcEBgYycuRIJk+enKXxSMIRQuQpEydOJCQkhDNnzhASEkLdunX55ptveO+990hKSmLgwIGcO3eORo0a8eWXX2JiYpLTIeeIcePG8eeffzJ//nx8fX356quvWLRoEb6+vkRERFC8eHG++OIL/Pz8tPsEBwcTEhLC6tWr+eWXXzAyMsrSmN6dVC+EyPNmzZrFjz/+SI8ePdi+fTsLFy4kISGBb7/9liNHjmBsbMycOXNwdnYmIiJC+8XPd1Hx4sUxMzNjzJgx7Nq1i9u3b7Nz507279/PvHnzsLCw4JtvvtH5PlKpUqVo1aoVGzZswMjIiOTk5CyNSUY4QohcT1EUFEWhffv22Nvbs2jRIu22TZs28fXXX1OsWDEmTZpEtWrVSE5ORq1WY2BgkOEy4HfFxo0bWbx4MRqNhsTERLZt24aNjQ0A+/fvZ86cOURFRTFp0iTq1q2b7fHICEcIkeupVCrUajU2Njba74Q806ZNG9q0acPff//NpEmTOH78OEZGRhgYGKDRaN7JZPNsHNG2bVu6d+9OQkICZ8+e5fr169o6H3zwAYMGDcLJyYnevXsTEBCQ7XFJwhFC5Bk+Pj4cOnQIf39/nXI3Nzfq1atHSkoKa9euJSUlBeCdWiDwIpVKpU06HTp0YMSIEXh7e/PVV18RGBiorVe3bl0+/fRT2rZtm6llzm8cl0ypCSHykjZt2uDv78/ixYvx9fXF3t6eTp060axZMx4+fMiECRO4fPkyTk5OOR1qjntxOnHDhg0sWrQICwsLJkyYkGGCSU1NxcDAINvikYQjhMgTXvww7NSpEwcPHkStVmNubo6iKISEhHDs2DG6du3K/v37KVKkSA5HnDu8mHTWr1/Pzz//jJWVFV999RWVK1fWayzv1kOEhBB51osLAH755Rf27t1LVFQUqampdO7cGYBVq1ZhY2ODhYVFDkebezybXlOpVLRv3x61Ws2kSZPYuHGj3hOOjHCEEHlKRtM+Z8+eZe7cuWzevJn9+/fj6+ubQ9HlXi+OdP766y/q1q2r93tc7+YdNSH0TK7r3pxGo8mw/FmyebY9MTGRu3fvcvPmzXcq2byqf15VrlKptNvq1auHWq3W+9+ljHCEyAZnzpzh/v37WFhYULNmTeDde6zKm9qwYQOJiYnY2trSvHlz4NU3sV+8Wr927RpFixYlNTU13aNs8quXFwNkpt+uXr1K0aJFMTU11WvgQogstHnzZsXGxkYpWrSo4unpqfTp00e7LTU1NQcjy71Gjhyp2NnZKSVKlFA8PDyUnj17arc9+4GwZ579kJqiKMrMmTOVEiVKKDdv3tRbrLlJZvtt1qxZOdJvknCEyCIajUZJSEhQmjRpoqxatUq5dOmSsmjRIsXV1VX5+OOPtfUk6Tyn0WiU+/fvK40bN1aCgoKU8PBwZfny5YqLi4tOnz378HzxQ3PhwoWKnZ2dsnbtWr3HnVOe/e3k1X6ThCNEFnn06JESExOjfPzxx8r169cVRVGUx48fK2vXrlWcnZ0l6WTgzp07yrlz55RWrVop9+/fVxRFt886duyorZuUlKT998KFCxVra2tl48aNeo85p7z4N5NX+00SjhBZYMuWLUrlypWV5s2bK66urjq/9f7sg6B48eJKs2bNcjDK3OWrr75SSpQooVSrVk3x8PBQYmJitNseP36s/Prrr4qbm5vSqFEjnf1++uknxcbG5p1KNi96sd/c3d2V6Oho7bbHjx8r69atU9zc3JSGDRvq7Jcb+k0SjhBv6fjx44qtra0yZMgQpW/fvkrBggWVFi1a6NRJSEhQli9frpQpU0a5detWDkWae6xatUpxdnZWFi5cqHzzzTdKwYIFlZYtW+rUSUhIUJYuXap89NFH2qv79evXKyqVStm8eXNOhJ0jXhzZrF69WnF2dlYWLVqkjB07VilUqFCGf2vLli3T6bcNGzbkin6TVWpCvIVz584REhLCpUuX+Prrr3n69ClHjx6lY8eOVK9enS1btmjrPnnyhJSUFKysrHIu4Fxgy5Yt3Lx5EwsLC3r06EFycjKHDx/OsM8SExN1fs8mPj6eY8eOUb9+/RyIPGfli37L0XQnRB4WHx+vODo6KiqVKt1KtP379yuFChVS2rRpk4MR5j737t1TLCwsFJVKpUyYMEFb/qzPHB0dlY8++ijDfZOTk/UVZq6TX/pNEo4QbyEoKEgpW7asUqlSJSUiIkJbnpqaqhw4cEAxMDBQOnfunIMR5j4XLlxQSpUqpdSsWVOJjIzUlms0GuXAgQOKSqVSRo4cmYMR5rwXV5U9kx/6TabUhHhDT548wdjYmJSUFExMTDh//jwNGzakXLlyrFq1CgcHByDti55HjhyhUKFCeHp65nDUOevBgwcYGhqiUqmwtrbm/PnzNGjQgPLly6frs7Nnz+Lr65utTy3OzV78gnB+6zdJOEK8gR07drB69WpCQ0OpWrUqTZs2pVmzZpw/f55GjRrh6+ur80EgYNKkSRw6dIirV69SvXp12rVrx4cffsiFCxe0iXrlypXp+iy7H5WfGykvPAkgX/Zbjo6vhMhDtm7dqpiamiqTJk1SFixYoHTo0EFRq9XKxYsXFUVRlPPnzysuLi5KjRo1tN+NeNd9/fXXir29vbJ582bljz/+UD744APFzs5O+w338+fPK87OzkqVKlWU2NjYHI4298iv/SYJR4jXEBMTo9SvX1+ZMWOGoiiKEhUVpRQpUkTp37+/Tr3AwEDF29tbuXHjRg5EmbvcuHFDqV69urJv3z5FURRl165dirW1tfLTTz8pivL8C4lnzpxRWrZsKV+G/Ud+7jdJOEK8hqioKMXd3V05ffq0EhERoRQtWlTp1auXdvvGjRuVkJAQRVEUJTExMafCzFUuX76sODk5KXfu3FG2bdumWFpaKgsWLFAUJe27IgsXLlSuXbums09e+vDMKi+3OT/3mzy6Voh/ERgYyM2bN7GxscHb25uAgABq1apF06ZNWbBgAQC3bt1i+/btXLx4EUVRMDY2zuGoc9Zvv/3GjRs3KFCgAN7e3ixcuJBPPvmEH3/8kT59+gAQEhLC3r17uXnzps6+79rTtF9cIPBO9FtOZzwhcqvffvtNKVKkiPL1118rqampSv/+/RWVSqXzDW5FUZRRo0YppUuX1nmczbvqyy+/VIoWLarMnj1bURRF6d27t6JSqZQRI0Zo68THxytNmzZVGjdunGeuzLPDi0uf35V+k5+YFiID27dvp1OnTsyePZvGjRujVquZO3cu8fHx7NixgylTpqBWq7l27Rpr167l77//xtnZOafDzlETJ07k559/ZseOHdpl4IsWLeLx48esXLmSJ0+eYGhoSGBgIPfv3ycgIAC1Wv3O/k7Qs9Vo71K/ScIR4iVPnz5lxYoVfP7553z66ackJCQQEhLCtm3baN26NbGxsRw6dIi7d+9StmxZjhw5QtmyZXM67BwVHR3NoUOHmDlzJlWqVCEiIoKAgADWrl1L48aNUalUPHjwgCdPnlCzZk3Gjx+PoaEhKSkpGBq+ux9D71q/5b2IhchmiqIQFhaGo6Mj0dHRjB07lnPnznHlyhWMjIwYNGgQvXv3Rq1WY2ho+M7fs4G0q/WLFy8SHBzMoUOHmD9/PmFhYWg0Gnbs2MHXX39Nnz59dL5nkpqamic/NLPSu9ZveWs8JoQemJmZMXDgQBYvXkzx4sWJiIigZ8+eRERE0KpVK/744w9MTU0xNzeXZPMPOzs7JkyYwPz582nRogWurq58++23nDx5krp163L06FHg+TQSkHu/nKhH71q/5c00KUQ269KlC5UrVyYiIoIGDRqg0WiAtKtLZ2fnPH2VmV169uxJgwYNSExMxMPDA0hbhXXnzh2qV6+ew9HlXu9Sv8mjbYR4DZcuXWLVqlXMmzePw4cPv/P3bP5LfHw8gYGBTJkyhRs3bhAQECAJ+jXk937LPy0RIpucPn2aadOmERgYyMGDByXZ/AdFUTh16hTTpk0jOTmZ06dPY2homLuf8ZULvAv9JiMcIf7DkydPOHXqFG5ubu/80ufXlZiYyMWLFylXrhxqtTrPrqrSt/zeb5JwhBDZKi9+XyQ3yI/9JglHCCGEXuSv9CmEECLXkoQjhBBCLyThCCGE0AtJOEIIIfRCEo4QQgi9kIQjhBBCLyThCCGE0AtJOEIIIfRCEo4QQgi9yD8P6RG5nuaOZ06HIN4Rn0dWyukQ8qRZFdZm6/FlhCOEEEIvJOEIIYTQC0k4Qggh9EISjhBCCL2QhCOEEEIvJOEIIYTQC0k4Qggh9EISjhBCCL2QhCOEEEIvJOEIIYTQC0k4Qggh9EISjhBCCL2QhJPHdOvWjQ8//DCnwxBCiDcmT4sWb6xbt27ExsayZcuWnA7lldb8Bkt/hfvRUKokjB4Mvt4Z1w0NgzlL4UII3L6jYtQAha7tdOvMXQbzlqt0yoq7KOxYlU0NyAFv0mfrf4dtu9P6DqC0F3zeS7f+nkOwbmtav8Y9VLF5sYK3R/a3Q99u7L5H2O93SYpLxsrFDO/uzti6W7yy/p1jMYSuv82Te0mYO5rg1akoBSvYaLenPE0l5Jfb3D0VS/KjFMwKmeDauCAuDQrqoznZSkY4uURSUlJOh5Bv7PgLpsyD/l1h08/gVRJ6DYcHMRnXf/oUnIvA0N7gUEB55XHdiysc2vz8tWZONjUgB7xpn50MhKZ+sHwmrJ0PTgXh0+Fw997zOk+eQEUfGPaZPlqQMyKPRHNp1S3c2zpRc3IprFzNODX5ColxyRnWj7kcz9nZYRT7wIGa35eicGVbAqZe49HNJ9o6l1be4v7Zh/j2d+O9aaVxa1KI4GU3iToVq6dWZR9JONnk0aNHdO7cGQsLC5ycnJgxYwZ169ZlyJAhALi5uTFx4kS6dOmCtbU1vXv3Zvny5dja2rJ79268vb2xtLSkcePGREZGpjv++PHjKViwINbW1vTp00cnYSUmJjJo0CAKFSqEqakp7733HidPntTZ/+DBg1StWhUTExOcnJwYNWoUKSkp2u0bN27Ex8cHMzMz7O3tqV+/Po8fP2bcuHGsWLGCrVu3olKpUKlUHDhwIFv6MLNWrId2zaF1U3B3g3HDwNQUNu/IuL6PN4zoC838wNj41cc1NICC9s9fdrbZEX3OeNM++3EMdPoIvD2ghCtM/AI0Gjh6+nmdVo2gfzeomY9/mub69iic6zlQrK49lsXMKPOpCwbGaiIOPMiw/o2dUTiUs6Z4i8JYFjXDo0MRrIubEb77eaaODXlMkdoFsC9jhXkhE5zrO2Dlakbs1QR9NSvbSMLJJkOHDsXf359t27axd+9e/v77bwICAnTqTJ06lXLlynHmzBnGjBkDQEJCAlOnTmXVqlUcOnSI8PBwhg8frrPfvn37CA4O5sCBA6xdu5bNmzczfvx47fYvvviCTZs2sWLFCgICAnB3d6dRo0ZER0cDEBERQdOmTalSpQpnz55lwYIFLFmyhEmTJgEQGRlJx44d6dGjh/Y8rVu3RlEUhg8fTvv27bWJMDIykpo1a2ZnV76RpOS0KZwaL3zIqdVp7wMvvN2xb9yC2q2hwccwYiLcvvt2x8stsqLPniZCSgrYWGdPjLmRJkXDw7AE7H2stGUqtQp7HytiQx5nuE9s6GPsfXQ7yaGctU59W08Lok7H8TQ6CUVReHDhEY8jE3HwtXr5cHmO3MPJBo8ePWLFihX88ssv+Pn5AbBs2TKKFCmiU69evXoMGzZM+/7vv/8mOTmZhQsXUrJkSQAGDBjAhAkTdPYzNjZm6dKlmJubU6ZMGSZMmMCIESOYOHEiT548YcGCBSxfvpwmTZoA8PPPP7N3716WLFnCiBEjmD9/Ps7OzsydOxeVSkWpUqW4ffs2I0eO5JtvviEyMpKUlBRat26Nq6srAD4+Ptrzm5mZkZiYiKOjY9Z33luKjYPUVBX2drpTY/Z2EBae+eP6esN3o6C4C9x7APOWw/8Gwu/LwcL8rULOcVnRZ1MXQiGH/D2aeVnSwxQUDRjb6H6MmtgY8jjiaYb7JMamZFDfSGcKrnR3Z87/HM6BfudRGQAqFWV7u1DAWxKOyMC1a9dITk6matWq2jIbGxu8vLx06lWuXDndvubm5tpkA+Dk5ERUVJROnXLlymFu/vxTrkaNGsTHx3Pz5k3i4uJITk6mVq1a2u1GRkZUrVqV4OBgAIKDg6lRowYq1fOb4LVq1SI+Pp5bt25Rrlw5/Pz88PHxoVGjRjRs2JC2bdtiZ2eXyR7J+2pXf/5vr5JpCcivA+zcD22b5VxcucHPa2DnX7BiFpiY5HQ0ed+NXfeIDX1MxRElMHMwJjo4notLb2JiZ4SDT94eQsqUWg6ysEi/ksXIyEjnvUqlQlFefSM7OxgYGLB371527txJ6dKlmTNnDl5eXoSFhek1jsywtQEDAyXdze4HMeBQIOvOY20FbsUgPCLrjplT3qbPlv4KP/8Ci6emJeJ3ibG1ISo1JMWl6JQnxqVgYmuU4T4mtoYZ1E/GxCatfmqShpBfb1Pqk2IUqmSLlas5ro0L4VTDjut/RGV0yDxFEk42KFGiBEZGRjo36uPi4ggJCcmS4589e5YnT56vajl27BiWlpY4OztTsmRJjI2N8ff3125PTk7m5MmTlC5dGgBvb2+OHj2qk8j8/f2xsrKiWLFiQFqiq1WrFuPHj+fMmTMYGxvz22+/AWlTeqmpqVnSlqxmbARlPOHYCzevNRo4FgDly2TdeR4nwM3bUDALk1hOyWyfLf4FFqyEn36AsqWyP87cRm2oxrq4OQ/OP9KWKRqFB+cfYeuZ8bJoWw8LHpx/qFP24Nzz+kqKgpKqoFK9tKNahaLR74VndpCEkw2srKzo2rUrI0aMYP/+/Vy4cIGePXuiVqt1prEyKykpiZ49e3Lx4kV27NjB2LFjGTBgAGq1GgsLC/r27cuIESPYtWsXFy9epFevXiQkJNCzZ08A+vXrx82bNxk4cCCXLl1i69atjB07lqFDh6JWqzl+/Djfffcdp06dIjw8nM2bN3Pv3j28vdO+ZOHm5sa5c+e4fPky9+/fJzk54yWgOaVre9iwHbbsgqvXYfz0tCW6H6Xd0mLktzD9p+f1k5IhODTtlZwMUffT/n3j1vM6P8yHE4EQEQlnzsPAr9NurDerr8+WZZ837bOff4HZS+HbkVDUMe2+1r0HaYn4mdiHaf145Uba+7Cbae/vZbyAK09ya1aIW3/dJ+LgA+IjnnBhyU1SEzUUrWMPwLl517m89vkw2LVJIe6ffUjYH3eJj3hK6IbbxF1LwKVR2ndsDM0NsPO25PKaCB5ceERCVCK3Djzg9qEHFK5imxNNzFJyDyebTJ8+nT59+tC8eXOsra354osvuHnzJqampm99bD8/Pzw8PKhduzaJiYl07NiRcePGabd///33aDQaPvnkEx49ekTlypXZvXu39h5M0aJF2bFjByNGjKBcuXIUKFCAnj178vXXXwNgbW3NoUOHmDlzJg8fPsTV1ZVp06ZpFyH06tWLAwcOULlyZeLj49m/fz9169Z963Zllab1ICY27QPxfjR4u8NPPz6fHoqMSksWz9y7D60/fX4hsPTXtFeV8gorZ6WV3bkHwyekfYgWsE37fsmvC9L+nR+8aZ/9uhWSk1UM/kb3OP27KQzonvbv/f7w1ffP+3XYeFW6OnmdU80CJD1MIXRDJImxyVi7mlF5lLt2Su3J/SR44RrTzsuScgOLE7LuNiG/3sbC0YSKw0tg5WymrVN+cHFC1kZwbu51kuNTMCtojMfHRXBu4KDv5mU5laLvGwTvqMePH1O0aFGmTZumHWm8azR3PHM6BPGO+DzyHVoul4VmVVibrceXEU42OXPmDJcuXaJq1arExcVplza3atUqhyMTQoicIQknG02dOpXLly9jbGxMpUqV+Pvvv3FwyPvDYiGEyAxJONmkQoUKnD59+r8rCiHEO0JWqQkhhNALSThCCCH0QhKOEEIIvZCEI4QQQi8k4QghhNALSThCCCH0QhKOEEIIvZCEI4QQQi8k4QghhNALSThCCCH0Qh5tI/TmQnLCf1cS6dirNTkdQp5z+E6JnA5BZEBGOEIIIfRCEo4QQgi9kIQjhBBCLyThCCGE0AtJOEIIIfRCEo4QQgi9kIQjhBBCLyThCCGE0AtJOEIIIfRCEo4QQgi9kIQjhBBCLyThCCGE0It3LuEcOHAAlUpFbGzsv9Zzc3Nj5syZeokpL5L+EUK8qXfuadE1a9YkMjISGxsbAJYvX86QIUPSJaCTJ09iYWGRAxFmj1e1M7/audWQbeuNiI1W4VpSQ88BSXiUyvipyzevq/h1uTHXQtXcu6umW99EmrdJeeWxf1trxJolxjRrnUz3fknZ1QS9+22LEevWGRMdraJkSQ2DBj7F2zvjPvvjDyP27DUkLMwAAE/PVD7tmahT/9AhQ37/3YiQUAMePlTx80+PcXfPf0++budSjS7F38fexJLQR3f44eIfXIi79cr69R3L0tejPk5mttxMeMDsy7vxvxei3X66ybcZ7jfz0k5WhR3O8vj16Z0b4RgbG+Po6IhKpfrXegULFsTc3FxPUeUeSUl5/wPUf78BKxYa0+6TZH5Y+AS3EhomjTIlLibj+olPVRR20tD50yRsC/z7B+KVS2r2bjfEtURqNkSec/7ab8iCBSZ07ZLIT4sSKFkylS9GmhMTk/F/J4FnDahXL4UZ0xOYNzeBQgUVRnxhzr17z+s/fQplfVLp3StRX83QuwaOPgz1bspPV/6i85F5hDy8w9wq3bAzzvhi1dfWhW/LtWfLrVN08p/HgbvBTKvYmZKWhbR1Gu6brPMad24TGkXDX3cu6KtZ2SZXJpy6desyYMAABgwYgI2NDQ4ODowZMwZFUQCIiYmhS5cu2NnZYW5uTpMmTQgNDdXuf+PGDVq0aIGdnR0WFhaUKVOGHTt2ALpTagcOHKB79+7ExcWhUqlQqVSMGzcO0J0y6tSpEx06dNCJMTk5GQcHB1auXAmARqNh8uTJFC9eHDMzM8qVK8fGjRtfu80XLlygefPmWFtbY2Vlxfvvv8/Vq1e1x54wYQLFihXDxMSE8uXLs2vXLu2+169fR6VSsXnzZj744APMzc0pV64cR48e1bb539o5ceJEunTpgrW1Nb179wZg06ZNlClTBhMTE9zc3Jg2bdprtyWn/b7JiPpNU6jXOAVnV4XeQ5IwMVH4a5dRhvXdS2no8lky732QilHGVQB48gRmTTahz+eJWFhmU/A5ZMMGY5o1TaZJkxTc3DQM/TwRUxOFnTsz7pCvRz/lw1bJuLtrcHHRMHz4UxQFAs4YaOs0bJhC1y5JVKr06tFiXve/4rX47eYpfo8IICz+Ht9d2MrT1GRaFauUYf2ObjU4ej+UVWGHuf74HgtC/+TSw9u0d62hrfMgKV7nVbewN6cehBHx5BVXTHlIrkw4ACtWrMDQ0JATJ04wa9Yspk+fzuLFiwHo1q0bp06dYtu2bRw9ehRFUWjatCnJyckA9O/fn8TERA4dOkRQUBBTpkzB0jL9J0TNmjWZOXMm1tbWREZGEhkZyfDhw9PV69y5M7///jvx8fHast27d5OQkMBHH30EwOTJk1m5ciULFy7kwoULfP755/zvf//j4MGD/9nWiIgIateujYmJCX/99RenT5+mR48epKSk/Yc6a9Yspk2bxtSpUzl37hyNGjWiZcuWOkkWYPTo0QwfPpzAwEA8PT3p2LEjKSkp/9nOqVOnUq5cOc6cOcOYMWM4ffo07du35+OPPyYoKIhx48YxZswYli9f/p9tyWnJyXAtRI1vxecjELUafCqmcvni2/25L55tTMVqqfhWyl/TQsnJEBKiplIl3T6rWCmVC6/ZZ4mJkJIC1lZKdoWZ6xiqDChlXYQT969oyxQUTty/go+tS4b7+Nq6cPzBVZ2yo/eu4GvrnGH9AsYWvFfQi623TmVd4Dko197DcXZ2ZsaMGahUKry8vAgKCmLGjBnUrVuXbdu24e/vT82aNQFYs2YNzs7ObNmyhXbt2hEeHk6bNm3w8fEBoESJjH/9z9jYGBsbG1QqFY6Ojq+MpVGjRlhYWPDbb7/xySefAPDLL7/QsmVLrKysSExM5LvvvuPPP/+kRo0a2nMePnyYRYsWUadOnX9t67x587CxseHXX3/F6J9LbE9PT+32qVOnMnLkSD7++GMApkyZwv79+5k5cybz5s3T1hs+fDjNmjUDYPz48ZQpU4YrV65QqlSpf21nvXr1GDZsmPZ9586d8fPzY8yYMdpYLl68yI8//ki3bt3+tS057VGcCo1GhY2d7gefrZ1CxM3MJ5zD+w0ICzXg+/lP3jbEXCfunz6zs9NNpHZ2CuHhBq/YS9ein0xwsFd0klZ+Z2tsjqHagAdJ8TrlD5LicbMsmOE+9iaWRCfq1o9OisfexCrD+s2LVuRxSiJ/3b2YNUHnsFw7wqlevbrOfZYaNWoQGhrKxYsXMTQ0pFq1atpt9vb2eHl5ERwcDMCgQYOYNGkStWrVYuzYsZw7d+6tYjE0NKR9+/asWbMGgMePH7N161Y6d+4MwJUrV0hISKBBgwZYWlpqXytXrtROi/2bwMBA3n//fW2yedHDhw+5ffs2tWrV0imvVauWtr3P+Pr6av/t5OQEQFRU1H+ev3Llyjrvg4ODMzxfaGgoqanvzgfKM/ejVCybZ8Kgr55ibJzT0eQ+v/xizP79RkyY8ET6J4u1KlaJnbfPkqTJH9OSuXaE8zY+/fRTGjVqxPbt29mzZw+TJ09m2rRpDBw4MNPH7Ny5M3Xq1CEqKoq9e/diZmZG48aNAbRTbdu3b6do0aI6+5mYmPznsc3MzDId14teTFjPkrVG89/TP/lpNZ6VjYJarRD30s3u2BgVtnaZm+65FqomLlbFF32e//+k0agIDlKzc4sha3cmYPB6A4FcyeafPouJUQPP/15iYlQU+I9FFOvWGfHLWmOmTU2gZMn8NdX4X2KTEkjRpGJvrDtdb29syf2XRjHPPEiMp4CJbv0CxpY8SHyUrm55O1fcLAsyKvDXrAs6h+XaEc7x48d13h87dgwPDw9Kly5NSkqKzvYHDx5w+fJlSpcurS1zdnamT58+bN68mWHDhvHzzz9neB5jY+PXumqvWbMmzs7OrFu3jjVr1tCuXTvtB3zp0qUxMTEhPDwcd3d3nZezc8Zzsy/y9fXl77//1t6DepG1tTVFihTB399fp9zf31+nvf/lddsJ4O3tneH5PD09Mcjln6xGRlDCU0NQwPM4NRoIOmOAV+nMfSD6VEhl+s8JTF30RPsq6ZnK+36pTF30JE8nG0jrM09PDQEv9VlAgAFl/qXP1v5qzKrVJvwwJQEvr3cr2QCkKKlcenibKvYltWUqVFRxKElQbHiG+5yLDafqC/UBqjmU5FzszXR1PyxWmYtxEYQ+upO1geegXDvCCQ8PZ+jQoXz22WcEBAQwZ84cpk2bhoeHB61ataJXr14sWrQIKysrRo0aRdGiRWnVqhUAQ4YMoUmTJnh6ehITE8P+/fvx9vbO8Dxubm7Ex8ezb98+ypUrh7m5+SuXQ3fq1ImFCxcSEhLC/v37teVWVlYMHz6czz//HI1Gw3vvvUdcXBz+/v5YW1vTtWvXf23rgAEDmDNnDh9//DFffvklNjY2HDt2jKpVq+Ll5cWIESMYO3YsJUuWpHz58ixbtozAwEDtFN/reJN2Dhs2jCpVqjBx4kQ6dOjA0aNHmTt3LvPnz3/t8+WkFm2SmfuDCSW9NLh7pbJ9sxGJT1V80Dgtoc/+3hh7B4XOn6a9T06GWzfSrr1SUiD6voqwK2pMzRSciiqYmYNLcd3RkYkpWFkr6crzqnbtkvj+e1M8vVLxLqVh4yYjnj5V0fifPvtusikFHTT06pW2bH7tWmOWLTdm9OinODoqREenjSjNzBSeDdgfPoSoKDX376dtC//nHlqBAgoFCuSPflsd5s943zYEP4zgfOwtOrnVxMzAmG23TgMw3rct954+ZG7IHgDWXj/Kz9U+5X9utTh87zINnXwpbVOUb89v0TmuhaEJ9R3LMuPSTn03KVvl2oTTpUsXnjx5QtWqVTEwMGDw4MHaJbvLli1j8ODBNG/enKSkJGrXrs2OHTu0I47U1FT69+/PrVu3sLa2pnHjxsyYMSPD89SsWZM+ffrQoUMHHjx4wNixY7VLhl/WuXNnvv32W1xdXdPd45g4cSIFCxZk8uTJXLt2DVtbWypWrMhXX331n221t7fnr7/+YsSIEdSpUwcDAwPKly+vPcegQYOIi4tj2LBhREVFUbp0abZt24aHh8frducbtbNixYqsX7+eb775hokTJ+Lk5MSECRNy/YKBZ2p9kMrDuCR+XW5EbIwxbiU1jJ78FFu7tO33o9So1S9MHT1QMeKF6bJtG4zZtgFK+6YyYfpTfYefI+p9kEJcbCLLl5kQHZP2xc8pUxK0iSEqSoVa/XxCZOs2I5KTVYwbpzsd3LVLIt26pSWlI0cMmfLD8+0TJ5qlq5PX7b0ThJ2xBX08/LA3sSLkYSQDTy4nOukxAI6mNtqvc0DaCGf02fX09ahPf6+GhD9+wLCANVyN173X2tDJF5UKdkee1Wt7sptKebE3com6detSvnx5eXRKPhN0s1hOh5An2avfvemqt9XiXPecDiFPetVTDrJKrr2HI4QQIn+RhKMHffr00Vku/eKrT58+OR2eEELoRa6cUstvoqKiePjwYYbbrK2tKVSoUIbb8huZUsscmVJ7czKlljnZPaWWaxcN5CeFChV6Z5KKEEK8ikypCSGE0AtJOEIIIfRCEo4QQgi9kIQjhBBCLyThCCGE0AtJOEIIIfRCEo4QQgi9kIQjhBBCLyThCCGE0AtJOEIIIfRCHm0jhMh3UjVyLZ0byf8rQggh9EISjhBCCL2QhCOEEEIvJOEIIYTQC0k4Qggh9EISjhBCCL2QhCOEEEIvJOEIIYTQC0k4Qggh9EISjhBCCL2QhCOEEEIvJOEIIYTQC0k4Qggh9EKeFi3ypZ1bDdm23ojYaBWuJTX0HJCERylNhnVvXlfx63JjroWquXdXTbe+iTRvk/LKY/+21og1S4xp1jqZ7v2SsqsJevfbFiPWrTMmOlpFyZIaBg18ird3xn32xx9G7NlrSFiYAQCenql82jNRp/6hQ4b8/rsRIaEGPHyo4uefHuPunvHx8rIOrlXpWuI97E0sCXl4hykXtnM+LuKV9Rs4lqGflx9FzGwJfxzNrEu7OXwvVLs9sNnEDPebEbyLFdf8szx+fZIRTjZISso/H0LPvKpNycnJeo7kv/nvN2DFQmPafZLMDwuf4FZCw6RRpsTFZFw/8amKwk4aOn+ahG2Bf/9AvHJJzd7thriWSM2GyHPOX/sNWbDAhK5dEvlpUQIlS6byxUhzYmJUGdYPPGtAvXopzJiewLy5CRQqqDDiC3Pu3Xte/+lTKOuTSu9eifpqht41dCrLMO8mLArdT8fDCwh5dIf51bpiZ2yRYf1yds5MrtCOLTdP8/HhBey/G8yMyp0oaVlIW8fvzyk6r7FnN6NRNPwZeVFfzco2+T7h1K1blwEDBjBgwABsbGxwcHBgzJgxKIoCQGJiIsOHD6do0aJYWFhQrVo1Dhw4oN3/wYMHdOzYkaJFi2Jubo6Pjw9r167N8BxDhgzBwcGBRo0aoSgK48aNw8XFBRMTE4oUKcKgQYO0+8TExNClSxfs7OwwNzenSZMmhIY+v8pZvnw5tra27N69G29vbywtLWncuDGRkZGv3falS5dSpkwZTExMcHJyYsCAAdpt4eHhtGrVCktLS6ytrWnfvj13797Vbh83bhzly5dn8eLFFC9eHFNTUwBUKhULFiygZcuWWFhY8O233752PPry+yYj6jdNoV7jFJxdFXoPScLEROGvXUYZ1ncvpaHLZ8m890EqRhlXAeDJE5g12YQ+nydiYZlNweeQDRuMadY0mSZNUnBz0zD080RMTRR27sy4Q74e/ZQPWyXj7q7BxUXD8OFPURQIOGOgrdOwYQpduyRRqdKrR4t53SfFa7L55im23jrDtfh7TAr6naepyXzoXDHD+p3canDk3hVWXPMnLP4e80P2ERwXycdu1bR1HiTG67zqFvbm5IMwIp684oopD8n3CQdgxYoVGBoacuLECWbNmsX06dNZvHgxAAMGDODo0aP8+uuvnDt3jnbt2tG4cWPth//Tp0+pVKkS27dv5/z58/Tu3ZtPPvmEEydOpDuHsbEx/v7+LFy4kE2bNjFjxgwWLVpEaGgoW7ZswcfHR1u/W7dunDp1im3btnH06FEURaFp06Y6I4aEhASmTp3KqlWrOHToEOHh4QwfPvy12rxgwQL69+9P7969CQoKYtu2bbi7uwOg0Who1aoV0dHRHDx4kL1793Lt2jU6dOigc4wrV66wadMmNm/eTGBgoLZ83LhxfPTRRwQFBdGjR4/X/z9CD5KT4VqIGt+Kz0cgajX4VEzl8sW3+3NfPNuYitVS8a2Uv6aFkpMhJERNpUq6fVaxUioXXrPPEhMhJQWsrZTsCjPXMVQZ4G1ThOP3r2nLFBSO37+Kr61zhvv42jlz/P5VnbKj967ga+eSYf0Cxha8V8iTLTcDsi7wHPRO3MNxdnZmxowZqFQqvLy8CAoKYsaMGTRq1Ihly5YRHh5OkSJFABg+fDi7du1i2bJlfPfddxQtWlTnQ37gwIHs3r2b9evXU7VqVW25h4cHP/zwg/b99u3bcXR0pH79+hgZGeHi4qKtHxoayrZt2/D396dmzZoArFmzBmdnZ7Zs2UK7du2AtOmqhQsXUrJkSSAtOU6YMOG12jxp0iSGDRvG4MGDtWVVqlQBYN++fQQFBREWFoazc9p/GCtXrqRMmTKcPHlSWy8pKYmVK1dSsGBBnWN36tSJ7t27v1Yc+vYoToVGo8LGTveDz9ZOIeJm5hPO4f0GhIUa8P38J28bYq4T90+f2dnpJlI7O4XwcINX7KVr0U8mONgrOkkrv7MzNsdQbcCDxHid8geJ8bhZOGS4j4OJJQ+SXqqfFI+DScZD5pbFKpCQksi+O3l/Og3ekRFO9erVUamezy3XqFGD0NBQgoKCSE1NxdPTE0tLS+3r4MGDXL2adhWSmprKxIkT8fHxoUCBAlhaWrJ7927Cw8N1zlGpUiWd9+3atePJkyeUKFGCXr168dtvv5GSkja1EBwcjKGhIdWqPR9G29vb4+XlRXBwsLbM3Nxcm2wAnJyciIqK+s/2RkVFcfv2bfz8/DLcHhwcjLOzszbZAJQuXRpbW1ud87u6uqZLNgCVK1f+zxjyk/tRKpbNM2HQV08xNs7paHKfX34xZv9+IyZMeCL9k8VaOVdkx+1zJGnyx7TkOzHCeZX4+HgMDAw4ffo0Bga6V3KWlmlXHD/++COzZs1i5syZ+Pj4YGFhwZAhQ9LdRLew0L1J6OzszOXLl/nzzz/Zu3cv/fr148cff+TgwYOvHZ/RSzcUVCqV9t7TvzEzM3vtc/ybl9v0X+W5gZWNglqtEPfSze7YGBW2dpmb7rkWqiYuVsUXfZ73q0ajIjhIzc4thqzdmYDB6w0EciWbf/osJkYNPB/lxMSoKPAfiyjWrTPil7XGTJuaQMmS+Wuq8b/EJCWQoknF/qXRib2JJfdfGvU8cz8xHnvjl+obZ1y/gp0rxS0LMjJgfdYFncPeiRHO8ePHdd4fO3YMDw8PKlSoQGpqKlFRUbi7u+u8HB0dAfD396dVq1b873//o1y5cpQoUYKQkJDXOq+ZmRktWrRg9uzZHDhwgKNHjxIUFIS3tzcpKSk6cT148IDLly9TunTpt26vlZUVbm5u7Nu3L8Pt3t7e3Lx5k5s3b2rLLl68SGxsbJacPycZGUEJTw1BAc8zgEYDQWcM8CqduQ9EnwqpTP85gamLnmhfJT1Ted8vlamLnuTpZANpfebpqSHgpT4LCDCgzL/02dpfjVm12oQfpiTg5fVuJRuAFCWV4LjbVHUooS1ToaKqfQnOxd7McJ9zMTd16gNUL1iSczHh6ep+5FyRC7ERhDy6k7WB56B3YoQTHh7O0KFD+eyzzwgICGDOnDlMmzYNT09POnfuTJcuXZg2bRoVKlTg3r177Nu3D19fX5o1a4aHhwcbN27kyJEj2NnZMX36dO7evfufH8zLly8nNTWVatWqYW5uzurVqzEzM8PV1RV7e3tatWpFr169WLRoEVZWVowaNYqiRYvSqlWrLGnzuHHj6NOnD4UKFaJJkyY8evQIf39/Bg4cSP369fHx8aFz587MnDmTlJQU+vXrR506dfLFdFmLNsnM/cGEkl4a3L1S2b7ZiMSnKj5onLYgY/b3xtg7KHT+NO19cjLcupF27ZWSAtH3VYRdUWNqpuBUVMHMHFyK646OTEzBylpJV55XtWuXxPffm+LplYp3KQ0bNxnx9KmKxv/02XeTTSnooKFXr7SR/dq1xixbbszo0U9xdFSIjk4bUZqZKTwbYD98CFFRau7fT9sW/s89tAIFFAoUyB/9tirsCBPLteZibATn4yLo7FYDM0Njtv5zk39iuTZEPX3InMt7Afjl+lEWV+/JJ8Vr8ndUCI2L+FDapggTzm3VOa6FoQkNnMoyLXiX3tuUnd6JhNOlSxeePHlC1apVMTAwYPDgwfTu3RuAZcuWaW+wR0RE4ODgQPXq1WnevDkAX3/9NdeuXaNRo0aYm5vTu3dvPvzwQ+Li4v71nLa2tnz//fcMHTqU1NRUfHx8+P3337G3t9eed/DgwTRv3pykpCRq167Njh070k2jZVbXrl15+vQpM2bMYPjw4Tg4ONC2bVsgbWpu69atDBw4kNq1a6NWq2ncuDFz5szJknPntFofpPIwLolflxsRG2OMW0kNoyc/xdYubfv9KDVq9QtTRw9UjHhhumzbBmO2bYDSvqlMmP5U3+HniHofpBAXm8jyZSZEx6R98XPKlARtYoiKUqFWP58Q2brNiORkFePG6U7fdu2SSLduaUnpyBFDpvzwfPvEiWbp6uR1eyLPY2dsQV9PPxxMLLn8MJJ+J1YSnfQYACczGxTl+d/a2ZibfHVmA/296jPQqwHhCQ/4/NQvXI3XvTfb2MkHVLDr9jm9tie7qZTXuSmQh9WtW5fy5cszc+bMnA7lnRd0s1hOh5An2avfvemqt9U0sGdOh5AnveopB1nlnbiHI4QQIudJwsmjXlzG/fLr77//zunwhBAinXx/D+fFx9TkJy9+8/9lRYsW1V8gQgjxmvJ9wsmvnj2mRggh8gqZUhNCCKEXknCEEELohSQcIYQQeiEJRwghhF5IwhFCCKEXknCEEELoxWsvi962bdtrH7Rly5aZCkYIIUT+9doJ58MPP9R5//Jvs7z4A2epqe/Or/4JIYR4Pa89pabRaLSvPXv2UL58eXbu3ElsbCyxsbHs2LGDihUrsmtX/nqcthBCiKyRqadFly1bloULF/Lee+/plP/999/07t1b52eKhXjm7+vydITMWBtdPadDyHPmFjmR0yHkSWrH1/txyUwfPzM7Xb16FVtb23TlNjY2XL9+/S1DEkIIkR9lKuFUqVKFoUOHcvfuXW3Z3bt3GTFiBFWrVs2y4IQQQuQfmUo4S5cuJTIyEhcXF9zd3XF3d8fFxYWIiAiWLFmS1TEKIYTIBzL1tGh3d3fOnTvH3r17uXTpEgDe3t7Ur19fZ7WaEEII8Uymf55ApVLRsGFDGjZsmJXxCCGEyKcy/aSBgwcP0qJFC+2UWsuWLeWXJoUQQrxSphLO6tWrqV+/Pubm5gwaNIhBgwZhamqKn58fv/zyS1bHKIQQIh/I1PdwvL296d27N59//rlO+fTp0/n555/lezgiQ/I9nMyR7+G8OfkeTubkyu/hXLt2jRYtWqQrb9myJWFhYW8dlBBCiPwnUwnH2dmZffv2pSv/888/cXZ2fuughBBC5D+ZWqU2bNgwBg0aRGBgIDVr1gTA39+f5cuXM2vWrCwNUAghRP6QqYTTt29fHB0dmTZtGuvXrwfS7uusW7eOVq1aZWmAQggh8odMfw/no48+4qOPPsrKWEQucf36dYoXL86ZM2coX758TocjhMgnMp1wAJKSkoiKikKj0eiUu7i4vFVQQn+6detGbGwsW7Zs0ZY5OzsTGRmJg4NDzgX2lv7apmL3RhVx0eBcAjr201CiVMZ1I67D1pVqblyBB3dVdPhMQ4PWr168uWOdis1L1dT/UMPHfd94kWeudXPPXcJ/v0NSXDKWLuZ4dnPBxt0yw7rxN59wbWMEj6495un9JDw+ccalqaNOHf+BZ3l6PyndvkUbFKJUD9dsaUNOWPMbLP0V7kdDqZIwejD4emdcNzQM5iyFCyFw+46KUQMUurZ7u2PmJZlaNBAaGsr777+PmZkZrq6uFC9enOLFi+Pm5kbx4sWzOkaRDVJTU9NdKDxjYGCAo6MjhoZvdT2SY04cULH+JxUtOit8M0+DcwmFmaPVPIzNuH5SIhR0UmjTQ8GmwL8nkLDLcGi7imLF80+iAbh79AGhq25SvE0RqnxXBktXcwK/DyEpLjnD+pqkVMwKmVCyYzGMbY0yrFPl29K8t6C89lXhK08ACle3y7Z26NuOv2DKPOjfFTb9DF4loddweBCTcf2nT8G5CAztDQ6v+Ft702PmJZlKON26dUOtVvPHH39w+vRpAgICCAgI4MyZMwQEBGR1jAKoW7cuAwYMYMCAAdjY2ODg4MCYMWO0v7oaExNDly5dsLOzw9zcnCZNmhAaGqrdf/ny5dja2rJt2zZKly6NiYkJPXr0YMWKFWzduhWVSoVKpeLAgQNcv34dlUpFYGAgkJacevbsSfHixTEzM8PLyytXLw7Zu1nF+40V3mukUMQV/jdIwdgEDu/O+Dl/xb2gXS+FqnUVDDP+7ATg6RNYPEVNlyEazK2yKfgcEr79LkXrFaRI3YJYFjOjVE9XDIzV3D5wP8P61iUt8ejsjGNNe9SGGfersbURJrbPX/cD4jArbIKtd/7pvBXroV1zaN0U3N1g3DAwNYXNOzKu7+MNI/pCMz8wNs6aY+YlmbqEDQwM5PTp05Qq9Yo5CpEtVqxYQc+ePTlx4gSnTp2id+/euLi40KtXL7p160ZoaCjbtm3D2tqakSNH0rRpUy5evIiRUdqnaEJCAlOmTGHx4sXY29vj5OTEkydPePjwIcuWLQOgQIEC3L59W+e8Go2GYsWKsWHDBuzt7Tly5Ai9e/fGycmJ9u3b670f/k1KMtwIhaYfP796VKvBu4LCtYsqIPMjkzVzVfhUVShdEf5YmwXB5hKaFA2Pwh7j1spJW6ZSq7Ara01caHyWnePO4Qe4NCucbx7wm5ScNjXWq/PzMrUaalSCwAu555i5SaYSTunSpbl/P+MrH5F9nJ2dmTFjBiqVCi8vL4KCgpgxYwZ169Zl27Zt+Pv7a5epr1mzBmdnZ7Zs2UK7dmmTxMnJycyfP59y5cppj2lmZkZiYiKOjo4ZnhPAyMiI8ePHa98XL16co0ePsn79+lyXcOIfgkajwtpWt9zaDu7czPxxTxxQEX5FxddzMp6GzMuSH6agaMDYRnd4Z2xjRMLtp1lyjnsnY0lJSMGpdt69L/iy2DhITVVhb6d7EWNvB2HhueeYuclrT6k9fPhQ+5oyZQpffPEFBw4c4MGDBzrbHj58mJ3xvtOqV6+uc3VYo0YNQkNDuXjxIoaGhlSrVk27zd7eHi8vL53HDBkbG+Pr65upc8+bN49KlSpRsGBBLC0t+emnnwgPzwf/BbyG6ChYu0DFpyM1GL1iGkT8u9sH7mFf3gaTAtKB77LXHuHY2trqfNgpioKfn59OHUVRUKlUpKamZl2EIsuYmZllajrj119/Zfjw4UybNo0aNWpgZWXFjz/+yPHjx7MhyrdjaQ1qtZJugcDDGLDJ5L3qG1fgUayKif2fX59pNCpCg9JWwy38Q4PaIPMx5zQja0NUatItEEiKS37lgoA38eReItFBD/Edmr+epWdrAwYGSrqb+Q9iwKFA7jlmbvLaCWf//v3ZGYd4DS9/wB87dgwPDw9Kly5NSkoKx48f106pPXjwgMuXL1O6dOl/PaaxsfF/XiA8m6rr16+ftuzq1auZbEX2MjQCVw8IPqOiQs20aQmNBi4FqvigZebu33iXh/GLdPto2TQ1js4KTdoreTrZAKgN1VgVtyD6/EMKVknLyopGIebCQ4o1LPzWx488eB9jGyPsK9i+9bFyE2MjKOMJx05D/ffTyjQaOBYAnTP5FcXsOGZu8toJp06dOtp/h4eH4+zsnO5qWVEUbt58i4ly8a/Cw8MZOnQon332GQEBAcyZM4dp06bh4eFBq1at6NWrF4sWLcLKyopRo0ZRtGjR/3zyg5ubG7t37+by5cvY29tjY2OTro6HhwcrV65k9+7dFC9enFWrVnHy5MlcuwS+QWuFpVNVuHpCcS+FP39TkfgUajVMSzhLflBh6wBteqS9T0mG2//MDqYkQ+wDCL8KJqZQuCiYmkNRN91zGJuCpVX68rzKpVlhLi4Iw7qEBdbuFoTvvEtqoganOmn3XC7Mv4aJnRHuHdOelahJ0fD41tN//q2QGJPMo+sJGJiqMXc01R5X0ShEHryPU2171Ab5Y7HAi7q2hy8nQ9lS4FMKVm6EJ0/goyZp20d+C4ULpi2DhrRFAVevp/07ORmi7kNwKJibgWux1ztmXpapRQPFixcnMjKSQoUK6ZRHR0dTvHhxmVLLJl26dOHJkydUrVoVAwMDBg8eTO/eaX/Jy5YtY/DgwTRv3pykpCRq167Njh07tCvUXqVXr14cOHCAypUrEx8fz/79+3Fzc9Op89lnn3HmzBk6dOiASqWiY8eO9OvXj507d2ZXU99K1boK8XGwdaWKhzEqnEvAkG812im1B/dUqNTPRzuxD2BCv+fDlN0bVezeCJ6+Cl/8mP8WCWSkcA17kh6mcG1jBImxyVi5mlN+lCcm/0ypPb2fxIvXl4kxyZz48vmyqfA/7hD+xx1sva2o9M3z1avR5x/y9H4SReoW1Ftb9KlpPYiJhdlL076k6e0OP/34fPorMiptldkz9+5D60+fd+TSX9NeVcorrJz1esfMyzL1ezhqtZq7d+9SsKDuH9GNGzcoXbo0jx8/zrIARZq6detSvnx5Zs6cmdOhZJr8Hk7myO/hvDn5PZzMye7fw3mjEc7QoUMBUKlUjBkzBnNzc+221NRUjh8/Ls/eEkIIkaE3SjhnzpwB0u7VBAUFYfzCV2WNjY0pV64cw4cPz9oIhRBC5AtvlHCerVTr3r07s2bNwtraOluCEukdOHAgp0MQQoi3kqlnqS1btgxra2uuXLnC7t27efLkCQCZuB0khBDiHZGphBMdHY2fnx+enp40bdqUyMhIAHr27MmwYcOyNEAhhBD5Q6YSzpAhQzAyMiI8PFxn4UCHDh3YtWtXlgUnhBAi/8jU93D27NnD7t27KVasmE65h4cHN27cyJLAhBBC5C+ZGuE8fvxYZ2TzTHR0NCYmJm8dlBBCiPwnUwnn/fffZ+XKldr3KpUKjUbDDz/8wAcffJBlwQkhhMg/MjWl9sMPP+Dn58epU6dISkriiy++4MKFC0RHR+Pv75/VMQohhMgHMjXCKVu2LJcvX+a9996jVatWPH78mNatW3PmzBlKliyZ1TEKIYTIBzI1wgEwNTWlQYMGlCtXDo0m7QGHJ0+eBKBly5ZZE50QQoh8I1MJZ9euXXzyySdER0en+7Kn/ACbEEKIjGRqSm3gwIG0b9+e27dvo9FodF6SbIQQQmQkUz9PYG1tLfdrxBur1+D7nA4hT0q2yOM/KZoDLI+F5XQIedLOO/Oz9fiZGuG0bdtWHiYphBDijWTqHs7cuXNp164df//9Nz4+Pul+VXLQoEFZEpwQQoj8I1MJZ+3atezZswdTU1MOHDiA6oXfnlWpVJJwhBBCpJOphDN69GjGjx/PqFGjUKszNSsnhBDiHZOpbJGUlESHDh0k2QghhHhtmcoYXbt2Zd26dVkdixBCiHwsU1Nqqamp/PDDD+zevRtfX990iwamT5+eJcEJIYTIPzKVcIKCgqhQoQIA58+f19n24gICIYQQ4plMJZz9+/dndRxCCCHyObnrL4QQQi8k4QghhNALSThCCCH0QhKOeKW6desyZMiQnA5DCJFPZPoH2ET+t3nz5nRL3vOKVi0r0qFdNQoUsODq1SjmzNvLpcuRr6xfp7YX3bvWxtHRhlsR0fy8+ADHT1zTbu/6yXt8UNebggWtSEnREBJ6hyXLDnLp0quPmdd81KQCHT+qQgFbC65ej2Lmz/sIDr3zyvp1a3ryaaf3cCxkw63IGBauPMix07pPaXYtVoA+XepQvowzBgYqrt98wNdTthJ1/1F2N0dvmnevTdt+DbAraM21i7dYMHo9IWduvLL+ey0q0OWLFhR2ticiLIplk7Zwct8F7faaTcvTrMv7uPs6Y13Akv5+33Htwi19NCXbyQhHvFKBAgWwsrLK6TDeWN06pej7WT1Wrj7MZ32XcfVaFFMmd8DW1jzD+mVKF+Xrr1qxc9dZevddhr9/KBPGtcHNzUFb5+ataGbP3cOnvZcw+PPV3Lkbxw/fd8DGxkxfzcpW9Wp5MaBHXZb/eoRPh67kyvV7TBvbDlubjPusrFcRxg5rwfY/g+g5dAV/Hw/lu1EfUdzleZ8VcbRl3nedCI+IZtDXv9JtyApWrD9KUnL++c2s2q0q0XtcG9ZM287AhpMJuxDBpLUDsXGwzLC+d+USjFrQg91rjzCgwWSO7jzLmGWf4VrKSVvH1NyYCyeusHTSFj21Qn8k4eRyGzduxMfHBzMzM+zt7alfvz6PHz8GYOnSpZQpUwYTExOcnJwYMGCAdr/Y2Fg+/fRTChYsiLW1NfXq1ePs2bPa7ePGjaN8+fKsWrUKNzc3bGxs+Pjjj3n06PmV58tTaomJiYwcORJnZ2dMTExwd3dnyZIl2d8Jb6hdm6rs2HmWXbuDuBH+gBmzdpGYmEyTRr4Z1m/9UWVOnLzGug0nCA9/wLIVfxN65Q4ftqqkrfPX/osEnLlB5J04rt+4z4KF+7C0MKVEiUL6ala26tCqMr/vOceOv85z/dYDpi7Yw9PEZJr5lc2wftsWlTgREMbaLSe5cSuaJb/4E3LtLq2bVtDW6d35PY4FXGPBioOEhkVx+04s/ievEhuXoK9mZbuPPqvHzjX+7P31GOEhd5jzxVoSnyTR8OOaGdZv1esDTu2/yKb5f3Iz9A6rfviDq0E3adG9rrbOXxtP8Mv0nZz5+5J+GqFHknByscjISDp27EiPHj0IDg7mwIEDtG7dGkVRWLBgAf3796d3794EBQWxbds23N3dtfu2a9eOqKgodu7cyenTp6lYsSJ+fn5ER0dr61y9epUtW7bwxx9/8Mcff3Dw4EG+//7VP5LWpUsX1q5dy+zZswkODmbRokVYWmZ8JZdTDA3VeHo6cjrgurZMUeB0wHVKly6a4T6lSxch4IX6ACdPhVHGO+P6hoZqmjctT3z8U65ejcqq0HOMoaEaz5KOnD73fBpIUeDU2RuU8SqS4T5lvYpw6pzutNGJM9cp+099lQpqVC7JzdsxTBvblm3L+7Hoh868X809o8PlSYZGBnj4uhB46LK2TFEUAv++hHfl4hnu412pOIGHdBPJ6QMXX1k/v5F7OLlYZGQkKSkptG7dGldXVwB8fHwAmDRpEsOGDWPw4MHa+lWqVAHg8OHDnDhxgqioKExMTACYOnUqW7ZsYePGjfTu3RsAjUbD8uXLtdNmn3zyCfv27ePbb79NF0tISAjr169n79691K9fH4ASJUpkU8szz8bGHAMDNTExj3XKY2Ie4+Jsn+E+BewsiYlNX9+ugIVOWfVqJRkzuhUmJkZER8czYuSvPHz4JGsbkANsrMwwNFATHas78oiJS8C1WIEM9ylga0H0S30WHfeYAnZpfWZnY4G5mTGdW1dl8ZrDLFh5iGoV3Jg08kMGj/mVwHxwT8K6gCUGhgbE3HuoUx5z7xHF3AtnuI9dIWti7j1KV9+ukHW2xZmbSMLJxcqVK4efnx8+Pj40atSIhg0b0rZtW5KTk7l9+zZ+fn4Z7nf27Fni4+Oxt9f9gH3y5AlXr17Vvndzc9O5R+Pk5ERUVMZX7IGBgRgYGFCnTp0saFneFHg2nF59lmJjY06zJuX45usP6T9oJbGx+WeKKKs8e8LV4RNXWP/7aQCuhEVRtlRRWjUqny8SjnhzknByMQMDA/bu3cuRI0fYs2cPc+bMYfTo0ezbt+9f94uPj8fJySnDnwG3tbXV/vvlFWgqlQqNRpPhMc3M8sbN8bi4BFJTNdjZ6Y5O7OwsiH5p1PNMdEw8drbp68dE69Z/+jSZ27djuX07luDg26xc3psmjX1Z++uxrG2EnsU9ekJKqoYCLy2qsLMx58Gr+iz2MQVe6rMCNs/7OO7RE1JSUrl+84FOnRu3HuDrXSwLo885D6PjSU1Jxa6g7ujErqAVMVEPM9wnJuohdgWtXrt+fiP3cHI5lUpFrVq1GD9+PGfOnMHY2Ji9e/fi5ub2ysRTsWJF7ty5g6GhIe7u7jovBweHDPf5Lz4+Pmg0Gg4ePPg2zcl2KSkaQkLuULGCm7ZMpYKKFVy5eDEiw30uXrytUx+gckU3LgRnXP8ZtUqFsVHev2ZLSdEQcvUOlXxdtWUqFVTydeXC5dsZ7nP+8m0q+brolFUu78r5f+qnpGgIvnIHl6K6U3LORQpw515cFrcgZ6QkpxJ6Lpzy73tpy1QqFeXf8yL4VFiG+wSfDqP8+6V0yirU9n5l/fxGEk4udvz4cb777jtOnTpFeHg4mzdv5t69e3h7ezNu3DimTZvG7NmzCQ0NJSAggDlz5gBQv359atSowYcffsiePXu4fv06R44cYfTo0Zw6dSpTsbi5udG1a1d69OjBli1bCAsL48CBA6xfvz4rm5wlNmw6QbOm5WjYoCwuLvYMGdQIU1Njdu0+B8CoL5rzaY/nU4ObfztFlSrFade2Ks7OBej6yXt4ejqxZWvaVJCpqRE9e9TG27sIhQtZ4+FRmBHDmuLgYMXBQ/ljJdG6rado3sCXxh+UwbVYAYb1aYiZqRE79qU9DX704KZ89r/3tfU3/n6aahWK06FVZVyKFqD7xzUpVdKRzTvOaOus/e0k9WqVokUDX4o62tK6aQVqVinJbzsD9d28bPPbor9o3LkW9dtXw9nDkQFTPsbE3IS9vx4FYNicrnT7qpW2/taf91Ppg9K07uNHMffCdB7eDI9yLvy+7IC2jqWtOSXKFMPVM22pdDH3wpQoUyzdSCovyvuXZ/mYtbU1hw4dYubMmTx8+BBXV1emTZtGkyZNAHj69CkzZsxg+PDhODg40LZtWyDtKmvHjh2MHj2a7t27c+/ePRwdHalduzaFC2d8M/N1LFiwgK+++op+/frx4MEDXFxc+Oqrr7KkrVnpwMFL2Nqa073r+9jZpX3xc+RX64j5515LoULWaBRFW//CxQi+nbyNHt1q07N7bSIiYvhm3CauX78PQGqqBhdnexo18MHa2oyHj55w+fIdBn++mus37udIG7PaX/6XsbUxp2fHWhSws+BKWBTDx28k5p8lzIULWqG80GfnL99m/PQ/6NX5fXr/731u3Y7hq+9/Iyz8eX/8fTyUqQv38L821Rn8aT3Cb8cwZspWgv5j5JiXHNp6Ght7S/73RXMKFLTm6oVbjOk4l9h/vthaqKgdygvT1MGnrjGl31K6jmxJty9bEhF2j4ndF3HjhS8QV2/ky7BZXbTvv1zUE4DVU7ezZup2PbUse6iUF/+KhMhG9Rq8esm1eLVkC4OcDiHPsTz2bkxRZbWdd+Zn6/FlSk0IIYReSMIRQgihF5JwhBBC6IUkHCGEEHohCUcIIYReSMIRQgihF5JwhBBC6IUkHCGEEHohCUcIIYReSMIRQgihF5JwhBBC6IUkHCGEEHohCUcIIYReyM8TCL0xOhqc0yHkSUapGf8Kq/gXBWxzOgKRARnhCCGE0AtJOEIIIfRCEo4QQgi9kIQjhBBCLyThCCGE0AtJOEIIIfRCEo4QQgi9kIQjhBBCLyThCCGE0AtJOEIIIfRCEo4QQgi9kIQjhBBCLyThCCGE0At5WrTIl1r09qPt4CYUKGzDtaCbzB++msunr72y/vsfVaHrmNYUdnEg4updloxZz8k957Tba7WsRLOe9fAo74a1vSV9a4zhWlC4PpqiNy0+q0/bz5s+77OhK7l86l/6rHVVun7ThsKuDkRcucuSr9dxcvdZ7fZarSrT7NN6eFRww9reir7VRnPtXP7qM4DmXd+nbV8/7Apac+1iBAvGbCQk8MYr67/XvDxdRjSncLECRITdY9l3Wzn510Xt9ppNytHsk1q4+7pgbWdB/4bfc+1ChD6aku1khCPeSlJSUk6HkE6dNlXpPbkjayZvpf97Y7l2/ibfbhmOTUGrDOuXrubOl8v6smvFIfrV+oYjfwQw9tfBuJYuqq1jam7ChaMhLPlmvb6aoVd12laj95ROrPn2N/rXGMO1c+F8u+0LbApaZ1i/dHUPvlzRj10rDtKv+hiO/H6aseuH4Fq6mLaOqbkJF46EsOTrdfpqht7VblmR3mM/Ys30nQxs/ANhFyOYtKYfNvaWGdb3rlycUfO6sXvtUQY0msLR3ecYs6QXrl5O2jqm5sZcOHGNpd9u1Vcz9EYSTh5St25dBgwYwIABA7CxscHBwYExY8agKAoAbm5uTJw4kY4dO2JhYUHRokWZN2+ezjFiY2P57LPPKFy4MKamppQtW5Y//vhDu33Tpk2UKVMGExMT3NzcmDZtms7+z87RpUsXrK2t6d27d/Y3/A21HtCYXcsPsmf134Rfus3sQctJfJJEo09qZ1j/w34NObU3iI2zdnLzciQrJ27mSuB1Wn1WX1tn369HWPP9Vs7sv6CvZuhV60FN2LXsAHtW/dNnA5eR+CSRRl1f0Wf9G3Jqzzk2ztjBzcu3WTlhU1qf9Xmhz9b6s2byFs78lT/7DOCjXh+w85ej7F1/nPDQO8wZtY7EJ0k0/LhGhvVb9azLqQPBbFq4j5tX7rLqx+1cPX+TFt2f9/Nfm07yy8xdnPn7sr6aoTeScPKYFStWYGhoyIkTJ5g1axbTp09n8eLF2u0//vgj5cqV48yZM4waNYrBgwezd+9eADQaDU2aNMHf35/Vq1dz8eJFvv/+ewwMDAA4ffo07du35+OPPyYoKIhx48YxZswYli9frhPD1KlTtecYM2aM3tr+OgyNDPCo4EbAC4lBURTO7L9A6aruGe7jXdU9XSI5ve883q+on99o++yvl/rsr3/ps2oZ9NneILyreWRrrLmJoZEBHr7OBL6QGBRFIfDwZbwruWW4j3clN536AKcPXMK7UvHsDDXXkHs4eYyzszMzZsxApVLh5eVFUFAQM2bMoFevXgDUqlWLUaNGAeDp6Ym/vz8zZsygQYMG/Pnnn5w4cYLg4GA8PT0BKFGihPbY06dPx8/PT5tEPD09uXjxIj/++CPdunXT1qtXrx7Dhg3TU4vfjLW9FQaGBsRGxemUx0TF4ezplOE+doVtiLn3MF19u8I22RZnbmLt8Ko+e4izV5EM97ErbEtMBn38rvQZgHUBCwwMDYi5/9Lfzr1HFCtZOMN97ApaE3PvkW79+4+we8V0b34jI5w8pnr16qhUKu37GjVqEBoaSmpqqvb9i2rUqEFwcNpPOwcGBlKsWDFtsnlZcHAwtWrV0imrVauWzvEBKleunCVtEUK8WyThvEPMzMyy5DgWFhZZcpzs8PDBI1JTUrEtpHulbVfIhpi7cRnuE3M3DruXbo7/W/385uH9V/WZNTF3YjPcJ+ZuLHZv0Mf50cPox6SmpGLn8NLfTkGrdCPmZ2LuPUw3mrFzsEo36smvJOHkMcePH9d5f+zYMTw8PLT3YY4dO5Zuu7e3NwC+vr7cunWLkJCQDI/t7e2Nv7+/Tpm/vz+enp7a4+d2KcmphJ65ToW6pbVlKpWK8nVLc/HElQz3CT5xhfIv1Aeo+EEZgl9RP7/R9tkHL/XZB2Ve3WfHr1C+bhmdsop+ZQk+HpqtseYmKcmphJ67Sfn3ns8YqFQqyr/nSfDp6xnuE3z6uk59gAq1vQg+HZadoeYaknDymPDwcIYOHcrly5dZu3Ytc+bMYfDgwdrt/v7+/PDDD4SEhDBv3jw2bNig3V6nTh1q165NmzZt2Lt3L2FhYezcuZNdu3YBMGzYMPbt28fEiRMJCQlhxYoVzJ07l+HDh+dIWzNr89xdNOlWh/qdauHs5cTAWV0xNTdhz+q/ARjxU2+6j2unrb9l/h4qN/ChzcDGOHs68b+vPsSjYnG2LvpTW8fKzoISPi64lEq7p+Hs6UgJH5d0V/l51ebZO2nSvS71O7+Hs1cRBs7ultZnKw8BMGLxZ3Sf0F5bf8u8PVRu6EObwU3S+mz0R2l9tvClPvN1wcU7bXm5s6cTJXxd8tV9nt9+3k/jTjWp364qzu6FGfB9e0zMTNi7Lu3Cb9isT+g2qoW2/tYlB6hUtzStP6tHsZKF6Ty0CR6+Lvy+7JC2jqWtOSXKFMXV0xGAYiULU6JM0Xxxn0cWDeQxXbp04cmTJ1StWhUDAwMGDx6sszR52LBhnDp1ivHjx2Ntbc306dNp1KiRdvumTZsYPnw4HTt25PHjx7i7u/P9998DULFiRdavX88333zDxIkTcXJyYsKECToLBvKCg5tOYONgTZevW2NX2IZr58IZ/dFUYqPSpjkKOhdAo9Fo6188foXveyyk65g2dBvXlttX7zL+41ncuPj8y3bVm1Zg+KJe2vdfregPwKrvfmP1d1v007BsdHDjcWwcrOjyTZvnfdbqxxf6zB6NRtHWv3gslO+7LaDr2LZ0G9+O21fuMr79TG5cvKWtU71ZRYb//Pxv86tVAwBYNWkzq7/9TU8ty16HtgVgU8CS/w1vRoGCVly9EMGY/80n9n7aFFmhInYoL/Rb8KkwpgxYTtcvmtNtZHMiwu4xsefP3Lgcqa1TvaEPw2b8T/v+ywXdAVg9bQdrpu/UU8uyh0p59iUOkevVrVuX8uXLM3PmzAy3u7m5MWTIEIYMGaLXuF5XI8uuOR1C3pSq+e86Qoe6gG1Oh5An7YyYk63Hlyk1IYQQeiEJRwghhF7IPZw85MCBA/+6/fr163qJQwghMkNGOEIIIfRCEo4QQgi9kIQjhBBCLyThCCGE0AtJOEIIIfRCEo4QQgi9kIQjhBBCLyThCCGE0AtJOEIIIfRCEo4QQgi9kIQjhBBCL+RZakJvNAkJOR2CeEdoIu/kdAgiAzLCEUIIoReScIQQQuiFJBwhhBB6IQlHCCGEXkjCEUIIoReScIQQQuiFJBwhhBB6IQlHCCGEXkjCEUIIoReScIQQQuiFJBwhhBB6IQlHCCGEXkjCeYVx48ZRvnz5nA4j15L+EUK8KUk4gEqlYsuWLTplw4cPZ9++fTkTUDbJqJ35Vct+jVh1bR7bE9Yw++h3eFVx/9f6tdtWZ8nFmWxPWMNPZ6dRtUmFdHW6ju/ArxE/8cfjNUzZM4ai7o7ZFX6OyOo+e++jqny/62s23VvKXs0GSpZzy8boc478rb0+STivYGlpib29fU6HoXdJSUk5HcJbq9O+Jp9N68rqCRvoW2kk187dYPKu0dgWtM6wfukannz1yxB2Lf2LvhW/wH/rCcb99gVuZZy1dTp80YoPBzZhVt+fGFj9S54+TmTyrq8xMjHSV7OyVXb0mamFKef9L7F41Gp9NUPv5G/tzeRowqlbty6DBg3iiy++oECBAjg6OjJu3Djt9tjYWD799FMKFiyItbU19erV4+zZszrHmDRpEoUKFcLKyopPP/2UUaNG6Uz1nDx5kgYNGuDg4ICNjQ116tQhICBAu93NzQ2Ajz76CJVKpX3/4pTRnj17MDU1JTY2VufcgwcPpl69etr3hw8f5v3338fMzAxnZ2cGDRrE48ePX6svEhMTGTlyJM7OzpiYmODu7s6SJUu02w8ePEjVqlUxMTHBycmJUaNGkZKS8tp9+V/tXLx4McWLF8fU1BSA8PBwWrVqhaWlJdbW1rRv3567d+++VltyWpvPm7Nz8T52Lz9AePAtZvX5icSEJBr1qJdh/Y8GNePkrkA2TN1G+KUIVnyzjisB12g1oPHzOoObsebbTRzddoqwoHCmdJ2LfRE7an1YRV/NylbZ0Wd/rj7E6okbCfgzSF/N0Dv5W3szOT7CWbFiBRYWFhw/fpwffviBCRMmsHfvXgDatWtHVFQUO3fu5PTp01SsWBE/Pz+io6MBWLNmDd9++y1Tpkzh9OnTuLi4sGDBAp3jP3r0iK5du3L48GGOHTuGh4cHTZs25dGjR0BaQgJYtmwZkZGR2vcv8vPzw9bWlk2bNmnLUlNTWbduHZ07dwbg6tWrNG7cmDZt2nDu3DnWrVvH4cOHGTBgwGv1Q5cuXVi7di2zZ88mODiYRYsWYWlpCUBERARNmzalSpUqnD17lgULFrBkyRImTZr02n35b+28cuUKmzZtYvPmzQQGBqLRaGjVqhXR0dEcPHiQvXv3cu3aNTp06PBabclJhkaGeFYqQcCf57RliqIQ8Oc5Slf3zHCf0jU8Cdh3Tqfs1J6zeP9T37F4Ieyd7DjzwgdnwsMELh2/QukaXtnQCv3Kjj57F8jf2pvL8V/89PX1ZezYsQB4eHgwd+5c9u3bh5mZGSdOnCAqKgoTExMApk6dypYtW9i4cSO9e/dmzpw59OzZk+7duwPwzTffsGfPHuLj47XHf3EEAvDTTz9ha2vLwYMHad68OQULFgTA1tYWR8eM50kNDAz4+OOP+eWXX+jZsycA+/btIzY2ljZt2gAwefJkOnfuzJAhQ7RtmT17NnXq1GHBggXakUNGQkJCWL9+PXv37qV+/foAlChRQrt9/vz5ODs7M3fuXFQqFaVKleL27duMHDmSb775BrVa/a992aBBg39tZ1JSEitXrtTW2bt3L0FBQYSFheHsnDbUX7lyJWXKlOHkyZNUqZJ7r7RsHKwwMDQg5m6cTnlMVBzOpYpmuI+doy2xL9e/G0sBR1sA7f/G3I1NV8eusG1WhJ2jsqPP3gXyt/bmcnyE4+vrq/PeycmJqKgozp49S3x8PPb29lhaWmpfYWFhXL16FYDLly9TtWpVnf1ffn/37l169eqFh4cHNjY2WFtbEx8fT3h4+BvF2blzZw4cOMDt27eBtNFVs2bNsLW1BeDs2bMsX75cJ9ZGjRqh0WgICwv712MHBgZiYGBAnTp1MtweHBxMjRo1UKlU2rJatWoRHx/PrVu3tGWv6sv/4urqqk02z87n7OysTTYApUuXxtbWluDg4P88nhBCZCTHRzhGRro3wlQqFRqNhvj4eJycnDhw4EC6fZ59yL+Orl278uDBA2bNmoWrqysmJibUqFHjjW+OV6lShZIlS/Lrr7/St29ffvvtN5YvX67dHh8fz2effcagQYPS7evi4vKvxzYzM3ujWF7lVX35XywsLLLk/LlB3P1HpKakYlfYRqfcrpANMXdiM9wn5k4sti/XL2xL9D/1n/3vi2XP3l89ez2LIs852dFn7wL5W3tzOT7CeZWKFSty584dDA0NcXd313k5ODgA4OXlle6ey8vv/f39GTRoEE2bNqVMmTKYmJhw//59nTpGRkakpqb+Z0ydO3dmzZo1/P7776jVapo1a6YT78WLF9PF6u7ujrGx8b8e18fHB41Gw8GDBzPc7u3tzdGjR1EURaddVlZWFCtW7D/jfuZ12+nt7c3Nmze5efOmtuzixYvExsZSunTp1z5fTkhJTiHk9DUq+Ploy1QqFRX8fLh4LCTDfS4eDaFCPR+dsor1fQn+p/6dsCgeRMZQwa+sdru5lRmlqrlz8ejlbGiFfmVHn70L5G/tzeXahFO/fn1q1KjBhx9+yJ49e7h+/TpHjhxh9OjRnDp1CoCBAweyZMkSVqxYQWhoKJMmTeLcuXM6U08eHh6sWrWK4OBgjh8/TufOndONKNzc3Ni3bx937twhJibmlTF17tyZgIAAvv32W9q2bau9twQwcuRIjhw5woABAwgMDCQ0NJStW7e+1qIBNzc3unbtSo8ePdiyZQthYWEcOHCA9evXA9CvXz9u3rzJwIEDuXTpElu3bmXs2LEMHTpUe//mdbxuO+vXr4+Pj4+2vSdOnKBLly7UqVOHypUrv/b5csqmGX/Q9FM/GnSpg0upogxa0AtTCxN2L9sPwBfLB9Dju07a+r/N3k6VxuVpO7Q5zl5F+GRsOzwrl2Tr3F3P68zaTqfRbajRojJuZV34YsUAHtyOwX9L+kUmeVF29JmVnSUly7nhWjrtoqiYVxFKlnPLF/cinpG/tTeT41Nqr6JSqdixYwejR4+me/fu3Lt3D0dHR2rXrk3hwoWBtARw7do1hg8fztOnT2nfvj3dunXjxIkT2uMsWbKE3r17U7FiRZydnfnuu+8YPny4zrmmTZvG0KFD+fnnnylatCjXr1/PMCZ3d3eqVq3KiRMnmDlzps42X19fDh48yOjRo3n//fdRFIWSJUu+9squBQsW8NVXX9GvXz8ePHiAi4sLX331FQBFixZlx44djBgxgnLlylGgQAF69uzJ119//Zq9+WbtVKlUbN26lYEDB1K7dm3UajWNGzdmzpw5b3S+nHJw/RFsC1rTdXwH7BxtuRp4na+afEtsVNrN2kIuDiia56PFi0dDmNx5Ft0mdqT7t52ICI1k3Ec/cP3C8xHeuh+2YmphypBFn2Fpa875w5f4ssm3JCcm67192SE7+qxGy8qMWNZf+/7rXz8HYOX49awav0FPLcte8rf2ZlTKi/M0+UCDBg1wdHRk1apVOR2KeEkDdbucDkEI8S/2arL3QiDXjnBeR0JCAgsXLqRRo0YYGBiwdu1a/vzzT+13T4QQQuQeeTrhPJt2+/bbb3n69CleXl5s2rRJ+12W3OLvv/+mSZMmr9z+4veGhBAiv8rTCcfMzIw///wzp8P4T5UrVyYwMDCnwxBCiByVpxNOXmFmZoa7+78/QVYIIfK7XLssWgghRP4iCUcIIYReSMIRQgihF5JwhBBC6IUkHCGEEHohCUcIIYReSMIRQgihF5JwhBBC6IUkHCGEEHqR754WLYQQIneSEY4QQgi9kIQjhBBCLyThCCGE0AtJOEIIIfRCEo4QQgi9kIQjhBBCLyThCCGE0AtJOEIIIfRCEo4QQgi9+D9ShfDFe6ot8wAAAABJRU5ErkJggg==", "text/plain": [ - "
      " + "
      " ] }, "metadata": {}, @@ -115,191 +115,6 @@ " plot_heatmap(df_subset, name='', cmap=\"viridis\", ax=ax)\n", " plt.title(f'{dataset}')" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Assemble the results from differnet runs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!rm -r resources/results/all_main/" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Copied identical file: method_configs.yaml\n", - "Copied identical file: metric_configs.yaml\n", - "Merged: dataset_uns.yaml\n", - "Merged: score_uns.yaml\n", - "Merged: trace.txt (duplicates removed)\n", - "Copied unique file: state.yaml → state.yaml\n", - "Copied unique file: op_.celloracle.celloracle.prediction.h5ad → op_.celloracle.celloracle.prediction.h5ad\n", - "Copied unique file: scplus_mdata.h5mu.2DAaAA8E → scplus_mdata.h5mu.2DAaAA8E\n", - "Copied unique file: op_.negative_control.negative_control.prediction.h5ad → op_.negative_control.negative_control.prediction.h5ad\n", - "Copied unique file: op_.scenicplus.scenicplus.prediction.h5ad → op_.scenicplus.scenicplus.prediction.h5ad\n", - "Copied unique file: op_.portia.portia.prediction.h5ad → op_.portia.portia.prediction.h5ad\n", - "Copied unique file: op_.granie.granie.prediction.h5ad → op_.granie.granie.prediction.h5ad\n", - "Copied unique file: op_.scprint.scprint.prediction.h5ad → op_.scprint.scprint.prediction.h5ad\n", - "Copied unique file: op_.pearson_corr.pearson_corr.prediction.h5ad → op_.pearson_corr.pearson_corr.prediction.h5ad\n", - "Copied unique directory: output → output\n", - "Copied unique file: op_.scglue.scglue.prediction.h5ad → op_.scglue.scglue.prediction.h5ad\n", - "Copied unique file: op_.ppcor.ppcor.prediction.h5ad → op_.ppcor.ppcor.prediction.h5ad\n", - "Copied unique file: op_.positive_control.positive_control.prediction.h5ad → op_.positive_control.positive_control.prediction.h5ad\n", - "Copied unique file: op_.scenic.scenic.prediction.h5ad → op_.scenic.scenic.prediction.h5ad\n", - "Copied unique file: op_.figr.figr.prediction.h5ad → op_.figr.figr.prediction.h5ad\n", - "Copied unique file: task_info.yaml → task_info.yaml\n", - "Copied unique file: op_.grnboost2.grnboost2.prediction.h5ad → op_.grnboost2.grnboost2.prediction.h5ad\n", - "Copied unique file: state.yaml → state_nakatake.yaml\n", - "Copied unique file: nakatake_.positive_control.positive_control.prediction.h5ad → nakatake_.positive_control.positive_control.prediction.h5ad\n", - "Copied unique file: nakatake_.portia.portia.prediction.h5ad → nakatake_.portia.portia.prediction.h5ad\n", - "Copied unique file: nakatake_.grnboost.grnboost.prediction.h5ad → nakatake_.grnboost.grnboost.prediction.h5ad\n", - "Copied unique file: nakatake_.ppcor.ppcor.prediction.h5ad → nakatake_.ppcor.ppcor.prediction.h5ad\n", - "Copied unique file: nakatake_.pearson_corr.pearson_corr.prediction.h5ad → nakatake_.pearson_corr.pearson_corr.prediction.h5ad\n", - "Copied unique file: nakatake_.negative_control.negative_control.prediction.h5ad → nakatake_.negative_control.negative_control.prediction.h5ad\n", - "Copied unique file: nakatake_.scenic.scenic.prediction.h5ad → nakatake_.scenic.scenic.prediction.h5ad\n", - "Copied unique file: task_info.yaml → task_info_nakatake.yaml\n", - "Copied unique file: norman_.grnboost2.grnboost2.prediction.h5ad → norman_.grnboost2.grnboost2.prediction.h5ad\n", - "Copied unique file: state.yaml → state_norman.yaml\n", - "Copied unique file: norman_.ppcor.ppcor.prediction.h5ad → norman_.ppcor.ppcor.prediction.h5ad\n", - "Copied unique file: norman_.scprint.scprint.prediction.h5ad → norman_.scprint.scprint.prediction.h5ad\n", - "Copied unique file: norman_.pearson_corr.pearson_corr.prediction.h5ad → norman_.pearson_corr.pearson_corr.prediction.h5ad\n", - "Copied unique file: norman_.portia.portia.prediction.h5ad → norman_.portia.portia.prediction.h5ad\n", - "Copied unique file: norman_.negative_control.negative_control.prediction.h5ad → norman_.negative_control.negative_control.prediction.h5ad\n", - "Copied unique file: norman_.scenic.scenic.prediction.h5ad → norman_.scenic.scenic.prediction.h5ad\n", - "Copied unique file: norman_.positive_control.positive_control.prediction.h5ad → norman_.positive_control.positive_control.prediction.h5ad\n", - "Copied unique file: task_info.yaml → task_info_norman.yaml\n", - "Copied unique file: state.yaml → state_replogle.yaml\n", - "Copied unique file: replogle_.pearson_corr.pearson_corr.prediction.h5ad → replogle_.pearson_corr.pearson_corr.prediction.h5ad\n", - "Copied unique file: replogle_.positive_control.positive_control.prediction.h5ad → replogle_.positive_control.positive_control.prediction.h5ad\n", - "Copied unique file: replogle_.portia.portia.prediction.h5ad → replogle_.portia.portia.prediction.h5ad\n", - "Copied unique file: replogle_.grnboost2.grnboost2.prediction.h5ad → replogle_.grnboost2.grnboost2.prediction.h5ad\n", - "Copied unique file: replogle_.negative_control.negative_control.prediction.h5ad → replogle_.negative_control.negative_control.prediction.h5ad\n", - "Copied unique file: replogle_.ppcor.ppcor.prediction.h5ad → replogle_.ppcor.ppcor.prediction.h5ad\n", - "Copied unique file: task_info.yaml → task_info_replogle.yaml\n", - "Copied unique file: replogle_.scenic.scenic.prediction.h5ad → replogle_.scenic.scenic.prediction.h5ad\n", - "Copied unique file: state.yaml → state_adamson.yaml\n", - "Copied unique file: adamson_.portia.portia.prediction.h5ad → adamson_.portia.portia.prediction.h5ad\n", - "Copied unique file: adamson_.negative_control.negative_control.prediction.h5ad → adamson_.negative_control.negative_control.prediction.h5ad\n", - "Copied unique file: adamson_.pearson_corr.pearson_corr.prediction.h5ad → adamson_.pearson_corr.pearson_corr.prediction.h5ad\n", - "Copied unique file: adamson_.ppcor.ppcor.prediction.h5ad → adamson_.ppcor.ppcor.prediction.h5ad\n", - "Copied unique file: adamson_.scenic.scenic.prediction.h5ad → adamson_.scenic.scenic.prediction.h5ad\n", - "Copied unique file: adamson_.grnboost2.grnboost2.prediction.h5ad → adamson_.grnboost2.grnboost2.prediction.h5ad\n", - "Copied unique file: adamson_.positive_control.positive_control.prediction.h5ad → adamson_.positive_control.positive_control.prediction.h5ad\n", - "Copied unique file: task_info.yaml → task_info_adamson.yaml\n" - ] - } - ], - "source": [ - "import os\n", - "import shutil\n", - "import yaml\n", - "from pathlib import Path\n", - "from collections import OrderedDict\n", - "\n", - "base_dir = 'resources/results/'\n", - "save_dir = 'resources/results/all_main/'\n", - "runs = ['op', 'nakatake', 'norman', 'replogle', 'adamson']\n", - "\n", - "os.makedirs(save_dir, exist_ok=True)\n", - "\n", - "# 1. Copy one version of the identical files\n", - "identical_files = ['method_configs.yaml', 'metric_configs.yaml']\n", - "for fname in identical_files:\n", - " src = os.path.join(base_dir, f'{runs[0]}_run', fname)\n", - " dst = os.path.join(save_dir, fname)\n", - " shutil.copyfile(src, dst)\n", - " print(f\"Copied identical file: {fname}\")\n", - "\n", - "# 2. Merge dataset_uns.yaml by appending all contents\n", - "merged_uns = []\n", - "for run in runs:\n", - " path = os.path.join(base_dir, f'{run}_run', 'dataset_uns.yaml')\n", - "\n", - " with open(path, 'r') as f:\n", - " data = yaml.safe_load(f)\n", - " merged_uns.extend(data)\n", - " \n", - "with open(os.path.join(save_dir, 'dataset_uns.yaml'), 'w') as f:\n", - " yaml.dump(merged_uns, f)\n", - "print(\"Merged: dataset_uns.yaml\")\n", - "\n", - "# 3. Merge score_uns.yaml similarly\n", - "merged_scores = []\n", - "for run in runs:\n", - " path = os.path.join(base_dir, f'{run}_run', 'score_uns.yaml')\n", - "\n", - " with open(path, 'r') as f:\n", - " data = yaml.safe_load(f)\n", - " # - remove those with missing (because of the metric)\n", - " data = [d for d in data if d is not None and 'missing' not in str(d)]\n", - " \n", - " # print(str(data[0]))\n", - " # aa\n", - " # missing\n", - " if data:\n", - " if isinstance(data, dict):\n", - " merged_scores.append(data)\n", - " elif isinstance(data, list):\n", - " merged_scores.extend(data)\n", - " else:\n", - " print(f\"Unexpected format in {path}: {type(data)}\")\n", - "\n", - "with open(os.path.join(save_dir, 'score_uns.yaml'), 'w') as f:\n", - " yaml.dump(merged_scores, f)\n", - "print(\"Merged: score_uns.yaml\")\n", - "\n", - "# 4. Merge trace.txt with deduplication\n", - "seen_lines = OrderedDict()\n", - "for run in runs:\n", - " path = os.path.join(base_dir, f'{run}_run', 'trace.txt')\n", - " \n", - " with open(path, 'r') as f:\n", - " for line in f:\n", - " seen_lines[line] = None\n", - "\n", - "with open(os.path.join(save_dir, 'trace.txt'), 'w') as f:\n", - " for line in seen_lines.keys():\n", - " f.write(line)\n", - "df = pd.read_csv(os.path.join(save_dir, 'trace.txt'), sep='\\t')\n", - "df = df.drop_duplicates(subset=['name'])\n", - "df.to_csv(os.path.join(save_dir, 'trace.txt'), sep='\\t')\n", - "print(\"Merged: trace.txt (duplicates removed)\")\n", - "\n", - "# 5. Copy other unknown files/directories\n", - "all_known = set(identical_files + ['dataset_uns.yaml', 'score_uns.yaml', 'trace.txt'])\n", - "\n", - "for run in runs:\n", - " run_dir = Path(base_dir) / f'{run}_run'\n", - " for file_path in run_dir.iterdir():\n", - " if file_path.name in all_known:\n", - " continue\n", - "\n", - " dest_path = Path(save_dir) / file_path.name\n", - "\n", - " if dest_path.exists():\n", - " dest_path = Path(save_dir) / f\"{file_path.stem}_{run}{file_path.suffix}\"\n", - "\n", - " if file_path.is_file():\n", - " shutil.copyfile(file_path, dest_path)\n", - " print(f\"Copied unique file: {file_path.name} → {dest_path.name}\")\n", - " elif file_path.is_dir():\n", - " shutil.copytree(file_path, dest_path)\n", - " print(f\"Copied unique directory: {file_path.name} → {dest_path.name}\")" - ] } ], "metadata": {