feat: clean up file download behavior and descriptions (#163)

Author: jsstevenson

.gitignore

@@ -135,4 +135,5 @@ pyproject.toml
 cool_seq_tool/data/seqrepo/
 cool_seq_tool/data/*.txt
 cool_seq_tool/data/LRG_RefSeqGene*
+cool_seq_tool/data/MANE*
 cool_seq_tool/data/notebooks/

README.md

@@ -93,13 +93,6 @@ sudo mv /usr/local/share/seqrepo/2021-01-29._fkuefgd /usr/local/share/seqrepo/20
 exit
 ```
 
-#### transcript_mappings.tsv
-`cool-seq-tool` uses [Ensembl BioMart](http://www.ensembl.org/biomart/martview) to retrieve `cool_seq_tool/data/transcript_mappings.tsv`. We currently use `Human Genes (GRCh38.p13)` for the dataset and the following attributes we use are: Gene stable ID, Gene stable ID version, Transcript stable ID, Transcript stable ID version, Protein stable ID, Protein stable ID version, RefSeq match transcript (MANE Select), Gene name.
-
-![image](biomart.png)
-
-Use the `TRANSCRIPT_MAPPINGS_PATH` environment variable to set the path of an already existing `transcript_mappings.tsv`. The default is `cool_seq_tool/data/transcript_mapping.tsv`.
-
 #### LRG_RefSeqGene
 
 `cool-seq-tool` fetches the latest version of `LRG_RefSeqGene` if the environment variable `LRG_REFSEQGENE_PATH` is not set. When `LRG_REFSEQGENE_PATH` is set, `cool-seq-tool` will look at this path and expect the LRG_RefSeqGene file. This file is found can be found [here](https://ftp.ncbi.nlm.nih.gov/refseq/H_sapiens/RefSeqGene).
@@ -108,6 +101,22 @@ Use the `TRANSCRIPT_MAPPINGS_PATH` environment variable to set the path of an al
 
 `cool-seq-tool` fetches the latest version of `MANE.GRCh38.*.summary.txt.gz` if the environment variable `MANE_SUMMARY_PATH` is not set. When `MANE_SUMMARY_PATH` is set, `cool-seq-tool` will look at this path and expect the MANE Summary Data file. This file is found can be found [here](https://ftp.ncbi.nlm.nih.gov/refseq/MANE/MANE_human/current/).
 
+#### transcript_mapping.tsv
+`cool-seq-tool` is packaged with transcript mapping data acquired from [Ensembl BioMart](http://www.ensembl.org/biomart/martview). If the environment variable `TRANSCRIPT_MAPPINGS_PATH` is not set, `cool-seq-tool` will use the built-in file. When `TRANSCRIPT_MAPPINGS_PATH` is set, `cool_seq_tool` will look at this path and expect to find the transcript mapping TSV file.
+
+To acquire this data manually from the [BioMart](https://www.ensembl.org/biomart/martview), select the `Human Genes (GRCh38.p13)` dataset and choose the following attributes:
+
+* Gene stable ID
+* Gene stable ID version
+* Transcript stable ID
+* Transcript stable ID version
+* Protein stable ID
+* Protein stable ID version
+* RefSeq match transcript (MANE Select)
+* Gene name
+
+![image](biomart.png)
+
 ## Starting the UTA Tools Service Locally
 
 To start the service, run the following:

cool_seq_tool/app.py

@@ -36,7 +36,7 @@ def __init__(
     ) -> None:
         """Initialize CoolSeqTool class
 
-        :param Path transcript_file_path: The path to transcript_mappings.tsv
+        :param Path transcript_file_path: The path to transcript_mapping.tsv
         :param Path lrg_refseqgene_path: The path to LRG_RefSeqGene
         :param Path mane_data_path: Path to RefSeq MANE summary data
         :param str db_url: PostgreSQL connection URL

cool_seq_tool/data/data_downloads.py

@@ -1,79 +1,83 @@
-"""Module for downloading data files."""
+"""Module for handling downloadable data files."""
 from ftplib import FTP
+import logging
 from os import remove
 import gzip
+from pathlib import Path
 import shutil
 import datetime
 
 from dateutil import parser
 
 from cool_seq_tool import APP_ROOT
 
+logger = logging.getLogger("cool_seq_tool")
+
 
 class DataDownload:
-    """Class for downloading data files."""
+    """Class for managing downloadable data files. Responsible for checking if files
+    are available under default locations, and fetching them if not.
+    """
 
     def __init__(self) -> None:
-        """Initialize DataDownload."""
-        self._make_data_dir()
-        self._download_data()
-
-    def _make_data_dir(self) -> None:
-        """Make data directory"""
+        """Initialize downloadable data locations."""
         self._data_dir = APP_ROOT / "data"
-        self._data_dir.mkdir(exist_ok=True, parents=True)
-
-    def _download_data(self) -> None:
-        """Download data files needed for cool_seq_tool."""
-        with FTP("ftp.ncbi.nlm.nih.gov") as ftp:
-            ftp.login()
-            self._download_mane_summary(ftp)
-            self._download_lrg_refseq_gene_data(ftp)
 
-    def _download_mane_summary(self, ftp: FTP) -> None:
-        """Download latest MANE summary data and set path
+    def get_mane_summary(self) -> Path:
+        """Identify latest MANE summary data. If unavailable locally, download from
+        source.
 
-        :param FTP ftp: FTP connection
+        :return: path to MANE summary file
         """
-        ftp.cwd("/refseq/MANE/MANE_human/current")
-        files = ftp.nlst()
-        mane_summary_file = \
-            [f for f in files if f.endswith(".summary.txt.gz")]
-        if not mane_summary_file:
-            raise Exception("Unable to download MANE summary data")
-        mane_summary_file = mane_summary_file[0]
-        self._mane_summary_path = \
-            self._data_dir / mane_summary_file[:-3]
-        mane_data_path = self._data_dir / mane_summary_file
-        if not self._mane_summary_path.exists():
-            with open(mane_data_path, "wb") as fp:
-                ftp.retrbinary(f"RETR {mane_summary_file}", fp.write)
-            with gzip.open(mane_data_path, "rb") as f_in:
-                with open(self._mane_summary_path, "wb") as f_out:
-                    shutil.copyfileobj(f_in, f_out)
-            remove(mane_data_path)
+        with FTP("ftp.ncbi.nlm.nih.gov") as ftp:
+            ftp.login()
+            ftp.cwd("/refseq/MANE/MANE_human/current")
+            files = ftp.nlst()
+            mane_summary_file = \
+                [f for f in files if f.endswith(".summary.txt.gz")]
+            if not mane_summary_file:
+                raise Exception("Unable to download MANE summary data")
+            mane_summary_file = mane_summary_file[0]
+            self._mane_summary_path = \
+                self._data_dir / mane_summary_file[:-3]
+            mane_data_path = self._data_dir / mane_summary_file
+            if not self._mane_summary_path.exists():
+                logger.info("Downloading MANE summary file from NCBI.")
+                with open(mane_data_path, "wb") as fp:
+                    ftp.retrbinary(f"RETR {mane_summary_file}", fp.write)
+                with gzip.open(mane_data_path, "rb") as f_in:
+                    with open(self._mane_summary_path, "wb") as f_out:
+                        shutil.copyfileobj(f_in, f_out)
+                remove(mane_data_path)
+                logger.info("MANE summary file download complete.")
+        return self._mane_summary_path
 
-    def _download_lrg_refseq_gene_data(self, ftp: FTP) -> None:
-        """Download latest LRG_RefSeqGene and set path
+    def get_lrg_refseq_gene_data(self) -> Path:
+        """Identify latest LRG RefSeq Gene file. If unavailable locally, download from
+        source.
 
-        :param FTP ftp: FTP connection
+        :return: path to acquired LRG RefSeq Gene data file
         """
-        lrg_refseqgene_file = "LRG_RefSeqGene"
-        ftp_dir_path = "/refseq/H_sapiens/RefSeqGene/"
-        ftp_file_path = f"{ftp_dir_path}{lrg_refseqgene_file}"
-        timestamp = ftp.voidcmd(f"MDTM {ftp_file_path}")[4:].strip()
-        date = str(parser.parse(timestamp)).split()[0]
-        version = datetime.datetime.strptime(date,
-                                             "%Y-%m-%d").strftime("%Y%m%d")
-
-        fn_versioned = f"{lrg_refseqgene_file}_{version}"
-        lrg_refseqgene_path = self._data_dir / lrg_refseqgene_file
-        self._lrg_refseqgene_path = self._data_dir / fn_versioned
-        if not self._lrg_refseqgene_path.exists():
-            ftp.cwd(ftp_dir_path)
-            with open(lrg_refseqgene_path, "wb") as fp:
-                ftp.retrbinary(f"RETR {lrg_refseqgene_file}", fp.write)
-            with open(lrg_refseqgene_path, "rb") as f_in:
-                with open(self._lrg_refseqgene_path, "wb") as f_out:
-                    shutil.copyfileobj(f_in, f_out)
-            remove(lrg_refseqgene_path)
+        with FTP("ftp.ncbi.nlm.nih.gov") as ftp:
+            ftp.login()
+            lrg_refseqgene_file = "LRG_RefSeqGene"
+            ftp_dir_path = "/refseq/H_sapiens/RefSeqGene/"
+            ftp_file_path = f"{ftp_dir_path}{lrg_refseqgene_file}"
+            timestamp = ftp.voidcmd(f"MDTM {ftp_file_path}")[4:].strip()
+            date = str(parser.parse(timestamp)).split()[0]
+            version = datetime.datetime.strptime(date,
+                                                 "%Y-%m-%d").strftime("%Y%m%d")
+            fn_versioned = f"{lrg_refseqgene_file}_{version}"
+            lrg_refseqgene_path = self._data_dir / lrg_refseqgene_file
+            self._lrg_refseqgene_path = self._data_dir / fn_versioned
+            if not self._lrg_refseqgene_path.exists():
+                logger.info("Downloading LRG RefSeq data from NCBI.")
+                ftp.cwd(ftp_dir_path)
+                with open(lrg_refseqgene_path, "wb") as fp:
+                    ftp.retrbinary(f"RETR {lrg_refseqgene_file}", fp.write)
+                with open(lrg_refseqgene_path, "rb") as f_in:
+                    with open(self._lrg_refseqgene_path, "wb") as f_out:
+                        shutil.copyfileobj(f_in, f_out)
+                remove(lrg_refseqgene_path)
+                logger.info("LRG RefSeq data download complete.")
+        return self._lrg_refseqgene_path

cool_seq_tool/paths.py

@@ -2,24 +2,27 @@
 from os import environ
 from pathlib import Path
 
+from cool_seq_tool.data.data_downloads import DataDownload
+
 
 APP_ROOT = Path(__file__).resolve().parents[0]
 
 TRANSCRIPT_MAPPINGS_PATH = Path(environ.get("TRANSCRIPT_MAPPINGS_PATH",
                                             f"{APP_ROOT}/data/transcript_mapping.tsv"))
 
-MANE_SUMMARY_PATH = environ.get("MANE_SUMMARY_PATH", "")
-LRG_REFSEQGENE_PATH = environ.get("LRG_REFSEQGENE_PATH", "")
-if not all((MANE_SUMMARY_PATH, LRG_REFSEQGENE_PATH)):
-    from cool_seq_tool.data import DataDownload  # noqa: E402, I202
-    d = DataDownload()
+d = DataDownload()
+
+provided_mane_summary_path = environ.get("MANE_SUMMARY_PATH", "")
+if provided_mane_summary_path:
+    MANE_SUMMARY_PATH = Path(provided_mane_summary_path)
+else:
+    MANE_SUMMARY_PATH = d.get_mane_summary()
 
-    if not MANE_SUMMARY_PATH:
-        MANE_SUMMARY_PATH = d._mane_summary_path
+provided_lrg_refseq_path = environ.get("LRG_REFSEQGENE_PATH", "")
+if provided_lrg_refseq_path:
+    LRG_REFSEQGENE_PATH = Path(provided_lrg_refseq_path)
+else:
+    LRG_REFSEQGENE_PATH = d.get_lrg_refseq_gene_data()
 
-    if not LRG_REFSEQGENE_PATH:
-        LRG_REFSEQGENE_PATH = d._lrg_refseqgene_path
-MANE_SUMMARY_PATH = Path(MANE_SUMMARY_PATH)
-LRG_REFSEQGENE_PATH = Path(LRG_REFSEQGENE_PATH)
 
 SEQREPO_ROOT_DIR = environ.get("SEQREPO_ROOT_DIR", "/usr/local/share/seqrepo/latest")