Metadata JSON design (#30)

* added metadataJSON.py and query_mapper.yaml

* addressing design comments:clean up init function, connect to db once, merged create_json_file and write_json_file, moving away from manual queries towards using pypika to generate SQL

* submitter details added to JSON.

* started to add project.

* set up unit tests

* moved sql queries _fetch functions. started on samples. unit tests added.

* test.config file

* added _get_analysis, _get_files function. changed placeholders to UNSPECIFIED. added validate to fetch functions

* added unit tests

* added gather_metadata function

* edit fetching file name

* remove TODO

* requirements.txt: add oracledb

* requirements.txt: add pypika

* change _get_sample_new

* removed query mapper

* fix(metadataJSON.py): correct the dot notation for sample_status

* fix(metadataJSON.py): change the return value

* fix(metadataJSON.py): change how unregistered samples are handled

* fix(metadataJSON.py): change how preregistered samples are handled

* fix(metadataJSON.py): change use of next() to calling validate_fetch_result

* fix(metadataJSON.py): change use of  next() to calling validate_fetch_result

* fix(convertGVFtoVCF.py): check if config is present before obtaining metadata

* fix(convertGVFtoVCF.py): make arguments optional

* fix(metadataJSON.py): validate the project accession

* fix(metadataJSON.py): change the sampleinvcf to match the sample id

* fix(test_metadataJSON.py): edit tests

* fix(test_metadataJSON.py): remove place holder and raise ValueError for missing results

* fix(test_metadataJSON.py): fetch methods changed to fetch what is named in the function name

* fix(test_metadataJSON.py): edit mock data to represent the real return type

* feat(gather_metadata.py): make gather_metadata its own executable

* edit comment

* fix(metadataJSON.py): improve error handling

* fix(metadataJSON.py): correct typo

* fix(metadataJSON.py): replace UNSPECIFIED placeholders with empty strings

* fix(metadataJSON.py): edit unit test

* fix(metadataJSON.py, test_metadataJSON.py): fix typos, remove dead code

* fix(metadataJSON.py, test_metadataJSON.py): fix typos

Author: khetherin

convert_gvf_to_vcf/convertGVFtoVCF.py

@@ -4,10 +4,11 @@
 from ebi_eva_common_pyutils.logger import logging_config as log_cfg
 
 from convert_gvf_to_vcf.conversionstatistics import FileStatistics
+from convert_gvf_to_vcf.gather_metadata import gather_metadata
 from convert_gvf_to_vcf.lookup import Lookup
 from convert_gvf_to_vcf.utils import read_in_gvf_header, read_in_gvf_data
 from convert_gvf_to_vcf.vcfline import VcfLineBuilder
-
+from convert_gvf_to_vcf.metadataJSON import DGVaMetadataRetriever
 
 logger = log_cfg.get_logger(__name__)
 
@@ -377,13 +378,19 @@ def cleanup_temp_files(list_of_temp_files):
         if os.path.exists(temp_file):
             os.remove(temp_file)
 
+
+
 def main():
     # Parse command line arguments
     parser = argparse.ArgumentParser()
     parser.add_argument("gvf_input", help="GVF input file.")
     parser.add_argument("vcf_output", help="VCF output file.")
+    parser.add_argument("--json_output", help="JSON output file.")
+    parser.add_argument("--study_accession", help="DGVa Study Accession")
     parser.add_argument("-a", "--assembly", help="FASTA assembly file")
     parser.add_argument("--log", help="Path to log file")
+    parser.add_argument("--config", help="Path to config file")
+
     args = parser.parse_args()
 
     # Set up logging functionality
@@ -393,8 +400,11 @@ def main():
     else:
         log_cfg.add_stdout_handler()
     convert(args.gvf_input, args.vcf_output, args.assembly)
-
-
+    if args.config:
+        logger.info(f"The config file is {args.config}. Gathering metadata")
+        gather_metadata(args.config, args.json_output, args.study_accession, args.vcf_output)
+    else:
+        logger.info(f"No config file provided. Unable to gather metadata.")
 
 if __name__ == "__main__":
     main()

convert_gvf_to_vcf/gather_metadata.py

@@ -0,0 +1,24 @@
+import argparse
+
+from convert_gvf_to_vcf.metadataJSON import DGVaMetadataRetriever
+
+
+def gather_metadata(config_input, json_output, study_accession, vcf_output):
+
+    retrieved_dgva_metadata = DGVaMetadataRetriever(config_input)
+    with retrieved_dgva_metadata:
+        retrieved_dgva_metadata.create_json_file(json_file_path=json_output, study_accession=study_accession, vcf_output=vcf_output)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config")
+    parser.add_argument("--json_output")
+    parser.add_argument("--study_accession")
+    parser.add_argument("--vcf_output")
+    args = parser.parse_args()
+
+    gather_metadata(args.config, args.json_output, args.study_accession, args.vcf_output)
+
+if __name__ == "__main__":
+    main()