release v. 3.1.1 bug fixes and numpy version restriction

Author: jrober84

mob_suite/constants.py

@@ -171,8 +171,4 @@
 
 default_database_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'databases')
 
-ETE3DBTAXAFILE = os.path.abspath(default_database_dir + "/taxa.sqlite")
-
-ETE3_LOCK_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "databases/ETE3_DB.lock")
-
 LOG_FORMAT = '%(asctime)s %(name)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'

mob_suite/mob_cluster.py

@@ -487,11 +487,11 @@ def main():
 
     if not os.path.isdir(out_dir):
         logging.info('Creating directory {}'.format(args.outdir))
-        os.mkdir(out_dir, 0o755)
+        os.makedirs(out_dir, 0o755)
     tmp_dir = os.path.join(out_dir, '__tmp')
     if not os.path.isdir(tmp_dir):
         logging.info('Creating directory {}'.format(args.outdir))
-        os.mkdir(tmp_dir, 0o755)
+        os.makedirs(tmp_dir, 0o755)
 
     taxonomy_file = args.taxonomy
 

mob_suite/mob_init.py

@@ -126,7 +126,7 @@ def main():
 
 
     if os.path.exists(database_directory) == False:
-        os.mkdir(database_directory)
+        os.makedirs(database_directory)
     else:
         logger.info("Database directory folder already exists at {}".format(database_directory))
 
@@ -164,8 +164,15 @@ def main():
 
     logger.info('Initializing databases...this will take some time')
     # Find available threads and use the maximum number available for mash sketch but cap it at 32
-    num_threads = min(multiprocessing.cpu_count(), 32)
-
+    try:
+        num_threads = len(os.sched_getaffinity(0))
+    except AttributeError:
+        num_threads = multiprocessing.cpu_count()
+
+    if num_threads > 32:
+        num_threads = 32
+    if num_threads < 1:
+        num_threads = 1
 
     if not os.path.exists(database_directory):
         os.makedirs(database_directory)

mob_suite/mob_recon.py

@@ -14,7 +14,6 @@
 from mob_suite.constants import \
     MOB_CLUSTER_INFO_HEADER, \
     MOB_RECON_INFO_HEADER, \
-    ETE3DBTAXAFILE, \
     default_database_dir, \
     LOG_FORMAT, \
     LIT_PLASMID_TAXONOMY_HEADER
@@ -29,7 +28,7 @@
     read_sequence_info, \
     fixStart, \
     calc_md5, \
-    GC, \
+    gc_fraction, \
     ETE3_db_status_check, \
     writeReport, \
     dict_from_alt_key_list, \
@@ -998,7 +997,6 @@ def main():
     orit_blast_results = os.path.join(tmp_dir, 'orit_blast_results.txt')
     repetitive_blast_results = os.path.join(tmp_dir, 'repetitive_blast_results.txt')
     contig_blast_results = os.path.join(tmp_dir, 'contig_blast_results.txt')
-    contig_blast_results = os.path.join(tmp_dir, 'contig_blast_results.txt')
     prefix = None
     if args.prefix is not None:
         prefix = args.prefix
@@ -1010,9 +1008,7 @@ def main():
     logger.info('Analysis directory {}'.format(args.outdir))
 
     database_dir = os.path.abspath(args.database_directory)
-
-
-
+    print
 
     if database_dir == default_database_dir:
         plasmid_ref_db = args.plasmid_db
@@ -1043,27 +1039,26 @@ def main():
     else:
         sample_id = args.sample_id
 
-    verify_init(logger, database_dir)
 
     run_overhang = args.run_overhang
     unicycler_contigs = args.unicycler_contigs
 
     # initialize analysis directory
     if not os.path.isdir(args.outdir):
-        os.mkdir(args.outdir, 0o755)
+        os.makedirs(args.outdir, 0o755)
 
     elif not args.force:
         logger.error("Error output directory exists, please specify a new directory or use --force to overwrite")
         sys.exit(-1)
     else:
         shutil.rmtree(args.outdir)
-        os.mkdir(args.outdir, 0o755)
+        os.makedirs(args.outdir, 0o755)
 
     if not os.path.isdir(tmp_dir):
-        os.mkdir(tmp_dir, 0o755)
+        os.makedirs(tmp_dir, 0o755)
     else:
         shutil.rmtree(tmp_dir)
-        os.mkdir(tmp_dir, 0o755)
+        os.makedirs(tmp_dir, 0o755)
 
     # Initialize clustering distance thresholds
     if not (args.primary_cluster_dist >= 0 and args.primary_cluster_dist <= 1):
@@ -1126,7 +1121,7 @@ def main():
         for feature in MOB_RECON_INFO_HEADER:
             contig_info[id][feature] = ''
         contig_info[id]['md5'] = calc_md5(seq)
-        contig_info[id]['gc'] = GC(seq)
+        contig_info[id]['gc'] = gc_fraction(seq)
         contig_info[id]['size'] = len(seq)
         contig_info[id]['contig_id'] = id
         contig_info[id]['sample_id'] = sample_id
@@ -1398,7 +1393,7 @@ def main():
             mobtyper_report = os.path.join(out_dir, "mobtyper_results.txt")
             if prefix is not None:
                 mobtyper_report = os.path.join(out_dir, "{}.mobtyper_results.txt".format(prefix))
-            build_mobtyper_report(contig_memberships['plasmid'], out_dir, mobtyper_report,contig_seqs, ncbi, lit)
+            build_mobtyper_report(contig_memberships['plasmid'], out_dir, mobtyper_report,contig_seqs, ncbi, lit,ETE3DBTAXAFILE)
 
         writeReport(results, MOB_RECON_INFO_HEADER, contig_report)
 

mob_suite/mob_typer.py

@@ -18,9 +18,8 @@
     read_sequence_info, \
     writeReport, \
     sort_biomarkers, \
-    ETE3_db_status_check, \
     calc_md5, \
-    GC, \
+    gc_fraction, \
     read_fasta_dict, \
     identify_biomarkers, \
     parseMash, \
@@ -31,11 +30,9 @@
     blast_mge, \
     writeMGEresults
 
-from mob_suite.constants import ETE3DBTAXAFILE, \
-    MOB_TYPER_REPORT_HEADER, \
+from mob_suite.constants import MOB_TYPER_REPORT_HEADER, \
     MOB_CLUSTER_INFO_HEADER, \
     default_database_dir, \
-    ETE3_LOCK_FILE, \
     LIT_PLASMID_TAXONOMY_HEADER
 
 
@@ -187,7 +184,7 @@ def main():
         tmp_dir = args.analysis_dir
 
     if not os.path.isdir(tmp_dir):
-        os.mkdir(tmp_dir, 0o755)
+        os.makedirs(tmp_dir, 0o755)
 
     if not isinstance(args.num_threads, int):
         logger.info('Error number of threads must be an integer, you specified "{}"'.format(args.num_threads))
@@ -218,6 +215,7 @@ def main():
         primary_distance = float(args.primary_cluster_dist)
 
     min_length = int(args.min_length)
+    ETE3DBTAXAFILE = os.path.abspath(database_dir + "/taxa.sqlite")
 
     if database_dir == default_database_dir:
         mob_ref = args.plasmid_mob
@@ -236,7 +234,7 @@ def main():
         mpf_ref = os.path.join(database_dir, 'mpf.proteins.faa')
         plasmid_orit = os.path.join(database_dir, 'orit.fas')
         repetitive_mask_file = os.path.join(database_dir, 'repetitive.dna.fas')
-        ETE3DBTAXAFILE = os.path.abspath(database_dir + "/taxa.sqlite")
+
 
     LIT_PLASMID_TAXONOMY_FILE = os.path.join(database_dir, "host_range_literature_plasmidDB.txt")
     NCBI_PLASMID_TAXONOMY_FILE = plasmid_meta
@@ -340,7 +338,7 @@ def main():
                 contig_info[id][feature] = ''
             seq = str(record.seq)
             contig_info[id]['md5'] = calc_md5(seq)
-            contig_info[id]['gc'] = GC(seq)
+            contig_info[id]['gc'] = gc_fraction(seq)
             contig_info[id]['size'] = len(seq)
             contig_info[id]['contig_id'] = id
             contig_info[id]['sample_id'] = sample_id
@@ -510,7 +508,7 @@ def main():
             record['relaxase_type_accession(s)'] = ",".join(record['relaxase_type_accession(s)'])
 
         host_range = hostrange(record['rep_type(s)'].split(','), record['relaxase_type_accession(s)'].split(','),
-                               mob_cluster_id, ncbi, lit)
+                               mob_cluster_id, ncbi, lit,ETE3DBTAXAFILE)
 
         for field in host_range:
             record[field] = host_range[field]

mob_suite/utils.py

@@ -1,5 +1,5 @@
 from Bio import SeqIO
-from Bio.SeqUtils import GC
+from Bio.SeqUtils import gc_fraction
 from mob_suite.blast import BlastRunner
 from mob_suite.blast import BlastReader
 import os, re, time
@@ -11,8 +11,7 @@
 from ete3 import NCBITaxa
 from mob_suite.constants import \
     MOB_TYPER_REPORT_HEADER, \
-    MGE_INFO_HEADER, \
-    ETE3DBTAXAFILE
+    MGE_INFO_HEADER
 
 
 def getAssocValues(query_list_values, look_up_key, value_key, data):
@@ -65,30 +64,6 @@ def parseMash(mash_results):
     return hits
 
 
-def getHeirarchy(taxid):
-    if not isETE3DBTAXAFILEexists(ETE3DBTAXAFILE):
-        logging.info("Did not find taxa.sqlite in {}. Initializaing ete3 taxonomy database".format(ETE3DBTAXAFILE))
-        initETE3Database()
-
-    ncbi = NCBITaxa(dbfile=ETE3DBTAXAFILE)
-    if not isETE3DBTAXAFILEexists(ETE3DBTAXAFILE):
-        logging.error(
-            "Tried ete3 init, but still was not able to find taxa.sqlite file for ete3 lib in {}. Aborting".format(
-                ETE3DBTAXAFILE))
-        return ['-', '-']
-
-    if not isinstance(taxid, int):
-        return {'names': [], 'ranks': []}
-
-    lineage = ncbi.get_lineage(taxid)
-    names = ncbi.get_taxid_translator(lineage)
-    ranks = []
-    for id in lineage:
-        ranks.append(ncbi.get_rank(id))
-
-    return {'names': names, 'ranks': names}
-
-
 def filter_invalid_taxids(taxids):
     filtered = []
     for i in taxids:
@@ -104,7 +79,7 @@ def filter_invalid_taxids(taxids):
     return filtered
 
 
-def getHeirarchy(taxid):
+def getHeirarchy(taxid,ETE3DBTAXAFILE):
     if not isETE3DBTAXAFILEexists(ETE3DBTAXAFILE):
         logging.info("Did not find taxa.sqlite in {}. Initializaing ete3 taxonomy database".format(ETE3DBTAXAFILE))
         initETE3Database()
@@ -128,7 +103,7 @@ def getHeirarchy(taxid):
     return {'names': names, 'ranks': names}
 
 
-def getTaxid(taxon):
+def getTaxid(taxon,ETE3DBTAXAFILE):
     if not isETE3DBTAXAFILEexists(ETE3DBTAXAFILE):
         logging.info("Did not find taxa.sqlite in {}. Initializaing ete3 taxonomy database".format(ETE3DBTAXAFILE))
         initETE3Database()
@@ -146,7 +121,7 @@ def getTaxid(taxon):
 
 
 
-def NamesToTaxIDs(names):
+def NamesToTaxIDs(names,ETE3DBTAXAFILE):
     if not isETE3DBTAXAFILEexists(ETE3DBTAXAFILE):
         logging.info("Did not find taxa.sqlite in {}. Initializaing ete3 taxonomy database".format(ETE3DBTAXAFILE))
         initETE3Database(ETE3DBTAXAFILE)
@@ -163,7 +138,7 @@ def NamesToTaxIDs(names):
 
 
 
-def getTaxonConvergence(taxids):
+def getTaxonConvergence(taxids,ETE3DBTAXAFILE):
     if not isETE3DBTAXAFILEexists(ETE3DBTAXAFILE):
         logging.info("Did not find taxa.sqlite in {}. Initializaing ete3 taxonomy database".format(ETE3DBTAXAFILE))
         initETE3Database(ETE3DBTAXAFILE)
@@ -231,7 +206,7 @@ def getTaxonConvergence(taxids):
     return (['-', '-'])
 
 
-def hostrange(replion_types, relaxase_types, mob_cluster_id, ncbi, lit):
+def hostrange(replion_types, relaxase_types, mob_cluster_id, ncbi, lit,ETE3DBTAXAFILE):
     host_range_predictions = {
         'observed_host_range_ncbi_name': '',
         'observed_host_range_ncbi_rank': '',
@@ -276,25 +251,25 @@ def hostrange(replion_types, relaxase_types, mob_cluster_id, ncbi, lit):
         ncbi_unique_taxids = filter_invalid_taxids(
             list(set(ncbi_replicon_taxids + ncbi_cluster_taxids + ncbi_relaxase_taxids)))
         host_range_predictions['observed_host_range_ncbi_rank'], host_range_predictions[
-            'observed_host_range_ncbi_name'] = getTaxonConvergence(ncbi_unique_taxids)
+            'observed_host_range_ncbi_name'] = getTaxonConvergence(ncbi_unique_taxids,ETE3DBTAXAFILE)
 
     # Determine taxids associated with literature
 
     lit_unique_taxids = filter_invalid_taxids(list(set(lit_replicon_taxids)))
 
     host_range_predictions['reported_host_range_lit_rank'], host_range_predictions[
-        'reported_host_range_lit_name'] = getTaxonConvergence(lit_unique_taxids)
+        'reported_host_range_lit_name'] = getTaxonConvergence(lit_unique_taxids,ETE3DBTAXAFILE)
 
     # determine overall host range
     overall_taxids = filter_invalid_taxids(list(set(ncbi_unique_taxids + lit_unique_taxids)))
     host_range_predictions['predicted_host_range_overall_rank'], host_range_predictions[
-        'predicted_host_range_overall_name'] = getTaxonConvergence(overall_taxids)
+        'predicted_host_range_overall_name'] = getTaxonConvergence(overall_taxids,ETE3DBTAXAFILE)
 
     # move host-range prediction up to family when it is at genus or species level
     if host_range_predictions['predicted_host_range_overall_rank'] == 'genus' or host_range_predictions[
         'predicted_host_range_overall_rank'] == 'species':
-        taxid = getTaxid(host_range_predictions['predicted_host_range_overall_name'])
-        heir = getHeirarchy(taxid)
+        taxid = getTaxid(host_range_predictions['predicted_host_range_overall_name'],ETE3DBTAXAFILE)
+        heir = getHeirarchy(taxid,ETE3DBTAXAFILE)
         names = heir['names']
         ranks = heir['ranks']
 
@@ -869,7 +844,7 @@ def calcFastaStatsIndividual(fasta):
         id = record.id
         seq = record.seq
         genome_size = len(seq)
-        gc = GC(seq)
+        gc = gc_fraction(seq)
         md5 = calc_md5(seq)
         stats[id] = {
             'total_length': genome_size,
@@ -892,7 +867,7 @@ def calcFastaStats(fasta):
         num_seqs += 1
         seq = seq + record.seq
     genome_size = len(seq)
-    gc = GC(seq)
+    gc = gc_fraction(seq)
     md5 = calc_md5(seq)
 
     return {
@@ -974,7 +949,7 @@ def determine_mpf_type(hits):
     return max(types, key=lambda i: types[i])
 
 
-def build_mobtyper_report(plasmid_contig_info, out_dir, outfile, seq_dict, ncbi, lit):
+def build_mobtyper_report(plasmid_contig_info, out_dir, outfile, seq_dict, ncbi, lit,ETE3DBTAXAFILE):
     mob_typer_results = {}
     for clust_id in plasmid_contig_info:
 
@@ -1008,7 +983,7 @@ def build_mobtyper_report(plasmid_contig_info, out_dir, outfile, seq_dict, ncbi,
         cluster_seq = sorted(cluster_seq,key=len)
         seq = "".join(cluster_seq)
         mob_typer_results[clust_id]['md5'] = [calc_md5(seq)]
-        mob_typer_results[clust_id]['gc'] = [GC(seq)]
+        mob_typer_results[clust_id]['gc'] = [gc_fraction(seq)]
         mob_typer_results[clust_id]['size'] = [len(seq)]
         mob_typer_results[clust_id]['num_contigs'] = len(cluster_seq)
 
@@ -1089,7 +1064,7 @@ def build_mobtyper_report(plasmid_contig_info, out_dir, outfile, seq_dict, ncbi,
         else:
             mob_cluster_id = '-'
 
-        host_range = hostrange(rep_types, relaxase_types, mob_cluster_id, ncbi, lit)
+        host_range = hostrange(rep_types, relaxase_types, mob_cluster_id, ncbi, lit,ETE3DBTAXAFILE)
 
         for field in host_range:
             mob_typer_results[clust_id][field] = host_range[field]

mob_suite/version.py

@@ -1,2 +1,2 @@
-__version__ = '3.1.0'
+__version__ = '3.1.1'
 

setup.py

@@ -29,7 +29,7 @@ def read(fname):
 setup(
     name='mob_suite',
     include_package_data=True,
-    version='3.1.0',
+    version='3.1.1',
     python_requires='>=3.7.0,<4',
     setup_requires=['pytest-runner'],
     tests_require=['pytest'],
@@ -46,7 +46,7 @@ def read(fname):
     package_data={'mob_suite': ['config.json']},
 
     install_requires=[
-        'numpy>=1.11.1,<2',
+        'numpy>=1.11.1,<1.23.5',
         'tables>=3.3.0,<4',
         'pandas>=0.22.0,<=1.0.5',
         'biopython>=1.70,<2',