Merge branch 'master' of github.com:MG-RAST/MG-RAST-Tools

Author: wilke

README.md

@@ -19,8 +19,6 @@ Repository of scripts and libraries for using the MG-RAST API and MG-RAST data
 
 -   Python libs
     -   prettytable - <https://pypi.python.org/pypi/PrettyTable>
-    -   poster - <https://pypi.python.org/pypi/poster>
-         For python3, you will need a python3 port of poster:  <https://github.com/mrd1no/poster-0.8.1-for-Python-3.4>
     -   requests - <http://docs.python-requests.org/en/latest>
     -   requests_toolbelt - <https://github.com/sigmavirus24/requests-toolbelt>
     -   scipy - <http://www.scipy.org>

examples/python/abundance_matrix.py

@@ -4,18 +4,20 @@
 
 from __future__ import print_function
 import sys
+import json
 from optparse import OptionParser
 from mglib import async_rest_api, sparse_to_dense, get_auth_token, API_URL
 
 DEBUG = 0
 
 if __name__ == '__main__':
-    usage = "usage: %prog -i <input sequence file> -o <output file>"
+    usage = "Usage: %prog [options]\nFunction: retrieves data from MG-RAST matrix API, unwraps into csv format"
     parser = OptionParser(usage)
     parser.add_option("-s", "--source", dest="source", default="RefSeq", help="Annotation source: RefSeq, GenBank, IMG, SEED, TrEMBL, SwissProt, PATRIC, KEG, RDP, Greengenes, LSU, SSU")
     parser.add_option("-g", "--grouplevel", dest="grouplevel", default="domain", help="Grouping level: strain, species, genus, family, order, class, phylum, domain / function, level1, level2, level3")
     parser.add_option("-i", "--hittype", dest="hittype", default="single", help="Hit type: all, single, lca")
     parser.add_option("-c", "--call", dest="call", default="organism", help="organism or function")
+    parser.add_option("-b", "--biom", dest="biom", action="store_true", help="biom output (csv defaut)")
     parser.add_option("-d", "--identity", dest="identity", default=1, help="% identity threshold")
     parser.add_option("-e", "--evalue", dest="evalue", default="1", help="organism or function")
     parser.add_option("-t", "--type", dest="resulttype", default="abundance", help="Result type: abundnaance, evalue, identity, or length")
@@ -36,35 +38,40 @@
     length = opts.length
     identity = opts.identity
     hittype = opts.hittype
+    biom = opts.biom
 # construct API call
     base_url = API_URL + "/matrix/organism"
     if opts.call == "function" or opts.source == "SubSystems":
         base_url = API_URL + "/matrix/function"
-    base_url = base_url + "?asynchronous=1&group_level=%s&result_type=%s&auth=%s&source=%s&evalue=%s&length=%s&identity=%s&hittype=%s&" % (group_level, result_type, key, source, evalue, length, identity, hittype)
+    base_url = base_url + "?asynchronous=1&group_level=%s&result_type=%s&source=%s&evalue=%s&length=%s&identity=%s&hittype=%s&" % (group_level, result_type, source, evalue, length, identity, hittype)
     URI = base_url + "&".join(["id=%s" % m for m in metagenomes.split(",")])
     print(URI, file=sys.stderr)
-    print("#"+ URI, file=sys.stdout)
 # retrieve the data by sending at HTTP GET request to the MG-RAST API
 
     jsonstructure = async_rest_api(URI, auth=key)
+    if biom:
+        print(json.dumps(jsonstructure))
+        sys.exit()
+    else:
+        print("#"+ URI, file=sys.stdout)
 
-# unpack and display the data table
-    cols = [x["id"] for x in jsonstructure["columns"]]
-    rows = [x["id"] for x in jsonstructure["rows"]]
-    matrixtype = jsonstructure["type"]
-
+    matrixtype = jsonstructure["data"]["matrix_type"]
     if matrixtype == "sparse":
         data = sparse_to_dense(jsonstructure["data"], len(rows), len(cols))
     else:
         data = jsonstructure["data"]
+# unpack and display the data table
+    cols = [x["id"] for x in data["columns"]]
+    rows = [x["id"] for x in data["rows"]]
+
 
     if DEBUG:
         print(jsonstructure)
         print("COLS", cols)
         print("ROWS", rows)
         print("TYPE", matrixtype)
         print("DATA", data)
-    h = data
+    h = data["data"]
 
     sys.stdout.write("Taxon\t")
     for j in range(0, len(cols)):

examples/python/annotation_table.py

@@ -9,9 +9,8 @@
 
 DEBUG = 0
 
-
 if __name__ == '__main__':
-    usage = "usage: %prog -i <input sequence file> -o <output file>"
+    usage = "usage: %prog [options]\nFunction: retrieves and presents table of sequence IDs and annotation table results"
     parser = OptionParser(usage)
     parser.add_option("-s", "--source", dest="source", default="RefSeq", help="Annotation source: RefSeq, GenBank, IMG, SEED, TrEMBL, SwissProt, PATRIC, KEG, RDP, Greengenes, LSU, SSU")
     parser.add_option("-g", "--grouplevel", dest="grouplevel", default="domain", help="Grouping level: strain, species, genus, family, order, class, phylum, domain / function, level1, level2, level3")
@@ -36,7 +35,7 @@
 
 # construct API call
     base_url = API_URL + "/profile/{}".format(metagenomes)
-    base_url = base_url + "?asynchronous=1&group_level=%s&result_type=%s&auth=%s&source=%s&evalue=%s&" % (group_level, result_type, key, source, evalue)
+    base_url = base_url + "?asynchronous=1&group_level=%s&result_type=%s&source=%s&evalue=%s&" % (group_level, result_type, source, evalue)
     URI = base_url + "&".join(["id=%s" % m for m in metagenomes.split(",")])
     URI = base_url 
     print(URI, file=sys.stderr)
@@ -48,6 +47,7 @@
 #    rows = [x["id"] for x in jsondata["rows"]]
 
     data = jsondata # ["data"]
+    data = jsondata["data"]
 
     if DEBUG:
         print(jsonstructure)

examples/python/list_all_mg.py

@@ -1,24 +1,30 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 '''This script retrieves a list of metagenomes from the MG-RAST API.'''
 from __future__ import print_function
-import urllib
+from __future__ import unicode_literals
 import sys
+import time
 
-from mglib import get_auth_token, obj_from_url, API_URL
+from mglib import get_auth_token, obj_from_url, API_URL, urlencode
 
 def printlist(js):
     '''prints essential fields from metagenome list'''
     for item in js["data"]:
         if "public" in item.keys():
-            public = repr(item["public"])
+            public = item["public"]
         else:
             public = "False"
-        sys.stdout.write( ("\t".join([item["metagenome_id"],
-#                         str(len(item.keys())),
-                          public, item["created_on"],
-                          item["name"]]) + "\n").encode("utf-8"))
+        try:
+            mg_name= item["name"]
+            project_id = item["project_id"]
+            project_name = item["project_name"]
+        except KeyError:
+            sys.stderr.write(repr(item) + "\n")
+        sys.stdout.write(("\t".join([item["metagenome_id"],
+#                        str(len(item.keys())),
+                         repr(public), item["created_on"],
+                         mg_name, project_id, project_name]) + "\n"))
 
-CALL = "/metagenome"
 CALL = "/search"
 
 key = get_auth_token()
@@ -27,23 +33,30 @@ def printlist(js):
 limit = 1000 # initial call
 
 # construct API call
+# public = 0 means "don't show public metagenomes"
+parameters = {"limit": limit, "order":"created_on", "direction": "asc", "public": "1"}
+API_URL= "https://api.mg-rast.org/"
 
-parameters = {"limit": limit, "auth": key, "order":"created_on", "direction": "asc"}
-base_url = API_URL + CALL + "?" + urllib.urlencode(parameters)
+base_url = API_URL + CALL + "?" + urlencode(parameters)
 
 # convert the data from a JSON structure to a python data type, a dict of dicts.
-jsonstructure = obj_from_url(base_url)
+jsonstructure = obj_from_url(base_url, auth=key)
 
 # unpack and display the data table
 total_count = int(jsonstructure["total_count"])
 sys.stderr.write("Total number of records: {:d}\n".format(total_count))
 
-for i in range(0, total_count / limit +1):
-    sys.stderr.write("Page {:d}\t".format(i))
-    jsonstructure = obj_from_url(base_url)
+for i in range(0, int(total_count / limit) +2):
+#    sys.stderr.write("Page {:d}\t".format(i))
+    sys.stderr.write("Page {:d}\t{}\n".format(i, base_url))
+    jsonstructure = obj_from_url(base_url, auth=key)
     printlist(jsonstructure)
-    try:
+    time.sleep(1)
+    if "next" in jsonstructure.keys():
         next_url = jsonstructure["next"]
         base_url = next_url
-    except KeyError:
+        continue
+    else:
+        sys.stderr.write("No next, page {} url {} \n".format(i, base_url))
+        sys.stderr.write(repr(jsonstructure)) 
         break

mglib/mglib.py

@@ -72,10 +72,26 @@ def body_from_url(url, accept, auth=None, data=None, debug=False, method=None):
 
 # return python struct from JSON output of MG-RAST or Shock API
 def obj_from_url(url, auth=None, data=None, debug=False, method=None):
-    result = body_from_url(url, 'application/json', auth=auth, data=data, debug=debug, method=method)
-    obj = json.loads(result.read().decode("utf8"))
+    if type(data) is str:
+        data=data.encode("utf8")
+    try:
+        result = body_from_url(url, 'application/json', auth=auth, data=data, debug=debug, method=method)
+        read = result.read()
+    except:  # try one more time  ConnectionResetError is incompatible with python2
+        result = body_from_url(url, 'application/json', auth=auth, data=data, debug=debug, method=method)
+        read = result.read()
+    if result.headers["content-type"] == "application/x-download" or result.headers["content-type"] == "application/octet-stream":
+        return(read)   # Watch out!
+    if result.headers["content-type"][0:9] == "text/html":  # json decoder won't work
+        return(read)   # Watch out!
+    if result.headers["content-type"] == "application/json":  # If header is set, this should work 
+        data = read.decode("utf8")
+        obj = json.loads(data)
+    else:
+        data = read.decode("utf8")
+        obj = json.loads(data)
     if obj is None:
-        sys.stderr.write("ERROR: return structure not valid json format\n")
+        sys.stderr.write("ERROR: return structure not valid json format\n" + repr(data))
         sys.exit(1)
     if len(list(obj.keys())) == 0:
         if debug:
@@ -86,7 +102,7 @@ def obj_from_url(url, auth=None, data=None, debug=False, method=None):
         sys.stderr.write("ERROR: %s\n" %obj['ERROR'])
         sys.exit(1)
     if ('error' in obj) and obj['error']:
-        if isinstance(obj['error'], basestring):
+        if isinstance(obj['error'], str):
             sys.stderr.write("ERROR:\n%s\n" %obj['error'])
         else:
             sys.stderr.write("ERROR: %s\n" %obj['error'][0])
@@ -118,28 +134,44 @@ def async_rest_api(url, auth=None, data=None, debug=False, delay=60):
     except:
         parameters = {"asynchronous": 1}
     submit = obj_from_url(url, auth=auth, data=data, debug=debug)
-# If "status" is nor present, or if "status" is somehow not "submitted" 
+# If "status" is nor present, or if "status" is somehow not "submitted"
 # assume this is not an asynchronous call and it's done.
-    if ('status' in submit) and (submit['status'] == 'done') and ('url' in submit):
-        return submit['data']
-    if not (('status' in submit) and (submit['status'] == 'submitted') and ('url' in submit)):
+    if type(submit) == bytes:   # can't decode
+        try: 
+            return decode("utf-8", submit)
+        except:
+            return submit
+    if ('status' in submit) and (submit['status'] != 'submitted') and (submit['status'] != "processing") and ('data' in submit):
         return submit
-    result = obj_from_url(submit['url'], debug=debug)
-    try:
-        while result['status'] != 'done':
+    if not ('url' in submit.keys()):
+        return submit
+#    if not (('status' in submit) and (submit['status'] == 'submitted') and ('url' in submit)):
+#        return submit  # No status, no url and no submitted
+    result = obj_from_url(submit['url'], auth=auth, debug=debug)
+    if type(result) is bytes:
+        return(result)
+    if 'status' in result.keys():
+        while result['status'] == 'submitted' or result['status'] == "processing":
             if debug:
                 print("waiting %d seconds ..."%delay)
             time.sleep(delay)
-            result = obj_from_url(submit['url'], debug=debug)
-    except KeyError:
+            result = obj_from_url(submit['url'], auth=auth, debug=debug)
+    if 'url' in result.keys() or 'next' in result.keys(): # does not need to wait
+        return(result)
+    try:
         print("Error in response to "+url, file=sys.stderr)
-        print("Does not contain 'status' field, likely API syntax error", file=sys.stderr)
-        print(json.dumps(result), file=sys.stderr)
-        sys.exit(1)
-    if 'ERROR' in result['data']:
-        sys.stderr.write("ERROR: %s\n" %result['data']['ERROR'])
+        print("Does not contain 'status' or 'next' field, likely API syntax error", file=sys.stderr)
         print(json.dumps(result), file=sys.stderr)
         sys.exit(1)
+    except TypeError:  # result isn't json, return it anyway
+        return(result.decode("utf8"))
+    try:
+        if 'ERROR' in result['data']:
+            sys.stderr.write("ERROR: %s\n" %result['data']['ERROR'])
+            print(json.dumps(result), file=sys.stderr)
+            sys.exit(1)
+    except KeyError:  # result doesn't have "data"
+        return result
     return result['data']
 
 # POST file to MG-RAST or Shock
@@ -206,14 +238,15 @@ def sparse_to_dense(sMatrix, rmax, cmax):
 # transform BIOM format to tabbed table
 # returns max value of matrix
 def biom_to_tab(biom, hdl, rows=None, use_id=True, col_name=False):
+    assert 'matrix_type' in biom.keys(), repr(biom)
     if biom['matrix_type'] == 'sparse':
         matrix = sparse_to_dense(biom['data'], biom['shape'][0], biom['shape'][1])
     else:
         matrix = biom['data']
     if col_name:
-        hdl.write( "\t%s\n" %"\t".join([c['name'] for c in biom['columns']]) )
+        hdl.write("\t%s\n" %"\t".join([c['name'] for c in biom['columns']]))
     else:
-        hdl.write( "\t%s\n" %"\t".join([c['id'] for c in biom['columns']]) )
+        hdl.write("\t%s\n" %"\t".join([c['id'] for c in biom['columns']]))
     rowmax = []
     for i, row in enumerate(matrix):
         name = biom['rows'][i]['id']
@@ -223,7 +256,7 @@ def biom_to_tab(biom, hdl, rows=None, use_id=True, col_name=False):
             continue
         try:
             rowmax.append(max(row))
-            hdl.write( "%s\t%s\n" %(name, "\t".join(map(str, row))) )
+            hdl.write("%s\t%s\n" %(name, "\t".join(map(str, row))))
         except:
             try:
                 hdl.close()
@@ -254,6 +287,7 @@ def profile_to_matrix(p):
     p['matrix_element_type'] = 'int'
     p['matrix_element_value'] = 'abundance'
     p['date'] = time.strftime("%Y-%m-%d %H:%M:%S")
+    assert 'matrix_type' in p.keys(), repr(p)
     if p['matrix_type'] == 'sparse':
         p['data'] = sparse_to_dense(p['data'], p['shape'][0], p['shape'][1])
     if trim:
@@ -293,6 +327,7 @@ def merge_biom(b1, b2):
                "id": b1['id']+'_'+b2['id'],
                "type": b1['type'] }
     # make sure we are dense
+    assert 'matrix_type' in b2.keys(), repr(b2)
     if b2['matrix_type'] == 'sparse':
         b2['data'] = sparse_to_dense(b2['data'], b2['shape'][0], b2['shape'][1])
     # get lists of ids
@@ -344,14 +379,15 @@ def biom_to_matrix(biom, col_name=False, sig_stats=False):
     except KeyError:
         rows = [r['id'] for r in biom['rows']]
 #        rows = [";".join(r['metadata']['hierarchy']) for r in biom['rows']]
+    assert "matrix_type" in biom.keys(), repr(biom)
     if biom['matrix_type'] == 'sparse':
         data = sparse_to_dense(biom['data'], len(rows), len(cols))
     else:
         data = biom['data']
     if sig_stats and ('significance' in biom['rows'][0]['metadata']) and (len(biom['rows'][0]['metadata']['significance']) > 0):
-        cols.extend( [s[0] for s in biom['rows'][0]['metadata']['significance']] )
+        cols.extend([s[0] for s in biom['rows'][0]['metadata']['significance']] )
         for i, r in enumerate(biom['rows']):
-            data[i].extend( [s[1] for s in r['metadata']['significance']] )
+            data[i].extend([s[1] for s in r['metadata']['significance']] )
     return rows, cols, data
 
 # transform tabbed table to matrix in json format
@@ -374,7 +410,7 @@ def sub_matrix(matrix, ncols):
         return matrix
     sub = list()
     for row in matrix:
-        sub.append( row[:ncols] )
+        sub.append(row[:ncols] )
     return sub
 
 # return KBase id for MG-RAST id
@@ -407,7 +443,7 @@ def kbids_to_mgids(kbids):
 def kbid_lookup(ids, reverse=False):
     request = 'mg2kb' if reverse else 'kb2mg'
     post = json.dumps({'ids': ids}, separators=(',',':'))
-    data = obj_from_url(API_URL+'/job/'+request, data=post)
+    data = obj_from_url(API_URL+'/job/'+request, auth=auth, data=post)
     return data['data']
 
 def get_auth_token(opts=None):

scripts/mg-abundant-functions.py

@@ -48,7 +48,7 @@ def main(args):
     parser.add_argument("--filter_level", dest="filter_level", default=None, help="function level to filter by")
     parser.add_argument("--top", dest="top", type=int, default=10, help="display only the top N taxa, default is 10")
     parser.add_argument("--evalue", dest="evalue", type=int, default=5, help="negative exponent value for maximum e-value cutoff, default is 5")
-    parser.add_argument("--identity", dest="identity", type=int, default=60, help="percent value for minimum % identity cutoff, default is 60")
+    parser.add_argument("--identity", dest="identity", type=int, default=60, help="percent value for minimum %% identity cutoff, default is 60")
     parser.add_argument("--length", dest="length", type=int, default=15, help="value for minimum alignment length cutoff, default is 15")
     parser.add_argument("--version", type=int, dest="version", default=1, help="M5NR annotation version to use, default is 1")
 
@@ -94,8 +94,10 @@ def main(args):
         data = obj_from_url(url)
         level = 'level4' if opts.level == 'function' else opts.level
         sub_ann = set(map(lambda x: x[level], data['data']))
-    
+    biomorig = biom
+    biom = biomorig["data"] 
     # sort data
+    assert "matrix_type" in biom.keys(), repr(biom)
     if biom["matrix_type"] == "sparse":
         for d in sorted(biom['data'], key=itemgetter(2), reverse=True):
             name = biom['rows'][d[0]]['id']  # if opts.source != 'Subsystems' else biom['rows'][d[0]]['metadata']['ontology'][-1]