adding api-endpoint client for singularity hub, still needs tests

Author: vsoch

examples/singularity_hub/compare_builds.py

@@ -0,0 +1,24 @@
+# Compare Singularity Hub containers
+
+# This is a simple script to use the singularity command line tool to download containers
+# (using Singularity, Section 1) and compare build specs (using Singularity Hub API, Section 2) and to
+# compare the containers themselves using singularity python (Section 3)
+
+container_names = ['vsoch/singularity-hello-world',
+                   'researchapps/quantum_state_diffusion',
+                   'vsoch/pefinder']
+
+from singularity.hub.client import Client
+
+client = Client()
+results = dict()
+
+for container_name in container_names:
+    
+    # Retrieve the container based on the name
+    collection = client.get_collection(container_name)
+    container_ids = collection['container_set']
+    containers = []
+    for container_id in container_ids:
+       container = client.get_container(container_id)
+       containers.append(container)

singularity/hub/__init__.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+'''
+auth.py: authentication functions for singularity hub api
+         currently no token / auth for private collections
+'''
+
+from singularity.logman import bot
+import os
+import sys
+
+
+def get_headers(token=None):
+    '''get_headers will return a simple default header for a json
+    post. This function will be adopted as needed.
+    :param token: an optional token to add for auth
+    '''
+    headers = dict()
+    headers["Content-Type"] = "application/json"
+    if token!=None:
+        headers["Authorization"] = "Bearer %s" %(token)
+
+    header_names = ",".join(list(headers.keys()))
+    bot.logger.debug("Headers found: %s",header_names)
+    return headers

singularity/hub/auth.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+'''
+base.py: base module for working with singularity hub api. Right
+         now serves to hold defaults.
+
+'''
+
+from singularity.hub.utils import (
+    parse_container_name,
+    is_number,
+    api_get,
+    api_post
+)
+
+api_base = "https://singularity-hub.org/api"
+
+def get_template(container_name,get_type=None):
+    '''get a container/collection or return None.
+    '''
+    if get_type == None:
+        get_type = "container"
+    get_type = get_type.lower().replace(' ','')
+
+    result = None
+    image = parse_container_name(container_name)
+    if is_number(image):
+        url = "%s/%ss/%s" %(api_base,get_type,image)
+
+    elif image['user'] is not None and image['repo_name'] is not None:        
+        url = "%s/%s/%s/%s" %(api_base,
+                              get_type,
+                              image['user'],
+                              image['repo_name'])
+
+        if image['repo_tag'] is not None and get_type is not "collection":
+            url = "%s:%s" %(url,image['repo_tag'])
+ 
+        result = api_get(url)
+
+    return result

singularity/hub/base.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+'''
+client.py: simple client for singularity hub api
+
+'''
+
+from singularity.hub.auth import (
+    get_headers
+)
+
+from singularity.logman import bot
+from singularity.hub.base import get_template
+
+
+class Client(object):
+
+
+    def __init__(self, token=None):
+ 
+        if token is not None:
+            self.token = token
+        # currently not used
+        self.headers = get_headers(token=token)
+
+
+    def update_headers(self, headers):
+        '''update_headers will add headers to the client
+        :param headers: should be a dictionary of key,value to update/add to header
+        '''
+        for key,value in headers.items():
+            self.client.headers[key] = item
+
+
+    def get_container(self,container_name):
+        '''get a container or return None.
+        '''
+        return get_template(container_name,"container")
+
+
+    def get_collection(self,container_name):
+        '''get a container collection or return None.
+        '''
+        return get_template(container_name,"collection")

singularity/hub/client.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+
+'''
+utils.py: general http functions (utils) for som api
+
+'''
+
+from singularity.logman import bot
+from singularity.hub.auth import get_headers
+
+import requests
+import os
+import sys
+
+
+def api_get(url,headers=None,token=None,data=None, return_json=True):
+    '''api_get will use requests to get a particular url
+    :param url: the url to send file to
+    :param headers: a dictionary with headers for the request
+    :param putdata: additional data to add to the request
+    :param return_json: return json if successful
+    '''
+    bot.logger.debug("GET %s",url)
+
+    if headers == None:
+        headers = get_headers(token=token)
+    if data == None:
+        response = requests.get(url,         
+                                headers=headers)
+    else:
+        response = requests.get(url,         
+                                headers=headers,
+                                json=data)
+
+    if response.status_code == 200 and return_json:
+        return response.json()
+
+    return response
+
+
+def api_put(url,headers=None,token=None,data=None, return_json=True):
+    '''api_put will send a read file (spec) to Singularity Hub with a particular set of headers
+    :param url: the url to send file to
+    :param headers: the headers to get
+    :param headers: a dictionary with headers for the request
+    :param data: additional data to add to the request
+    :param return_json: return json if successful
+    '''
+    bot.logger.debug("PUT %s",url)
+
+    if headers == None:
+        headers = get_headers(token=token)
+    if data == None:
+        response = requests.put(url,         
+                                headers=headers)
+    else:
+        response = requests.put(url,         
+                                headers=headers,
+                                json=data)
+    
+    if response.status_code == 200 and return_json:
+        return response.json()
+
+    return response
+
+
+def api_post(url,headers=None,data=None,token=None,return_json=True):
+    '''api_get will use requests to get a particular url
+    :param url: the url to send file to
+    :param headers: a dictionary with headers for the request
+    :param data: additional data to add to the request
+    :param return_json: return json if successful
+    '''
+    bot.logger.debug("POST %s",url)
+
+    if headers == None:
+        headers = get_headers(token=token)
+    if data == None:
+        response = requests.post(url,         
+                                 headers=headers)
+    else:
+        response = requests.post(url,         
+                                 headers=headers,
+                                 json=data)
+
+    if response.status_code == 200 and return_json:
+        return response.json()
+
+    return response
+
+
+######################################################################
+# OS/IO and Formatting Functions
+######################################################################
+
+
+def is_number(container_name):
+    '''is_number determines if the user is providing a singularity hub
+    number (meaning the id of an image to download) vs a full name)
+    '''
+    if isinstance(container_name,dict):
+        return False
+    try:
+        float(container_name)
+        return True
+    except ValueError:
+        return False
+
+
+def parse_container_name(image):
+    '''parse_container_name will return a json structure with a repo name, tag, user.
+    '''
+    container_name = image
+    if not is_number(image):
+        image = image.replace(' ','')
+
+    # If the user provided a number (unique id for an image), return it
+    if is_number(image) == True:
+        logger.info("Numeric image ID %s found.", image)
+        return int(image)
+
+    image = image.split('/')
+
+    # If there are two parts, we have username with repo (and maybe tag)
+    if len(image) >= 2:
+        user = image[0]
+        image = image[1]
+
+    # Otherwise, we trigger error (not supported just usernames yet)
+    else:
+        bot.logger.error('You must specify a repo name and username, %s is not valid',container_name)
+        sys.exit(1)
+
+    # Now split the name by : in case there is a tag
+    image = image.split(':')
+    if len(image) == 2:
+        repo_name = image[0]
+        repo_tag = image[1]
+
+    # Otherwise, assume latest of an image
+    else:
+        repo_name = image[0]
+        repo_tag = "latest"
+
+    bot.logger.info("User: %s", user)
+    bot.logger.info("Repo Name: %s", repo_name)
+    bot.logger.info("Repo Tag: %s", repo_tag)
+
+    parsed = {'repo_name':repo_name,
+              'repo_tag':repo_tag,
+              'user':user }
+
+    return parsed

singularity/hub/utils.py

@@ -1,4 +1,4 @@
-__version__ = "1.1.1"
+__version__ = "1.1.2"
 AUTHOR = 'Vanessa Sochat'
 AUTHOR_EMAIL = '[email protected]'
 NAME = 'singularity'