First Blood

Author: cosmincatalin

.editorconfig

@@ -0,0 +1,9 @@
+root = true
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+indent_style = space
+indent_size = 4
+# Idealy we want to respect the official limit of 100 characters per line
+ij_visual_guides=100, 160

.gitignore

@@ -0,0 +1,6 @@
+build
+dist
+env
+ENVIRONMENT
+.wercker
+.idea

.werckerignore

@@ -0,0 +1,3 @@
+build
+dist
+env

README.md

@@ -0,0 +1,3 @@
+## artifacts-metadata
+
+This library provides a generic and reusable solution to the problem of storing and retrieving complex metadata associated with artifacts, such as those that are the result of machine learning.

artifacts_metadata/__init__.py

@@ -0,0 +1,4 @@
+"""
+Copyright (c) 2020 AudienceProject ApS (https://audienceproject.com)
+"""
+from artifacts_metadata.artifacts_metadata_client import ArtifactsMetadataClient

artifacts_metadata/artifacts_metadata_client.py

@@ -0,0 +1,58 @@
+"""
+Copyright (c) 2020 AudienceProject ApS (https://audienceproject.com)
+"""
+import time
+import uuid
+from typing import Dict, Optional
+
+import boto3
+
+
+class ArtifactsMetadataClient:
+    """
+    Class which provides functionality for:
+         - logging metadata information about artifacts to DynamoDB
+         - retrieving metadata information about artifacts from DynamoDB
+         - searching for artifacts based on various criteria being matched against metadata
+    """
+
+    def __init__(self, table_name: str = "artifacts_metadata", dynamodb_resource=None):
+        if dynamodb_resource is None:
+            session = boto3.session.Session()
+            dynamodb_resource = session.resource("dynamodb")
+        self.table = dynamodb_resource.Table(table_name)
+
+    def log(self, artifact_type: str, metadata: Optional[Dict[str, object]] = None) -> str:
+        """
+        Stores a dictionary of metadata into DynamoDB. Will always create a record,
+        it does not update.
+        If the location key is part of the dictionary, it will look for a {artifact-id}
+        template variable that will be replaced by the id of the record to be inserted.
+
+        @param artifact_type: A string categorization for artifacts.
+        @param metadata: A dictionary of values. Can be deep, but must conform to the
+            constraints of a DynamoDB PUT operation.
+
+        @return: The id of the artifact.
+        """
+        if metadata is None:
+            metadata = {}
+        metadata["id"] = str(uuid.uuid1())
+        metadata["artifactType"] = artifact_type
+        metadata["timestamp"] = time.time_ns() // 1000000
+        if "location" in metadata:
+            metadata["location"] = metadata["location"].replace("{artifact-id}", metadata["id"])
+        self.table.put_item(
+            Item=metadata
+        )
+        return metadata["id"]
+
+    def get(self, artifact_id: str) -> Dict[str, object]:
+        """
+        Returns the medata associated with a give id.
+
+        @param artifact_id: The id of the artifact's metadata as it is stored in DynamoDB
+        @return: A dictionary of values representing the whole metadata associated with
+            a particular artifact id.
+        """
+        return self.table.get_item(Key={"id": artifact_id})["Item"]

requirements.in

@@ -0,0 +1,7 @@
+boto3==1.0.0
+
+# Operational packages
+pip==20.0.2      # The ubiquitous dependency manager
+pylint==2.4.4    # For detecting Python errors early and keep consistency of the code
+pip-tools==4.4.1 # dependency management helper
+twine==3.1.1     # For publishing to PyPi

requirements.lock

@@ -0,0 +1,39 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file=requirements.lock --pre requirements.in
+#
+astroid==2.3.3            # via pylint
+bleach==3.1.0             # via readme-renderer
+boto3==1.0.0
+botocore==1.0.0           # via boto3
+certifi==2019.11.28       # via requests
+chardet==3.0.4            # via requests
+click==7.0                # via pip-tools
+docutils==0.15.2          # via botocore, readme-renderer
+futures==2.2.0            # via boto3
+idna==2.8                 # via requests
+isort==4.3.21             # via pylint
+jmespath==0.7.1           # via boto3, botocore
+keyring==21.1.0           # via twine
+lazy-object-proxy==1.4.3  # via astroid
+mccabe==0.6.1             # via pylint
+pip-tools==4.4.1
+pkginfo==1.5.0.1          # via twine
+pygments==2.5.2           # via readme-renderer
+pylint==2.4.4
+python-dateutil==2.8.1    # via botocore
+readme-renderer==24.0     # via twine
+requests-toolbelt==0.9.1  # via twine
+requests==2.22.0          # via requests-toolbelt, twine
+six==1.14.0               # via astroid, bleach, pip-tools, python-dateutil, readme-renderer
+tqdm==4.42.1              # via twine
+twine==3.1.1
+urllib3==1.25.8           # via requests
+webencodings==0.5.1       # via bleach
+wrapt==1.11.2             # via astroid
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools

setup.py

@@ -0,0 +1,16 @@
+import setuptools
+
+setuptools.setup(
+    name="artifacts-metadata",
+    version="0.0.1",
+    author="Cosmin Catalin Sanda",
+    author_email="[email protected]",
+    description="A library for managing artifacts metadata in DynamoDB",
+    long_description="A library that allows generating and storing metadata for artifacts with"
+                     "complex dependencies, parameters and configuration, such as machine learning "
+                     "models.",
+    url="https://github.com/audienceproject/artifacts-metadata-py",
+    packages=setuptools.find_packages(),
+    python_requires=">=3.6",
+    install_requires=["boto3>=1.0.0"]
+)

wercker.yml

@@ -0,0 +1,32 @@
+box:
+    id: audienceproject/python
+    username: $DOCKERHUB_ACCOUNT
+    password: $DOCKERHUB_PASSWORD
+    tag: 3.6
+no-response-timeout: 25
+command-timeout: 25
+
+build:
+    steps:
+        - script:
+            name: Setup virtualenv
+            code: |
+                virtualenv env --python=python3.6 --clear
+                env/bin/pip install pylint==2.4.4
+        - script:
+            name: PyLint
+            code: env/bin/pylint artifacts_metadata
+
+deploy:
+    steps:
+        - script:
+            name: Setup virtualenv
+            code: |
+                virtualenv env --python=python3.6 --clear
+                env/bin/pip install twine==3.1.1
+        - script:
+            name: Build library
+            code: env/bin/python setup.py sdist bdist_wheel
+        - script:
+            name: Release library
+            code: env/bin/twine upload --non-interactive dist/*