Initial pipeline code

Author: PiotrCzapla

.gitignore

@@ -0,0 +1,100 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# mypy
+.mypy_cache/
+.idea/*
+.vscode/settings.json

setup.py

@@ -0,0 +1,47 @@
+import os
+import re
+
+import setuptools
+
+directory = os.path.dirname(os.path.abspath(__file__))
+
+# Extract version information
+# path = os.path.join(directory, 'sota_extractor2', '__init__.py')
+# with open(path) as read_file:
+#     text = read_file.read()
+# pattern = re.compile(r"^__version__ = ['\"]([^'\"]*)['\"]", re.MULTILINE)
+# version = pattern.search(text).group(1)
+version="2.0-alpha"
+
+# # Extract long_description
+# path = os.path.join(directory, 'README.md')
+# with open(path) as read_file:
+#     long_description = read_file.read()
+long_description = ""
+setuptools.setup(
+    name='sota_extractor2',
+    version=version,
+    url='https://...',
+    description='System for extracting data from arxiv papers',
+    long_description_content_type='text/markdown',
+    long_description=long_description,
+    license='???',
+    packages=setuptools.find_packages(),
+    include_package_data=True,
+
+    keywords='machine-learning ai information-extraction weak-supervision',
+    classifiers=[
+        'Intended Audience :: Science/Research',
+        'Topic :: Scientific/Engineering :: Bio-Informatics',
+        'Topic :: Scientific/Engineering :: Information Analysis',
+        'License :: OSI Approved :: Apache Software License',
+        'Programming Language :: Python :: 3',
+    ],
+
+    project_urls={  # Optional
+        'Homepage': 'https://...',
+        'Source': 'https://...',
+        'Bug Reports': 'https://...',
+        'Citation': 'https://...',
+    },
+)

sota_extractor2/__init__.py

@@ -0,0 +1,16 @@
+import logging
+from pathlib import  Path
+
+logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
+                    datefmt='%m/%d/%Y %H:%M:%S',
+                    level=logging.WARN)
+
+# used only to dynamically fetch graph ql data
+graphql_url = 'http://10.0.1.145:8001/graphql/'
+
+# otherwise use this files
+data = Path("/mnt/efs/pwc/data")
+goldtags_dump = data / "dumps" / "goldtags-2019.06.28_0916.json.gz"
+
+
+elastic = dict(hosts=['localhost'], timeout=20)

sota_extractor2/config.py

@@ -0,0 +1,10 @@
+import logging
+from .. import config # to get logging init
+
+logger = logging.getLogger(__name__)
+
+try:
+    from db import *
+except:
+    logger.info("Unable to intialise django falling back to json data")
+    from json import *

sota_extractor2/data/__init__.py

@@ -0,0 +1 @@
+raise NotImplementedError()