travis README notebook

Author: OriHoch

.travis.sh

@@ -0,0 +1,17 @@
+if [ "${1}" == "install" ]; then
+    python3 -m pip install \
+        jupyter jupyterlab ipython \
+        plyvel \
+        'https://github.com/OriHoch/dataflows/archive/specify-encoding-for-load.zip#egg=dataflows[speedup]' \
+        'https://github.com/frictionlessdata/datapackage-pipelines/archive/2.0.0.zip#egg=datapackage-pipelines[speedup]'
+
+elif [ "${1}" == "script" ]; then
+    ./render_notebook.sh QUICKSTART
+
+elif [ "${1}" == "deploy" ]; then
+    travis_ci_operator.sh github-update self master "
+        cp -f $PWD/QUICKSTART.md $PWD/QUICKSTART.ipynb ./ &&\
+        git add QUICKSTART.md QUICKSTART.ipynb
+    " "update QUICKSTART notebook"
+
+fi

.travis.yml

@@ -0,0 +1,21 @@
+language: python
+sudo: required
+python: ["3.6"]
+env:
+  global:
+  - TRAVIS_CI_OPERATOR=1
+  - secure: "SkjWPaBjlZMQYnpZo3beBexwQZ6/FrnQL6s+pRC5Hb+rMPcuvyZj7iQiW4FeUIBDXeOwuVu8iy3VY4gHaZgwViEDeGLZPoCLf2eLhQSG55AC8S2GXn1dMX2//v2sIuxctCb/jHN66tvdqN6t+45qHbFnGwsQcciN2b8TRiSSnNGjEZg90pP0ZKCwyVstCkBgfMSvnp4MuT3EoEOnmhwrdhPVk4OO7mQp606N5csVB8ZNryXOiBSlKKAElFqrOeDmT+mmMuwhZQ1ftfRqvChxltWdUEkYkMTEzureWabl8PdaIe5xXoz6YPqaHqpgKm9LNblQn5k6FRltPu+Pp+7j8lEg8584Shi62hv+kUZHb3XpnUSoJYyfGqz2/fSH90zVZpvkVac7b+lRpHMv/CxrhYs9+UeNQHxDNTxDMQEDLWd300a6D0FVwOR1vLRPrlfOIG0KfkhhBbLxsAE4Q0wTYJFI8PBlX+bzYjPpP0KpJEL0oQxbWnWaHh+CgWKqQFLRl6z9JSQBiX9FOu0TTawspXTWMUrYu13KJiEbiEsBMFijLJ65rXwZZVOIz8YnmJ+ZYqLoQpBs/hkMth9jV3R201P4z6vNDg56A5j5N/S+11cA2hQ0X5B1r6y2JkJPEDaiZZcCBoAMdV2tQxucRU6AZRxgaAGpfimgaK50f1uspQs="
+  - secure: "bzC20wujS9IpZ8/XMd1O2UUpCRZNvM++eO+TISk264uFoBA1zHvO0af3QxYULxexoMG1H5JHKurKnsJ/W7sWiQLC8aVylc8XtGp+ov4JqVivfOglBcYu2mCRhvYcFRKbRxyRTdcznXg2E35gjNih57waSEayc256R4LcHIxZTYYBtHrx9V6s6D4CFTf/cQtppGm7xZmeW5QQuZ9MZDjbp6uFtb53bsJes8EPt4vFLiTmygo+pSvCITjigE4K/hHUZVYyFJbYwMZk5tc+TXx4/MeIe3sYH8R6fMLtf1g9kBygBHGk/EZycW+CBTWFarenSzBUc9Bo6m1qGGgC5HiQS9QGLt99aGO7mm0A5M6iHWNIvz0URzaKGgAWf3OMMnlYsQGgrqeGoGUhPy/qP8JGIwlztHaP42GFuSq4h+cIEkZX5yflmIKQWfTNIWYo7pnsogWTrMKqZIwGC63ulzU9AP0pWnIjwiH0WjLu+r+LjmsGhXIdTrcI+xVz1s0J7k4F8txrpOol2JrvqvYKe89ydm3rp86uaL8yZGWbNLeZxonFEVrVOJrDf4qExGlRkL8n+wCsEgEPmrphc8sLUgDWNv4euld2xWknDrDr7mcmMDjhQsZKddgWP4dUTvDs+rdntjqWp8B19I+xROxfc3M2CMBZOMD0T96mE2YgIAVZW4A="
+services:
+- docker
+install:
+- curl -L https://raw.githubusercontent.com/OriHoch/travis-ci-operator/master/travis_ci_operator.sh > $HOME/bin/travis_ci_operator.sh
+- bash $HOME/bin/travis_ci_operator.sh init
+- travis_ci_operator.sh docker-login
+- bash .travis.sh install
+script:
+- bash .travis.sh script
+deploy:
+  provider: script
+  script: bash .travis.sh deploy
+  skip_cleanup: true

QUICKSTART.ipynb

@@ -1,5 +1,8 @@
 # Migdar Data Pipelines
 
+## Quickstart
+
+[QUICKSTART.ipynb](QUICKSTART.ipynb)
 
 ## Running the pipelines locally
 

QUICKSTART.md

@@ -0,0 +1,15 @@
+# Create a conda environment using this file by running:
+# conda env create -f environment.yaml
+name: migdar-data-pipelines
+channels:
+  - defaults
+dependencies:
+  - ipython>=7.0.1
+  - jupyter>=1.0.0
+  - jupyterlab>=0.35.0
+  - leveldb>=1.20
+  - python=3.6
+  - pip:
+    - plyvel
+    - https://github.com/OriHoch/dataflows/archive/specify-encoding-for-load.zip#egg=dataflows[speedup]
+    - https://github.com/frictionlessdata/datapackage-pipelines/archive/2.0.0.zip#egg=datapackage-pipelines[speedup]

README.md

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+( [ -z "${MIGDAR_USERNAME}" ] || [ -z "${MIGDAR_PASSWORD}" ] ) && echo missing required env vars && exit 1
+
+env MIGDAR_USERNAME="${MIGDAR_USERNAME}" MIGDAR_PASSWORD="${MIGDAR_PASSWORD}" \
+    jupyter nbconvert --execute "${1}.ipynb" --to notebook --inplace --ExecutePreprocessor.timeout=-1 &&\
+    jupyter nbconvert "${1}.ipynb" --to markdown