add docker image, continuous deployment, update documentation

OriHoch · OriHoch · commit 275182a8ee32 · 2019-01-06T11:43:08.000+02:00
diff --git a/.travis.sh b/.travis.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+if [ "${1}" == "install" ]; then
+    ! docker pull viderum/ckan-cloud-operator:latest && echo Failed to pull image && exit 1
+    echo Great Success! && exit 0
+
+elif [ "${1}" == "script" ]; then
+    ! docker build --cache-from viderum/ckan-cloud-operator:latest -t ckan-cloud-operator . && echo Failed to build image && exit 1
+    echo Great Success! && exit 0
+
+elif [ "${1}" == "deploy" ]; then
+    TAG="${TRAVIS_TAG:-${TRAVIS_COMMIT}}"
+    docker tag ckan-cloud-operator "viderum/ckan-cloud-operator:${TAG}" &&\
+    docker push "viderum/ckan-cloud-operator:${TAG}"
+    [ "$?" != "0" ] && echo Failed to tag and push && exit 1
+    if [ "${TRAVIS_BRANCH}" == "master" ]; then
+        docker tag ckan-cloud-operator viderum/ckan-cloud-operator:latest &&\
+        docker push viderum/ckan-cloud-operator:latest
+        [ "$?" != "0" ] && echo Failed to tag and push && exit 1
+    fi
+    echo Great Success! && exit 0
+
+else
+    echo invalid arguments && exit 1
+
+fi
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,23 @@
+language: bash
+sudo: required
+env:
+  global:
+  - TRAVIS_CI_OPERATOR=1
+  - secure: "L2+magDtNAe7hKqOfCIkFEFkC01gjMhQsQbChwFd1ZEPa4tqX0yNgCoflghYIVfORsol/Uw3LRumiQNnJLGH4ft3NS7/azBWYLCbQkZduqJhFqYGbBuVIYlQTe1ZjrIxxV/McP0Fadha7fSS2AiyrVuPbvZSAPU0mCLz7LZazisYJg7dqNNZZTkvLEKbi0t6u+saDGUdvFqWlxG93QRJFpd4oCJ7l1K12MDouMik1b1KA5E6NWnU+LbT1wPJx3komqWE5qaM3uKs+3d2/7KTrXkXbPMExipAMGzXB4Z89hNwglFzdo0vCdM2NXrUv32O+9WoGdblfq6qQ8LU6XkyDwWP/QYz0mbAYAEtNiNAfGu1JaQ1ntH/mxyseY/ec1FnAS7TNr7zx/CUAvrs2giPB2z46nOF2KNnXlc7b04DP3OYzZIj3kmLkiF7QN2ERYNiknzHIv961DRs2k8jg5YFmz92smBqeYoi7268VtL6/LIErOubDe4jfVQPMHoP8Nn5+jcOn4MoVjh9GmduTXzQYcOCZKlJehOC7kaYPJkage8xSSjf1bMZqfstQgrhIg8R1aYw+leGW4hzegDV6qPcgToGVLloerMVbdGR9k1T6X6XsIlU7FXs5U8IJS4xiwwq8Qjc/ZMvMsgD697wyG1Yf6hUSK//kZVqkbEiGF/QLFA="
+  - secure: "Hz/RBOEQRra7n2AI6r8jo0bN2QHBtZO8GVb2gMNuAZJ7SwBVT2RazZ9MxoHPrKIZ77ThVutwIk/afyjaVHLN6KO74iT597Fw8onKpX88TXfD9/01yE6elMasmlrGG7Z2VrqOZmqz1g2wTwRNXtMv0L7bDv2M94bcjW+b67g9ddaN58kX6xSjizjAmKOkxuW+F804EpEpmceu1VWwntSUukpZCCV82UiwOJekAhTNVpp7J4DEDaJBch8l1C3fNaPeADRSwo50eCdICsQ34rgkHevWVeY2L7er+VpCIfAMQO/wKazc+zBkcb9gH78gVEGjgix2WVmUDfdjaJjAYwPSkj+TvCG9OXjLcgdjCOWqO+BxqpmAIKpjDk2FsfwRBqHMolc+J/q/IhCqVD6Z57CzfM4QfAxNmWd+zxL9GVdINtJWiDyh8HWxPVqskrO+zTzM/YIyp/7x/1b16URi74f/KzwV0cdi3UI+ktn3FRYs3wVbI6z8vOT40YsOQoXsZVtcpVYHKkY6qgcJzT4OVc+PUbFWM2C/j1B67WmOyljRKDx+FFP98e3gZsk+Uqg+inYpm41Z6XXJnoZ96/qobIBmXIPqiQm92EpokEDe6+HAMW4nO1YoK65SKkbK5TP4RjoJAeuLA0g38XvlVAYk5ZNJEeNw9YR0jOXWTq/GQJM3C1I="
+services:
+- docker
+install:
+- curl -L https://raw.githubusercontent.com/OriHoch/travis-ci-operator/master/travis_ci_operator.sh > $HOME/bin/travis_ci_operator.sh
+- bash $HOME/bin/travis_ci_operator.sh init
+- travis_ci_operator.sh docker-login
+- bash .travis.sh install
+script:
+- bash .travis.sh script
+deploy:
+  skip_cleanup: true
+  provider: script
+  script: bash .travis.sh deploy
+  on:
+    all_branches: true
+    condition: $TRAVIS_PULL_REQUEST = "false"
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -0,0 +1,13 @@
+# Contributing to CKAN Cloud
+
+* Welcome to CKAN Cloud!
+* Contributions of any kind are welcome.
+* Please search for issues in this repository and [across the different CKAN Cloud repositories](https://github.com/search?q=repo%3AViderumGlobal%2Fckan-cloud-docker+repo%3AViderumGlobal%2Fckan-cloud-helm+repo%3AViderumGlobal%2Fckan-cloud-cluster&type=Issues)
+
+## Continuous Deployment
+
+Travis builds and pushes a docker image on every push.
+
+Pushes are tagged with the commit SHA or published tag name.
+
+Pushes to master are also tagged with `latest`.
diff --git a/Dockerfile b/Dockerfile
@@ -1,19 +1,17 @@
 FROM continuumio/miniconda3:4.5.11
 RUN conda update -n base -c defaults conda
+RUN apt-get update && apt-get install -y gnupg bash-completion
+RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-stretch main" >> /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
+    apt-get update -y && apt-get install -y google-cloud-sdk kubectl postgresql nano
 COPY environment.yaml /environment.yaml
 RUN conda env create -f /environment.yaml
-RUN apt-get update && apt-get install -y lsb-release gnupg
-RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
-    echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
-    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
-    apt-get update -y && apt-get install google-cloud-sdk -y &&\
-    apt-get update -y && apt-get install -y kubectl
 RUN echo conda activate ckan-cloud-operator >> ~/.bashrc &&\
     echo '[ -e /etc/ckan-cloud/.kube-config ] && export KUBECONFIG=/etc/ckan-cloud/.kube-config' >> ~/.bashrc &&\
-    echo '[ -e /etc/ckan-cloud/gcloud-service-account.json ] && ! [ -z "${GCLOUD_SERVICE_ACCOUNT_EMAIL}" ] && gcloud --project="${GCLOUD_AUTH_PROJECT}" auth activate-service-account "${GCLOUD_SERVICE_ACCOUNT_EMAIL}" --key-file=/etc/ckan-cloud/gcloud-service-account.json' >> ~/.bashrc &&\
+    echo '! [ -z "${KUBE_CONTEXT}" ] && kubectl config use-context "${KUBE_CONTEXT}" >/dev/null 2>&1' >> ~/.bashrc &&\
     mkdir /usr/src/ckan-cloud-operator
 COPY ckan_cloud_operator /usr/src/ckan-cloud-operator/ckan_cloud_operator
-COPY entrypoint.sh LICENSE MANIFEST.in README.md setup.py VERSION.txt /usr/src/ckan-cloud-operator/
+COPY entrypoint.sh setup.py /usr/src/ckan-cloud-operator/
 RUN . /opt/conda/etc/profile.d/conda.sh && conda activate ckan-cloud-operator &&\
     cd /usr/src/ckan-cloud-operator && python3 -m pip install -e .
 ENTRYPOINT ["/usr/src/ckan-cloud-operator/entrypoint.sh"]
diff --git a/MANIFEST.in b/MANIFEST.in
diff --git a/README.md b/README.md
@@ -4,42 +4,88 @@ CKAN Cloud operator manages, provisions and configures Ckan Cloud instances and
 
 ## Install
 
-Create secret `ckan-infra` under namespace `ckan-cloud` with the following values:
-
-* GCLOUD_SQL_INSTANCE_NAME
-* GCLOUD_SQL_PROJECT
-* POSTGRES_HOST
-* POSTGRES_USER
-* POSTGRES_PASSWORD
-* SOLR_HTTP_ENDPOINT
-* SOLR_NUM_SHARDS
-* SOLR_REPLICATION_FACTOR
-* DOCKER_REGISTRY_SERVER
-* DOCKER_REGISTRY_USERNAME
-* DOCKER_REGISTRY_PASSWORD
-* DOCKER_REGISTRY_EMAIL
-
-## Run using Docker
-
-You will need the following details:
-
-* Path to .kube-config file with permissions to the relevant Kubernetes cluster
-* Path to Google Compute Cloud service account json with required permissions
-* The Google service account email associated with the service account json
-* The Google Project ID associated with the infrastructure and GKE cluster
+ckan-cloud-operator-env is used to install and manage CKAN Cloud operator environments on your local PC
 
-Run ckan-cloud-operator without arguments to get a help message:
+The only requirement is a .kube-config file with permissions to the relevant cluster
+
+Install ckan-cloud-operator-env
+
+```
+curl https://raw.githubusercontent.com/ViderumGlobal/ckan-cloud-operator/master/ckan-cloud-operator-env.sh \
+    | sudo tee /usr/local/bin/ckan-cloud-operator-env &&\
+sudo chmod +x /usr/local/bin/ckan-cloud-operator-env
+```
+
+Pull the latest Docker image
 
 ```
-docker run \
-       -v /path/to/.kube-config:/etc/ckan-cloud/.kube-config \
-       -v /path/to/glcoud-service-account.json:/etc/ckan-cloud/gcloud-service-account.json \
-       -e GCLOUD_SERVICE_ACCOUNT_EMAIL= \
-       -e GCLOUD_AUTH_PROJECT= \
-       -it viderum/ckan-cloud-operator
+ckan-cloud-operator-env pull
 ```
 
-## Run locally
+Add an environment (to run on Minikube set PATH_TO_KUBECONFIG_FILE to minikube)
+
+```
+sudo ckan-cloud-operator-env add <ENVIRONMENT_NAME> <PATH_TO_KUBECONFIG_FILE>
+```
+
+Verify you are connected to the correct cluster
+
+```
+ckan-cloud-operator cluster-info
+```
+
+## Usage
+
+Use the help message of the different commands for the reference documentation and usage examples
+
+```
+ckan-cloud-operator --help
+```
+
+Start a bash shell with completion
+
+```
+ckan-cloud-operator bash
+```
+
+This start a bash shell inside the ckan-cloud-operator Docker container, you can use bash completion inside this shell
+
+```
+ckan-cloud-operator <TAB><TAB>
+```
+
+## Managing multiple environments
+
+ckan-cloud-operator-env supports managing multiple environments
+
+Add environments using `ckan-cloud-operator-env add <ENVIRONMENT_NAME> <PATH_TO_KUBECONFIG_FILE>`
+
+Each environment is accessible using executable `ckan-cloud-operator-<ENVIRONMENT_NAME>`
+
+Activating an environment sets the `ckan-cloud-operator` executable to use to the relevant environment executable
+
+```
+ckan-cloud-operator-env activate <ENVIRONMENT_NAME>
+```
+
+## Initializing a cluster for ckan-cloud-operator
+
+Kubernetes custom resource definitions are used for management of the CKAN Cloud resources
+
+Run the following command to ensure the crds are installed on the cluster
+
+```
+ckan-cloud-operator install-crds
+```
+
+ckan-cloud-operator using a secret named `ckan-infra` under namespace `ckan-cloud` to get infrastrutcute secrets.
+
+See the list of values in [ckan_cloud_operator/infra.py](ckan_cloud_operator/infra.py)
+
+You can clone an infrastructure secret using `ckan-infra clone` command
+and some infrastructure secrets can be set using `ckan-infra set` command
+
+## Run ckan-cloud-operator locally
 
 Ensure you have `kubectl` and `gcloud` binaries, authenticated to the relevant gcloud account / kubernetes cluster.
 
@@ -55,19 +101,20 @@ Install the Python package:
 python3 -m pip install -e .
 ```
 
-Run ckan-cloud-operator without arguments to get a help message:
+Authenticate the gcloud CLI to the relevant account:
 
 ```
-ckan-cloud-operator
+ckan-cloud-operator activate-gcloud-auth
 ```
 
-## Install custom resource definitions
+Run ckan-cloud-operator without arguments to get a help message:
 
-Kubernetes custom resource definitions are used for management of the CKAN Cloud resources
+```
+ckan-cloud-operator
+```
 
-Run the following command to ensure the crds are installed on the cluster
+Enable Bash completion
 
 ```
-ckan-cloud-operator install-crds
+eval "$(_CKAN_CLOUD_OPERATOR_COMPLETE=source ckan-cloud-operator)"
 ```
-
diff --git a/VERSION.txt b/VERSION.txt
diff --git a/entrypoint.sh b/entrypoint.sh
@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
-source ~/.bashrc
-
-exec ckan-cloud-operator "$@"
+if [ "${1}" == "bash" ]; then
+    exec bash --init-file <(echo 'source ~/.bashrc; eval "$(_CKAN_CLOUD_OPERATOR_COMPLETE=source ckan-cloud-operator)"')
+else
+    source ~/.bashrc
+    exec ckan-cloud-operator "$@"
+fi
diff --git a/environment.yaml b/environment.yaml
@@ -6,3 +6,4 @@ dependencies:
   - pip:
     - pyyaml
     - kubernetes
+    - click
diff --git a/release.sh b/release.sh
diff --git a/setup.py b/setup.py
@@ -4,10 +4,6 @@
 
 here = path.abspath(path.dirname(__file__))
 
-# Get the long description from the relevant file
-with open(path.join(here, 'README.md')) as f:
-    long_description = f.read()
-
 if path.exists("VERSION.txt"):
     # this file can be written by CI tools (e.g. Travis)
     with open("VERSION.txt") as version_file:
@@ -19,7 +15,6 @@
     name='ckan_cloud_operator',
     version=version,
     description='''CKAN Cloud Kubernetes operator''',
-    long_description=long_description,
     url='https://github.com/ViderumGlobal/ckan-cloud-operator',
     author='''Viderum''',
     license='MIT',

-Original file line number
+Diff line change
   - pip:
     - pyyaml
     - kubernetes
 +    - click