Feature/deployment logs (#150)

* Intorduce Drone logging

* Introduce deployments status

* Intoroduce deployment version

* Introduce image get and set

Co-authored-by: Irakli Mchedlishvili <irakli.mchedlishvili@datopian.com>
Co-authored-by: Patricio Del Boca <patriciodelboca@gmail.com>

Author: zelima

ckan_cloud_operator/drivers/helm/driver.py

@@ -81,5 +81,13 @@ def delete(tiller_namespace, release_name):
     tiller_cmd = '' if _check_helm_version() == 3 else f' --tiller-namespace {tiller_namespace}'
     subprocess.check_call(f'helm delete --purge --timeout 5 {release_name}' + tiller_cmd, shell=True)
 
+
+def check_status(instance_id):
+    subprocess.check_call(f'helm status ckan-cloud-{instance_id} -n {instance_id}', shell=True)
+
+def get_values(instance_id):
+    return subprocess.check_output(f'helm get values ckan-cloud-{instance_id} -n {instance_id} -o json', shell=True)
+
+
 def _check_helm_version():
     return 3 if 'v3.' in str(subprocess.check_output('helm version -c', shell=True)) else 2

ckan_cloud_operator/providers/ckan/deployment/cli.py

@@ -1,13 +1,61 @@
 import click
 
-
 from .helm import cli as helm_cli
+from .drone import cli as drone_cli
+from .drone import manager
+from ckan_cloud_operator.providers.ckan.deployment import manager as deployment_manager
 
+drone_manager = manager.Drone()
 
 @click.group()
 def deployment():
     """Manage CKAN instance deployments"""
     pass
 
+@click.group()
+def image():
+    """Manage CKAN instance deployments"""
+    pass
+
+@deployment.command()
+@click.option('--branch', default='develop', help='Source Branch for build [default: develop]')
+def logs(branch):
+    """See CKAN instances deployment Logs"""
+    drone_manager.initialize()
+    drone_manager.builds_logs(branch)
+
+
+@deployment.command()
+@click.argument('instance-id')
+def status(instance_id):
+    """See CKAN instances deployment status"""
+    helm_driver.check_status(instance_id)
+
+
+@deployment.command()
+@click.argument('instance-id')
+def version(instance_id):
+    """See CKAN instances deployment version"""
+    deployment_manager.get_deployment_version(instance_id)
+
+
+@image.command()
+@click.argument('instance-id')
+@click.option('--service', default='ckan', help='Source Branch for build [default: develop]')
+def get(instance_id, service):
+    """Get instances container image"""
+    deployment_manager.get_image(instance_id, service=service)
+
+
+@image.command()
+@click.argument('instance-id')
+@click.argument('image-name')
+@click.option('--service', default='ckan', help='Source Branch for build [default: develop]')
+def set(instance_id, image_name, service):
+    """Set instances container image"""
+    deployment_manager.set_image(instance_id, image_name, service=service)
+
 
 deployment.add_command(helm_cli.helm)
+deployment.add_command(drone_cli.drone)
+deployment.add_command(image)

ckan_cloud_operator/providers/ckan/deployment/drone/__init__.py

@@ -0,0 +1,20 @@
+import click
+
+from ckan_cloud_operator import logs
+
+from . import manager
+
+
+drone_manager = manager.Drone()
+
+@click.group()
+def drone():
+    """Manage Drone CI/CD"""
+    pass
+
+
+@drone.command()
+@click.option('--force-update', default=False, help='Force update drone configurations [default: develop]', is_flag=True)
+def initialize(force_update):
+    """Initialize drone"""
+    drone_manager.initialize(force_update)

ckan_cloud_operator/providers/ckan/deployment/drone/cli.py

@@ -0,0 +1 @@
+PROVIDER_ID='drone'

ckan_cloud_operator/providers/ckan/deployment/drone/constants.py

@@ -0,0 +1,91 @@
+import requests
+from urllib.parse import urljoin
+
+from ...env import manager as env_manager
+
+
+class Drone(object):
+
+    def initialize(self, force_prompt=False):
+        self.conf = env_manager._read_yaml().get('cicd', {})
+        self.server_url = self.conf.get('serverUrl')
+        self.api_token = self.conf.get('token')
+        self.org = self.conf.get('organization')
+        self.repo = self.conf.get('repo')
+        if force_prompt or not self.conf:
+            self.server_url = input('Please enter Drone Server URL: ')
+            self.api_token = input(f'Please enter Drone API Token. See {self. server_url} acount: ')
+            self.org = input('Please enter Github organization name: ')
+            self.repo = input('Please enter Github repository name: ')
+            self.conf['cicd'] = {
+                 'serverUrl': self.server_url,
+                 'token': self.api_token,
+                 'organization': self.org,
+                 'repo': self.repo
+            }
+            env_manager._write_yaml(self.conf)
+        self.api_url = urljoin(self.server_url, '/'.join(['api/repos', f'{self.org}/{self.repo}', 'builds/']))
+        self.header = {'Authorization': f'Bearer {self.api_token}'}
+        print('CCO is configured for Drone ')
+
+
+    def builds_list(self):
+        resp = requests.get(self.api_url, headers=self.header)
+        if not _check_for_200(resp):
+            return []
+        return resp.json()
+
+
+    def builds_info(self, branch='develop'):
+        build_number = self.get_build_number(branch=branch)
+        if build_number is None:
+            print(f'Build {build_number} not found')
+            return None
+        url = urljoin(self.api_url, build_number)
+        builds_info_resp = requests.get(url, headers=self.header)
+        if not _check_for_200(builds_info_resp):
+            return {}
+        return builds_info_resp.json()
+
+
+    def builds_logs(self, branch='develop'):
+        build_info = self.builds_info(branch=branch)
+        if build_info is None:
+            return None
+        stages = build_info.get('stages', [])
+        for stage in stages:
+            steps = stage.get('steps', [])
+            stage_name = stage.get('name')
+            for step in steps:
+                step_name = step.get('name')
+                print(f'--- Build Stage: {stage_name} | Builds Step: {step_name}')
+                url = urljoin(self.api_url, '/'.join([
+                    self.build_number,
+                    'logs',
+                    str(stage.get('number')),
+                    str(step.get('number'))])
+                )
+                log_resp = requests.get(url, headers=self.header)
+                log_list = log_resp.json()
+                for _log in log_list:
+                    print(_log.get('out').rstrip('\n'))
+        print(f'Showing logs for "{branch}" Branch')
+
+
+    def get_build_number(self, branch='develop'):
+        for build in self.builds_list():
+            if build.get('source') == branch:
+                self.build_number = str(build.get('number'))
+                return str(build.get('number'))
+        print(f'No build found for branch {branch}')
+        return None
+
+
+def _check_for_200(resp):
+    if resp.status_code == 200:
+        return True
+    if resp.status_code == 401:
+        print("Seems like you are not authorized")
+        print("Please run `cco ckan deployment drone initialize --force-update` and rerun")
+        return False
+    return False

ckan_cloud_operator/providers/ckan/deployment/drone/manager.py

@@ -1,3 +1,11 @@
+import json
+import requests
+from urllib.parse import urljoin
+
+from ckan_cloud_operator import kubectl
+from ckan_cloud_operator.drivers.helm import driver as helm_driver
+
+
 def initialize(interactive=False):
     from .helm.manager import initialize as ckan_helm_initialize
     ckan_helm_initialize(interactive=interactive)
@@ -32,6 +40,34 @@ def pre_update_hook(instance_id, instance_type, instance, override_spec, skip_ro
                                                                    dry_run=dry_run)
 
 
+def get_deployment_version(instance_id):
+    values = helm_driver.get_values(instance_id)
+    values = json.loads(values)
+    site_url = values.get('siteUrl')
+    print('Current deployment version: ', requests.get(urljoin(site_url, 'version')).text)
+
+
+def get_image(instance_id, service='ckan'):
+    deployment_info = kubectl.get('deployment', namespace=instance_id)
+    image_name = None
+    for item in deployment_info.get('items', []):
+        if item['metadata'].get('name') == service:
+            containers = item.get('spec', {}).get('template', {}).get('spec', {}).get('containers', [])
+            for container in containers:
+                image_name = container.get('image')
+                print(image_name)
+    if image_name is None:
+        print(f'Not able to find image for service "{service}", please make sure service name spelled correctly')
+
+
+def set_image(instance_id, image_name, service='ckan', container_name=None):
+    cont_name = container_name or service
+    deployment_info = kubectl.call(
+        f'set image deployment/{service} {cont_name}={image_name}',
+        namespace=instance_id
+    )
+
+
 def create_ckan_admin_user(instance_id, instance_type, instance, user):
     _get_deployment_provider(instance_type).create_ckan_admin_user(instance_id, instance, user)
 

ckan_cloud_operator/providers/ckan/deployment/manager.py

@@ -163,6 +163,12 @@ def _write_yaml(data):
     _file.close()
 
 
+def _read_yaml():
+    with open(_get_config_file_name()) as _file:
+        yaml_conf = yaml.load(_file, Loader=yaml.FullLoader)
+    return yaml_conf
+
+
 def _mkconfdir():
     dir = _get_config_dir()
     Path(f'{dir}').mkdir(parents=True, exist_ok=True)