Expanded fetch commits info from git api

supermeng · supermeng · commit b77f2176eb9e · 2017-08-24T11:37:40.000+08:00
diff --git a/apis/base_app.py b/apis/base_app.py
@@ -177,16 +177,17 @@ def _get_lain_config(self):
             return None
         try:
             config = LainConf()
-            image_config = get_image_config_from_registry(self.appname, self.meta_version)
+            image_config = get_image_config_from_registry(
+                self.appname, self.meta_version)
             config.load(self.meta, self.meta_version, self.default_image,
-                    image_config=image_config, registry=PRIVATE_REGISTRY,
-                    domains=get_domains())
+                        image_config=image_config, registry=PRIVATE_REGISTRY,
+                        domains=get_domains())
             return config
         except Exception as e:
             logger.error('_get_lain_config() failed, error: %s' % e)
             config = LainConf()
             config.load(self.meta, self.meta_version, self.default_image,
-                    registry=PRIVATE_REGISTRY, domains=get_domains())
+                        registry=PRIVATE_REGISTRY, domains=get_domains())
             return config
 
     @property
@@ -350,6 +351,8 @@ def check_giturl(self, meta, update=False):
             giturl = "http://" + giturl
         if update and self.update_git_url(giturl):
             return
+        if self.giturl == '':
+            raise InvalidLainYaml("No Giturl bound")
         if giturl != self.giturl:
             raise InvalidLainYaml(
                 "app giturl '%s' doesn't match with bound url '%s'" % (giturl, self.giturl))
diff --git a/apis/utils.py b/apis/utils.py
@@ -180,16 +180,19 @@ def get_image_config_from_registry(app, meta_version, registry=None):
 
     headers = _get_registry_access_header(app, registry)
     headers['Accept'] = 'application/vnd.docker.distribution.manifest.v2+json'
-    url = 'http://%s/v2/%s/manifests/release-%s' % (registry, app, meta_version)
+    url = 'http://%s/v2/%s/manifests/release-%s' % (
+        registry, app, meta_version)
     resp = requests.get(url, headers=headers)
     if resp.status_code != 200:
-        raise Exception("requests.get(%s, %s) failed, resp: %s" % (url, headers, resp))
+        raise Exception("requests.get(%s, %s) failed, resp: %s" %
+                        (url, headers, resp))
     config_digest = resp.json()['config']['digest']
     headers = _get_registry_access_header(app, registry)
     url = 'http://%s/v2/%s/blobs/%s' % (registry, app, config_digest)
     resp = requests.get(url, headers=headers)
     if resp.status_code != 200:
-        raise Exception("requests.get(%s, %s) failed, resp: %s" % (url, headers, resp))
+        raise Exception("requests.get(%s, %s) failed, resp: %s" %
+                        (url, headers, resp))
     return resp.json()['config']
 
 
diff --git a/apis/views.py b/apis/views.py
@@ -16,6 +16,7 @@
 from raven.contrib.django.raven_compat.models import client
 from log import logger, op_logger
 from oplog.models import add_oplog
+from git.client import fetch_project_commits
 
 
 def render_op_result(op_result):
@@ -735,16 +736,30 @@ def handle(app):
                 app.check_latest_giturl()
             except InvalidLainYaml, e:
                 return (400, None, '%s' % e, reverse('api_image_push', kwargs={'appname': appname}))
+
+            uniq_authors = authors
+            total_commits = commits
+            if authors is None:
+                timestamp_len = 10
+                commits_info = fetch_project_commits(
+                    app.giturl, int(app.meta_version[:timestamp_len]))
+                if commits_info is not None:
+                    uniq_authors, total_commits = commits_info
+                else:
+                    return (400, None, 'Fetch commmits information failed', reverse('api_docs'))
+            if len(total_commits) == 0 or len(uniq_authors) == 0:
+                return (400, None, 'Nothing Changed', reverse('api_docs'))
             commitid_len = 40
             datas = {
                 "appname": appname,
-                "commits": commits,
+                "commits": total_commits,
                 "operator": AuthApi.operater,
                 "lastid": app.meta_version[-commitid_len:],
                 "nextid": app.latest_meta_version[-commitid_len:],
                 "giturl": app.giturl,
-                "authors": authors,
+                "authors": uniq_authors,
             }
+            logger.info('notify datas:%s', str(datas))
             image_push_notify(datas)
             return (200, None, 'ok', reverse('api_image_push', kwargs={'appname': appname}))
         return cls.deal_with_appname(appname, handle)
diff --git a/commons/miscs.py b/commons/miscs.py
@@ -19,3 +19,4 @@ class DoesNotExist(Exception):
 
 class NoAvailableImages(Exception):
     pass
+
diff --git a/commons/settings.py b/commons/settings.py
@@ -75,3 +75,6 @@
 IMAGE_PUSH_KEY = environ.get(
     "IMAGE_PUSH_KEY", "image_push")
 NOTIFIES_TYPES = {'imagepush': IMAGE_PUSH_KEY}
+
+# git configs
+GITLAB_TOKEN = environ.get("GITLAB_TOKEN", "1234567890")
diff --git a/config b/config
@@ -9,4 +9,5 @@ mysql_dbname="console"
 mysql_port="3306"
 mysql_user="console"
 mysql_passwd="console"
+gitlab_token="1234567890"
 console_sentry_dsn="http://da:b2b@sentry.app.com/8"
diff --git a/console/views.py b/console/views.py
@@ -8,6 +8,7 @@
 from apis.views import is_deployable
 from commons.settings import SERVER_NAME, AUTH_TYPES
 from functools import wraps
+from log import logger
 
 
 def permission_required(permission=None):
@@ -397,12 +398,17 @@ def api_detail_get(request, appname):
 
 @permission_required('maintain')
 def api_image_push_post(request, appname):
-    try:
-        options = json.loads(request.body)
-        authors = options['authors']
-        commits = options['commits']
-    except Exception:
-        return render_json_response(400, 'proc', None, 'invalid request: should be json body as {"authors": [string], "commits": [{"id": string, "message": string}]}', reverse('api_docs'))
+    authors = None
+    commits = None
+    if request.body != '':
+        try:
+            options = json.loads(request.body)
+            if options.get('authors') is not None and options.get('commits') is not None:
+                authors = options['authors']
+                commits = options['commits']
+        except Exception as e:
+            logger.error(
+                'loads api_image_push_post body failed with error %s' % str(e))
     status_code, view_object, msg, url = AppApi.post_image_push(
         appname, authors, commits)
     return render_json_response(status_code, 'image_push', view_object, msg, url)
diff --git a/docs/API.md b/docs/API.md
@@ -162,3 +162,23 @@ curl -X DELETE /api/v1/apps/:appName/procs/:procName/
 ```sh
 curl /api/v1/resources/:resourceName/instances/
 ```
+
+### `/api/v1/notify/:notify_type`
+
+1. `GET`
+>获取 notify_type 所有 notifies 列表
+```sh
+curl /api/v1/notify/imagepush
+```
+
+2. `POST`
+>向 notify_type 注册 notifier
+```sh
+curl -XPOST -H 'Content-type: application/json' /api/v1/notify/imagepush/ -d '{"notify_url": "http://lxcmond.lain:3001/api/v1/imagepush"}'
+```
+
+2. `DELETE`
+>向 notify_type 注销 notifier
+```sh
+curl -XDELETE -H 'Content-type: application/json' /api/v1/notify/imagepush/ -d '{"notify_url": "http://lxcmond.lain:3001/api/v1/imagepush"}'
+```
diff --git a/entry.sh b/entry.sh
@@ -15,6 +15,7 @@ export CONSOLE_DB_PORT=$mysql_port
 export CONSOLE_DB_USER=$mysql_user
 export CONSOLE_DB_PASSWORD=$mysql_passwd
 export CONSOLE_SENTRY_DSN=$console_sentry_dsn
+export GITLAB_TOKEN=$gitlab_token
 
 mkdir -p /lain/logs
 exec gunicorn -w 3 -b 0.0.0.0:8000 --max-requests=100 --preload --error-logfile /lain/logs/error.log --access-logfile /lain/logs/access.log console.wsgi 
diff --git a/git/__init__.py b/git/__init__.py
diff --git a/git/client.py b/git/client.py
@@ -0,0 +1,25 @@
+# coding:utf-8
+
+
+import time
+import requests
+
+from commons.settings import GITLAB_TOKEN
+from gitlab import GitLabApi
+from util import parse_giturl
+
+
+SUPPORT_GIT_TYPE = {'gitlab': GitLabApi(GITLAB_TOKEN)}
+TIMEFORMAT_ISO8601 = '%Y-%m-%dT%H:%M:%S%z'
+
+
+def fetch_project_commits(giturl, from_timestamp):
+    scheme, host, namespace, project = parse_giturl(giturl)
+    if scheme is None:
+        return
+    git_type = host.split('.')[0]
+    git_api = SUPPORT_GIT_TYPE.get(git_type, None)
+    if git_api is None:
+        return
+    since = time.strftime(TIMEFORMAT_ISO8601, time.gmtime(from_timestamp+1))
+    return git_api.fetch_project_commits(scheme, host, namespace, project, since)
diff --git a/git/gitlab.py b/git/gitlab.py
@@ -0,0 +1,44 @@
+# coding:utf-8
+
+from util import api_get
+import requests
+
+
+class GitLabApi:
+
+    def __init__(self, token):
+        self.session = requests.Session()
+        self.token = token
+
+    def _fetch_project_id(self, scheme, host, namespace, project):
+        project_url = '{scheme}://{host}/api/v4/projects/{namespace}%2F{project}'.format(
+            scheme=scheme, host=host, namespace=namespace, project=project)
+        headers = {'PRIVATE-TOKEN': self.token}
+        project = api_get(self.session, project_url, headers)
+        if project is None:
+            return
+        return project['id']
+
+    def fetch_project_commits(self, scheme, host, namespace, project, since=None, until=None):
+        pid = self._fetch_project_id(scheme, host, namespace, project)
+        if id is None:
+            return
+        commits_url = '{scheme}://{host}/api/v4/projects/{pid}/repository/commits'.format(
+            scheme=scheme, host=host, pid=pid)
+        headers = {'PRIVATE-TOKEN': self.token}
+        payload = {}
+        if since is not None:
+            payload['since'] = since
+        if until is not None:
+            payload['until'] = until
+        commits_detail = api_get(self.session, commits_url,
+                          headers=headers, params=payload)
+        if commits_detail is None:
+            return
+        unique_authors = set()
+        commits = []
+        for commit in commits_detail:
+            unique_authors.add(commit['committer_email'])
+            commits.append({'id': commit['short_id'], 'message': '%s [%s]' % (
+                commit['committer_name'], commit['message'].strip())})
+        return list(unique_authors), commits
diff --git a/git/util.py b/git/util.py
@@ -0,0 +1,36 @@
+# coding:utf-8
+
+import re
+from log import logger
+
+
+GITURL_PATTEN = re.compile(
+    r'^(http|https)://([\w|\.]+)/([\w|\-|\-]+)/([\w|\-|\_]+)$')
+
+
+TIME_OUT = 5
+
+
+def parse_giturl(giturl):
+    m = GITURL_PATTEN.match(giturl)
+    if m is None:
+        return (None, None, None, None)
+    return m.groups()
+
+
+def api_get(session, url, headers=None, params=None):
+    try:
+        resp = session.request(
+            'GET', url, headers=headers, timeout=TIME_OUT, params=params)
+    except Exception as e:
+        logger.error('request %s failed with error:%s' %
+                     (url, str(e)))
+        return
+    if resp.status_code >= 300 or resp.status_code < 200:
+        logger.error('request %s failed with code:%d, info:%s' %
+                     (url, resp.status_code, resp.text))
+        return
+    try:
+        return resp.json()
+    except Exception as e:
+        logger.error('Invalid Response!!')

Original file line number	Diff line number	Diff line change
`@@ -19,3 +19,4 @@ class DoesNotExist(Exception):`
`19`	`19`
`20`	`20`	`class NoAvailableImages(Exception):`
`21`	`21`	`pass`
	`22`	`+`