Merge pull request #574 from meolu/2.0.0/feature/project

2.0.0/feature/project

Author: meolu

migrations/versions/5ff964e844a7_07_project_include.py

@@ -0,0 +1,23 @@
+"""07_project_include
+
+Revision ID: 5ff964e844a7
+Revises: 0af33c7b8832
+Create Date: 2019-01-04 18:00:58.941866
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = '5ff964e844a7'
+down_revision = '0af33c7b8832'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('projects', sa.Column('is_include', sa.Integer(), nullable=True, server_default='0'))
+
+
+def downgrade():
+    op.drop_column('projects', 'is_include')

walle/form/project.py

@@ -23,6 +23,7 @@ class ProjectForm(FlaskForm):
     space_id = StringField('space_id', [validators.Length(min=1, max=10)])
     status = StringField('status', [])
     excludes = StringField('excludes', [])
+    is_include = StringField('excludes', [])
     master = StringField('master', [])
     server_ids = StringField('server_ids', [validators.Length(min=1)])
     keep_version_num = StringField('keep_version_num', [])
@@ -70,6 +71,7 @@ def form2dict(self):
             'environment_id': self.environment_id.data if self.environment_id.data else '',
             'space_id': self.space_id.data if self.space_id.data else current_user.space_id(),
             'excludes': self.excludes.data if self.excludes.data else '',
+            'is_include': self.is_include.data,
             'server_ids': self.server_ids.data if self.server_ids.data else '',
             'keep_version_num': self.keep_version_num.data if self.keep_version_num.data else 5,
 

walle/model/project.py

@@ -37,6 +37,7 @@ class ProjectModel(SurrogatePK, Model):
     master = db.Column(String(100))
     version = db.Column(String(40))
     excludes = db.Column(Text)
+    is_include = db.Column(Integer)
     target_root = db.Column(String(200))
     target_releases = db.Column(String(200))
     server_ids = db.Column(Text)
@@ -154,6 +155,7 @@ def to_json(self):
             'master': UserModel.fetch_by_uid(self.master.split(',')) if self.master else '',
             'version': self.version,
             'excludes': self.excludes,
+            'is_include': self.is_include,
             'target_root': self.target_root,
             'target_releases': self.target_releases,
             'server_ids': self.server_ids,

walle/service/deployer.py

@@ -18,8 +18,8 @@
 from walle.model.task import TaskModel
 from walle.service.code import Code
 from walle.service.error import WalleError
-from walle.service.utils import color_clean, suffix_format
-from walle.service.utils import excludes_format
+from walle.service.utils import color_clean
+from walle.service.utils import excludes_format, includes_format
 from walle.service.notice import Notice
 from walle.service.waller import Waller
 from flask_login import current_user
@@ -45,6 +45,7 @@ class Deployer:
     TaskRecord = None
 
     console = False
+    custom_global_env = {}
 
     version = datetime.now().strftime('%Y%m%d%H%M%S')
 
@@ -56,7 +57,7 @@ class Deployer:
     local = None
 
     def __init__(self, task_id=None, project_id=None, console=False):
-        self.local_codebase = current_app.config.get('CODE_BASE')
+        self.local_codebase = current_app.config.get('CODE_BASE').rstrip('/') + '/'
         self.localhost = Waller(host='127.0.0.1')
         self.TaskRecord = RecordModel()
 
@@ -69,6 +70,37 @@ def __init__(self, task_id=None, project_id=None, console=False):
             self.servers = self.taskMdl.get('servers_info')
             self.project_info = self.taskMdl.get('project_info')
 
+            # copy to a local version
+            self.release_version = '{project_id}_{task_id}_{timestamp}'.format(
+                project_id=self.project_info['id'],
+                task_id=self.task_id,
+                timestamp=time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())),
+            )
+            current_app.logger.info(self.taskMdl)
+
+            self.custom_global_env = {
+                'WEBROOT': '"{}"'.format(self.project_info['target_root']),
+                'CURRENT_RELEASE': '"{}"'.format(self.release_version),
+                'BRANCH': '"{}"'.format(self.taskMdl.get('branch')),
+                'TAG': '"{}"'.format(self.taskMdl.get('tag')),
+                'COMMIT_ID': '"{}"'.format(self.taskMdl.get('commit_id')),
+                'PROJECT_NAME': '"{}"'.format(self.project_info['name']),
+                'PROJECT_ID': '"{}"'.format(self.project_info['id']),
+                'TASK_NAME': '"{}"'.format(self.taskMdl.get('name')),
+                'TASK_ID': '"{}"'.format(self.task_id),
+                'DEPLOY_USER': '"{}"'.format(self.taskMdl.get('user_name')),
+                'DEPLOY_TIME': '"{}"'.format(time.strftime('%Y%m%d-%H:%M:%S', time.localtime(time.time()))),
+            }
+            if self.project_info['task_vars']:
+                task_vars = [i.strip() for i in self.project_info['task_vars'].split('\n') if i.strip() and not i.strip().startswith('#')]
+                for var in task_vars:
+                    var_list = var.split('=', 1)
+                    if len(var_list) != 2:
+                        continue
+                    self.custom_global_env[var_list[0]] = var_list[1]
+
+            self.localhost.init_env(env=self.custom_global_env)
+
         if project_id:
             self.project_id = project_id
             self.project_info = ProjectModel(id=project_id).item()
@@ -118,11 +150,13 @@ def prev_deploy(self):
         self.init_repo()
 
         # 用户自定义命令
-        command = self.project_info['prev_deploy']
-        if command:
-            current_app.logger.info(command)
-            with self.localhost.cd(self.dir_codebase_project):
-                result = self.localhost.local(command, wenv=self.config())
+        commands = self.project_info['prev_deploy']
+        if commands:
+            for command in commands.split('\n'):
+                if command.strip().startswith('#') or not command.strip():
+                    continue
+                with self.localhost.cd(self.dir_codebase_project):
+                    result = self.localhost.local(command, wenv=self.config())
 
     def deploy(self):
         '''
@@ -133,10 +167,10 @@ def deploy(self):
         '''
         self.stage = self.stage_deploy
         self.sequence = 2
-
-        # copy to a local version
-        self.release_version = '%s_%s_%s' % (
-            self.project_name, self.task_id, time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
+#
+#        # copy to a local version
+#        self.release_version = '%s_%s_%s' % (
+#            self.project_name, self.task_id, time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
 
         with self.localhost.cd(self.local_codebase):
             command = 'cp -rf %s %s' % (self.dir_codebase_project, self.release_version)
@@ -168,26 +202,25 @@ def post_deploy(self):
         self.sequence = 3
 
         # 用户自定义命令
-        command = self.project_info['post_deploy']
-        if command:
-            with self.localhost.cd(self.local_codebase + self.release_version):
-                result = self.localhost.local(command, wenv=self.config())
+        commands = self.project_info['post_deploy']
+        if commands:
+            for command in commands.split('\n'):
+                if command.strip().startswith('#') or not command.strip():
+                    continue
+                with self.localhost.cd(self.local_codebase + self.release_version):
+                    result = self.localhost.local(command, wenv=self.config())
 
         # 压缩打包
         # 排除文件发布
         self.release_version_tar = '%s.tgz' % (self.release_version)
         with self.localhost.cd(self.local_codebase):
-            excludes = excludes_format(self.project_info['excludes'])
-            command = 'tar zcf  %s %s %s' % (self.release_version_tar, excludes, self.release_version)
+            if self.project_info['is_include']:
+                files = includes_format(self.release_version, self.project_info['excludes'])
+            else:
+                files = excludes_format(self.release_version, self.project_info['excludes'])
+            command = 'tar zcf %s/%s %s' % (self.local_codebase.rstrip('/'), self.release_version_tar, files)
             result = self.localhost.local(command, wenv=self.config())
 
-        # # 指定文件发布
-        # self.release_version_tar = '%s.tgz' % (self.release_version)
-        # with self.localhost.cd(self.local_codebase):
-        #     excludes = suffix_format(self.dir_codebase_project, self.project_info['excludes'])
-        #     command = 'tar zcf  %s %s %s' % (self.release_version_tar, excludes, self.release_version)
-        #     result = self.local.run(command, wenv=self.config())
-
     def prev_release(self, waller):
         '''
         4.部署代码到目标机器前做的任务
@@ -215,13 +248,15 @@ def prev_release(self, waller):
 
     def prev_release_custom(self, waller):
         # 用户自定义命令
-        command = self.project_info['prev_release']
-        if command:
-            current_app.logger.info(command)
-            # TODO
-            target_release_version = "%s/%s" % (self.project_info['target_releases'], self.release_version)
-            with waller.cd(target_release_version):
-                result = waller.run(command, wenv=self.config())
+        commands = self.project_info['prev_release']
+        if commands:
+            for command in commands.split('\n'):
+                if command.strip().startswith('#') or not command.strip():
+                    continue
+                # TODO
+                target_release_version = "%s/%s" % (self.project_info['target_releases'], self.release_version)
+                with waller.cd(target_release_version):
+                    result = waller.run(command, wenv=self.config())
 
     def release(self, waller):
         '''
@@ -299,14 +334,16 @@ def post_release(self, waller):
         '''
         self.stage = self.stage_post_release
         self.sequence = 6
-
         # 用户自定义命令
-        command = self.project_info['post_release']
-        if command:
-            current_app.logger.info(command)
-            with waller.cd(self.project_info['target_root']):
-                result = waller.run(command, wenv=self.config())
-
+        commands = self.project_info['post_release']
+        if commands:
+            for command in commands.split('\n'):
+                if command.strip().startswith('#') or not command.strip():
+                    continue
+                # TODO
+                with waller.cd(self.project_info['target_root']):
+                    pty = False if command.find('nohup') >= 0 else True
+                    result = waller.run(command, wenv=self.config(), pty=pty)
         # 个性化，用户重启的不一定是NGINX，可能是tomcat, apache, php-fpm等
         # self.post_release_service(waller)
 
@@ -495,7 +532,10 @@ def walle_deploy(self):
             for server_info in self.servers:
                 host = server_info['host']
                 try:
-                    self.connections[host] = Waller(host=host, user=server_info['user'], port=server_info['port'])
+                    waller = Waller(host=host, user=server_info['user'], port=server_info['port'], inline_ssh_env=True)
+                    waller.init_env(env=self.custom_global_env)
+
+                    self.connections[host] = waller
                     self.prev_release(self.connections[host])
                     self.release(self.connections[host])
                     self.post_release(self.connections[host])
@@ -527,7 +567,10 @@ def walle_rollback(self):
             for server_info in self.servers:
                 host = server_info['host']
                 try:
-                    self.connections[host] = Waller(host=host, user=server_info['user'], port=server_info['port'])
+                    waller = Waller(host=host, user=server_info['user'], port=server_info['port'], inline_ssh_env=True)
+                    waller.init_env(env=self.custom_global_env)
+
+                    self.connections[host] = waller                   
                     self.prev_release_custom(self.connections[host])
                     self.release(self.connections[host])
                     self.post_release(self.connections[host])

walle/service/utils.py

@@ -81,16 +81,39 @@ def say_yes():
     )
 
 
-def excludes_format(excludes_string):
-    excludes = [i for i in excludes_string.split('\n') if i.strip()]
-    if not excludes:
-        return ''
-    excludes = ' --exclude='.join(excludes)
-    return ' --exclude=' + excludes
-
-
-# 指定发布文件，支持模糊匹配，如：*.war
-def suffix_format(path, suffix_file):
-    for suffix_file in os.listdir('%s' % path):
-        if fnmatch.fnmatch(suffix_file, '%s' % suffix_file):
-            return suffix_file
+def excludes_format(path, excludes_string=None):
+    '''
+    排除文件，支持正则匹配，支持多选字符串
+    @param path:
+    @param excludes_string:
+    @return:
+    '''
+    path = os.path.basename(path) + '/'
+    if not excludes_string:
+        return path
+
+    prefix = '--exclude='
+    excludes = [prefix + i for i in excludes_string.split('\n') if i.strip()]
+
+    return ' {excludes} {path} '.format(excludes=' '.join(excludes), path=path)
+
+
+def includes_format(path, includes_string=None):
+    '''
+    指定发布文件，支持正则匹配，如：*.war。支持多行字符串。
+
+    @param path: release目录，非路径
+    @param includes_string:
+    @return:
+    '''
+    path = os.path.basename(path) + '/'
+    if not includes_string:
+        return path
+
+    prefix = path
+    includes = [prefix + i for i in includes_string.split('\n') if i.strip()]
+
+    if not includes:
+        return path
+
+    return ' '.join(includes)

walle/service/waller.py

@@ -23,6 +23,11 @@ class Waller(Connection):
     connections, success, errors = {}, {}, {}
     release_version_tar, release_version = None, None
 
+    custom_global_env = {}
+
+    def init_env(self, env):
+        self.custom_global_env = env
+
     def run(self, command, wenv=None, run_mode=run_mode_remote, pty=True, exception=True, **kwargs):
         '''
         pty=True/False是直接影响到输出.False较适合在获取文本,True更适合websocket
@@ -40,11 +45,12 @@ def run(self, command, wenv=None, run_mode=run_mode_remote, pty=True, exception=
         current_app.logger.info(message)
         try:
             if run_mode == self.run_mode_sudo:
-                result = super(Waller, self).sudo(command, pty=pty, **kwargs)
+                result = super(Waller, self).sudo(command, pty=pty, env=self.custom_global_env, **kwargs)
             elif run_mode == self.run_mode_local:
-                result = super(Waller, self).local(command, pty=pty, warn=True, watchers=[say_yes()], **kwargs)
+                current_app.logger.info(self.custom_global_env)
+                result = super(Waller, self).local(command, pty=pty, warn=True, watchers=[say_yes()], env=self.custom_global_env, **kwargs)
             else:
-                result = super(Waller, self).run(command, pty=pty, warn=True, watchers=[say_yes()], **kwargs)
+                result = super(Waller, self).run(command, pty=pty, warn=True, watchers=[say_yes()], env=self.custom_global_env, **kwargs)
 
             if result.failed:
                 exitcode, stdout, stderr = result.exited, '', result.stdout
@@ -174,17 +180,24 @@ def sync(self, wtype, remote=None, local=None, wenv=None):
         except Exception as e:
             # TODO 收尾下
             current_app.logger.info('put: %s, %s', e, dir(e))
+            if wtype == 'put':
+                op_user = pwd.getpwuid(os.getuid())[0]
+                op_host = '127.0.0.1'
+            else:
+                op_user = self.user
+                op_host = self.host
 
             # TODO command
             ws_dict = {
-                'user': self.user,
-                'host': self.host,
+                'user': op_user,
+                'host': op_host,
                 'cmd': command,
                 'status': 1,
                 'stage': wenv['stage'],
                 'sequence': wenv['sequence'],
                 'success': '',
-                'error': e.message,
+                'error': str(e),
             }
+            current_app.logger.info(ws_dict)
             if wenv['console']:
                 emit('console', {'event': 'task:console', 'data': ws_dict}, room=wenv['task_id'])