Merge branch '300_release' into 'master'

300 release

See merge request oceanbase/ob-deploy!267

Author: gys-git

Docker/Dockerfile

@@ -0,0 +1,54 @@
+FROM registry.openanolis.cn/openanolis/anolisos:8.9
+ENV PATH="/opt/miniconda/bin:$PATH"
+ENV TZ=UTC-8
+ENV CRYPTOGRAPHY_OPENSSL_NO_LEGACY=1
+
+ADD Docker/ob_build /usr/bin/
+ADD Docker/python-env-activate.sh /usr/bin/py-env-activate
+
+RUN yum install -y wget  \
+    && wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo \
+    && yum -y install libffi-devel bzip2-devel readline readline-devel jq which bison ncurses-devel libaio autoconf automake libtool perl-CPAN gettext-devel perl-devel openssl-devel zlib-devel curl-devel  xz-devel \
+        expat-devel asciidoc xmlto rpm-build cmake make gcc gcc-c++ python2-devel python3-devel sudo git mysql-devel \
+    && yum clean all
+
+RUN wget https://repo.anaconda.com/miniconda/Miniconda2-py27_4.8.3-Linux-x86_64.sh --no-check-certificate\
+    && bash Miniconda2-py27_4.8.3-Linux-x86_64.sh -p /opt/miniconda -b \
+    && rm -rf Miniconda2-py27_4.8.3-Linux-x86_64.sh \
+    && conda clean --all -y
+
+COPY requirements3.txt /workspaces/obd/requirements3.txt
+COPY service/service-requirements.txt   workspaces/obd/service-requirements.txt
+
+RUN conda create -n py27 -y python=2.7 \
+    && source /opt/miniconda/bin/activate base \
+    && pip config set global.index-url https://mirrors.aliyun.com/pypi/simple \
+    && pip config set global.trusted-host  mirrors.aliyun.com \
+    && pip install pyinstaller==3.6 \
+    && rm -rf ~/.cache/pip \
+    && conda clean -a -y
+
+RUN conda create -n py38 -y python=3.8 \
+    && source /opt/miniconda/bin/activate py38 \
+    && pip config set global.index-url https://mirrors.aliyun.com/pypi/simple \
+    && pip config set global.trusted-host  mirrors.aliyun.com \
+    && pip install -r /workspaces/obd/requirements3.txt \
+    && pip install -r /workspaces/obd/service-requirements.txt \
+    && pip install pyinstaller==3.6 \
+    && pip cache purge \
+    && echo "source /opt/miniconda/bin/activate py38" >> ~/.bashrc \
+    && /opt/miniconda/bin/conda clean -a -y \
+    && rm -rf /workspaces/obd
+
+# install nodejs and yarn
+RUN wget https://rpm.nodesource.com/setup_16.x  \
+    && touch /etc/redhat-release \
+    && bash  setup_16.x \
+    && rm -f  /etc/redhat-release \
+    && yum install -y nodejs \
+    && yum clean all
+
+RUN npm install -g yarn \
+    && yarn config delete registry --global \
+    && yarn config set registry https://registry.npmmirror.com/  --global \
+    && npm cache clean --force

Docker/ob_build

@@ -0,0 +1,76 @@
+#!/bin/bash
+set -x
+set -e
+
+if [ $# -lt 4 ]; then
+    echo "no enough parameters. Please provide project_name, version and release."
+    exit 1
+fi
+
+# Get system release
+os_release=$(grep -Po '(?<=^ID=")[^"]*' /etc/os-release || true)
+
+if [ -z "$os_release" ]; then
+    os_release=$(grep -Po '^ID=\K[^ ]+' /etc/os-release)
+fi
+
+set +e
+source /etc/profile
+set -e
+
+project_dir=$1
+project_name=$2
+version=$3
+release=$4
+rpm_work_dir=${5:-rpm}
+ob_build_script=${project_name}-build.sh
+ob_build_spec=${project_name}.spec
+ob_build_deps=${project_name}.deps
+cur_dir=`pwd`
+echo "cur dir: $cur_dir"
+# check rpm work dir
+if [ ! -d "${cur_dir}/${rpm_work_dir}" ]; then
+    echo "rpm work dir (${rpm_work_dir}) missing! Please create ${rpm_work_dir} in source code dir and place the packaging related files in the ${rpm_work_dir} dir."
+    exit 1
+fi
+
+# trigger building
+echo "trigger building, current dir: "
+pwd
+cd $rpm_work_dir
+ABS_PATH=`pwd`
+
+if [[ x"$os_release" == x"alios" && x"$AONE_COMPATIBLE_AUTO_DEP_CREATE" == x"on" ]]; then
+    if [ -e "$ob_build_deps" ]; then
+        echo "execute dep_create for alios"
+        dep_create $ob_build_deps
+        echo "execute sw for alios"
+        sw
+    fi
+fi
+
+if [ -e "$ob_build_script" ]; then
+    bash $ob_build_script $project_dir $project_name $version $release
+elif [ -e "$ob_build_spec" ]; then
+    if [[ x"$os_release" == x"alios" ]]; then
+        rpm_create $ob_build_spec -v $version -r $release
+    else
+        TOP_DIR=".rpm_create"
+        RPM_MACROS=$HOME/.rpmmacros
+        if [ -e $RPM_MACROS ]; then
+            mv -f $RPM_MACROS $RPM_MACROS.bak
+        fi
+
+        # prepare rpm build dirs
+        mkdir -p $TOP_DIR/BUILD
+        mkdir -p $TOP_DIR/RPMS
+        mkdir -p $TOP_DIR/SRPMS
+
+        echo "%_topdir $ABS_PATH/$TOP_DIR" > $RPM_MACROS
+        rpmbuild -bb $ob_build_spec
+        find $TOP_DIR/ -name "*.rpm"  -exec mv {} . 2>/dev/null \;
+    fi
+else
+    echo "packaging files missing! Please provide $ob_build_script or $ob_build_spec"
+    exit 1
+fi

Docker/python-env-activate.sh

@@ -0,0 +1,6 @@
+eval "$(conda shell.bash hook)"
+if [[ -z $1 ]]; then
+    echo "Please input proper python env"
+    exit 1
+fi
+conda activate $1

_cmd.py

@@ -26,6 +26,8 @@
 import time
 import textwrap
 import json
+import glob
+import datetime
 from uuid import uuid1 as uuid, UUID
 from optparse import OptionParser, BadOptionError, Option, IndentedHelpFormatter
 
@@ -39,8 +41,8 @@
 from ssh import LocalClient
 from const import (
     CONST_OBD_HOME,
-    VERSION, REVISION, BUILD_BRANCH, BUILD_TIME, FORBIDDEN_VARS,
-    COMP_OCEANBASE_DIAGNOSTIC_TOOL, PKG_RPM_FILE,PKG_REPO_FILE
+    VERSION, REVISION, BUILD_BRANCH, BUILD_TIME, FORBIDDEN_VARS, COMP_OB_CE, COMP_ODP_CE, COMP_ODP, COMP_OCP_SERVER_CE,
+    COMP_OCEANBASE_DIAGNOSTIC_TOOL, PKG_RPM_FILE,PKG_REPO_FILE, BUILD_PLUGIN_LIST
 )
 
 
@@ -197,12 +199,50 @@ def init_home(self):
         if not COMMAND_ENV.get(ENV.ENV_OBD_ID):
             COMMAND_ENV.set(ENV.ENV_OBD_ID, uuid())
         if VERSION != version:
-            for part in ['plugins', 'config_parser', 'optimize', 'mirror/remote']:
+            for part in ['workflows', 'plugins', 'config_parser', 'optimize', 'mirror/remote']:
                 obd_part_dir = os.path.join(self.OBD_PATH, part)
+                root_part_path = os.path.join(self.OBD_INSTALL_PATH, part)
                 if DirectoryUtil.mkdir(self.OBD_PATH):
-                    root_part_path = os.path.join(self.OBD_INSTALL_PATH, part)
-                    if os.path.exists(root_part_path):
-                        DirectoryUtil.copy(root_part_path, obd_part_dir, ROOT_IO)
+                    if part != 'mirror/remote':
+                        if os.path.exists(obd_part_dir):
+                            backup_path = os.path.join(self.OBD_PATH, '.backup_plugin', VERSION + '_' + REVISION, part)
+                            if DirectoryUtil.mkdir(backup_path):
+                                DirectoryUtil.copy(obd_part_dir, backup_path, ROOT_IO)
+                            os.chdir(obd_part_dir)
+                            if part != 'config_parser':
+                                for file in glob.glob('*/*/*'):
+                                    if COMP_OB_CE in file or COMP_OCP_SERVER_CE in file or (COMP_ODP_CE in file and 'file_map.yaml' in file):
+                                        continue
+                                    if file not in BUILD_PLUGIN_LIST:
+                                        FileUtil.rm(file)
+                            else:
+                                for file in glob.glob('*/*'):
+                                    if file not in BUILD_PLUGIN_LIST:
+                                        FileUtil.rm(file)
+
+                        if os.path.exists(root_part_path):
+                            os.chdir(root_part_path)
+                            if part != 'config_parser':
+                                for file in glob.glob('*/*/*'):
+                                    if COMP_OB_CE in file or COMP_OCP_SERVER_CE in file or (COMP_ODP_CE in file and 'file_map.yaml' in file):
+                                        continue
+                                    if file not in BUILD_PLUGIN_LIST:
+                                        if file.replace(COMP_ODP_CE, COMP_ODP).replace('3.2.1', '3.1.0') in BUILD_PLUGIN_LIST:
+                                            continue
+                                        FileUtil.rm(file)
+                            else:
+                                for file in glob.glob('*/*'):
+                                    if file not in BUILD_PLUGIN_LIST:
+                                        FileUtil.rm(file)
+                if os.path.exists(root_part_path):
+                    DirectoryUtil.copy(root_part_path, obd_part_dir, ROOT_IO)
+
+            backup_path = os.path.join(self.OBD_PATH, '.backup_plugin')
+            if os.path.exists(backup_path):
+                dir_list = [os.path.join(backup_path, f) for f in os.listdir(backup_path)]
+                sorted_dir_list = sorted(dir_list, key=lambda dir: datetime.datetime.fromtimestamp(os.path.getmtime(dir)))
+                for dir in sorted_dir_list[12:]:
+                    DirectoryUtil.rm(dir)
             version_fobj.seek(0)
             version_fobj.truncate()
             version_fobj.write(VERSION)
@@ -1208,7 +1248,8 @@ def init(self, cmd, args):
 
     def _do_command(self, obd):
         if len(self.cmds) == 2:
-            return obd.failover_decouple_tenant(self.cmds[0], self.cmds[1], 'failover')
+            self.cmds.append('failover')
+            return obd.failover_decouple_tenant(self.cmds[0], self.cmds[1], self.cmds[2])
         else:
             return self._show_help()
 
@@ -1226,7 +1267,8 @@ def init(self, cmd, args):
 
     def _do_command(self, obd):
         if len(self.cmds) == 2:
-            return obd.failover_decouple_tenant(self.cmds[0], self.cmds[1], 'decouple')
+            self.cmds.append('decouple')
+            return obd.failover_decouple_tenant(self.cmds[0], self.cmds[1], self.cmds[2])
         else:
             return self._show_help()
 
@@ -1652,7 +1694,7 @@ def do_command(self):
         return super(UpdateCommand, self).do_command()
 
     def _do_command(self, obd):
-        return obd.update_obd(VERSION, self.OBD_INSTALL_PRE)
+        return obd.update_obd(VERSION, self.OBD_INSTALL_PRE, self.OBD_INSTALL_PATH)
 
 
 class DisplayTraceCommand(ObdCommand):

_deploy.py

@@ -1096,6 +1096,27 @@ def enable_mem_mode(self):
     def disable_mem_mode(self):
         self._mem_mode = False
 
+    @property
+    def sorted_components(self):
+        available_depends = list(self.components.keys())
+        unsort_components = available_depends
+        sorted_components = []
+        while unsort_components:
+            components = unsort_components
+            unsort_components = []
+            for component in components:
+                cluster_config = self.components[component]
+                for component_name in cluster_config.depends:
+                    if component_name not in available_depends:
+                        continue
+                    if component_name not in sorted_components:
+                        unsort_components.append(component)
+                        break
+                else:
+                    sorted_components.append(component)
+        return sorted_components
+
+
     @property
     def user(self):
         return self._user

_plugin.py

@@ -77,6 +77,10 @@ def __init__(self, spacename):
         self._variables = {}
         self._return = {}
 
+    @property
+    def all_plugin_ret(self):
+        return self._return
+
     @property
     def variables(self):
         return self._variables
@@ -198,9 +202,6 @@ def __init__(self, stdio):
         self.stdio = getattr(stdio, 'sub_io', lambda: None)()
         self._func = {}
 
-    def __del__(self):
-        self.before_close()
-
     def _temp_function(self, *arg, **kwargs):
         pass
 
@@ -262,6 +263,9 @@ def before_do(
 
     def after_do(self, stdio, *arg, **kwargs):
         self._export(stdio)
+        if self.context.stdio.sync_obj:
+            self.context.stdio.warn("%s has animation not been closed" % self)
+        del self.context.stdio
         self.context = None
 
 
@@ -385,6 +389,7 @@ class Null(object):
     def __init__(self):
         pass
 
+
 class ParamPlugin(Plugin):
 
 
@@ -807,13 +812,14 @@ def requirement_list(self, package_info):
 class ComponentPluginLoader(object):
 
     PLUGIN_TYPE = None
+    MODULE_NAME = __name__
 
     def __init__(self, home_path, plugin_type=PLUGIN_TYPE, dev_mode=False, stdio=None):
         if plugin_type:
             self.PLUGIN_TYPE = plugin_type
         if not self.PLUGIN_TYPE:
             raise NotImplementedError
-        self.plguin_cls = getattr(sys.modules[__name__], self.PLUGIN_TYPE.value, False)
+        self.plguin_cls = getattr(sys.modules[self.MODULE_NAME], self.PLUGIN_TYPE.value, False)
         if not self.plguin_cls:
             raise ImportError(self.PLUGIN_TYPE.value)
         self.dev_mode = dev_mode