Merge remote-tracking branch 'upstream/main' into feat/add_disk_status_from_main

Author: litongxin

.devcontainer/Dockerfile

@@ -0,0 +1,3 @@
+FROM 4pdosc/hybridsql:latest
+RUN yum update -y
+RUN yum install -y vim

.devcontainer/devcontainer.json

@@ -0,0 +1,18 @@
+{
+	"name": "OpenMLDB codespaces",
+	"build": {
+		"dockerfile": "Dockerfile"
+	},
+	"settings": {
+		"C_Cpp.default.includePath": [
+			"${workspaceFolder}/**",
+            "/deps/usr/include"
+		],
+		"C_Cpp.default.compilerPath": "/opt/rh/devtoolset-8/root/usr/bin/gcc",
+		"C_Cpp.default.cppStandard": "c++17"
+	},
+	// Visual Studio Code extensions (plugins) to be installed.
+	"extensions": [
+		"ms-vscode.cpptools"
+	]
+}

.github/workflows/cicd.yaml

@@ -141,10 +141,66 @@ jobs:
           check_name: Linux Test Report
           comment_title: Linux Test Report
 
+  macos-cpp:
+    runs-on: macos-latest
+    if: github.repository == '4paradigm/OpenMLDB' && startsWith(github.ref, 'refs/tags/v')
+    env:
+      OS: darwin
+      ARCH: x86_64
+      NPROC: 3
+      CTEST_PARALLEL_LEVEL: 1 # parallel test level for ctest (make test)
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: prepare release
+        run: |
+          VERSION="snapshot"
+          if [[ ${{ github.ref }} == "refs/tags/v"* ]]; then
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+            VERSION=${VERSION#v}
+            bash steps/prepare_release.sh "$VERSION"
+          fi
+          echo "OPENMLDB_PREFIX=openmldb-$VERSION-darwin" >> $GITHUB_ENV
+
+      - name: enable java on release
+        if: github.event_name == 'push'
+        run: |
+          echo "SQL_JAVASDK_ENABLE=ON" >> $GITHUB_ENV
+
+      - name: configure
+        env:
+          SQL_JAVASDK_ENABLE: ${{ env.SQL_JAVASDK_ENABLE }}
+        run: |
+          make configure CMAKE_INSTALL_PREFIX=${{ env.OPENMLDB_PREFIX }}
+
+      - name: build
+        run: |
+          make build
+          # GitHub runner disk space is limited
+          # delete thirdparty build directory($ROOT/.deps/) to save disk space
+          # make thirdpartybuild-clean
+
+      - name: install
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          make install
+
+      - name: package
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          tar czf ${{ env.OPENMLDB_PREFIX }}.tar.gz ${{ env.OPENMLDB_PREFIX }}/
+
+      - name: upload artifacts
+        if: ${{ github.event_name == 'push' }}
+        uses: actions/upload-artifact@v2
+        with:
+          path: openmldb-*.tar.gz
+          name: release-artifacts
+
   release:
     runs-on: ubuntu-latest
     # if test failed, shouldn't release
-    needs: [ "cpp" ]
+    needs: [ "cpp", "macos-cpp" ]
     if: >
       success() && startsWith(github.ref, 'refs/tags/v')
     steps:

.github/workflows/coverage.yml

@@ -1,3 +1,8 @@
+# The coverage jobs for cpp code
+#
+# This sperate job exists because C/CPP coverage require build type to be Debug.
+# Other coverage jobs e.g java/scala's coverage lives alongside with their test jobs.
+
 name: coverage
 
 on:
@@ -25,59 +30,52 @@ on:
 
 jobs:
   coverage:
-    runs-on: self-hosted
-    container:
-      image: ghcr.io/4paradigm/hybridsql:latest
+    runs-on: ubuntu-latest
     env:
       CTEST_PARALLEL_LEVEL: 1 # parallel test level for ctest (make test)
       CMAKE_BUILD_TYPE: Debug
-      SQL_PYSDK_ENABLE: OFF
-      SQL_JAVASDK_ENABLE: ON
       TESTING_ENABLE: ON
+      SQL_PYSDK_ENABLE: OFF
+      SQL_JAVASDK_ENABLE: OFF
       NPROC: 8
-      SPARK_HOME: /tmp/spark/
+      BUILD_SHARED_LIBS: ON
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: true
 
-      - name: Cache local Maven repository
-        uses: actions/cache@v2
+      - name: download lcov
+        uses: actions/checkout@v3
         with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('java/**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
+          repository: linux-test-project/lcov
+          ref: master
+          path: lcov
 
-      - name: coverage configure
+      - name: install lcov
+        working-directory: lcov
         run: |
-          make coverage-configure
+          sudo make install
 
-      - name: build openmldb
+      - name: coverage configure
         run: |
-          cmake --build build --target openmldb -- -j${{ env.NPROC }}
+          make coverage-configure
 
       - name: start service
         run: |
-          sh steps/ut_zookeeper.sh start
-          sh steps/download_openmldb_spark.sh $SPARK_HOME
-          cd onebox
-          bash start_onebox.sh
-          sh start_onebox.sh standalone
+          ./steps/ut_zookeeper.sh start
 
-      - name: Build and Coverage
+      - name: Coverage CPP
         run: |
-          make coverage
+          make coverage-cpp
 
       - name: Upload Coverage Report
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
-          files: build/coverage.info,java/**/target/site/jacoco/jacoco.xml,java/**/target/scoverage.xml
+          files: build/coverage.info
           name: coverage
           fail_ci_if_error: true
           verbose: true
 
       - name: stop service
         run: |
-          ./onebox/stop_all.sh
-          sh steps/ut_zookeeper.sh stop
+          ./steps/ut_zookeeper.sh stop

.github/workflows/hybridse-ci.yml

@@ -18,7 +18,6 @@ on:
   workflow_dispatch:
 
 env:
-  HYBRIDSE_PATH: hybridse
   CTEST_PARALLEL_LEVEL: 2 # parallel test level for ctest (make test)
   CTEST_OUTPUT_ON_FAILURE: ON
   NPROC: 2 # default Parallel build number for GitHub's Linux runner
@@ -29,14 +28,11 @@ jobs:
     container:
       image: ghcr.io/4paradigm/hybridsql:latest
     env:
-      OS: linux
-      ARCH: x86_64
+      TESTING_ENABLE_STRIP: ON
     steps:
       - uses: actions/checkout@v2
-        with:
-          submodules: true
 
-      - name: Build Core
+      - name: Build HybridSE
         run: |
           make hybridse-build
 
@@ -50,14 +46,12 @@ jobs:
         with:
           name: linux-ut-result-cpp-${{ github.sha }}
           path: |
-            ${{ env.HYBRIDSE_PATH }}/build/*.xml
+            build/hybridse/**/*.xml
 
   macos-build:
     name: Build in MacOS
     runs-on: macos-latest
     env:
-      OS: darwin
-      ARCH: x86_64
       NPROC: 3
       CTEST_PARALLEL_LEVEL: 3 # parallel test level for ctest (make test)
     steps:
@@ -89,7 +83,7 @@ jobs:
         with:
           name: macos-ut-result-cpp-${{ github.sha }}
           path: |
-            ${{ env.HYBRIDSE_PATH }}/build/*.xml
+            build/hybridse/**/*.xml
 
   publish-test-results:
     runs-on: ubuntu-latest

.github/workflows/openmldb-docker.yml

@@ -18,7 +18,7 @@ on:
       OPENMLDB_VERSION:
         description: 'The openmldb version installed in docker. ex: 0.3.0'
         required: true
-        default: '0.3.0'
+        default: '0.5.0'
 
 env:
   IMAGE_NAME: openmldb
@@ -40,7 +40,7 @@ jobs:
       - name: Docker Compose Test
         working-directory: demo
         run: |
-          docker-compose -f docker-compose.test.yml -- up --exit-code-from sut talking
+          docker-compose -f docker-compose.test.yml -- up --exit-code-from sut
 
       - name: Decide Push
         run: |

.github/workflows/sdk.yml

@@ -119,6 +119,20 @@ jobs:
           MAVEN_TOKEN: ${{ secrets.OSSRH_TOKEN }}
           GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
 
+      - name: maven coverage
+        working-directory: java
+        run: |
+            ./mvnw --batch-mode prepare-package
+            ./mvnw --batch-mode scoverage:report
+
+      - name: upload maven coverage
+        uses: codecov/codecov-action@v2
+        with:
+          files: java/**/target/site/jacoco/jacoco.xml,java/**/target/scoverage.xml
+          name: coverage
+          fail_ci_if_error: true
+          verbose: true
+
       - name: stop services
         run: |
           cd onebox && sh stop_all.sh && cd - || exit
@@ -138,18 +152,18 @@ jobs:
       OPENMLDB_BUILD_TARGET: 'cp_native_so openmldb'
       NPROC: 3
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('java/**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
 
       - name: Cache thirdparty
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             .deps/
@@ -249,7 +263,7 @@ jobs:
           pip install setuptools wheel twine
           yum install -y net-tools
 
-      - name: test sqlalchemy
+      - name: test sqlalchemy and generate coverage report
         run: |
           bash steps/test_python.sh
 
@@ -261,6 +275,14 @@ jobs:
           path: |
             python/openmldb/test/pytest.xml
 
+      - name: upload python coverage to codecov
+        uses: codecov/codecov-action@v2
+        with:
+          name: coverage
+          files: python/test/coverage.xml
+          fail_ci_if_error: true
+          verbose: true
+
       - name: upload to pypi
         if: >
           github.repository == '4paradigm/OpenMLDB' && startsWith(github.ref, 'refs/tags/v')
@@ -273,14 +295,15 @@ jobs:
 
   python-sdk-mac:
     runs-on: macos-latest
+    if: github.event_name == 'push'
     env:
       SQL_PYSDK_ENABLE: ON
       OPENMLDB_BUILD_TARGET: 'cp_python_sdk_so openmldb'
     steps:
-      - uses: actions/checkout@v2
-      
+      - uses: actions/checkout@v3
+
       - name: Cache thirdparty
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             .deps/
@@ -328,7 +351,7 @@ jobs:
 
   publish-test-results:
     runs-on: ubuntu-latest
-    needs: [ "java-sdk", "python-sdk", "python-sdk-mac" ]
+    needs: [ "java-sdk", "python-sdk" ]
 
     # the action will only run on 4paradigm/OpenMLDB's context, not for fork repo or dependabot
     if: >

.github/workflows/style.yml

@@ -48,3 +48,24 @@ jobs:
           filter_mode: diff_context
           fail_on_error: true
           exclude: "./.git/*" # Optional.
+
+  pylint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pylint
+
+      - name: Analysing the code with pylint
+        # FIXME: fix the pylint errors in project and rm continue-on-error flag
+        continue-on-error: true
+        run: |
+          make pylint

.gitignore

@@ -71,8 +71,6 @@ java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/proto/
 java/hybridse-native/src/main/java
 java/hybridse-proto/src
 **/.mvn/wrapper/maven-wrapper.jar
-python/logs
-/python/test/*.xml
 
 **/target/
 **/.settings/
@@ -100,3 +98,10 @@ python/logs
 # test
 logs
 
+# python builds
+/python/dist/
+/python/*.egg-info/
+/python/openmldb/native/**
+!/python/openmldb/native/__init__.pyt
+/python/test/*.xml
+