Merge remote-tracking branch 'upstream/main' into feat/add_disk_status_from_main

Author: litongxin

.github/labeler.yml

@@ -0,0 +1,43 @@
+sdk:
+  - 'python/**/*'
+  - 'java/openmldb-jdbc/**/*'
+  - 'java/openmldb-native/**/*'
+
+execute-engine:
+  - 'hybridse/**/*'
+
+storage-engine:
+  - any:
+      - 'src/**/*'
+    all:
+      - '!src/apiserver/**/*'
+
+api-server:
+  - 'src/apiserver/**/*'
+
+task-manager:
+  - 'java/openmldb-taskmanager/**/*'
+
+batch-engine:
+  - 'java/openmldb-batch/**/*'
+  - 'java/openmldb-batchjob/**/*'
+
+monitoring:
+  - 'monitoring/**/*'
+
+docker:
+  - 'docker/**/*'
+  - 'demo/**/*'
+
+documentation:
+  - 'docs/**/*'
+  - '*.md'
+
+build:
+  - '**/CMakeLists.txt'
+  - '**/*.cmake'
+  - 'third-party/**/*'
+  - 'Makefile'
+
+workflow:
+  - '.github/**/*.yml'

.github/workflows/cicd.yaml

@@ -37,7 +37,7 @@ jobs:
     runs-on: self-hosted
     if: github.repository == '4paradigm/OpenMLDB'
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.5
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
       SQL_PYSDK_ENABLE: OFF

.github/workflows/coverage.yml

@@ -27,7 +27,7 @@ jobs:
   coverage:
     runs-on: self-hosted
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.5
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       CTEST_PARALLEL_LEVEL: 1 # parallel test level for ctest (make test)
       CMAKE_BUILD_TYPE: Debug

.github/workflows/hybridse-ci.yml

@@ -27,7 +27,7 @@ jobs:
   linux-build:
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.5
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
       ARCH: x86_64

.github/workflows/integration-test-pkg.yml

@@ -81,7 +81,7 @@ jobs:
     if: ${{ github.event.inputs.EXEC_TEST_TYPE == 'all' || github.event.inputs.EXEC_TEST_TYPE == 'java' }}
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.5
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
     steps:
@@ -103,7 +103,7 @@ jobs:
     if: ${{ github.event.inputs.EXEC_TEST_TYPE == 'all' || github.event.inputs.EXEC_TEST_TYPE == 'java' }}
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.5
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
     steps:
@@ -166,7 +166,7 @@ jobs:
     if: ${{ github.event.inputs.EXEC_TEST_TYPE == 'all' || github.event.inputs.EXEC_TEST_TYPE == 'python' }}
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.5
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
     steps:
@@ -203,7 +203,7 @@ jobs:
     if: ${{ github.event.inputs.EXEC_TEST_TYPE == 'all' || github.event.inputs.EXEC_TEST_TYPE == 'python' }}
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.5
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
     steps:
@@ -224,7 +224,7 @@ jobs:
     if: ${{ github.event.inputs.EXEC_TEST_TYPE == 'all' || github.event.inputs.EXEC_TEST_TYPE == 'apiserver' }}
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.5
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
     steps:
@@ -245,7 +245,7 @@ jobs:
     if: ${{ github.event.inputs.EXEC_TEST_TYPE == 'all' || github.event.inputs.EXEC_TEST_TYPE == 'batch' }}
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.5
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
     steps:

.github/workflows/integration-test-src.yml

@@ -64,7 +64,7 @@ jobs:
     if: ${{ github.event.inputs.EXEC_TEST_TYPE == 'all' || github.event.inputs.EXEC_TEST_TYPE == 'java' }}
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.4.1
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
     steps:
@@ -89,7 +89,7 @@ jobs:
     if: ${{ github.event.inputs.EXEC_TEST_TYPE == 'all' || github.event.inputs.EXEC_TEST_TYPE == 'java' }}
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.4.1
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
     steps:
@@ -161,7 +161,7 @@ jobs:
     if: ${{ github.event.inputs.EXEC_TEST_TYPE == 'all' || github.event.inputs.EXEC_TEST_TYPE == 'python' }}
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.4.1
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
     steps:
@@ -201,7 +201,7 @@ jobs:
     if: ${{ github.event.inputs.EXEC_TEST_TYPE == 'all' || github.event.inputs.EXEC_TEST_TYPE == 'python' }}
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.4.1
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
     steps:
@@ -225,7 +225,7 @@ jobs:
     if: ${{ github.event.inputs.EXEC_TEST_TYPE == 'all' || github.event.inputs.EXEC_TEST_TYPE == 'apiserver' }}
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.4.1
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
     steps:
@@ -249,7 +249,7 @@ jobs:
     if: ${{ github.event.inputs.EXEC_TEST_TYPE == 'all' || github.event.inputs.EXEC_TEST_TYPE == 'batch' }}
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.4.1
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OS: linux
     steps:

.github/workflows/label.yml

@@ -0,0 +1,23 @@
+# This workflow will triage pull requests and apply a label based on the
+# paths that are modified in the pull request.
+#
+# To use this workflow, you will need to set up a .github/labeler.yml
+# file with configuration.  For more information, see:
+# https://github.com/actions/labeler
+
+name: Labeler
+on: [pull_request_target]
+
+jobs:
+  label:
+    permissions:
+      contents: read
+      pull-requests: write
+      # all other permissions set to none
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/labeler@v4
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/openmldb-docker.yml

@@ -6,13 +6,13 @@ on:
       - main
     paths:
       - .github/workflows/openmldb-docker.yml
-      - demo/predict-taxi-trip-duration-nb/**/*
+      - demo/**/*
     tags:
       - v*
   pull_request:
     paths:
       - .github/workflows/openmldb-docker.yml
-      - demo/predict-taxi-trip-duration-nb/**/*
+      - demo/**/*
   workflow_dispatch:
     inputs:
       OPENMLDB_VERSION:
@@ -38,9 +38,9 @@ jobs:
         uses: docker/setup-buildx-action@v1
 
       - name: Docker Compose Test
-        working-directory: demo/predict-taxi-trip-duration-nb
+        working-directory: demo
         run: |
-          docker-compose -f docker-compose.test.yml -- up
+          docker-compose -f docker-compose.test.yml -- up --exit-code-from sut talking
 
       - name: Decide Push
         run: |
@@ -68,7 +68,7 @@ jobs:
         id: docker_build
         uses: docker/build-push-action@v2
         with:
-          context: demo/predict-taxi-trip-duration-nb
+          context: demo
           push: ${{ env.PUSH }}
           platforms: linux/amd64
           tags: |

.github/workflows/sdk.yml

@@ -36,7 +36,7 @@ jobs:
   java-sdk:
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.5
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       SQL_JAVASDK_ENABLE: ON
       OPENMLDB_BUILD_TARGET: 'cp_native_so openmldb'
@@ -225,7 +225,7 @@ jobs:
   python-sdk:
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/4paradigm/hybridsql:0.5
+      image: ghcr.io/4paradigm/hybridsql:latest
     env:
       SQL_PYSDK_ENABLE: ON
       OPENMLDB_BUILD_TARGET: 'cp_python_sdk_so openmldb'

CHANGELOG.md

@@ -1,5 +1,41 @@
 # Changelog
 
+## [0.5.0] - 2022-05-07
+
+### Highlights
+
+- We have introduced an important performance optimization technique of pre-aggregation, which can significantly improve the performance for a query with time windows containing massive amount of rows, e.g., a few millions. (#1532 #1573 #1583 #1622 #1627 #1672 # 1712 @zhanghaohit @nautaa)
+- We have added a new storage engine that supports persistent storage (such as HDD and SSD) for the online SQL engine. Such a storage engine is helpful when a user wants to reduce the cost with acceptable performance degradation. (#1483 @Leowner)
+- We have supported C/C++ based User-Defined Functions (UDFs) with dynamic registration to enhance the development experience.  (#1509 #1733 #1700 @dl239 @tobegit3hub)
+
+### Other Features
+
+- Enhance the OpenMLDB Prometheus exporter ( #1584, #1645, #1754 @aceforeverd )
+- Support collecting statistics of query response time for online queries ( #1497, #1521 @aceforeverd )
+- Support new SQL commands: `SHOW COMPONENTS`, `SHOW TABLE STATUS` (#1380 #1431 #1704 @aceforeverd)
+- Support setting global variables (#1310 #1359 #1364 @keyu813 @aceforeverd)
+- Support reading Spark configuration files from the CLI (#1600 @tobegit3hub)
+- Support using multiple threads for the Spark local mode (#1675 @tobegit3hub)
+- Enhance the performance of join by using the Spark's native expression (#1502 tobegit3hub)
+- Support the validation for TaskManager configuration (#1262 @tobegit3hub)
+- Support tracking unfinished jobs in the TaskManager (#1474 @tobegit3hub)
+- Other minor features (#1601 @dl239; #1574 @vagetablechicken; #1546 @keyu813; #1729 @vagetablechicken; #1460 @tobegit3hub)
+
+### Bug Fixes
+- Incorrect results when the order of conditions specified in `where` is different from that of the index (#1709 @aceforeverd)
+- Incorrect results of `lag`/`at`/`lead` under certain circumstances (#1605 #1739 @aceforeverd)
+- Memory leakage in `zk_client` (#1660 @wuxiaobai24)
+- No catalog update if the role of a tablet is changed (#1655 @dl239)
+- Related bugs about `UnsafeRow` for the offline engine (#1298, #1312, #1326, #1362, #1637, #1381, #1731 @tobegit3hub)
+- Incorrect results after adding a new index in the standalone mode (#1721 @keyu813)
+- Incorrect results of `SHOW JOBS` under certain circumstances (#1453 @tobegit3hub)
+- Incorrect results of the date columns with `UnsafeRowOpt`(#1469 @tobegit3hub)
+- Other minor bug fixes (#1698 @kfiring; #1651 @kutlayacar; #1621 @KaidoWang; #1150, #1243 @tobegit3hub; )
+
+### Code Refactoring
+#1616 @dl239; #1743 @zjx1319
+
+
 ## [0.4.4] - 2022-04-01
 
 ### Features