chore: switch to SBT and fix docs ci (#52)

* Switch to SBT and fix docs ci

* Fix CI

* Fix "bad substitution error"

* docs CI

* docs CI

* docs CI

Author: SemyonSinchenko

.github/workflows/ci-jvm.yml

@@ -13,14 +13,20 @@ jobs:
           java-version: '11'
           distribution: 'temurin'
           architecture: x64
+          cache: sbt
+
+      - name: Set up SBT
+        uses: sbt/setup-sbt@v1
+        with:
+          sbt-runner-version: 1.9.9
 
       - name: Check codestyle
         run: |
-          mvn --no-transfer-progress spotless:check
+          sbt root/scalafmtCheckAll
         working-directory: tsumugi-server
 
       - name: Run tests
         run: |
-          mvn --no-transfer-progress test
+          sbt root/test
         working-directory: tsumugi-server
 

.github/workflows/docs.yml

@@ -35,5 +35,8 @@ jobs:
           git config user.email 'github-actions[bot]@users.noreply.github.com'
           
       - name: Build and Deploy
-        run:
-          poetry run -C ${{ github.workspace }}/tsumugi-python mkdocs gh-deploy --force
+        run: |
+          cd tsumugi-python
+          export VENV_ROOT=$(poetry env info --path)
+          cd ..
+          $VENV_ROOT/bin/mkdocs gh-deploy --force

.github/workflows/release.yml

@@ -21,6 +21,11 @@ jobs:
           java-version: '11'
           distribution: 'adopt'
 
+      - name: Set up SBT
+        uses: sbt/setup-sbt@v1
+        with:
+          sbt-runner-version: 1.9.9
+          
       - name: Set up Python
         if: startsWith(github.ref, 'refs/tags/')
         uses: actions/setup-python@v5
@@ -31,10 +36,10 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/')
         uses: snok/install-poetry@v1
 
-      - name: Build server with Maven
+      - name: Build server with SBT
         if: startsWith(github.ref, 'refs/tags/')
-        # Inspired by https://stackoverflow.com/a/51559900
-        run: mvn -Drevision=${{ github.ref_name }} -DskipTests --no-transfer-progress clean package
+        run: |
+          sbt -Drevision=${{ github.ref_name }} assemblyWithDeequ
         working-directory: tsumugi-server
 
       - name: Build client with poetry
@@ -51,7 +56,7 @@ jobs:
           draft: false
           prerelease: false
           files: |
-            tsumugi-server/target/tsumugi-server-*.jar
+            tsumugi-server/withDeequ/scala-2.12/target/tsumugi-server-*.jar
             tsumugi-python/dist/tsumugi-*.tar.gz
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

dev/run-connect.py

@@ -5,10 +5,10 @@
 import sys
 from pathlib import Path
 
-MVN_BUILD_COMMAND = ["mvn", "clean", "-DskipTests", "package"]
-SPARK_VERSION = "3.5.2"
+SBT_BUILD_COMMAND = ["sbt", "assemblyWithDeequ"]
+SPARK_VERSION = "3.5.4"
 SCALA_VERSION = "2.12"
-TSUMUGI_VERSION = "1.0-SNAPSHOT"
+TSUMUGI_VERSION = "0.1.0-SNAPSHOT"
 
 
 if __name__ == "__main__":
@@ -17,18 +17,18 @@
 
     print("Build Tsumugi...")
     os.chdir(scala_root)
-    build_mvn = subprocess.run(
-        MVN_BUILD_COMMAND,
+    build_sbt = subprocess.run(
+        SBT_BUILD_COMMAND,
         stdout=subprocess.PIPE,
         universal_newlines=True,
     )
 
-    if build_mvn.returncode == 0:
+    if build_sbt.returncode == 0:
         print("Done.")
     else:
-        print(f"MVN build return an error: {build_mvn.returncode}")
-        print("stdout: ", build_mvn.stdout)
-        print("stderr: ", build_mvn.stderr)
+        print(f"SBT build return an error: {build_sbt.returncode}")
+        print("stdout: ", build_sbt.stdout)
+        print("stderr: ", build_sbt.stderr)
         sys.exit(1)
 
     tmp_dir = prj_root.joinpath("tmp")
@@ -82,7 +82,9 @@
     os.chdir(spark_home)
 
     tsumugi_jar = (
-        scala_root.joinpath("target")
+        scala_root.joinpath("withDeequ")
+        .joinpath("target")
+        .joinpath(f"scala-{SCALA_VERSION}")
         .joinpath(f"tsumugi-server-{TSUMUGI_VERSION}.jar")
     )
     shutil.copyfile(tsumugi_jar, spark_home.joinpath(tsumugi_jar.name))

tsumugi-python/tsumugi/__init__.py

@@ -4,3 +4,9 @@
 proto_root = Path(__file__).parent.joinpath("proto")
 if str(proto_root.absolute()) not in sys.path:
     sys.path.append(str(proto_root.absolute()))
+
+# We must me sure that all the protobuf packages are loaded!
+# An order is also important!
+from tsumugi.proto import repository_pb2  # noqa: E402, F401
+from tsumugi.proto import analyzers_pb2  # noqa: E402, F401
+from tsumugi.proto import strategies_pb2  # noqa: E402, F401