Support ScalarDB Analytics 3.17 (#82)

Author: choplin

scalardb-analytics-sample/README.md

@@ -20,10 +20,16 @@ docker compose run --rm scalardb-analytics-cli catalog create --catalog sample_c
 
 ```bash
 # Register ScalarDB data source
-docker compose run --rm scalardb-analytics-cli data-source register --data-source-json /config/data-sources/scalardb.json
+docker compose run --rm scalardb-analytics-cli data-source register \
+  --catalog sample_catalog \
+  --data-source scalardb \
+  --provider-file /config/data-sources/scalardb.json
 
 # Register PostgreSQL data source
-docker compose run --rm scalardb-analytics-cli data-source register --data-source-json /config/data-sources/postgres.json
+docker compose run --rm scalardb-analytics-cli data-source register \
+  --catalog sample_catalog \
+  --data-source postgres \
+  --provider-file /config/data-sources/postgres.json
 ```
 
 ### 4. Run Spark SQL

scalardb-analytics-sample/config/data-sources/postgres.json

@@ -1,12 +1,8 @@
 {
-  "catalog": "sample_catalog",
-  "name": "postgres",
-  "type": "postgres",
-  "provider": {
-    "host": "postgres",
-    "port": 5432,
-    "username": "postgres",
-    "password": "postgres",
-    "database": "sampledb"
-  }
+  "type": "postgresql",
+  "host": "postgres",
+  "port": 5432,
+  "username": "postgres",
+  "password": "postgres",
+  "database": "sampledb"
 }

scalardb-analytics-sample/config/data-sources/scalardb.json

@@ -1,8 +1,4 @@
 {
-  "catalog": "sample_catalog",
-  "name": "scalardb",
   "type": "scalardb",
-  "provider": {
-    "configPath": "/etc/scalardb.properties"
-  }
+  "configs": "${file:/etc/scalardb.properties}"
 }

scalardb-analytics-sample/config/scalardb-analytics-server.properties

@@ -5,7 +5,7 @@ scalar.db.analytics.server.catalog.port=11051
 scalar.db.analytics.server.metering.port=11052
 
 # Server database configuration (for catalog metadata)
-scalar.db.analytics.server.db.url=jdbc:postgresql://analytics-catalog-postgres:5432/catalogdb
+scalar.db.analytics.server.db.contact_points=jdbc:postgresql://analytics-catalog-postgres:5432/catalogdb
 scalar.db.analytics.server.db.username=analytics
 scalar.db.analytics.server.db.password=analytics
 

scalardb-analytics-sample/config/spark-defaults.conf

@@ -1,4 +1,4 @@
-spark.jars.packages com.scalar-labs:scalardb-analytics-spark-all-3.5_2.12:3.16.2
+spark.jars.packages com.scalar-labs:scalardb-analytics-spark-all-3.5_2.12:3.17.0
 spark.extraListeners com.scalar.db.analytics.spark.metering.ScalarDbAnalyticsListener
 
 # Use the ScalarDB Analytics catalog as `sample_catalog`

scalardb-analytics-sample/docker-compose.yml

@@ -26,14 +26,13 @@ services:
 
   # ScalarDB Analytics Server
   scalardb-analytics-server:
-    image: ghcr.io/scalar-labs/scalardb-analytics-server-without-licensing:3.16.2
+    image: ghcr.io/scalar-labs/scalardb-analytics-server-without-licensing:3.17.0
     platform: linux/amd64
     expose:
       - 11051 # Catalog service port
       - 11052 # Metering service port
     volumes:
       - ./config/scalardb-analytics-server.properties:/scalardb-analytics-server/server.properties:ro
-      - ./config/scalardb.properties:/etc/scalardb.properties:ro
     networks:
       - scalar-network
     depends_on:
@@ -47,10 +46,11 @@ services:
 
   # ScalarDB Analytics CLI
   scalardb-analytics-cli:
-    image: ghcr.io/scalar-labs/scalardb-analytics-cli:3.16.2
+    image: ghcr.io/scalar-labs/scalardb-analytics-cli:3.17.0
     volumes:
       - ./config/scalardb-analytics-cli.properties:/config/client.properties:ro
       - ./config/data-sources:/config/data-sources:ro
+      - ./config/scalardb.properties:/etc/scalardb.properties
     networks:
       - scalar-network
     profiles:
@@ -170,7 +170,6 @@ services:
       context: ./docker
       dockerfile: Dockerfile.spark
     volumes:
-      - ./config/scalardb.properties:/etc/scalardb.properties
       - ./config/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
       - .scala_history:/root/.scala_history
       - spark-ivy-cache:/root/.ivy2

scalardb-analytics-sample/docker/Dockerfile.spark

@@ -18,11 +18,12 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/*
 
 # Download and verify Spark
-RUN curl -fsSL -o spark.tgz "https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
-    curl -fsSL -o spark.tgz.sha512 "https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz.sha512" && \
-    sha512sum -c spark.tgz.sha512 && \
-    tar -xzf spark.tgz -C /opt && \
+RUN SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
+    curl -fsSL -o "${SPARK_ARCHIVE}" "https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}" && \
+    curl -fsSL -o "${SPARK_ARCHIVE}.sha512" "https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.sha512" && \
+    sha512sum -c "${SPARK_ARCHIVE}.sha512" && \
+    tar -xzf "${SPARK_ARCHIVE}" -C /opt && \
     mv "/opt/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" "${SPARK_HOME}" && \
-    rm -rf spark.tgz spark.tgz.sha512
+    rm -rf "${SPARK_ARCHIVE}" "${SPARK_ARCHIVE}.sha512"
 
 WORKDIR /opt/spark