Skip to content

Commit 4f7cb47

Browse files
inv-jishnuinv-jishnu
authored
Merge branch 'master' into fix/data-loader/deprecate-success-log
2 parents e9e42ee + 38fdc5a commit 4f7cb47

File tree

55 files changed

+6443
-2025
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+6443
-2025
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2122,6 +2122,76 @@ jobs:
21222122
name: tidb_8_5_integration_test_reports_${{ matrix.mode.label }}
21232123
path: core/build/reports/tests/integrationTestJdbc
21242124

2125+
integration-test-for-blob-storage:
2126+
name: Blob Storage integration test (${{ matrix.mode.label }})
2127+
runs-on: ubuntu-latest
2128+
2129+
strategy:
2130+
fail-fast: false
2131+
matrix:
2132+
mode:
2133+
- label: default
2134+
group_commit_enabled: false
2135+
- label: with_group_commit
2136+
group_commit_enabled: true
2137+
2138+
services:
2139+
postgres:
2140+
image: mcr.microsoft.com/azure-storage/azurite
2141+
env:
2142+
AZURITE_ACCOUNTS: "test:test"
2143+
ports:
2144+
- 10000:10000
2145+
2146+
steps:
2147+
- uses: actions/checkout@v5
2148+
2149+
- name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
2150+
uses: actions/setup-java@v5
2151+
with:
2152+
java-version: ${{ env.JAVA_VERSION }}
2153+
distribution: ${{ env.JAVA_VENDOR }}
2154+
2155+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
2156+
uses: actions/setup-java@v5
2157+
if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
2158+
with:
2159+
java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
2160+
distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
2161+
2162+
- name: Login to Oracle container registry
2163+
uses: docker/login-action@v3
2164+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
2165+
with:
2166+
registry: container-registry.oracle.com
2167+
username: ${{ secrets.OCR_USERNAME }}
2168+
password: ${{ secrets.OCR_TOKEN }}
2169+
2170+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
2171+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
2172+
run: |
2173+
container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
2174+
docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
2175+
2176+
- name: Setup Gradle
2177+
uses: gradle/actions/setup-gradle@v5
2178+
2179+
- name: Create Blob Storage container
2180+
run: |
2181+
az storage container create \
2182+
--name test-container \
2183+
--connection-string "DefaultEndpointsProtocol=http;AccountName=test;AccountKey=test;BlobEndpoint=http://localhost:10000/test;"
2184+
2185+
- name: Execute Gradle 'integrationTestObjectStorage' task
2186+
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.endpoint=http://localhost:10000/test/test-container -Dscalardb.object_storage.username=test -Dscalardb.object_storage.password=test ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
2187+
2188+
- name: Upload Gradle test reports
2189+
if: always()
2190+
uses: actions/upload-artifact@v5
2191+
with:
2192+
name: blob_storage_integration_test_reports_${{ matrix.mode.label }}
2193+
path: core/build/reports/tests/integrationTestObjectStorage
2194+
21252195
integration-test-for-multi-storage:
21262196
name: Multi-storage integration test (${{ matrix.mode.label }})
21272197
runs-on: ubuntu-latest

‎build.gradle‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,15 @@ subprojects {
2828
slf4jVersion = '1.7.36'
2929
cassandraDriverVersion = '3.11.5'
3030
azureCosmosVersion = '4.75.0'
31+
azureBlobStorageVersion = '12.32.0'
3132
jooqVersion = '3.14.16'
32-
awssdkVersion = '2.37.3'
33+
awssdkVersion = '2.38.2'
3334
commonsDbcp2Version = '2.13.0'
3435
mysqlDriverVersion = '8.4.0'
3536
postgresqlDriverVersion = '42.7.8'
3637
oracleDriverVersion = '23.26.0.0.0'
3738
sqlserverDriverVersion = '12.8.2.jre8'
38-
sqliteDriverVersion = '3.50.3.0'
39+
sqliteDriverVersion = '3.51.0.0'
3940
yugabyteDriverVersion = '42.7.3-yb-4'
4041
db2DriverVersion = '12.1.2.0'
4142
mariadDbDriverVersion = '3.5.6'
@@ -108,7 +109,8 @@ jreleaser {
108109
def stagingRepositories = ['core/build/staging-deploy',
109110
'integration-test/build/staging-deploy',
110111
'schema-loader/build/staging-deploy',
111-
'data-loader/core/build/staging-deploy']
112+
'data-loader/core/build/staging-deploy',
113+
'data-loader/cli/build/staging-deploy']
112114
mavenCentral {
113115
sonatype {
114116
active = 'RELEASE'

‎core/build.gradle‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,16 @@ sourceSets {
6464
}
6565
resources.srcDir file('src/integration-test/resources')
6666
}
67+
integrationTestObjectStorage {
68+
java {
69+
compileClasspath += main.output + test.output
70+
runtimeClasspath += main.output + test.output
71+
srcDir file('src/integration-test/java')
72+
include '**/com/scalar/db/common/*.java'
73+
include '**/com/scalar/db/storage/objectstorage/*.java'
74+
}
75+
resources.srcDir file('src/integration-test/resources')
76+
}
6777
integrationTestMultiStorage {
6878
java {
6979
compileClasspath += main.output + test.output
@@ -136,6 +146,9 @@ configurations {
136146
integrationTestJdbcImplementation.extendsFrom testImplementation
137147
integrationTestJdbcRuntimeOnly.extendsFrom testRuntimeOnly
138148
integrationTestJdbcCompileOnly.extendsFrom testCompileOnly
149+
integrationTestObjectStorageImplementation.extendsFrom testImplementation
150+
integrationTestObjectStorageRuntimeOnly.extendsFrom testRuntimeOnly
151+
integrationTestObjectStorageCompileOnly.extendsFrom testCompileOnly
139152
integrationTestMultiStorageImplementation.extendsFrom testImplementation
140153
integrationTestMultiStorageRuntimeOnly.extendsFrom testRuntimeOnly
141154
integrationTestMultiStorageCompileOnly.extendsFrom testCompileOnly
@@ -156,6 +169,7 @@ dependencies {
156169
implementation "org.slf4j:slf4j-api:${slf4jVersion}"
157170
implementation "com.datastax.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
158171
implementation "com.azure:azure-cosmos:${azureCosmosVersion}"
172+
implementation "com.azure:azure-storage-blob:${azureBlobStorageVersion}"
159173
implementation "org.jooq:jooq:${jooqVersion}"
160174
implementation platform("software.amazon.awssdk:bom:${awssdkVersion}")
161175
implementation 'software.amazon.awssdk:applicationautoscaling'
@@ -174,7 +188,9 @@ dependencies {
174188
exclude group: 'org.slf4j', module: 'slf4j-api'
175189
}
176190
// The Java connector library used to establish simply a secure connection to AlloyDB on GCP
177-
implementation("com.google.cloud:alloydb-jdbc-connector:${alloyDbJdbcConnectorVersion}")
191+
implementation("com.google.cloud:alloydb-jdbc-connector:${alloyDbJdbcConnectorVersion}") {
192+
exclude group: 'org.slf4j', module: 'slf4j-api'
193+
}
178194
implementation "org.apache.commons:commons-text:${commonsTextVersion}"
179195
testImplementation platform("org.junit:junit-bom:${junitVersion}")
180196
testImplementation 'org.junit.jupiter:junit-jupiter'
@@ -254,6 +270,17 @@ task integrationTestJdbc(type: Test) {
254270
maxHeapSize = "4g"
255271
}
256272

273+
task integrationTestObjectStorage(type: Test) {
274+
description = 'Runs the integration tests for Object Storages.'
275+
group = 'verification'
276+
testClassesDirs = sourceSets.integrationTestObjectStorage.output.classesDirs
277+
classpath = sourceSets.integrationTestObjectStorage.runtimeClasspath
278+
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
279+
options {
280+
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
281+
}
282+
}
283+
257284
task integrationTestMultiStorage(type: Test) {
258285
description = 'Runs the integration tests for multi-storage.'
259286
group = 'verification'

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminTestUtils.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,25 +30,26 @@ public JdbcAdminTestUtils(Properties properties) {
3030
@Override
3131
public void dropNamespacesTable() throws Exception {
3232
execute(
33-
"DROP TABLE " + rdbEngine.encloseFullTableName(metadataSchema, JdbcAdmin.NAMESPACES_TABLE));
33+
"DROP TABLE "
34+
+ rdbEngine.encloseFullTableName(metadataSchema, NamespaceMetadataService.TABLE_NAME));
3435
}
3536

3637
@Override
3738
public void dropMetadataTable() throws Exception {
38-
dropTable(metadataSchema, JdbcAdmin.METADATA_TABLE);
39+
dropTable(metadataSchema, TableMetadataService.TABLE_NAME);
3940
}
4041

4142
@Override
4243
public void truncateNamespacesTable() throws Exception {
4344
String truncateTableStatement =
44-
rdbEngine.truncateTableSql(metadataSchema, JdbcAdmin.NAMESPACES_TABLE);
45+
rdbEngine.truncateTableSql(metadataSchema, NamespaceMetadataService.TABLE_NAME);
4546
execute(truncateTableStatement);
4647
}
4748

4849
@Override
4950
public void truncateMetadataTable() throws Exception {
5051
String truncateTableStatement =
51-
rdbEngine.truncateTableSql(metadataSchema, JdbcAdmin.METADATA_TABLE);
52+
rdbEngine.truncateTableSql(metadataSchema, TableMetadataService.TABLE_NAME);
5253
execute(truncateTableStatement);
5354
}
5455

@@ -57,7 +58,7 @@ public void truncateMetadataTable() throws Exception {
5758
public void corruptMetadata(String namespace, String table) throws Exception {
5859
String insertCorruptedMetadataStatement =
5960
"INSERT INTO "
60-
+ rdbEngine.encloseFullTableName(metadataSchema, JdbcAdmin.METADATA_TABLE)
61+
+ rdbEngine.encloseFullTableName(metadataSchema, TableMetadataService.TABLE_NAME)
6162
+ " VALUES ('"
6263
+ getFullTableName(namespace, table)
6364
+ "','corrupted','corrupted','corrupted','corrupted','0','0')";
@@ -68,9 +69,9 @@ public void corruptMetadata(String namespace, String table) throws Exception {
6869
public void deleteMetadata(String namespace, String table) throws Exception {
6970
String deleteMetadataStatement =
7071
"DELETE FROM "
71-
+ rdbEngine.encloseFullTableName(metadataSchema, JdbcAdmin.METADATA_TABLE)
72+
+ rdbEngine.encloseFullTableName(metadataSchema, TableMetadataService.TABLE_NAME)
7273
+ " WHERE "
73-
+ rdbEngine.enclose(JdbcAdmin.METADATA_COL_FULL_TABLE_NAME)
74+
+ rdbEngine.enclose(TableMetadataService.COL_FULL_TABLE_NAME)
7475
+ " = ?";
7576
try (Connection connection = dataSource.getConnection();
7677
PreparedStatement preparedStatement =

‎core/src/integration-test/java/com/scalar/db/storage/multistorage/MultiStorageAdminTestUtils.java‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,13 @@
99
import com.scalar.db.storage.cassandra.CassandraAdmin;
1010
import com.scalar.db.storage.cassandra.CassandraConfig;
1111
import com.scalar.db.storage.cassandra.ClusterManager;
12-
import com.scalar.db.storage.jdbc.JdbcAdmin;
1312
import com.scalar.db.storage.jdbc.JdbcConfig;
1413
import com.scalar.db.storage.jdbc.JdbcTestUtils;
1514
import com.scalar.db.storage.jdbc.JdbcUtils;
15+
import com.scalar.db.storage.jdbc.NamespaceMetadataService;
1616
import com.scalar.db.storage.jdbc.RdbEngineFactory;
1717
import com.scalar.db.storage.jdbc.RdbEngineStrategy;
18+
import com.scalar.db.storage.jdbc.TableMetadataService;
1819
import com.scalar.db.util.AdminTestUtils;
1920
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
2021
import java.sql.Connection;
@@ -61,7 +62,8 @@ public void dropNamespacesTable() throws SQLException {
6162
// for JDBC
6263
execute(
6364
"DROP TABLE "
64-
+ rdbEngine.encloseFullTableName(jdbcMetadataSchema, JdbcAdmin.NAMESPACES_TABLE));
65+
+ rdbEngine.encloseFullTableName(
66+
jdbcMetadataSchema, NamespaceMetadataService.TABLE_NAME));
6567
}
6668

6769
@Override
@@ -71,7 +73,7 @@ public void dropMetadataTable() throws SQLException {
7173
// for JDBC
7274
execute(
7375
"DROP TABLE "
74-
+ rdbEngine.encloseFullTableName(jdbcMetadataSchema, JdbcAdmin.METADATA_TABLE));
76+
+ rdbEngine.encloseFullTableName(jdbcMetadataSchema, TableMetadataService.TABLE_NAME));
7577
}
7678

7779
@Override
@@ -86,7 +88,7 @@ public void truncateNamespacesTable() throws SQLException {
8688

8789
// for JDBC
8890
String truncateTableStatement =
89-
rdbEngine.truncateTableSql(jdbcMetadataSchema, JdbcAdmin.NAMESPACES_TABLE);
91+
rdbEngine.truncateTableSql(jdbcMetadataSchema, NamespaceMetadataService.TABLE_NAME);
9092
execute(truncateTableStatement);
9193
}
9294

@@ -97,7 +99,7 @@ public void truncateMetadataTable() throws Exception {
9799
// for JDBC
98100
String truncateTableStatement =
99101
"TRUNCATE TABLE "
100-
+ rdbEngine.encloseFullTableName(jdbcMetadataSchema, JdbcAdmin.METADATA_TABLE);
102+
+ rdbEngine.encloseFullTableName(jdbcMetadataSchema, TableMetadataService.TABLE_NAME);
101103
execute(truncateTableStatement);
102104
}
103105

@@ -109,7 +111,7 @@ public void corruptMetadata(String namespace, String table) throws Exception {
109111
// for JDBC
110112
String insertCorruptedMetadataStatement =
111113
"INSERT INTO "
112-
+ rdbEngine.encloseFullTableName(jdbcMetadataSchema, JdbcAdmin.METADATA_TABLE)
114+
+ rdbEngine.encloseFullTableName(jdbcMetadataSchema, TableMetadataService.TABLE_NAME)
113115
+ " VALUES ('"
114116
+ getFullTableName(namespace, table)
115117
+ "','corrupted','corrupted','corrupted','corrupted','0','0')";
@@ -123,9 +125,9 @@ public void deleteMetadata(String namespace, String table) throws Exception {
123125
// for JDBC
124126
String deleteMetadataStatement =
125127
"DELETE FROM "
126-
+ rdbEngine.encloseFullTableName(jdbcMetadataSchema, JdbcAdmin.METADATA_TABLE)
128+
+ rdbEngine.encloseFullTableName(jdbcMetadataSchema, TableMetadataService.TABLE_NAME)
127129
+ " WHERE "
128-
+ rdbEngine.enclose(JdbcAdmin.METADATA_COL_FULL_TABLE_NAME)
130+
+ rdbEngine.enclose(TableMetadataService.COL_FULL_TABLE_NAME)
129131
+ " = ?";
130132
try (Connection connection = dataSource.getConnection();
131133
PreparedStatement preparedStatement =

0 commit comments

Comments
 (0)