diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d61bc27a..f4f6924c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -10,9 +10,31 @@ on: type: [opened, reopened, edited, synchronize] jobs: + prepare: + name: Prepare Maven cache + runs-on: ubuntu-latest + + env: + MAVEN_ARGS: --batch-mode -Dstyle.color=always + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: '8' + distribution: 'temurin' + cache: 'maven' + + - name: Download dependencies for JDBC + run: mvn $MAVEN_ARGS dependency:resolve-plugins dependency:go-offline + build: name: Build JDBC Driver runs-on: ubuntu-latest + needs: prepare strategy: matrix: @@ -75,9 +97,6 @@ jobs: if: env.NEED_SDK == 'true' run: rm -rf yc - - name: Download dependencies - run: mvn $MAVEN_ARGS dependency:resolve-plugins dependency:go-offline - - name: Build with Maven run: mvn $MAVEN_ARGS package @@ -144,9 +163,6 @@ jobs: if: env.NEED_SDK == 'true' run: rm -rf yc - - name: Download dependencies - run: mvn $MAVEN_ARGS dependency:resolve-plugins dependency:go-offline - - name: Build with Maven run: mvn $MAVEN_ARGS test diff --git a/README.md b/README.md index fb0a1b02..9ac8702b 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ 1) Drop in [JDBC driver](https://github.com/ydb-platform/ydb-jdbc-driver/releases) to classpath or pick this file in IDE 2) Connect to YDB * Local or remote Docker (anonymous authentication):
`jdbc:ydb:grpc://localhost:2136/local` - * Self-hosted cluster:
`jdbc:ydb:grpcs://:2135/Root/testdb?secureConnectionCertificate=file:~/myca.cer` - * Connect with token to the cloud instance:
`jdbc:ydb:grpcs://:2135/path/to/database?token=file:~/my_token` - * Connect with service account to the cloud instance:
`jdbc:ydb:grpcs://:2135/path/to/database?saFile=file:~/sa_key.json` + * Self-hosted cluster:
`jdbc:ydb:grpcs://:2135/Root/testdb?secureConnectionCertificate=~/myca.cer` + * Connect with token to the cloud instance:
`jdbc:ydb:grpcs://:2135/path/to/database?tokenFile=~/my_token` + * Connect with service account to the cloud instance:
`jdbc:ydb:grpcs://:2135/path/to/database?saKeyFile=~/sa_key.json` 3) Execute queries, see example in [YdbDriverExampleTest.java](jdbc/src/test/java/tech/ydb/jdbc/YdbDriverExampleTest.java) ### Usage with Maven @@ -53,20 +53,15 @@ YDB JDBC Driver supports the following [authentication modes](https://ydb.tech/e ### Driver properties reference Driver supports the following configuration properties, which can be specified in the URL or passed via extra properties: -* `saFile` - service account key for authentication, can be passed either as literal JSON value or as a file reference; +* `saKeyFile` - file location of service account key for authentication; +* `tokenFile` - file location with token value for token based authentication; * `iamEndpoint` - custom IAM endpoint for authentication via service account key; -* `token` - token value for authentication, can be passed either as literal value or as a file reference; * `useMetadata` - boolean value, true if metadata authentication should be used, false otherwise (and default); * `metadataURL` - custom metadata endpoint; * `localDatacenter` - name of the datacenter local to the application being connected; * `secureConnection` - boolean value, true if TLS should be enforced (normally configured via `grpc://` or `grpcs://` scheme in the JDBC URL); * `secureConnectionCertificate` - custom CA certificate for TLS connections, can be passed either as literal value or as a file reference. -File references for `saFile`, `token` or `secureConnectionCertificate` must be prefixed with the `file:` URL scheme, for example: -* `saFile=file:~/mysaley1.json` -* `token=file:/opt/secret/token-file` -* `secureConnectionCertificate=file:/etc/ssl/cacert.cer` - ### Building By default all tests are run using a local YDB instance in Docker (if host has Docker or Docker Machine installed) To disable these tests run `mvn test -DYDB_DISABLE_INTEGRATION_TESTS=true` diff --git a/jdbc/src/main/java/tech/ydb/jdbc/YdbConst.java b/jdbc/src/main/java/tech/ydb/jdbc/YdbConst.java index c44fd832..9121387b 100644 --- a/jdbc/src/main/java/tech/ydb/jdbc/YdbConst.java +++ b/jdbc/src/main/java/tech/ydb/jdbc/YdbConst.java @@ -31,6 +31,9 @@ public final class YdbConst { public static final String DRIVER_IS_NOT_REGISTERED = "Driver is not registered " + "(or it has not been registered using YdbDriver.register() method)"; + public static final String MISSING_DRIVER_OPTION = "Missing value for option "; + public static final String INVALID_DRIVER_OPTION_VALUE = "Cannot process value %s for option %s: %s"; + public static final String PREPARED_CALLS_UNSUPPORTED = "Prepared calls are not supported"; public static final String ARRAYS_UNSUPPORTED = "Arrays are not supported"; public static final String STRUCTS_UNSUPPORTED = "Structs are not supported"; @@ -140,8 +143,6 @@ public final class YdbConst { public static final String INDEXED_PARAMETER_PREFIX = "p"; public static final String VARIABLE_PARAMETER_PREFIX = "$"; public static final String AUTO_GENERATED_PARAMETER_PREFIX = VARIABLE_PARAMETER_PREFIX + "jp"; - public static final String DEFAULT_BATCH_PARAMETER = "$values"; - public static final String OPTIONAL_TYPE_SUFFIX = "?"; private YdbConst() { // diff --git a/jdbc/src/main/java/tech/ydb/jdbc/YdbDriver.java b/jdbc/src/main/java/tech/ydb/jdbc/YdbDriver.java index a3cb405c..bce20e4e 100644 --- a/jdbc/src/main/java/tech/ydb/jdbc/YdbDriver.java +++ b/jdbc/src/main/java/tech/ydb/jdbc/YdbDriver.java @@ -134,9 +134,11 @@ public int getConnectionCount() { } public void close() { - LOGGER.log(Level.INFO, "Closing {0} cached connection(s)...", cache.size()); - cache.values().forEach(YdbContext::close); - cache.clear(); + if (!cache.isEmpty()) { + LOGGER.log(Level.FINE, "Closing {0} cached connection(s)...", cache.size()); + cache.values().forEach(YdbContext::close); + cache.clear(); + } } public static boolean isRegistered() { diff --git a/jdbc/src/main/java/tech/ydb/jdbc/context/StreamQueryResult.java b/jdbc/src/main/java/tech/ydb/jdbc/context/StreamQueryResult.java index 7870d96a..d7167d3c 100644 --- a/jdbc/src/main/java/tech/ydb/jdbc/context/StreamQueryResult.java +++ b/jdbc/src/main/java/tech/ydb/jdbc/context/StreamQueryResult.java @@ -80,9 +80,14 @@ public StreamQueryResult(String msg, YdbStatement statement, YdbQuery query, Run public void onStreamResultSet(int index, ResultSetReader rsr) { CompletableFuture> future = resultFutures.get(index); + if (!future.isDone()) { ColumnInfo[] columns = ColumnInfo.fromResultSetReader(rsr); - future.complete(Result.success(new LazyResultSet(statement, columns))); + LazyResultSet rs = new LazyResultSet(statement, columns); + rs.addResultSet(rsr); + if (future.complete(Result.success(rs))) { + return; + } } Result res = future.join(); diff --git a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConfig.java b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConfig.java index 2dd320e7..a2879de2 100644 --- a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConfig.java +++ b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConfig.java @@ -164,7 +164,8 @@ public DriverPropertyInfo[] toPropertyInfo() throws SQLException { YdbConnectionProperties.USE_SECURE_CONNECTION.toInfo(properties), YdbConnectionProperties.SECURE_CONNECTION_CERTIFICATE.toInfo(properties), YdbConnectionProperties.TOKEN.toInfo(properties), - YdbConnectionProperties.SERVICE_ACCOUNT_FILE.toInfo(properties), + YdbConnectionProperties.TOKEN_FILE.toInfo(properties), + YdbConnectionProperties.SA_KEY_FILE.toInfo(properties), YdbConnectionProperties.USE_METADATA.toInfo(properties), YdbConnectionProperties.IAM_ENDPOINT.toInfo(properties), YdbConnectionProperties.METADATA_URL.toInfo(properties), diff --git a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConnectionProperties.java b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConnectionProperties.java index ccfcc0fa..92b7a6ac 100644 --- a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConnectionProperties.java +++ b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConnectionProperties.java @@ -2,29 +2,42 @@ import java.sql.SQLException; import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; import tech.ydb.auth.TokenAuthProvider; import tech.ydb.auth.iam.CloudAuthHelper; import tech.ydb.core.auth.StaticCredentials; import tech.ydb.core.grpc.BalancingSettings; import tech.ydb.core.grpc.GrpcTransportBuilder; +import tech.ydb.jdbc.YdbDriver; public class YdbConnectionProperties { + private static final Logger LOGGER = Logger.getLogger(YdbDriver.class.getName()); + static final YdbProperty TOKEN = YdbProperty.content(YdbConfig.TOKEN_KEY, "Authentication token"); + static final YdbProperty TOKEN_FILE = YdbProperty.file("tokenFile", + "Path to token file for the token-based authentication"); + static final YdbProperty LOCAL_DATACENTER = YdbProperty.string("localDatacenter", "Local Datacenter"); static final YdbProperty USE_SECURE_CONNECTION = YdbProperty.bool("secureConnection", "Use TLS connection"); - static final YdbProperty SECURE_CONNECTION_CERTIFICATE = YdbProperty.bytes("secureConnectionCertificate", - "Use TLS connection with certificate from provided path"); + static final YdbProperty SECURE_CONNECTION_CERTIFICATE = YdbProperty.fileBytes( + "secureConnectionCertificate", "Use TLS connection with certificate from provided path" + ); + @Deprecated static final YdbProperty SERVICE_ACCOUNT_FILE = YdbProperty.content("saFile", "Service account file based authentication"); + static final YdbProperty SA_KEY_FILE = YdbProperty.file("saKeyFile", + "Path to key file for the service account authentication"); + static final YdbProperty USE_METADATA = YdbProperty.bool("useMetadata", "Use metadata service for authentication"); @@ -41,7 +54,9 @@ public class YdbConnectionProperties { private final YdbValue useSecureConnection; private final YdbValue secureConnectionCertificate; private final YdbValue token; + private final YdbValue tokenFile; private final YdbValue serviceAccountFile; + private final YdbValue saKeyFile; private final YdbValue useMetadata; private final YdbValue iamEndpoint; private final YdbValue metadataUrl; @@ -56,7 +71,9 @@ public YdbConnectionProperties(YdbConfig config) throws SQLException { this.useSecureConnection = USE_SECURE_CONNECTION.readValue(props); this.secureConnectionCertificate = SECURE_CONNECTION_CERTIFICATE.readValue(props); this.token = TOKEN.readValue(props); + this.tokenFile = TOKEN_FILE.readValue(props); this.serviceAccountFile = SERVICE_ACCOUNT_FILE.readValue(props); + this.saKeyFile = SA_KEY_FILE.readValue(props); this.useMetadata = USE_METADATA.readValue(props); this.iamEndpoint = IAM_ENDPOINT.readValue(props); this.metadataUrl = METADATA_URL.readValue(props); @@ -87,33 +104,78 @@ public GrpcTransportBuilder applyToGrpcTransport(GrpcTransportBuilder builder) { builder = builder.withSecureConnection(secureConnectionCertificate.getValue()); } + String usedProvider = null; + + if (username != null && !username.isEmpty()) { + builder = builder.withAuthProvider(new StaticCredentials(username, password)); + usedProvider = "username & password credentials"; + } + + if (useMetadata.hasValue()) { + if (usedProvider != null) { + LOGGER.log(Level.WARNING, "Dublicate authentication config! Metadata credentials replaces {0}", + usedProvider); + } + + if (metadataUrl.hasValue()) { + String url = metadataUrl.getValue(); + builder = builder.withAuthProvider(CloudAuthHelper.getMetadataAuthProvider(url)); + } else { + builder = builder.withAuthProvider(CloudAuthHelper.getMetadataAuthProvider()); + } + usedProvider = "metadata credentials"; + } + + if (tokenFile.hasValue()) { + if (usedProvider != null) { + LOGGER.log(Level.WARNING, "Dublicate authentication config! Token credentials replaces {0}", + usedProvider); + } + builder = builder.withAuthProvider(new TokenAuthProvider(tokenFile.getValue())); + usedProvider = "token file credentitals"; + } + if (token.hasValue()) { + if (usedProvider != null) { + LOGGER.log(Level.WARNING, "Dublicate authentication config! Token credentials replaces {0}", + usedProvider); + } builder = builder.withAuthProvider(new TokenAuthProvider(token.getValue())); + usedProvider = "token value credentitals"; } - if (serviceAccountFile.hasValue()) { - String json = serviceAccountFile.getValue(); + if (saKeyFile.hasValue()) { + if (usedProvider != null) { + LOGGER.log(Level.WARNING, "Dublicate authentication config! Token credentials replaces {0}", + usedProvider); + } + String json = saKeyFile.getValue(); if (iamEndpoint.hasValue()) { String endpoint = iamEndpoint.getValue(); builder = builder.withAuthProvider(CloudAuthHelper.getServiceAccountJsonAuthProvider(json, endpoint)); } else { builder = builder.withAuthProvider(CloudAuthHelper.getServiceAccountJsonAuthProvider(json)); } + builder = builder.withAuthProvider(new TokenAuthProvider(token.getValue())); + usedProvider = "service account credentitals"; } - if (useMetadata.hasValue()) { - if (metadataUrl.hasValue()) { - String url = metadataUrl.getValue(); - builder = builder.withAuthProvider(CloudAuthHelper.getMetadataAuthProvider(url)); + if (serviceAccountFile.hasValue()) { + LOGGER.warning("Option 'saFile' is deprecated and will be removed in next versions. " + + "Use options 'saKeyFile' instead"); + if (usedProvider != null) { + LOGGER.log(Level.WARNING, "Dublicate authentication config! Token credentials replaces {0}", + usedProvider); + } + String json = serviceAccountFile.getValue(); + if (iamEndpoint.hasValue()) { + String endpoint = iamEndpoint.getValue(); + builder = builder.withAuthProvider(CloudAuthHelper.getServiceAccountJsonAuthProvider(json, endpoint)); } else { - builder = builder.withAuthProvider(CloudAuthHelper.getMetadataAuthProvider()); + builder = builder.withAuthProvider(CloudAuthHelper.getServiceAccountJsonAuthProvider(json)); } } - if (username != null && !username.isEmpty()) { - builder = builder.withAuthProvider(new StaticCredentials(username, password)); - } - return builder; } } diff --git a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbLookup.java b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbLookup.java index 1caf4941..4293be70 100644 --- a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbLookup.java +++ b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbLookup.java @@ -1,16 +1,20 @@ package tech.ydb.jdbc.settings; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.net.MalformedURLException; import java.net.URL; +import java.sql.SQLException; import java.util.Optional; import com.google.common.io.ByteStreams; import com.google.common.io.CharStreams; +import tech.ydb.jdbc.YdbConst; + /** * * @author Aleksandr Gorshenin @@ -18,11 +22,57 @@ public class YdbLookup { private static final String FILE_REF = "file:"; private static final String CLASSPATH_REF = "classpath:"; - private static final String HOME_REF = "~"; + private static final String HOME_REF = "~/"; private static final String FILE_HOME_REF = FILE_REF + HOME_REF; private YdbLookup() { } + public static String readFileAsString(String name, String value) throws SQLException { + try (Reader reader = new InputStreamReader(readFile(name, value))) { + return CharStreams.toString(reader).trim(); + } catch (IOException e) { + String msg = String.format(YdbConst.INVALID_DRIVER_OPTION_VALUE, value, name, e.getMessage()); + throw new SQLException(msg); + } + } + + public static byte[] readFileAsBytes(String name, String value) throws SQLException { + try (InputStream is = readFile(name, value)) { + return ByteStreams.toByteArray(is); + } catch (IOException e) { + String msg = String.format(YdbConst.INVALID_DRIVER_OPTION_VALUE, value, name, e.getMessage()); + throw new SQLException(msg); + } + } + + private static InputStream readFile(String name, String value) throws SQLException, IOException { + if (value == null || value.trim().isEmpty()) { + throw new SQLException(YdbConst.MISSING_DRIVER_OPTION + name); + } + + String path = value.trim(); + if (path.toLowerCase().startsWith(CLASSPATH_REF)) { + URL resource = ClassLoader.getSystemResource(path.substring(CLASSPATH_REF.length())); + if (resource == null) { + String msg = String.format(YdbConst.INVALID_DRIVER_OPTION_VALUE, value, name, "resource not found"); + throw new SQLException(msg); + } + return resource.openStream(); + } + + if (path.toLowerCase().startsWith(FILE_REF)) { + path = path.substring(FILE_REF.length()); + } + + if (path.startsWith(HOME_REF)) { + String home = System.getProperty("user.home"); + path = path.substring(HOME_REF.length() - 1) + home; + } + + return new FileInputStream(path); + } + + public static String stringFileReference(String ref) { Optional urlOpt = resolvePath(ref); if (urlOpt.isPresent()) { @@ -56,8 +106,8 @@ static Optional resolvePath(String ref) { try { String home = System.getProperty("user.home"); String fixedRef = ref.startsWith(HOME_REF) - ? ref.substring(HOME_REF.length()) - : ref.substring(FILE_HOME_REF.length()); + ? ref.substring(HOME_REF.length() - 1) + : ref.substring(FILE_HOME_REF.length() - 1); return Optional.of(new URL(FILE_REF + home + fixedRef)); } catch (MalformedURLException e) { throw new RuntimeException("Unable to parse ref from home: " + ref, e); diff --git a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbProperty.java b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbProperty.java index 8c772e38..5b4d70c8 100644 --- a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbProperty.java +++ b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbProperty.java @@ -143,6 +143,7 @@ public static YdbProperty duration(String name, String description, St }); } + @Deprecated public static YdbProperty bytes(String name, String description) { return new YdbProperty<>(name, description, null, byte[].class, YdbLookup::byteFileReference); } @@ -150,4 +151,12 @@ public static YdbProperty bytes(String name, String description) { public static YdbProperty content(String name, String description) { return new YdbProperty<>(name, description, null, String.class, YdbLookup::stringFileReference); } + + public static YdbProperty file(String name, String description) { + return new YdbProperty<>(name, description, null, String.class, v -> YdbLookup.readFileAsString(name, v)); + } + + public static YdbProperty fileBytes(String name, String description) { + return new YdbProperty<>(name, description, null, byte[].class, v -> YdbLookup.readFileAsBytes(name, v)); + } } diff --git a/jdbc/src/test/java/tech/ydb/jdbc/settings/YdbDriverProperitesTest.java b/jdbc/src/test/java/tech/ydb/jdbc/settings/YdbDriverProperitesTest.java index 31f1388c..472c1258 100644 --- a/jdbc/src/test/java/tech/ydb/jdbc/settings/YdbDriverProperitesTest.java +++ b/jdbc/src/test/java/tech/ydb/jdbc/settings/YdbDriverProperitesTest.java @@ -234,13 +234,14 @@ public void getCaCertificateAs(String certificate, String expectValue) throws SQ @ParameterizedTest(name = "[{index}] {0} => {1}") @CsvSource(delimiter = ',', value = { - "classpath:data/unknown-file.txt,Unable to find classpath resource: classpath:data/unknown-file.txt", - "file:data/unknown-file.txt,Unable to read resource from file:data/unknown-file.txt", + "classpath:data/unknown-file.txt,resource not found", + "file:data/unknown-file.txt,data/unknown-file.txt (No such file or directory)", }) - public void getCaCertificateAsInvalid(String certificate, String expectException) { + public void getCaCertificateAsInvalid(String value, String message) { String url = "jdbc:ydb:ydb-demo.testhost.org:2135/test/db" + - "?secureConnectionCertificate=" + certificate; - ExceptionAssert.sqlException("Unable to convert property secureConnectionCertificate: " + expectException, + "?secureConnectionCertificate=" + value; + ExceptionAssert.sqlException( + "Cannot process value " + value + " for option secureConnectionCertificate: " + message, () -> driver.getPropertyInfo(url, new Properties()) ); } @@ -309,7 +310,8 @@ static DriverPropertyInfo[] defaultPropertyInfo(@Nullable String localDatacenter new DriverPropertyInfo("secureConnection", ""), new DriverPropertyInfo("secureConnectionCertificate", ""), new DriverPropertyInfo("token", ""), - new DriverPropertyInfo("saFile", ""), + new DriverPropertyInfo("tokenFile", ""), + new DriverPropertyInfo("saKeyFile", ""), new DriverPropertyInfo("useMetadata", ""), new DriverPropertyInfo("iamEndpoint", ""), new DriverPropertyInfo("metadataURL", ""), @@ -350,7 +352,8 @@ static DriverPropertyInfo[] customizedPropertyInfo() { new DriverPropertyInfo("secureConnection", "true"), new DriverPropertyInfo("secureConnectionCertificate", "classpath:data/certificate.txt"), new DriverPropertyInfo("token", "x-secured-token"), - new DriverPropertyInfo("saFile", "x-secured-json"), + new DriverPropertyInfo("tokenFile", "classpath:data/token.txt"), + new DriverPropertyInfo("saKeyFile", "classpath:data/token.txt"), new DriverPropertyInfo("useMetadata", "true"), new DriverPropertyInfo("iamEndpoint", "iam.endpoint.com"), new DriverPropertyInfo("metadataURL", "https://metadata.com"),