From 342c7a6b08f207a2d3b08cbcecc0915e6af4e48e Mon Sep 17 00:00:00 2001 From: Miguel Branco Date: Thu, 6 Mar 2025 13:40:33 +0100 Subject: [PATCH] First working implementation --- .github/workflows/ci.yaml | 51 ++ .github/workflows/docker-ci.yaml | 25 + .github/workflows/publish.yaml | 60 ++ .gitignore | 7 + .sbtopts | 7 + .scala-steward.conf | 17 + .scalafmt.conf | 32 + LICENSE | 3 + README.md | 32 +- build.sbt | 189 ++++ licenses/APL.txt | 202 +++++ licenses/BSL.txt | 103 +++ project/CopyrightHeader.scala | 65 ++ project/build.properties | 1 + project/plugins.sbt | 30 + .../com.rawlabs.das.sdk.DASSdkBuilder | 1 + .../com/rawlabs/das/sqlite/DASSqlite.scala | 127 +++ .../rawlabs/das/sqlite/DASSqliteBackend.scala | 851 ++++++++++++++++++ .../rawlabs/das/sqlite/DASSqliteBuilder.scala | 25 + .../rawlabs/das/sqlite/DASSqliteTable.scala | 399 ++++++++ .../das/sqlite/DASSqliteConnectionTest.scala | 78 ++ .../das/sqlite/DASSqliteTableTest.scala | 814 +++++++++++++++++ 22 files changed, 3118 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/docker-ci.yaml create mode 100644 .github/workflows/publish.yaml create mode 100644 .gitignore create mode 100644 .sbtopts create mode 100644 .scala-steward.conf create mode 100644 .scalafmt.conf create mode 100644 LICENSE create mode 100644 build.sbt create mode 100644 licenses/APL.txt create mode 100644 licenses/BSL.txt create mode 100644 project/CopyrightHeader.scala create mode 100755 project/build.properties create mode 100755 project/plugins.sbt create mode 100644 src/main/resources/META-INF/services/com.rawlabs.das.sdk.DASSdkBuilder create mode 100644 src/main/scala/com/rawlabs/das/sqlite/DASSqlite.scala create mode 100644 src/main/scala/com/rawlabs/das/sqlite/DASSqliteBackend.scala create mode 100644 src/main/scala/com/rawlabs/das/sqlite/DASSqliteBuilder.scala create mode 100644 src/main/scala/com/rawlabs/das/sqlite/DASSqliteTable.scala create mode 100644 src/test/scala/com/rawlabs/das/sqlite/DASSqliteConnectionTest.scala create mode 100644 src/test/scala/com/rawlabs/das/sqlite/DASSqliteTableTest.scala diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..1f0e8f9 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,51 @@ +name: CI + +on: + pull_request: + paths: + - '**/*.scala' + - '**/*.sbt' + - '.scalafmt.conf' + - 'project/**' + - '.github/workflows/ci.yaml' + +env: + SBT_OPTS: "-Xmx2G -XX:+UseG1GC -Xss2M" + GITHUB_TOKEN: ${{ secrets.READ_PACKAGES }} + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + - uses: sbt/setup-sbt@v1 + with: + sbt-runner-version: 1.9.9 + - run: sbt scalafmtCheckAll + - run: sbt headerCheckAll + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + - uses: sbt/setup-sbt@v1 + with: + sbt-runner-version: 1.9.9 + - name: Run tests + run: sbt clean test + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results + path: target/test-results diff --git a/.github/workflows/docker-ci.yaml b/.github/workflows/docker-ci.yaml new file mode 100644 index 0000000..741f2b5 --- /dev/null +++ b/.github/workflows/docker-ci.yaml @@ -0,0 +1,25 @@ +name: Docker CI + +on: + pull_request: + paths: + - .github/workflows/docker-ci.yaml + - build.sbt + +jobs: + docker-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + - uses: sbt/setup-sbt@v1 + with: + sbt-runner-version: 1.9.9 + - name: Build Docker image + env: + GITHUB_TOKEN: ${{ secrets.READ_PACKAGES }} + run: sbt docker/Docker/publishLocal diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000..1ddee5c --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,60 @@ +name: publish +on: + workflow_dispatch: + push: + tags: + - "v*.*.*" + - "v*.*.*-*" + +env: + GITHUB_TOKEN: ${{ secrets.WRITE_PACKAGES }} + +jobs: + publish-jars: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + - uses: sbt/setup-sbt@v1 + with: + sbt-runner-version: 1.9.9 + - name: publish + run: sbt clean publish + publish-docker-image: + runs-on: self-hosted + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + - uses: sbt/setup-sbt@v1 + with: + sbt-runner-version: 1.9.9 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.WRITE_PACKAGES }} + logout: false + - name: publish docker images + run: sbt docker/Docker/publish + gh-release: + needs: [publish-jars, publish-docker-image] + runs-on: self-hosted + steps: + - uses: softprops/action-gh-release@v2 + with: + token: ${{ secrets.RAW_CI_PAT }} + generate_release_notes: true + draft: false + prerelease: ${{ contains(github.ref_name, '-') }} + tag_name: ${{ github.ref_name }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6fdef46 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.class +.metals/ +.git/ +.vscode/ +target/ +.bsp/ +.idea/ diff --git a/.sbtopts b/.sbtopts new file mode 100644 index 0000000..71adad5 --- /dev/null +++ b/.sbtopts @@ -0,0 +1,7 @@ +-J-Xmx4G +-J-Xms1G +-J-Xss4M +-J-XX:+UseG1GC +-J--add-opens=java.base/java.lang=ALL-UNNAMED +-J--add-opens=java.base/java.util=ALL-UNNAMED +-J-XX:+CrashOnOutOfMemoryError diff --git a/.scala-steward.conf b/.scala-steward.conf new file mode 100644 index 0000000..3fd1a6f --- /dev/null +++ b/.scala-steward.conf @@ -0,0 +1,17 @@ +updates.allow = [ + { + groupId = "com.raw-labs" + } +] + +pullRequests = { + frequency = "@asap" + customLabels = ["dependencies", "com.raw-labs"] + includeMatchedLabels = "(.*semver.*)" +} + +pullRequests.grouping = [ + { name = "patches", "title" = "Patch updates", "filter" = [{"version" = "patch"}] } +] + +assignees = ["datYori"] diff --git a/.scalafmt.conf b/.scalafmt.conf new file mode 100644 index 0000000..93576b1 --- /dev/null +++ b/.scalafmt.conf @@ -0,0 +1,32 @@ +version = "3.8.3" +runner.dialect = "scala213" +style = "IntelliJ" +maxColumn = 120 +indentOperator.include = "^.*=$" +rewrite.rules = [Imports, RedundantParens, SortModifiers] +rewrite.imports.sort = ascii +rewrite.imports.groups = [ + ["java.?\\..*"], + ["sbt\\..*"], + ["scala\\..*"], + ["org\\..*"], + ["com\\..*"], +] +newlines.alwaysBeforeElseAfterCurlyIf = false +danglingParentheses { + callSite = false + defnSite = false +} +align { + preset = some + openParenDefnSite = false + openParenCallSite = false +} +docstrings.style = Asterisk +docstrings.wrap = fold +optIn.configStyleArguments = false +spaces { + inImportCurlyBraces = false +} +continuationIndent.defnSite = 4 +runner.optimizer.maxVisitsPerToken = 15000 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..262a9e8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,3 @@ +Source code in this repository is variously licensed under the Business Source License 1.1 (BSL) and the Apache 2.0 license. +A copy of each license can be found in the licenses directory. +Source code in a given file is licensed under the BSL and the copyright belongs to RAW Labs S.A. unless otherwise noted at the beginning of the file. diff --git a/README.md b/README.md index 46fbbdb..7c5bd09 100644 --- a/README.md +++ b/README.md @@ -1 +1,31 @@ -# das-sqlite \ No newline at end of file +# DAS SQLite +[![License](https://img.shields.io/:license-BSL%201.1-blue.svg)](/licenses/BSL.txt) + +[Data Access Service](https://github.com/raw-labs/protocol-das) for SQLite. + +## Options + +| Name | Description | Default | Required | +|------------|---------------------------------------|----------|----------| +| `database` | The path to the SQLite database | | Yes | + +## How to use + +First you need to build the project: +```bash +$ sbt "project docker" "docker:publishLocal" +``` + +This will create a docker image with the name `das-excel`. + +Then you can run the image with the following command: +```bash +$ docker run -p 50051:50051 +``` +... where `` is the id of the image created in the previous step. +This will start the server, typically on port 50051. + +You can find the image id by looking at the sbt output or by running: +```bash +$ docker images +``` diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..e2d7b2b --- /dev/null +++ b/build.sbt @@ -0,0 +1,189 @@ +import java.nio.file.Paths + +import sbt.* +import sbt.Keys.* + +import com.typesafe.sbt.packager.docker.{Cmd, LayeredMapping} + +ThisBuild / credentials += Credentials( + "GitHub Package Registry", + "maven.pkg.github.com", + "raw-labs", + sys.env.getOrElse("GITHUB_TOKEN", "")) + +lazy val commonSettings = Seq( + homepage := Some(url("https://www.raw-labs.com/")), + organization := "com.raw-labs", + organizationName := "RAW Labs SA", + organizationHomepage := Some(url("https://www.raw-labs.com/")), + // Use cached resolution of dependencies + // http://www.scala-sbt.org/0.13/docs/Cached-Resolution.html + updateOptions := updateOptions.in(Global).value.withCachedResolution(true), + resolvers += "RAW Labs GitHub Packages" at "https://maven.pkg.github.com/raw-labs/_") + +lazy val buildSettings = Seq( + scalaVersion := "2.13.15", + javacOptions ++= Seq("-source", "21", "-target", "21"), + scalacOptions ++= Seq( + "-feature", + "-unchecked", + "-deprecation", + "-Xlint:-stars-align,_", + "-Ywarn-dead-code", + "-Ywarn-macros:after", // Fix for false warning of unused implicit arguments in traits/interfaces. + "-Ypatmat-exhaust-depth", + "160")) + +lazy val compileSettings = Seq( + Compile / doc / sources := Seq.empty, + Compile / packageDoc / mappings := Seq(), + Compile / packageSrc / publishArtifact := true, + Compile / packageDoc / publishArtifact := false, + Compile / packageBin / packageOptions += Package.ManifestAttributes( + "Automatic-Module-Name" -> name.value.replace('-', '.')), + // Ensure Java annotations get compiled first, so that they are accessible from Scala. + compileOrder := CompileOrder.JavaThenScala) + +lazy val testSettings = Seq( + // Ensuring tests are run in a forked JVM for isolation. + Test / fork := true, + // Disabling parallel execution of tests. + // Test / parallelExecution := false, + // Pass system properties starting with "raw." to the forked JVMs. + Test / javaOptions ++= { + import scala.collection.JavaConverters.* + val props = System.getProperties + props + .stringPropertyNames() + .asScala + .filter(_.startsWith("raw.")) + .map(key => s"-D$key=${props.getProperty(key)}") + .toSeq + }, + // Set up heap dump options for out-of-memory errors. + Test / javaOptions ++= Seq( + "-XX:+HeapDumpOnOutOfMemoryError", + s"-XX:HeapDumpPath=${Paths.get(sys.env.getOrElse("SBT_FORK_OUTPUT_DIR", "target/test-results")).resolve("heap-dumps")}"), + Test / publishArtifact := true) + +val isCI = sys.env.getOrElse("CI", "false").toBoolean + +lazy val publishSettings = Seq( + versionScheme := Some("early-semver"), + publish / skip := false, + publishMavenStyle := true, + publishTo := Some("GitHub raw-labs Apache Maven Packages" at "https://maven.pkg.github.com/raw-labs/das-sqlite"), + publishConfiguration := publishConfiguration.value.withOverwrite(isCI)) + +lazy val strictBuildSettings = + commonSettings ++ compileSettings ++ buildSettings ++ testSettings ++ Seq(scalacOptions ++= Seq("-Xfatal-warnings")) + +lazy val root = (project in file(".")) + .settings( + name := "das-sqlite", + strictBuildSettings, + publishSettings, + libraryDependencies ++= Seq( + // DAS + "com.raw-labs" %% "das-server-scala" % "0.5.0" % "compile->compile;test->test", + "com.raw-labs" %% "protocol-das" % "1.0.0" % "compile->compile;test->test", + // Sqlite + "org.xerial" % "sqlite-jdbc" % "3.49.1.0", + // Jackson (for JSON handling) + "com.fasterxml.jackson.core" % "jackson-databind" % "2.18.3", + // Hikari Connection Pool, + "com.zaxxer" % "HikariCP" % "6.2.1", + // Mockito + "org.mockito" % "mockito-core" % "5.12.0" % Test, + "org.scalatestplus" %% "mockito-5-12" % "3.2.19.0" % Test, + // ScalaTest / containers + "com.dimafeng" %% "testcontainers-scala-scalatest" % "0.41.8" % Test)) + +val amzn_jdk_version = "21.0.4.7-1" +val amzn_corretto_bin = s"java-21-amazon-corretto-jdk_${amzn_jdk_version}_amd64.deb" +val amzn_corretto_bin_dl_url = s"https://corretto.aws/downloads/resources/${amzn_jdk_version.replace('-', '.')}" + +lazy val dockerSettings = strictBuildSettings ++ Seq( + name := "das-sqlite-server", + dockerBaseImage := s"--platform=amd64 debian:bookworm-slim", + dockerLabels ++= Map( + "vendor" -> "RAW Labs SA", + "product" -> "das-sqlite-server", + "image-type" -> "final", + "org.opencontainers.image.source" -> "https://github.com/raw-labs/das-sqlite"), + Docker / daemonUser := "raw", + dockerExposedVolumes := Seq("/var/log/raw"), + dockerExposedPorts := Seq(50051), + dockerEnvVars := Map("PATH" -> s"${(Docker / defaultLinuxInstallLocation).value}/bin:$$PATH"), + // We remove the automatic switch to USER 1001:0. + // We we want to run as root to install the JDK, also later we will switch to a non-root user. + dockerCommands := dockerCommands.value.filterNot { + case Cmd("USER", args @ _*) => args.contains("1001:0") + case cmd => false + }, + dockerCommands ++= Seq( + Cmd( + "RUN", + s"""set -eux \\ + && apt-get update \\ + && apt-get install -y --no-install-recommends \\ + curl wget ca-certificates gnupg software-properties-common fontconfig java-common \\ + && wget $amzn_corretto_bin_dl_url/$amzn_corretto_bin \\ + && dpkg --install $amzn_corretto_bin \\ + && rm -f $amzn_corretto_bin \\ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \\ + wget gnupg software-properties-common"""), + Cmd("USER", "raw")), + dockerEnvVars += "LANG" -> "C.UTF-8", + dockerEnvVars += "JAVA_HOME" -> "/usr/lib/jvm/java-21-amazon-corretto", + Compile / doc / sources := Seq.empty, // Do not generate scaladocs + // Skip docs to speed up build + Compile / packageDoc / mappings := Seq(), + updateOptions := updateOptions.value.withLatestSnapshots(true), + Linux / linuxPackageMappings += packageTemplateMapping(s"/var/lib/${packageName.value}")(), + bashScriptDefines := { + val ClasspathPattern = "declare -r app_classpath=\"(.*)\"\n".r + bashScriptDefines.value.map { + case ClasspathPattern(classpath) => s""" + |declare -r app_classpath="$${app_home}/../conf:$classpath" + |""".stripMargin + case _ @entry => entry + } + }, + Docker / dockerLayerMappings := (Docker / dockerLayerMappings).value.map { + case lm @ LayeredMapping(Some(1), file, path) => { + val fileName = java.nio.file.Paths.get(path).getFileName.toString + if (!fileName.endsWith(".jar")) { + // If it is not a jar, put it on the top layer. Configuration files and other small files. + LayeredMapping(Some(2), file, path) + } else if (fileName.startsWith("com.raw-labs") && fileName.endsWith(".jar")) { + // If it is one of our jars, also top layer. These will change often. + LayeredMapping(Some(2), file, path) + } else { + // Otherwise it is a 3rd party library, which only changes when we change dependencies, so leave it in layer 1 + lm + } + } + case lm @ _ => lm + }, + Compile / mainClass := Some("com.rawlabs.das.server.DASServer"), + Docker / dockerAutoremoveMultiStageIntermediateImages := false, + dockerAlias := dockerAlias.value.withTag(Option(version.value.replace("+", "-"))), + dockerAliases := { + val devRegistry = sys.env.getOrElse("DEV_REGISTRY", "ghcr.io/raw-labs/das-sqlite") + val releaseRegistry = sys.env.get("RELEASE_DOCKER_REGISTRY") + val baseAlias = dockerAlias.value.withRegistryHost(Some(devRegistry)) + + releaseRegistry match { + case Some(releaseReg) => Seq(baseAlias, dockerAlias.value.withRegistryHost(Some(releaseReg))) + case None => Seq(baseAlias) + } + }) + +lazy val docker = (project in file("docker")) + .dependsOn(root % "compile->compile;test->test") + .enablePlugins(JavaAppPackaging, DockerPlugin) + .settings( + strictBuildSettings, + dockerSettings, + libraryDependencies += "com.raw-labs" %% "das-server-scala" % "0.5.0" % "compile->compile;test->test") diff --git a/licenses/APL.txt b/licenses/APL.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/licenses/APL.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/licenses/BSL.txt b/licenses/BSL.txt new file mode 100644 index 0000000..99588db --- /dev/null +++ b/licenses/BSL.txt @@ -0,0 +1,103 @@ +Business Source License 1.1 + +Parameters + +Licensor: RAW Labs S.A. +Licensed Work: RAW + The Licensed Work is (c) 2015 RAW Labs S.A. +Additional Use Grant: You may make use of the Licensed Work, provided that + you may not use the Licensed Work for a Cloud + Service. + + A “Cloud Service” is a commercial offering that + allows third parties (other than your employees and + contractors) to access the functionality of the + Licensed Work to create programs or APIs whose source are + controlled by such third parties. + +Change Date: 2027-01-01 + +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please visit: https://cockroachlabs.com/ + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/project/CopyrightHeader.scala b/project/CopyrightHeader.scala new file mode 100644 index 0000000..e23e16a --- /dev/null +++ b/project/CopyrightHeader.scala @@ -0,0 +1,65 @@ +import de.heikoseeberger.sbtheader.HeaderPlugin.autoImport._ +import de.heikoseeberger.sbtheader.{CommentCreator, HeaderPlugin} +import sbt.Keys._ +import sbt._ + +object CopyrightHeader extends AutoPlugin { + + override def requires: Plugins = HeaderPlugin + + override def trigger: PluginTrigger = allRequirements + + protected def headerMappingSettings: Seq[Def.Setting[_]] = Seq(Compile, Test).flatMap { config => + inConfig(config)( + Seq( + headerLicense := Some(HeaderLicense.Custom(header)), + headerMappings := headerMappings.value ++ Map( + HeaderFileType.scala -> cStyleComment, + HeaderFileType.java -> cStyleComment + ) + ) + ) + } + + override def projectSettings: Seq[Def.Setting[_]] = Def.settings(headerMappingSettings, additional) + + def additional: Seq[Def.Setting[_]] = Def.settings( + Compile / compile := { + (Compile / headerCreate).value + (Compile / compile).value + }, + Test / compile := { + (Test / headerCreate).value + (Test / compile).value + } + ) + + def header: String = { + val currentYear = "2024" + s"""|/* + | * Copyright $currentYear RAW Labs S.A. + | * + | * Use of this software is governed by the Business Source License + | * included in the file licenses/BSL.txt. + | * + | * As of the Change Date specified in that file, in accordance with + | * the Business Source License, use of this software will be governed + | * by the Apache License, Version 2.0, included in the file + | * licenses/APL.txt. + | */""".stripMargin + } + + val cStyleComment = HeaderCommentStyle.cStyleBlockComment.copy(commentCreator = new CommentCreator() { + val CopyrightPattern = "Copyright (\\d{4}) RAW Labs S.A.".r + + override def apply(text: String, existingText: Option[String]): String = { + existingText match { + case Some(existingHeader) if CopyrightPattern.findFirstIn(existingHeader).isDefined => + // matches the pattern with any year, return it unchanged + existingHeader.trim + case _ => header + } + } + }) + +} diff --git a/project/build.properties b/project/build.properties new file mode 100755 index 0000000..04267b1 --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.9.9 diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100755 index 0000000..2794600 --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,30 @@ +resolvers += Classpaths.sbtPluginReleases + +autoCompilerPlugins := true + +addDependencyTreePlugin + +libraryDependencies += "commons-io" % "commons-io" % "2.11.0" + +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.0") + +addSbtPlugin("nl.gn0s1s" % "sbt-dotenv" % "3.1.1") + +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.10.4") + +addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0") + +addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.12.0") + +addSbtPlugin("com.github.sbt" % "sbt-protobuf" % "0.8.0") + +credentials += Credentials( + "GitHub Package Registry", + "maven.pkg.github.com", + "raw-labs", + sys.env.getOrElse("GITHUB_TOKEN", "") +) + +resolvers += "RAW Labs GitHub Packages" at "https://maven.pkg.github.com/raw-labs/_" + +addSbtPlugin("com.raw-labs" % "sbt-versioner" % "0.1.0") diff --git a/src/main/resources/META-INF/services/com.rawlabs.das.sdk.DASSdkBuilder b/src/main/resources/META-INF/services/com.rawlabs.das.sdk.DASSdkBuilder new file mode 100644 index 0000000..3ee259c --- /dev/null +++ b/src/main/resources/META-INF/services/com.rawlabs.das.sdk.DASSdkBuilder @@ -0,0 +1 @@ +com.rawlabs.das.sqlite.DASSqliteBuilder diff --git a/src/main/scala/com/rawlabs/das/sqlite/DASSqlite.scala b/src/main/scala/com/rawlabs/das/sqlite/DASSqlite.scala new file mode 100644 index 0000000..4731c3c --- /dev/null +++ b/src/main/scala/com/rawlabs/das/sqlite/DASSqlite.scala @@ -0,0 +1,127 @@ +/* + * Copyright 2025 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.das.sqlite + +import java.sql.SQLException + +import com.rawlabs.das.sdk.DASSdkInvalidArgumentException +import com.rawlabs.das.sdk.scala.{DASFunction, DASSdk, DASTable} +import com.rawlabs.protocol.das.v1.functions.FunctionDefinition +import com.rawlabs.protocol.das.v1.tables.TableDefinition +import com.zaxxer.hikari.pool.HikariPool.PoolInitializationException +import com.zaxxer.hikari.{HikariConfig, HikariDataSource} + +/** + * A DASSdk implementation for SQLite. + * + * Usage: + * - Provide a "database" option to specify the SQLite file path (or ":memory:" for in-memory). + * + * @param options Configuration map for the SDK, e.g.: "database" -> "/path/to/my.db" + */ +class DASSqlite(options: Map[String, String]) extends DASSdk { + + // ------------------------------------------------------------------------------------------------------ + // Parsing of options + // ------------------------------------------------------------------------------------------------------ + + /** + * SQLite connections typically only need a file path or special tokens like ":memory:". + */ + private val databasePath = + options.getOrElse("database", throw new DASSdkInvalidArgumentException("database is required")) + + // Construct a JDBC URL for SQLite. Ex: "jdbc:sqlite:/path/to/dbFile" or "jdbc:sqlite::memory:". + private val url = s"jdbc:sqlite:$databasePath" + + // ------------------------------------------------------------------------------------------------------ + // HikariCP Connection Pool Setup + // ------------------------------------------------------------------------------------------------------ + try { + // If using the default SQLite JDBC driver, ensure it's available + Class.forName("org.sqlite.JDBC") + } catch { + case _: ClassNotFoundException => + throw new DASSdkInvalidArgumentException("SQLite JDBC driver not found on the classpath.") + } + + private val hikariConfig = new HikariConfig() + hikariConfig.setJdbcUrl(url) + + // Tune Hikari as needed + hikariConfig.setMaximumPoolSize(10) + hikariConfig.setConnectionTimeout(3000) + + // 1) Create HikariDataSource with error handling + protected[sqlite] val dataSource = createDataSourceOrThrow(hikariConfig, url) + + // 2) Create backend, then attempt to retrieve tables + private val backend = new DASSqliteBackend(dataSource) + private val tables = + try { + backend.tables() // Immediately queries DB to list tables + } catch { + case sqlEx: SQLException => + // For SQLite, there's not a standard SQLState for "invalid credentials", but let's handle gracefully anyway + throw new DASSdkInvalidArgumentException(s"Could not connect: ${sqlEx.getMessage}", sqlEx) + } + + // ------------------------------------------------------------------------------------------------------ + // DASSdk interface + // ------------------------------------------------------------------------------------------------------ + + /** + * @return a list of table definitions discovered by the backend. + */ + override def tableDefinitions: Seq[TableDefinition] = tables.values.map(_.definition).toSeq + + /** + * @return a list of function definitions. SQLite example returns empty by default. + */ + override def functionDefinitions: Seq[FunctionDefinition] = Seq.empty + + /** + * Retrieve a table by name (case-sensitive or lowercased, depending on how you stored them). + */ + override def getTable(name: String): Option[DASTable] = tables.get(name) + + /** + * Retrieve a function by name, if implemented. For now, none. + */ + override def getFunction(name: String): Option[DASFunction] = None + + /** + * Close the SDK and underlying connections. + */ + override def close(): Unit = { + dataSource.close() + } + + /** + * Helper that instantiates the HikariDataSource and unwraps pool init errors. + */ + private def createDataSourceOrThrow(hc: HikariConfig, url: String): HikariDataSource = { + try { + new HikariDataSource(hc) + } catch { + case e: PoolInitializationException => + e.getCause match { + case sqlEx: SQLException => + // Could be a bad path or locked file, etc. + throw new DASSdkInvalidArgumentException(s"Could not connect: ${sqlEx.getMessage}", sqlEx) + case other => + throw new RuntimeException(s"Unexpected error initializing connection: ${other.getMessage}", other) + } + } + } +} diff --git a/src/main/scala/com/rawlabs/das/sqlite/DASSqliteBackend.scala b/src/main/scala/com/rawlabs/das/sqlite/DASSqliteBackend.scala new file mode 100644 index 0000000..f4c9f51 --- /dev/null +++ b/src/main/scala/com/rawlabs/das/sqlite/DASSqliteBackend.scala @@ -0,0 +1,851 @@ +/* + * Copyright 2025 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.das.sqlite + +import java.sql.{Connection, ResultSet, SQLException} +import javax.sql.DataSource + +import scala.util.{Try, Using} + +import com.fasterxml.jackson.databind.node.{NullNode, TextNode} +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.google.protobuf.ByteString +import com.rawlabs.das.sdk.DASExecuteResult +import com.rawlabs.das.sdk.scala.DASTable.TableEstimate +import com.rawlabs.protocol.das.v1.tables._ +import com.rawlabs.protocol.das.v1.types._ +import com.typesafe.scalalogging.StrictLogging + +/** + * A DAS backend for SQLite, providing: + * - Table discovery (`tables()`) + * - Basic SQL query execution (`execute(query)`) + * - Query cost estimates (very naive in SQLite) (`estimate(query)`) + * - Batch query execution with parameters (`batchQuery(...)`) + * - (Optional) returning-rows execution (`executeReturningRows(...)`) if using SQLite >= 3.35 that supports RETURNING + * + * @param dataSource A pre-configured DataSource (e.g., HikariCP for SQLite JDBC URL) + */ +private class DASSqliteBackend(dataSource: DataSource) extends StrictLogging { + + // ------------------------------------------------------------------------------------------------------ + // ObjectMapper for JSON + // ------------------------------------------------------------------------------------------------------ + private val mapper = new ObjectMapper() + private val nodeFactory = mapper.getNodeFactory + + // ------------------------------------------------------------------------------------------------------ + // Public API + // ------------------------------------------------------------------------------------------------------ + + /** + * Returns a map of tableName -> DASSqliteTable, each containing a TableDefinition. Enumerates all user-defined tables + * in the SQLite database, skipping internal tables named `sqlite_*`. + * + * @throws SQLException if there's an error querying the DB + */ + def tables(): Map[String, DASSqliteTable] = { + logger.debug("Fetching tables from the SQLite database") + + Using + .Manager { use => + val conn = use(dataSource.getConnection) + + // Retrieve the list of tables from sqlite_master + val listTablesSQL = + """ + |SELECT name + |FROM sqlite_master + |WHERE type='table' + | AND name NOT LIKE 'sqlite_%' + |ORDER BY name + """.stripMargin + + val stmt = use(conn.createStatement()) + val rs = use(stmt.executeQuery(listTablesSQL)) + + val tableMap = scala.collection.mutable.Map.empty[String, DASSqliteTable] + + while (rs.next()) { + val tableName = rs.getString("name").toLowerCase + val pkOpt = primaryKey(conn, tableName) + val tdef = tableDefinition(conn, tableName) + tableMap += tableName -> new DASSqliteTable(this, tdef, pkOpt) + } + + tableMap.toMap + } + .recover { case ex: SQLException => + logger.error(s"Error while fetching tables: ${ex.getMessage}", ex) + throw ex + } + .get + } + + /** + * Executes a SQL query and returns a DASExecuteResult that streams rows from the ResultSet. Caller is responsible for + * consuming the result and eventually calling `.close()` on it. + * + * @param query SQL query to execute + * @return DASExecuteResult that allows row-by-row iteration + * @throws SQLException if there's a DB error + */ + def execute(query: String): DASExecuteResult = { + logger.debug(s"Preparing to execute query: $query") + val conn = dataSource.getConnection + try { + val stmt = conn.createStatement() + val rs = stmt.executeQuery(query) + + // Return a DASExecuteResult with manual streaming + new DASExecuteResult { + private var nextFetched = false + private var hasMore = false + + override def close(): Unit = { + Try(rs.close()).failed.foreach { t => + logger.warn("Failed to close ResultSet cleanly", t) + } + Try(stmt.close()).failed.foreach { t => + logger.warn("Failed to close Statement cleanly", t) + } + Try(conn.close()).failed.foreach { t => + logger.warn("Failed to close Connection cleanly", t) + } + } + + override def hasNext: Boolean = { + if (!nextFetched) { + hasMore = rs.next() + nextFetched = true + } + hasMore + } + + override def next(): Row = { + if (!hasNext) throw new NoSuchElementException("No more rows available.") + nextFetched = false + + val rowBuilder = Row.newBuilder() + val meta = rs.getMetaData + val colCount = meta.getColumnCount + + for (i <- 1 to colCount) { + val colName = meta.getColumnName(i).toLowerCase + val value = toDASValue(rs, i) + rowBuilder.addColumns(Column.newBuilder().setName(colName).setData(value).build()) + } + rowBuilder.build() + } + } + } catch { + case ex: SQLException => + logger.error(s"Error executing query '$query': ${ex.getMessage}", ex) + // Make sure to close the connection if we fail before returning the result + Try(conn.close()).failed.foreach { t => + logger.warn("Failed to close Connection in error scenario", t) + } + throw ex + } + } + + /** + * Returns an estimate of the row count and average row width for the given query. Note: SQLite doesn't provide + * straightforward estimates via EXPLAIN (or EXPLAIN QUERY PLAN). We fallback to a naive default (100 rows @ 100 + * width). + * + * @param query SQL query to estimate + * @return TableEstimate(rowCount, avgRowWidthInBytes) + * @throws SQLException if there's a DB error + */ + def estimate(query: String): TableEstimate = { + logger.debug(s"Estimating cost for query (very naive in SQLite): $query") + + // Attempt a rudimentary parse from EXPLAIN QUERY PLAN or fallback + val explainSql = s"EXPLAIN QUERY PLAN $query" + + Using + .Manager { use => + val conn = use(dataSource.getConnection) + val stmt = use(conn.createStatement()) + val rs = use(stmt.executeQuery(explainSql)) + + // In SQLite, EXPLAIN QUERY PLAN returns lines describing the plan, but no direct row/width stats + // We'll just do a naive fallback approach: + if (!rs.next()) { + logger.warn("EXPLAIN QUERY PLAN returned no rows; using default estimate (100 rows @ 100 width).") + TableEstimate(100, 100) + } else { + // We at least return some naive defaults + TableEstimate(100, 100) + } + } + .recover { case ex: SQLException => + logger.error(s"Error while estimating query '$query': ${ex.getMessage}", ex) + throw ex + } + .get + } + + /** + * Executes a batch query (e.g. INSERT/UPDATE with parameters) and returns an array of update counts for each batch + * item. + * + * @param query SQL query with placeholders (e.g. "INSERT INTO ... VALUES (?, ?)") + * @param batchParameters Sequence of parameter sequences, one per batch + * @return array of update counts + * @throws SQLException if there's a DB error + */ + def batchQuery(query: String, batchParameters: Seq[Seq[Value]]): Array[Int] = { + logger.debug(s"Preparing to execute batch: $query (with ${batchParameters.size} batches)") + + Using + .Manager { use => + val conn = use(dataSource.getConnection) + val ps = use(conn.prepareStatement(query)) + + // Some SQLite drivers may return "UNKNOWN" or fail on param metadata, so we handle gracefully + val meta = ps.getParameterMetaData + + for (params <- batchParameters) { + params.zipWithIndex.foreach { case (dasValue, idx) => + val paramTypeName = + try { + meta.getParameterTypeName(idx + 1) + } catch { + case _: SQLException => "UNKNOWN" + } + ps.setObject(idx + 1, convertDASValueToJdbcValue(dasValue, paramTypeName, conn)) + } + ps.addBatch() + } + ps.executeBatch() + } + .recover { case ex: SQLException => + logger.error(s"Error in batch execution for query '$query': ${ex.getMessage}", ex) + throw ex + } + .get + } + + /** + * Executes a parameterized INSERT/UPDATE/DELETE that returns rows via `RETURNING *` (SQLite 3.35+ only). If the + * SQLite version does not support RETURNING, this may fail. + * + * @param sql The SQL statement containing placeholders, e.g. "INSERT INTO mytable (col1, col2) VALUES (?, ?) + * RETURNING *" + * @param params The flattened list of parameter values in the correct order + * @return a DASExecuteResult that can be used to iterate over returned rows + */ + def executeReturningRows(sql: String, params: Seq[Value]): DASExecuteResult = { + logger.debug(s"Executing returning-rows query: $sql with ${params.size} params (SQLite 3.35+ feature)") + + val conn = dataSource.getConnection + try { + val ps = conn.prepareStatement(sql) + val meta = ps.getParameterMetaData + + // Bind each param in order + params.zipWithIndex.foreach { case (dasValue, idx) => + val paramTypeName = + try { + meta.getParameterTypeName(idx + 1) + } catch { + case _: SQLException => "UNKNOWN" + } + ps.setObject(idx + 1, convertDASValueToJdbcValue(dasValue, paramTypeName, conn)) + } + + // Execute the statement and get a ResultSet + val rs = ps.executeQuery() + + new DASExecuteResult { + private var nextFetched = false + private var hasMore = false + + override def close(): Unit = { + scala.util.Try(rs.close()).failed.foreach { t => + logger.warn("Failed to close ResultSet cleanly", t) + } + scala.util.Try(ps.close()).failed.foreach { t => + logger.warn("Failed to close PreparedStatement cleanly", t) + } + scala.util.Try(conn.close()).failed.foreach { t => + logger.warn("Failed to close Connection cleanly", t) + } + } + + override def hasNext: Boolean = { + if (!nextFetched) { + hasMore = rs.next() + nextFetched = true + } + hasMore + } + + override def next(): Row = { + if (!hasNext) throw new NoSuchElementException("No more rows returned.") + nextFetched = false + + val rowBuilder = Row.newBuilder() + val meta = rs.getMetaData + val colCount = meta.getColumnCount + for (i <- 1 to colCount) { + val colName = meta.getColumnName(i).toLowerCase + val value = toDASValue(rs, i) + rowBuilder.addColumns( + Column + .newBuilder() + .setName(colName) + .setData(value) + .build()) + } + rowBuilder.build() + } + } + } catch { + case ex: SQLException => + logger.error(s"Error in executeReturningRows for query '$sql': ${ex.getMessage}", ex) + scala.util.Try(conn.close()) // best effort + throw ex + } + } + + // ------------------------------------------------------------------------------------------------------ + // Private Helpers for Table definitions, PK extraction + // ------------------------------------------------------------------------------------------------------ + + /** + * Retrieves exactly one primary key column name for the given table (if it has exactly one PK column). Uses "PRAGMA + * table_info(tableName)", which returns: cid, name, type, notnull, dflt_value, pk + */ + private def primaryKey(conn: Connection, tableName: String): Option[String] = { + val pragmaSQL = s"PRAGMA table_info('$tableName')" + + Using.resource(conn.createStatement()) { stmt => + Using.resource(stmt.executeQuery(pragmaSQL)) { rs => + val pkColumns = scala.collection.mutable.ListBuffer.empty[String] + while (rs.next()) { + val pk = rs.getInt("pk") // 1 means part of PK, 0 means not + if (pk == 1) { + pkColumns += rs.getString("name") + } + } + if (pkColumns.size == 1) Some(pkColumns.head.toLowerCase) + else None + } + } + } + + /** + * Builds a TableDefinition for the given table by inspecting PRAGMA table_info. + */ + private def tableDefinition(conn: Connection, tableName: String): TableDefinition = { + val descriptionBuilder = TableDefinition + .newBuilder() + .setTableId(TableId.newBuilder().setName(tableName)) + .setDescription(s"Table '$tableName' in SQLite database") + + val pragmaSQL = s"PRAGMA table_info('$tableName')" + + Using.resource(conn.createStatement()) { stmt => + Using.resource(stmt.executeQuery(pragmaSQL)) { rs => + while (rs.next()) { + val colName = rs.getString("name") + val declaredType = rs.getString("type") // e.g. "INTEGER", "TEXT", "REAL", "BLOB", "NUMERIC", ... + val notNull = rs.getInt("notnull") == 1 + + toDASType(declaredType, !notNull) match { + case Right(t) => + val colDef = ColumnDefinition.newBuilder().setName(colName).setType(t).build() + descriptionBuilder.addColumns(colDef) + case Left(err) => + logger.warn(s"Skipping column '$colName' in table '$tableName': $err") + } + } + } + } + descriptionBuilder.build() + } + + // ------------------------------------------------------------------------------------------------------ + // Private Conversion Helpers: SQLite -> DAS Value + // ------------------------------------------------------------------------------------------------------ + + /** + * Convert the column at index `i` in the ResultSet into a DAS Value. We rely on either + * `getMetaData.getColumnTypeName(i)` or direct `rs.getObject(i)` checks to map to DAS. We also attempt to handle some + * additional declared type keywords (BOOLEAN, DATE, DATETIME, etc.) if present. + */ + private def toDASValue(rs: ResultSet, i: Int): Value = { + val jdbcTypeName = + try { + rs.getMetaData.getColumnTypeName(i).toLowerCase + } catch { + // Some JDBC drivers for SQLite might not return a meaningful type name + case _: SQLException => "unknown" + } + + val obj = rs.getObject(i) + val builder = Value.newBuilder() + + if (obj == null) { + builder.setNull(ValueNull.newBuilder()) + return builder.build() + } + + // We'll do best-effort based on common SQLite type affinities and extended keywords + // - "INT" or "INTEGER" + // - "REAL", "FLOAT", "DOUBLE" + // - "BLOB" + // - "TEXT", "CHAR", "CLOB" + // - "NUMERIC", "DECIMAL" + // - "BOOLEAN" + // - "DATE", "TIME", "DATETIME", "TIMESTAMP" (heuristic handling) + // + // If none match, we fallback to object-based detection. + + jdbcTypeName match { + case t if t.contains("int") => + // Could be a normal integer or a long + val longVal = rs.getLong(i) + if (rs.wasNull()) { + builder.setNull(ValueNull.newBuilder()) + } else { + // if it fits in Int + if (longVal >= Int.MinValue && longVal <= Int.MaxValue) { + builder.setInt(ValueInt.newBuilder().setV(longVal.toInt)) + } else { + builder.setLong(ValueLong.newBuilder().setV(longVal)) + } + } + + case t if t.contains("real") || t.contains("floa") || t.contains("doub") => + val doubleVal = rs.getDouble(i) + if (rs.wasNull()) { + builder.setNull(ValueNull.newBuilder()) + } else { + builder.setDouble(ValueDouble.newBuilder().setV(doubleVal)) + } + + case t if t.contains("blob") => + val bytes = rs.getBytes(i) + builder.setBinary(ValueBinary.newBuilder().setV(ByteString.copyFrom(bytes))) + + case t if t.contains("text") || t.contains("char") || t.contains("clob") => + val strVal = rs.getString(i) + builder.setString(ValueString.newBuilder().setV(strVal)) + + case t if t.contains("boolean") => + // SQLite has no real boolean, but let's interpret 1=>true, 0=>false if numeric + // or 'true'/'false' if string, etc. + obj match { + case boolVal: java.lang.Boolean => + builder.setBool(ValueBool.newBuilder().setV(boolVal)) + case num: java.lang.Number => + builder.setBool(ValueBool.newBuilder().setV(num.intValue() != 0)) + case s: String => + val lower = s.trim.toLowerCase + val boolParsed = lower == "true" || lower == "1" + builder.setBool(ValueBool.newBuilder().setV(boolParsed)) + case _ => + // fallback + builder.setString(ValueString.newBuilder().setV(obj.toString)) + } + + case t if t.contains("numeric") || t.contains("dec") => + // This might be decimal or some numeric + // We'll attempt getBigDecimal; if not, fallback to string + val bd = Try(rs.getBigDecimal(i)).toOption + bd match { + case Some(numVal) => + builder.setDecimal(ValueDecimal.newBuilder().setV(numVal.toPlainString)) + case None => + // fallback + val strVal = rs.getString(i) + if (strVal == null) builder.setNull(ValueNull.newBuilder()) + else { + // Attempt parse + Try(new java.math.BigDecimal(strVal)).toOption match { + case Some(parsed) => + builder.setDecimal(ValueDecimal.newBuilder().setV(parsed.toPlainString)) + case None => + builder.setString(ValueString.newBuilder().setV(strVal)) + } + } + } + + case t if t.contains("date") || t.contains("time") => + // Heuristic approach: let's see if the actual data in the column can parse as date/time + // We'll attempt to parse as a timestamp, date, or time if it looks valid. Otherwise fallback to string. + val strVal = rs.getString(i) + if (strVal == null) { + builder.setNull(ValueNull.newBuilder()) + } else { + // Attempt multiple parse patterns + val trimmed = strVal.trim + // We'll do a simple check if it has a time portion + if (trimmed.matches("\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}.*")) { + // Attempt parse as local date-time + val maybeTs = parseAsTimestamp(trimmed) + if (maybeTs.isDefined) { + val ldt = maybeTs.get + builder.setTimestamp( + ValueTimestamp + .newBuilder() + .setYear(ldt.getYear) + .setMonth(ldt.getMonthValue) + .setDay(ldt.getDayOfMonth) + .setHour(ldt.getHour) + .setMinute(ldt.getMinute) + .setSecond(ldt.getSecond) + .setNano(ldt.getNano)) + } else { + // fallback string + builder.setString(ValueString.newBuilder().setV(strVal)) + } + } else if (trimmed.matches("\\d{4}-\\d{2}-\\d{2}")) { + // Attempt parse as date only + val maybeDate = parseAsDate(trimmed) + if (maybeDate.isDefined) { + val ld = maybeDate.get + builder.setDate( + ValueDate.newBuilder().setYear(ld.getYear).setMonth(ld.getMonthValue).setDay(ld.getDayOfMonth)) + } else { + builder.setString(ValueString.newBuilder().setV(strVal)) + } + } else if (trimmed.matches("\\d{2}:\\d{2}:\\d{2}.*")) { + // Attempt parse as time + val maybeTime = parseAsTime(trimmed) + if (maybeTime.isDefined) { + val lt = maybeTime.get + builder.setTime( + ValueTime.newBuilder().setHour(lt.getHour).setMinute(lt.getMinute).setSecond(lt.getSecond)) + } else { + builder.setString(ValueString.newBuilder().setV(strVal)) + } + } else { + // fallback to string + builder.setString(ValueString.newBuilder().setV(strVal)) + } + } + + case "unknown" => + // Some JDBC drivers do not return a column type name. We'll fallback to object-based detection + convertFallbackObject(obj, rs, i, builder) + + case _ => + // Fallback if the declared type is not recognized + convertFallbackObject(obj, rs, i, builder) + } + + builder.build() + } + + /** + * Attempts to interpret an SQLite object in a fallback manner if the column type name isn't reliable. + */ + private def convertFallbackObject(obj: Any, rs: ResultSet, i: Int, builder: Value.Builder): Unit = { + obj match { + case n: java.lang.Number => + // Could be int, long, double, etc. + val d = n.doubleValue() + val l = n.longValue() + // if integral + if (d == l.toDouble) { + // within int range? + if (l >= Int.MinValue && l <= Int.MaxValue) { + builder.setInt(ValueInt.newBuilder().setV(l.toInt)) + } else { + builder.setLong(ValueLong.newBuilder().setV(l)) + } + } else { + builder.setDouble(ValueDouble.newBuilder().setV(d)) + } + case s: String => + builder.setString(ValueString.newBuilder().setV(s)) + case b: Array[Byte] => + builder.setBinary(ValueBinary.newBuilder().setV(ByteString.copyFrom(b))) + case boolVal: java.lang.Boolean => + builder.setBool(ValueBool.newBuilder().setV(boolVal)) + case dateVal: java.sql.Date => + val ld = dateVal.toLocalDate + builder.setDate(ValueDate.newBuilder().setYear(ld.getYear).setMonth(ld.getMonthValue).setDay(ld.getDayOfMonth)) + case timeVal: java.sql.Time => + val lt = timeVal.toLocalTime + builder.setTime(ValueTime.newBuilder().setHour(lt.getHour).setMinute(lt.getMinute).setSecond(lt.getSecond)) + case tsVal: java.sql.Timestamp => + val ldt = tsVal.toLocalDateTime + builder.setTimestamp( + ValueTimestamp + .newBuilder() + .setYear(ldt.getYear) + .setMonth(ldt.getMonthValue) + .setDay(ldt.getDayOfMonth) + .setHour(ldt.getHour) + .setMinute(ldt.getMinute) + .setSecond(ldt.getSecond) + .setNano(ldt.getNano)) + case _ => + // fallback to string + builder.setString(ValueString.newBuilder().setV(rs.getString(i))) + } + } + + import java.time.format.DateTimeFormatter + // ------------------------------------------------------------------------------------------------------ + // Private Date/Time parse helpers + // ------------------------------------------------------------------------------------------------------ + import java.time.{LocalDate, LocalDateTime, LocalTime} + + import scala.util.Try + + private val datePatterns = List("yyyy-MM-dd", "yyyy/MM/dd").map(DateTimeFormatter.ofPattern) + + private val dateTimePatterns = List( + "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd HH:mm:ss.SSS", + "yyyy/MM/dd HH:mm:ss", + "yyyy/MM/dd HH:mm:ss.SSS", + "yyyy-MM-dd'T'HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ss.SSS").map(DateTimeFormatter.ofPattern) + + private val timePatterns = List("HH:mm:ss", "HH:mm:ss.SSS").map(DateTimeFormatter.ofPattern) + + private def parseAsDate(str: String): Option[LocalDate] = { + datePatterns.view + .map(p => Try(LocalDate.parse(str, p)).toOption) + .collectFirst { case Some(d) => d } + } + + private def parseAsTimestamp(str: String): Option[LocalDateTime] = { + dateTimePatterns.view + .map(p => Try(LocalDateTime.parse(str, p)).toOption) + .collectFirst { case Some(dt) => dt } + } + + private def parseAsTime(str: String): Option[LocalTime] = { + timePatterns.view + .map(p => Try(LocalTime.parse(str, p)).toOption) + .collectFirst { case Some(t) => t } + } + + // ------------------------------------------------------------------------------------------------------ + // Private Conversion Helpers: DAS Value -> JDBC + // ------------------------------------------------------------------------------------------------------ + + /** + * Converts a DAS Value into a JDBC-compatible value, guided by the (possibly unreliable) SQLite column type name + * `colType`. + */ + private def convertDASValueToJdbcValue(value: Value, colType: String, conn: Connection): Any = { + val lowerType = Option(colType).map(_.toLowerCase).getOrElse("unknown") + + // If it's NULL in DAS, it's NULL to JDBC + if (value.hasNull) { + return null + } + + // In SQLite, columns are dynamically typed. We'll do best-effort: + // - If it's integer/long/short, store as a numeric + // - If it's double/float/decimal, store as a real or numeric + // - If it's string, store as text + // - If it's bool, we might store as 1 or 0 if declared "BOOLEAN" + // - If it's date/time/timestamp, we store as string in ISO form unless we detect an integer column + // - Binaries store as blob + // - Otherwise fallback to a JSON string representation + lowerType match { + case t if t.contains("int") => + // Store numeric + fallbackToNumber(value) + + case t if t.contains("real") || t.contains("floa") || t.contains("doub") => + // Store floating + if (value.hasDouble) value.getDouble.getV + else if (value.hasFloat) value.getFloat.getV.toDouble + else if (value.hasDecimal) Try(new java.math.BigDecimal(value.getDecimal.getV).doubleValue()).getOrElse(0.0) + else fallbackToNumber(value) + + case t if t.contains("blob") => + // Store as blob + fallbackToBytes(value) + + case t if t.contains("boolean") => + // We store true => 1, false => 0 + if (value.hasBool) { + if (value.getBool.getV) 1 else 0 + } else { + // fallback numeric + fallbackToNumber(value) + } + + case t if t.contains("numeric") || t.contains("dec") => + // Attempt BigDecimal + if (value.hasDecimal) new java.math.BigDecimal(value.getDecimal.getV) + else fallbackToNumber(value) + + case t + if (t.contains("text") || t.contains("char") || t.contains("clob")) || t.contains("date") || t.contains( + "time") => + // We'll store as string representation + fallbackToString(value) + + case "unknown" => + // fallback approach + fallbackToString(value) + + case _ => + // fallback approach + fallbackToString(value) + } + } + + private def fallbackToNumber(value: Value): Any = { + if (value.hasLong) { + value.getLong.getV + } else if (value.hasInt) { + value.getInt.getV + } else if (value.hasShort) { + value.getShort.getV.toInt + } else if (value.hasDouble) { + value.getDouble.getV + } else if (value.hasFloat) { + value.getFloat.getV.toDouble + } else if (value.hasDecimal) { + // Try returning a BigDecimal + new java.math.BigDecimal(value.getDecimal.getV) + } else if (value.hasBool) { + if (value.getBool.getV) 1 else 0 + } else { + // fallback + 0 + } + } + + private def fallbackToBytes(value: Value): Any = { + if (value.hasBinary) value.getBinary.getV.toByteArray + else fallbackToString(value).toString.getBytes + } + + private def fallbackToString(value: Value): Any = { + // If there's a string form, use it, else serialize the value to JSON + if (value.hasString) value.getString.getV + else dasValueToJsonString(value) + } + + /** + * Serializes a DAS Value into JSON (string). Useful when we have no direct typed mapping. + */ + private def dasValueToJsonString(value: Value): String = { + val node = buildJsonNode(value) + mapper.writeValueAsString(node) + } + + /** + * Recursively builds a Jackson `JsonNode` from a DAS Value. + */ + private def buildJsonNode(value: Value): JsonNode = { + if (value.hasNull) { + NullNode.instance + } else if (value.hasBool) { + nodeFactory.booleanNode(value.getBool.getV) + } else if (value.hasInt) { + nodeFactory.numberNode(value.getInt.getV) + } else if (value.hasLong) { + nodeFactory.numberNode(value.getLong.getV) + } else if (value.hasShort) { + nodeFactory.numberNode(value.getShort.getV) + } else if (value.hasFloat) { + nodeFactory.numberNode(value.getFloat.getV) + } else if (value.hasDouble) { + nodeFactory.numberNode(value.getDouble.getV) + } else if (value.hasDecimal) { + nodeFactory.numberNode(new java.math.BigDecimal(value.getDecimal.getV)) + } else if (value.hasString) { + new TextNode(value.getString.getV) + } else if (value.hasList) { + val arr = nodeFactory.arrayNode() + value.getList.getValuesList.forEach { v => + arr.add(buildJsonNode(v)) + } + arr + } else if (value.hasRecord) { + val obj = nodeFactory.objectNode() + value.getRecord.getAttsList.forEach { attr => + obj.set(attr.getName, buildJsonNode(attr.getValue)) + } + obj + } else { + // fallback + new TextNode(value.toString) + } + } + + /** + * Converts an SQLite declared column type into a DAS Type. We try to handle all common SQLite types: + * - INTEGER (and synonyms, including TINYINT, MEDIUMINT, BIGINT) + * - REAL / FLOAT / DOUBLE + * - TEXT / CHAR / CLOB + * - BLOB + * - NUMERIC / DECIMAL + * - BOOLEAN + * - DATE / DATETIME / TIME / TIMESTAMP + * and fallback if not recognized. + */ + private def toDASType(sqliteType: String, nullable: Boolean): Either[String, Type] = { + val builder = Type.newBuilder() + val trimmedType = Option(sqliteType).map(_.trim.toLowerCase).getOrElse("") + + try { + val dasType: Type = + if (trimmedType.contains("int")) { + // Covers "integer", "int", "tinyint", "smallint", "mediumint", "bigint", etc. + builder.setLong(LongType.newBuilder().setNullable(nullable)).build() + } else if (trimmedType.contains("real") || trimmedType.contains("floa") || trimmedType.contains("doub")) { + // REAL, FLOAT, DOUBLE + builder.setDouble(DoubleType.newBuilder().setNullable(nullable)).build() + } else if (trimmedType.contains("blob")) { + builder.setBinary(BinaryType.newBuilder().setNullable(nullable)).build() + } else if (trimmedType.contains("text") || trimmedType.contains("char") || trimmedType.contains("clob")) { + builder.setString(StringType.newBuilder().setNullable(nullable)).build() + } else if (trimmedType.contains("numeric") || trimmedType.contains("dec")) { + builder.setDecimal(DecimalType.newBuilder().setNullable(nullable)).build() + } else if (trimmedType.contains("boolean")) { + builder.setBool(BoolType.newBuilder().setNullable(nullable)).build() + } else if ( + trimmedType.contains("date") || + trimmedType.contains("time") || + trimmedType.contains("timestamp") || + trimmedType.contains("datetime") + ) { + // This can be tricky, but we attempt to interpret as Timestamp if "datetime"/"timestamp", + // as Date if "date" alone, as Time if "time" alone. If it's ambiguous, fallback to String. + // + // For the column definition, let's treat "date"/"datetime"/"timestamp" as Timestamp for the sake of + // enabling more general usage. Or we can break it down further if needed. + builder.setTimestamp(TimestampType.newBuilder().setNullable(nullable)).build() + } else if (trimmedType.isEmpty || trimmedType == "null") { + // No declared type + builder.setString(StringType.newBuilder().setNullable(nullable)).build() + } else { + // fallback if we can't interpret + throw new IllegalArgumentException(s"Unsupported or unknown type: $sqliteType") + } + + Right(dasType) + } catch { + case e: IllegalArgumentException => Left(e.getMessage) + } + } + +} diff --git a/src/main/scala/com/rawlabs/das/sqlite/DASSqliteBuilder.scala b/src/main/scala/com/rawlabs/das/sqlite/DASSqliteBuilder.scala new file mode 100644 index 0000000..1880a73 --- /dev/null +++ b/src/main/scala/com/rawlabs/das/sqlite/DASSqliteBuilder.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2025 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.das.sqlite + +import com.rawlabs.das.sdk.DASSettings +import com.rawlabs.das.sdk.scala.{DASSdk, DASSdkBuilder} + +class DASSqliteBuilder extends DASSdkBuilder { + + override def dasType: String = "sqlite" + + override def build(options: Map[String, String])(implicit settings: DASSettings): DASSdk = { + new DASSqlite(options) + } +} diff --git a/src/main/scala/com/rawlabs/das/sqlite/DASSqliteTable.scala b/src/main/scala/com/rawlabs/das/sqlite/DASSqliteTable.scala new file mode 100644 index 0000000..d7a7a8b --- /dev/null +++ b/src/main/scala/com/rawlabs/das/sqlite/DASSqliteTable.scala @@ -0,0 +1,399 @@ +/* + * Copyright 2025 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.das.sqlite + +import scala.jdk.CollectionConverters._ + +import com.rawlabs.das.sdk.scala.DASTable +import com.rawlabs.das.sdk.scala.DASTable.TableEstimate +import com.rawlabs.das.sdk.{DASExecuteResult, DASSdkUnsupportedException} +import com.rawlabs.protocol.das.v1.query._ +import com.rawlabs.protocol.das.v1.tables.{Row, TableDefinition} +import com.rawlabs.protocol.das.v1.types.Value +import com.rawlabs.protocol.das.v1.types.Value.ValueCase + +/** + * A DASTable implementation for SQLite. + * + * @param backend The DASSqliteBackend that provides actual DB access methods + * @param defn The Protobuf-based TableDefinition describing column schema, etc. + * @param maybePrimaryKey The optional single-column primary key discovered for this table + */ +class DASSqliteTable(backend: DASSqliteBackend, defn: TableDefinition, maybePrimaryKey: Option[String]) + extends DASTable { + + def definition: TableDefinition = defn + + /** + * SQLite table might not have a single unique column. If it does not, we throw an exception. If it does, we return + * that PK name. + */ + override def uniqueColumn: String = + maybePrimaryKey.getOrElse(throw new DASSdkUnsupportedException()) + + /** + * A naive table-size estimate. Builds a SQL query with the same WHERE conditions used in `execute(...)` and calls + * backend.estimate(...) on it. SQLite's estimate is fairly simplistic. + */ + override def tableEstimate(quals: Seq[Qual], columns: Seq[String]): TableEstimate = { + // 1) Build the same WHERE clause used in `execute(...)`. + val whereClause = + if (quals.isEmpty) "" + else "\nWHERE " + quals.map(qualToSql).mkString(" AND ") + + // 2) Possibly use columns if you want to estimate only the subset of columns, + // or just use "*" or "1" to get an overall row count approximation. + val selectClause = + if (columns.isEmpty) "1" + else columns.map(quoteIdentifier).mkString(", ") + + val tableName = quoteIdentifier(defn.getTableId.getName) + val sql = + s"SELECT $selectClause FROM $tableName$whereClause" + + backend.estimate(sql) + } + + /** + * Returns the raw generated SQL as "explain" lines. In SQLite, we simply split the final query by newlines. + */ + override def explain( + quals: Seq[Qual], + columns: Seq[String], + sortKeys: Seq[SortKey], + maybeLimit: Option[Long]): Seq[String] = { + mkSQL(quals, columns, sortKeys, maybeLimit).split("\n").toSeq + } + + /** + * Executes a select-like query in SQLite, returning the raw results via a DASExecuteResult. + */ + override def execute( + quals: Seq[Qual], + columns: Seq[String], + sortKeys: Seq[SortKey], + maybeLimit: Option[Long]): DASExecuteResult = { + backend.execute(mkSQL(quals, columns, sortKeys, maybeLimit)) + } + + /** + * Inserts a single row and returns the inserted row (including DB defaults, if any). In SQLite 3.35+, we can use + * RETURNING * to fetch the inserted row(s). + */ + override def insert(row: Row): Row = { + bulkInsert(Seq(row)).head + } + + /** + * Inserts multiple rows, returning them all. If rows is empty, returns an empty sequence. + */ + override def bulkInsert(rows: Seq[Row]): Seq[Row] = { + if (rows.isEmpty) { + return Seq.empty + } + + // 1) Gather columns from the first row + val firstRow = rows.head + val columns = firstRow.getColumnsList.asScala.map(_.getName).toList + // Typically, ensure all subsequent rows have the same columns in the same order + + // 2) Build placeholders for each row + // e.g., rowPlaceholder => "(?, ?, ...)" for the number of columns + val numCols = columns.size + val rowPlaceholder = "(" + List.fill(numCols)("?").mkString(", ") + ")" + val placeholdersAll = List.fill(rows.size)(rowPlaceholder).mkString(", ") + + val columnNames = columns.map(quoteIdentifier).mkString(", ") + + val tableName = quoteIdentifier(defn.getTableId.getName) + // 3.35+ syntax in SQLite for RETURNING: + val insertSql = + s"INSERT INTO $tableName ($columnNames) VALUES $placeholdersAll RETURNING *" + + // 3) Flatten all param values + // For each row, for each col in the same order, gather the Value + val params: Seq[Value] = rows.flatMap { row => + row.getColumnsList.asScala.map(_.getData) + } + + // 4) Execute and read the result set + val result = backend.executeReturningRows(insertSql, params) + + val insertedRows = scala.collection.mutable.ListBuffer[Row]() + try { + while (result.hasNext) { + insertedRows += result.next() + } + } finally { + result.close() + } + insertedRows.toSeq + } + + /** + * Updates the row with the given rowId, returning the updated Row. If no row is matched, throws an exception. + */ + override def update(rowId: Value, newValues: Row): Row = { + // 1) Build the SET clause from columns in `newValues` + val setClause = newValues.getColumnsList.asScala + .map { col => s"${quoteIdentifier(col.getName)} = ?" } + .mkString(", ") + + // 2) Build the SQL with RETURNING * (3.35+) + val tableName = quoteIdentifier(defn.getTableId.getName) + val updateSql = + s""" + |UPDATE $tableName + |SET $setClause + |WHERE ${quoteIdentifier(uniqueColumn)} = ? + |RETURNING * + |""".stripMargin + + // 3) Gather parameters: first all the newValues columns, then rowId last + val params: Seq[Value] = + newValues.getColumnsList.asScala.map(_.getData).toSeq :+ rowId + + // 4) Execute + val result = backend.executeReturningRows(updateSql, params) + try { + if (!result.hasNext) { + // Possibly means no row matched that id? + result.close() + throw new RuntimeException(s"Update failed: no row matched id=$rowId") + } + val updatedRow = result.next() + + // If you want to ensure only one row was updated, you could do: + // if (result.hasNext) { ... log a warning or throw an error ... } + + updatedRow + } finally { + result.close() + } + } + + /** + * Deletes the row with the given rowId. + */ + override def delete(rowId: Value): Unit = { + val tableName = quoteIdentifier(defn.getTableId.getName) + val deleteSql = + s"DELETE FROM $tableName WHERE ${quoteIdentifier(uniqueColumn)} = ?" + + // We can reuse the batchQuery interface to pass a single parameter set + backend.batchQuery(deleteSql, Seq(Seq(rowId))) + } + + // ------------------------------------------------------------------------------------------ + // SQL Construction + // ------------------------------------------------------------------------------------------ + + private def mkSQL( + quals: Seq[Qual], + columns: Seq[String], + sortKeys: Seq[SortKey], + maybeLimit: Option[Long]): String = { + + // Build SELECT list + val selectClause = + if (columns.isEmpty) "1" + else columns.map(quoteIdentifier).mkString(", ") + + // Build WHERE from `quals` + val whereClause = + if (quals.isEmpty) "" + else "\nWHERE " + quals.map(qualToSql).mkString(" AND ") + + // Build ORDER BY + val orderByClause = + if (sortKeys.isEmpty) "" + else "\nORDER BY " + sortKeys.map(sortKeyToSql).mkString(", ") + + // Build LIMIT + val limitClause = maybeLimit.map(l => s"\nLIMIT $l").getOrElse("") + + val tableName = quoteIdentifier(defn.getTableId.getName) + s"SELECT $selectClause FROM $tableName$whereClause$orderByClause$limitClause" + } + + /** + * SQLite requires quotes around identifiers that might include unusual characters or match keywords. We double up + * internal quotes, then wrap in double-quotes. + */ + private def quoteIdentifier(ident: String): String = { + val escaped = ident.replace("\"", "\"\"") + s""""$escaped"""" + } + + /** + * Convert a SortKey to an ORDER BY snippet. SQLite does not inherently support "NULLS FIRST/LAST", so you may ignore + * or simulate it. We show a naive approach here. + */ + private def sortKeyToSql(sk: SortKey): String = { + val col = quoteIdentifier(sk.getName) + val direction = if (sk.getIsReversed) "DESC" else "ASC" + + // SQLite doesn't have a built-in "NULLS FIRST/LAST" syntax, but we keep the code for demonstration + val nullsPart = + if (sk.getNullsFirst) " -- NULLS FIRST (not directly supported)" else " -- NULLS LAST (not directly supported)" + + // We can also handle collate if needed; SQLite supports "COLLATE nocase", etc. + val collation = if (sk.getCollate.nonEmpty) s" COLLATE ${sk.getCollate}" else "" + + s"$col$collation $direction$nullsPart" + } + + /** + * Convert a single Value into an inline SQL literal. For dynamic queries used in e.g. EXPLAIN, or simplistic queries. + * For actual data-binding, we typically use parameters, so be cautious about SQL injection for real usage. + */ + private def valueToSql(v: Value): String = { + v.getValueCase match { + case ValueCase.NULL => "NULL" + case ValueCase.BYTE => v.getByte.getV.toString + case ValueCase.SHORT => v.getShort.getV.toString + case ValueCase.INT => v.getInt.getV.toString + case ValueCase.LONG => v.getLong.getV.toString + case ValueCase.FLOAT => v.getFloat.getV.toString + case ValueCase.DOUBLE => v.getDouble.getV.toString + case ValueCase.DECIMAL => + // Typically a numeric string, we just inline it + v.getDecimal.getV + case ValueCase.BOOL => + // In SQLite, "TRUE"/"FALSE" are not strictly special, but let's keep them as uppercase for clarity + if (v.getBool.getV) "TRUE" else "FALSE" + case ValueCase.STRING => + s"'${escape(v.getString.getV)}'" + case ValueCase.BINARY => + val bytes = v.getBinary.getV.toByteArray + // Convert to an SQLite-style blob literal: X'DEADBEEF' + s"${byteArrayToSQLiteHex(bytes)}" + + case ValueCase.DATE => + // E.g., '2024-01-15' + val d = v.getDate + f"'${d.getYear}%04d-${d.getMonth}%02d-${d.getDay}%02d'" + + case ValueCase.TIME => + // E.g., '10:30:25' + val t = v.getTime + f"'${t.getHour}%02d:${t.getMinute}%02d:${t.getSecond}%02d'" + + case ValueCase.TIMESTAMP => + val ts = v.getTimestamp + // E.g., '2024-01-15 10:30:25' + f"'${ts.getYear}%04d-${ts.getMonth}%02d-${ts.getDay}%02d ${ts.getHour}%02d:${ts.getMinute}%02d:${ts.getSecond}%02d'" + + case ValueCase.INTERVAL | ValueCase.RECORD | ValueCase.LIST => + // For a simple example, store them as string or skip + s"'${escape(v.toString)}'" + + case ValueCase.VALUE_NOT_SET => + "NULL" + } + } + + /** + * SQLite-style hexadecimal blob literal: X'DEADBEEF' + */ + private def byteArrayToSQLiteHex(byteArray: Array[Byte]): String = { + val hexString = byteArray.map("%02x".format(_)).mkString + s"X'$hexString'" + } + + private def escape(str: String): String = + str.replace("'", "''") // naive approach for single quotes + + /** + * Maps an Operator enum to the corresponding SQL string. Some operators like ILIKE are not native to SQLite, so we + * provide a naive fallback or throw an exception. + */ + private def operatorToSql(op: Operator): String = { + op match { + case Operator.EQUALS => "=" + case Operator.NOT_EQUALS => "<>" + case Operator.LESS_THAN => "<" + case Operator.LESS_THAN_OR_EQUAL => "<=" + case Operator.GREATER_THAN => ">" + case Operator.GREATER_THAN_OR_EQUAL => ">=" + case Operator.LIKE => "LIKE" + case Operator.NOT_LIKE => "NOT LIKE" + + // SQLite does not have native ILIKE support. We can fallback to "LIKE" or fail. + case Operator.ILIKE => throw new IllegalArgumentException("SQLite does not support ILIKE.") + case Operator.NOT_ILIKE => throw new IllegalArgumentException("SQLite does not support NOT ILIKE.") + + // Arithmetic operators might not be typical in a WHERE Qual + case Operator.PLUS => "+" + case Operator.MINUS => "-" + case Operator.TIMES => "*" + case Operator.DIV => "/" + case Operator.MOD => "%" + case Operator.AND => "AND" + case Operator.OR => "OR" + + case _ => throw new IllegalArgumentException(s"Unsupported operator: $op") + } + } + + /** + * `IsAllQual` means "col op ALL these values", we interpret as multiple AND clauses + */ + private def isAllQualToSql(colName: String, iq: IsAllQual): String = { + val opStr = operatorToSql(iq.getOperator) + val clauses = iq.getValuesList.asScala.map(v => s"$colName $opStr ${valueToSql(v)}") + // Combine with AND + clauses.mkString("(", " AND ", ")") + } + + /** + * `IsAnyQual` means "col op ANY of these values", we interpret as multiple OR clauses + */ + private def isAnyQualToSql(colName: String, iq: IsAnyQual): String = { + val opStr = operatorToSql(iq.getOperator) + val clauses = iq.getValuesList.asScala.map(v => s"$colName $opStr ${valueToSql(v)}") + // Combine with OR + clauses.mkString("(", " OR ", ")") + } + + /** + * `SimpleQual` is a single condition: "col op value" + */ + private def simpleQualToSql(colName: String, sq: SimpleQual): String = { + if (sq.getValue.hasNull && sq.getOperator == Operator.EQUALS) { + s"$colName IS NULL" + } else if (sq.getValue.hasNull && sq.getOperator == Operator.NOT_EQUALS) { + s"$colName IS NOT NULL" + } else { + val opStr = operatorToSql(sq.getOperator) + val valStr = valueToSql(sq.getValue) + s"$colName $opStr $valStr" + } + } + + /** + * Converts any `Qual` to a SQL snippet. We handle `SimpleQual`, `IsAnyQual`, or `IsAllQual`. + */ + private def qualToSql(q: Qual): String = { + val colName = quoteIdentifier(q.getName) + if (q.hasSimpleQual) { + simpleQualToSql(colName, q.getSimpleQual) + } else + q.getQualCase match { + case Qual.QualCase.IS_ANY_QUAL => isAnyQualToSql(colName, q.getIsAnyQual) + case Qual.QualCase.IS_ALL_QUAL => isAllQualToSql(colName, q.getIsAllQual) + case _ => throw new IllegalArgumentException(s"Unsupported qual: $q") + } + } + +} diff --git a/src/test/scala/com/rawlabs/das/sqlite/DASSqliteConnectionTest.scala b/src/test/scala/com/rawlabs/das/sqlite/DASSqliteConnectionTest.scala new file mode 100644 index 0000000..f57748d --- /dev/null +++ b/src/test/scala/com/rawlabs/das/sqlite/DASSqliteConnectionTest.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2025 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.das.sqlite + +import java.nio.file.Files + +import org.scalatest.funsuite.AnyFunSuite + +import com.rawlabs.das.sdk.{DASSdkInvalidArgumentException, DASSdkUnauthenticatedException} + +/** + * SQLite connection tests. + */ +class DASSqliteConnectionTest extends AnyFunSuite { + + test("Missing database => DASSdkInvalidArgumentException") { + // Our DASSqlite code requires a "database" option. + val ex = intercept[DASSdkInvalidArgumentException] { + new DASSqlite(Map.empty) // no "database" key + } + assert(ex.getMessage.contains("database")) + } + + test("Bad database path => DASSdkInvalidArgumentException (unwritable)") { + // We'll try to create a DB in a location we (hopefully) cannot write to, e.g. root "/no_access_here" + // Depending on your OS, this might fail in different ways. Adjust as needed for your environment. + val ex = intercept[DASSdkInvalidArgumentException] { + new DASSqlite(Map("database" -> "/no_access_here/some.db")) + } + assert(ex.getMessage.contains("Could not connect")) + } + + test("Valid in-memory => no exception") { + // SQLite in-memory: just pass ":memory:" as database path + val sdk = new DASSqlite(Map("database" -> ":memory:")) + // Basic check + assert(sdk.tableDefinitions.size >= 0) + sdk.close() + } + + test("Valid file-based => no exception") { + // Create a temporary file for SQLite. Typically, the driver will create or open this file as needed. + val tempFile = Files.createTempFile("sqlite_test_", ".db") + val sdk = new DASSqlite(Map("database" -> tempFile.toString)) + // Basic check + assert(sdk.tableDefinitions.size >= 0) + sdk.close() + } + + test("User/password are ignored => no exception") { + // Normally, SQLite doesn't use user/password, but DASSqlite code might accept them without error. + val sdk = new DASSqlite(Map("database" -> ":memory:", "user" -> "ignored_user", "password" -> "ignored_pass")) + // Basic check + assert(sdk.tableDefinitions.size >= 0) + sdk.close() + } + + test("If you want to forcibly fail authentication => DASSdkUnauthenticatedException example") { + // By default, standard SQLite doesn't enforce authentication, + // but if you have a custom driver or extension that does, you can mimic it here. + // For demonstration, we'll show how you *might* trigger or check it: + val ex = intercept[DASSdkUnauthenticatedException] { + throw new DASSdkUnauthenticatedException("Invalid username/password") + } + assert(ex.getMessage.contains("Invalid username/password")) + } + +} diff --git a/src/test/scala/com/rawlabs/das/sqlite/DASSqliteTableTest.scala b/src/test/scala/com/rawlabs/das/sqlite/DASSqliteTableTest.scala new file mode 100644 index 0000000..6f08c93 --- /dev/null +++ b/src/test/scala/com/rawlabs/das/sqlite/DASSqliteTableTest.scala @@ -0,0 +1,814 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.das.sqlite + +import java.nio.file.{Files, Path} +import java.sql.DriverManager + +import scala.jdk.CollectionConverters._ +import scala.util.Using + +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite + +import com.google.protobuf.ByteString +import com.rawlabs.das.sdk.DASSdkInvalidArgumentException +import com.rawlabs.protocol.das.v1.query._ +import com.rawlabs.protocol.das.v1.tables._ +import com.rawlabs.protocol.das.v1.types._ +import com.typesafe.scalalogging.StrictLogging + +/** + * Integration test suite covering the DASSqlite and DASSqliteTable classes, using a SQLite database. + */ +class DASSqliteTableTest extends AnyFunSuite with BeforeAndAfterAll with StrictLogging { + + private var tempFile: Path = _ + private var sdk: DASSqlite = _ + private var table: DASSqliteTable = _ + + override def beforeAll(): Unit = { + super.beforeAll() + + // 1) Create a temporary file + tempFile = Files.createTempFile("test_all_types_db", ".db") + + // 2) Connect manually (raw JDBC) and create the table + val createTableDDL = + """ + |CREATE TABLE IF NOT EXISTS all_types ( + | id INTEGER PRIMARY KEY AUTOINCREMENT, + | col_integer INTEGER, + | col_bigint INTEGER, + | col_boolean BOOLEAN, + | col_numeric DECIMAL(20,2), + | col_real REAL, + | col_double DOUBLE, + | col_timestamp TEXT, + | col_timestamptz TEXT, + | col_date TEXT, + | col_time TEXT, + | col_timetz TEXT, + | col_text TEXT, + | col_varchar TEXT, + | col_bytea BLOB, + | col_json TEXT, + | col_jsonb TEXT, + | col_hstore TEXT, + | + | col_int_array TEXT, + | col_text_array TEXT, + | col_date_array TEXT, + | col_json_array TEXT, + | col_jsonb_array TEXT, + | col_hstore_array TEXT + |); + """.stripMargin + + val jdbcUrl = s"jdbc:sqlite:${tempFile.toString}" + try { + Class.forName("org.sqlite.JDBC") + } catch { + case _: ClassNotFoundException => + throw new RuntimeException("SQLite JDBC driver not found on the classpath.") + } + Using.resource(DriverManager.getConnection(jdbcUrl)) { conn => + val st = conn.createStatement() + st.execute(createTableDDL) + st.close() + } + + // 3) Now that the DB file is populated, instantiate DASSqlite + // We pass the path minus "jdbc:sqlite:" because DASSqlite adds that prefix internally + val sdkOptions = Map("database" -> tempFile.toString) + sdk = new DASSqlite(sdkOptions) + + // 4) Get the "all_types" table from the DASSqlite + val maybeTable = sdk.getTable("all_types") + assert(maybeTable.isDefined, "Expected 'all_types' to exist in the file-based DB.") + table = maybeTable.get.asInstanceOf[DASSqliteTable] + } + + override def afterAll(): Unit = { + // 1) Close the SDK + if (sdk != null) sdk.close() + + // 2) Remove the temp file + if (tempFile != null) { + Files.deleteIfExists(tempFile) + } + super.afterAll() + } + + // =========================================================================== + // Tests for DASSqlite (top-level) methods + // =========================================================================== + test("DASSqlite.tableDefinitions returns our test table") { + val defs = sdk.tableDefinitions + assert(defs.nonEmpty, "tableDefinitions should not be empty.") + val names = defs.map(_.getTableId.getName.toLowerCase) + assert(names.contains("all_types"), "We expect 'all_types' table in the DB.") + } + + test("DASSqlite.functionDefinitions is empty") { + val funcs = sdk.functionDefinitions + assert(funcs.isEmpty, "We do not define any functions, so functionDefinitions should be empty.") + } + + test("DASSqlite.getTable returns a valid table, getFunction returns None") { + val fakeTable = sdk.getTable("non_existent_table") + assert(fakeTable.isEmpty, "getTable should return None for an invalid table name.") + + // getFunction always returns None in the current DASSqlite implementation + val func = sdk.getFunction("someFunctionName") + assert(func.isEmpty) + } + + // =========================================================================== + // Tests for DASSqliteTable public methods + // =========================================================================== + + test("DASSqliteTable.definition is consistent with our table schema") { + val defn = table.definition + assert(defn.getTableId.getName.equalsIgnoreCase("all_types"), "Expected table name 'all_types'") + assert(defn.getColumnsCount > 0, "We expect at least 1 column in the schema.") + + val columnNames = defn.getColumnsList.asScala.map(_.getName.toLowerCase).toSet + val expectedSomeColumns = Set( + "id", + "col_integer", + "col_bigint", + "col_boolean", + "col_numeric", + "col_real", + "col_double", + "col_timestamp", + "col_timestamptz", + "col_bytea", + "col_json", + "col_jsonb", + "col_hstore") + assert(expectedSomeColumns.subsetOf(columnNames), s"Expected columns $expectedSomeColumns to be present.") + } + + test("DASSqliteTable.uniqueColumn returns 'id' if single primary key") { + // If we discovered a single PK named "id", it should be returned. Otherwise, we might + // not have a single PK. (By default, 'id INTEGER PRIMARY KEY AUTOINCREMENT' is the PK.) + val uniqueCol = table.uniqueColumn + assert(uniqueCol.equalsIgnoreCase("id"), "We expect 'id' to be the single PK for all_types.") + } + + test("tableEstimate - naive in SQLite") { + // Insert one row + val inserted = insertSingleRowWithInt(999999) + val idVal = getRowId(inserted) + + // Create a Qual that checks col_integer = 999999 + val qual = Qual + .newBuilder() + .setName("col_integer") + .setSimpleQual( + SimpleQual + .newBuilder() + .setOperator(Operator.EQUALS) + .setValue(Value.newBuilder().setInt(ValueInt.newBuilder().setV(999999)))) + .build() + + val estimate = table.tableEstimate(Seq(qual), Seq("col_integer")) + logger.info(s"Estimate => rowCount=${estimate.expectedNumberOfRows}, avgRowWidth=${estimate.avgRowWidthBytes}") + // SQLite doesn't produce real estimates by default, so we may just see a naive (100,100). + assert(estimate.expectedNumberOfRows > 0) + table.delete(idVal) + } + + test("explain - we simply split the final SQL in lines") { + val qual = Qual + .newBuilder() + .setName("col_integer") + .setSimpleQual( + SimpleQual + .newBuilder() + .setOperator(Operator.LESS_THAN) + .setValue(Value.newBuilder().setInt(ValueInt.newBuilder().setV(1000000)))) + .build() + + val sortKey = SortKey + .newBuilder() + .setName("col_integer") + .setIsReversed(true) + .build() + + val explainLines = table.explain( + quals = Seq(qual), + columns = Seq("id", "col_integer"), + sortKeys = Seq(sortKey), + maybeLimit = Some(10)) + + logger.info("Explain lines for SQLite:") + explainLines.foreach(ln => logger.info(ln)) + + assert(explainLines.nonEmpty) + assert(explainLines.head.startsWith("SELECT "), "Expected a SELECT statement.") + } + + test("execute with projections") { + // Insert a row with col_varchar + val rowToInsert = Row + .newBuilder() + .addColumns( + Column + .newBuilder() + .setName("col_varchar") + .setData(Value.newBuilder().setString(ValueString.newBuilder().setV("ProjectionTestRow")))) + .build() + table.insert(rowToInsert) + + // Qual to filter on col_varchar + val qual = Qual + .newBuilder() + .setName("col_varchar") + .setSimpleQual( + SimpleQual + .newBuilder() + .setOperator(Operator.EQUALS) + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("ProjectionTestRow")))) + .build() + + // Ask only for 'id' and 'col_varchar' + val result = table.execute(Seq(qual), Seq("id", "col_varchar"), Seq.empty, None) + assert(result.hasNext, "Expected to find the inserted row.") + val found = result.next() + result.close() + val colMap = found.getColumnsList.asScala.map(c => c.getName -> c.getData).toMap + assert(colMap.size == 2) + assert(colMap.contains("id")) + assert(colMap.contains("col_varchar")) + + // Cleanup + table.delete(colMap("id")) + } + + test("execute with filtering") { + val rowA = insertSingleRowWithVarchar("FilteringTestA") + val rowB = insertSingleRowWithVarchar("FilteringTestB") + + // Qual B + val qualB = Qual + .newBuilder() + .setName("col_varchar") + .setSimpleQual( + SimpleQual + .newBuilder() + .setOperator(Operator.EQUALS) + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("FilteringTestB")))) + .build() + + val resultB = table.execute(Seq(qualB), Seq("id", "col_varchar"), Seq.empty, None) + assert(resultB.hasNext) + val foundB = resultB.next() + resultB.close() + val mapB = foundB.getColumnsList.asScala.map(c => c.getName -> c.getData).toMap + assert(mapB("col_varchar").hasString && mapB("col_varchar").getString.getV == "FilteringTestB") + + // No match + val qualZ = Qual + .newBuilder() + .setName("col_varchar") + .setSimpleQual( + SimpleQual + .newBuilder() + .setOperator(Operator.EQUALS) + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("FilteringTestZ")))) + .build() + val resultZ = table.execute(Seq(qualZ), Seq("id"), Seq.empty, None) + assert(!resultZ.hasNext) + resultZ.close() + + table.delete(getRowId(rowA)) + table.delete(getRowId(rowB)) + } + + test("execute with sorting") { + val row10 = insertSingleRowWithInt(10) + val row20 = insertSingleRowWithInt(20) + val row30 = insertSingleRowWithInt(30) + + // Desc sort + val sortKey = SortKey + .newBuilder() + .setName("col_integer") + .setIsReversed(true) + .build() + + val result = table.execute(Seq.empty, Seq("col_integer"), Seq(sortKey), None) + val returnedInts = consumeAllRows(result).flatMap { r => + r.getColumnsList.asScala.find(_.getName == "col_integer").map(_.getData.getInt.getV) + } + val ours = returnedInts.filter(Set(10, 20, 30)) + assert(ours == List(30, 20, 10), s"Expected [30, 20, 10], got $ours") + + table.delete(getRowId(row10)) + table.delete(getRowId(row20)) + table.delete(getRowId(row30)) + } + + test("execute with limiting") { + val inserted = (1 to 5).map { i => insertSingleRowWithInt(1000 + i) } + val result = table.execute(Seq.empty, Seq("id", "col_integer"), Seq.empty, Some(2)) + val rowsFetched = consumeAllRows(result) + assert(rowsFetched.size == 2) + inserted.foreach(r => table.delete(getRowId(r))) + } + + test("insert a row with multiple columns and verify (some columns not natively typed in SQLite)") { + val row = buildAllTypesRow() + table.insert(row) + + // Now select back by col_integer=555 + val qual555 = Qual + .newBuilder() + .setName("col_integer") + .setSimpleQual( + SimpleQual + .newBuilder() + .setOperator(Operator.EQUALS) + .setValue(Value.newBuilder().setInt(ValueInt.newBuilder().setV(555)))) + .build() + + val columnsToFetch = Seq( + "id", + "col_integer", + "col_bigint", + "col_numeric", + "col_real", + "col_double", + "col_date", + "col_timestamp", + "col_timestamptz", + "col_time", + "col_timetz", + "col_text", + "col_varchar", + "col_boolean", + "col_bytea", + "col_json", + "col_jsonb", + "col_hstore", + "col_int_array", + "col_text_array", + "col_date_array", + "col_json_array", + "col_jsonb_array", + "col_hstore_array") + + val selectResult = table.execute(Seq(qual555), columnsToFetch, Seq.empty, None) + assert(selectResult.hasNext) + val rowRead = selectResult.next() + selectResult.close() + val readMap = rowRead.getColumnsList.asScala.map(col => col.getName -> col.getData).toMap + + // Spot-check a few important columns + assert(readMap("col_integer").hasInt && readMap("col_integer").getInt.getV == 555) + assert(readMap("col_bigint").hasLong && readMap("col_bigint").getLong.getV == 9876543210123L) + assert(readMap("col_numeric").hasDecimal && readMap("col_numeric").getDecimal.getV == "54321.01") + assert(readMap("col_boolean").hasBool && readMap("col_boolean").getBool.getV) + + // Clean up + table.delete(readMap("id")) + } + + test("update works") { + val rowToInsert = Row + .newBuilder() + .addColumns( + Column + .newBuilder() + .setName("col_varchar") + .setData(Value.newBuilder().setString(ValueString.newBuilder().setV("UpdateTestRow")))) + .addColumns( + Column + .newBuilder() + .setName("col_text") + .setData(Value.newBuilder().setString(ValueString.newBuilder().setV("Original text")))) + .build() + + val inserted = table.insert(rowToInsert) + val insertedMap = inserted.getColumnsList.asScala.map(c => c.getName -> c.getData).toMap + val rowId = insertedMap("id") + + // Update col_text + val newTextRow = Row + .newBuilder() + .addColumns( + Column + .newBuilder() + .setName("col_text") + .setData(Value.newBuilder().setString(ValueString.newBuilder().setV("Updated text")))) + .build() + table.update(rowId, newTextRow) + + // Check + val qualById = Qual + .newBuilder() + .setName("id") + .setSimpleQual(SimpleQual.newBuilder().setOperator(Operator.EQUALS).setValue(rowId)) + .build() + val updatedRes = table.execute(Seq(qualById), Seq("col_text"), Seq.empty, None) + assert(updatedRes.hasNext) + val updated = updatedRes.next() + updatedRes.close() + val updatedMap = updated.getColumnsList.asScala.map(c => c.getName -> c.getData).toMap + assert(updatedMap("col_text").hasString && updatedMap("col_text").getString.getV == "Updated text") + + // cleanup + table.delete(rowId) + } + + test("delete works") { + val rowToInsert = Row + .newBuilder() + .addColumns( + Column + .newBuilder() + .setName("col_varchar") + .setData(Value.newBuilder().setString(ValueString.newBuilder().setV("DeleteTestRow")))) + .build() + val inserted = table.insert(rowToInsert) + val insertedMap = inserted.getColumnsList.asScala.map(c => c.getName -> c.getData).toMap + val rowId = insertedMap("id") + + // Confirm presence + val q = Qual + .newBuilder() + .setName("col_varchar") + .setSimpleQual( + SimpleQual + .newBuilder() + .setOperator(Operator.EQUALS) + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("DeleteTestRow")))) + .build() + val resBefore = table.execute(Seq(q), Seq("id"), Seq.empty, None) + assert(resBefore.hasNext) + resBefore.close() + + table.delete(rowId) + + val resAfter = table.execute(Seq(q), Seq("id"), Seq.empty, None) + assert(!resAfter.hasNext) + resAfter.close() + } + + test("bulkInsert works") { + val rows = (1 to 3).map { i => + Row + .newBuilder() + .addColumns( + Column + .newBuilder() + .setName("col_varchar") + .setData(Value.newBuilder().setString(ValueString.newBuilder().setV(s"BulkTestRow_$i")))) + .build() + } + + table.bulkInsert(rows) + + // Verify + val qual = Qual + .newBuilder() + .setName("col_varchar") + .setIsAnyQual( + IsAnyQual + .newBuilder() + .setOperator(Operator.LIKE) + .addValues(Value.newBuilder().setString(ValueString.newBuilder().setV("BulkTestRow_%")))) + .build() + + val result = table.execute(Seq(qual), Seq("id", "col_varchar"), Seq.empty, None) + val foundRows = consumeAllRows(result) + val foundTexts = foundRows.flatMap { r => + r.getColumnsList.asScala.find(_.getName == "col_varchar").map(_.getData.getString.getV) + } + val expected = (1 to 3).map(i => s"BulkTestRow_$i").toSet + val intersects = foundTexts.toSet intersect expected + assert(intersects.size >= 3, s"Should find all 3 inserted rows: $foundTexts") + + // Cleanup + foundRows.foreach { row => + val map = row.getColumnsList.asScala.map(c => c.getName -> c.getData).toMap + if (map.get("col_varchar").exists(_.getString.getV.startsWith("BulkTestRow_"))) { + table.delete(map("id")) + } + } + } + + // =========================================================================== + // Helper Functions + // =========================================================================== + + /** Insert a row with col_integer = `value`. Return the row returned by table.insert. */ + private def insertSingleRowWithInt(value: Int): Row = { + val row = Row + .newBuilder() + .addColumns( + Column + .newBuilder() + .setName("col_integer") + .setData(Value.newBuilder().setInt(ValueInt.newBuilder().setV(value)))) + .build() + table.insert(row) + } + + /** Insert a row with col_varchar = `value`. Return the row from table.insert. */ + private def insertSingleRowWithVarchar(value: String): Row = { + val row = Row + .newBuilder() + .addColumns( + Column + .newBuilder() + .setName("col_varchar") + .setData(Value.newBuilder().setString(ValueString.newBuilder().setV(value)))) + .build() + table.insert(row) + } + + /** Retrieve the 'id' column from an inserted Row (we treat it as PK). */ + private def getRowId(inserted: Row): Value = { + val colOpt = inserted.getColumnsList.asScala.find(_.getName.equalsIgnoreCase("id")) + colOpt.getOrElse { + throw new DASSdkInvalidArgumentException("No 'id' column found in inserted row.") + }.getData + } + + /** Utility to consume all rows from a DASExecuteResult. */ + private def consumeAllRows(execResult: com.rawlabs.das.sdk.DASExecuteResult): List[Row] = { + val buf = scala.collection.mutable.ListBuffer[Row]() + while (execResult.hasNext) { + buf += execResult.next() + } + execResult.close() + buf.toList + } + + /** + * Build a row with all the columns from your snippet set with sample values. Some columns (like hstore, JSONB, etc.) + * will be stored as TEXT in SQLite. + */ + private def buildAllTypesRow(): Row = { + Row + .newBuilder() + .addColumns(Column + .newBuilder() + .setName("col_integer") + .setData(Value.newBuilder().setInt(ValueInt.newBuilder().setV(555)))) + .addColumns(Column + .newBuilder() + .setName("col_bigint") + .setData(Value.newBuilder().setLong(ValueLong.newBuilder().setV(9876543210123L)))) + .addColumns(Column + .newBuilder() + .setName("col_numeric") + .setData(Value.newBuilder().setDecimal(ValueDecimal.newBuilder().setV("54321.01")))) + .addColumns(Column + .newBuilder() + .setName("col_real") + .setData(Value.newBuilder().setFloat(ValueFloat.newBuilder().setV(1.234f)))) + .addColumns(Column + .newBuilder() + .setName("col_double") + .setData(Value.newBuilder().setDouble(ValueDouble.newBuilder().setV(9.876)))) + .addColumns(Column + .newBuilder() + .setName("col_date") + .setData(Value.newBuilder().setDate(ValueDate.newBuilder().setYear(2025).setMonth(1).setDay(1)))) + .addColumns(Column + .newBuilder() + .setName("col_timestamp") + .setData(Value + .newBuilder() + .setTimestamp( + ValueTimestamp.newBuilder().setYear(2025).setMonth(1).setDay(1).setHour(23).setMinute(59).setSecond(59)))) + .addColumns(Column + .newBuilder() + .setName("col_timestamptz") + .setData(Value + .newBuilder() + .setTimestamp( + ValueTimestamp.newBuilder().setYear(2025).setMonth(1).setDay(1).setHour(23).setMinute(59).setSecond(59)))) + .addColumns(Column + .newBuilder() + .setName("col_time") + .setData(Value.newBuilder().setTime(ValueTime.newBuilder().setHour(23).setMinute(59).setSecond(59)))) + .addColumns(Column + .newBuilder() + .setName("col_timetz") + .setData(Value.newBuilder().setTime(ValueTime.newBuilder().setHour(23).setMinute(59).setSecond(59)))) + .addColumns(Column + .newBuilder() + .setName("col_text") + .setData(Value.newBuilder().setString(ValueString.newBuilder().setV("Test all types")))) + .addColumns(Column + .newBuilder() + .setName("col_varchar") + .setData(Value.newBuilder().setString(ValueString.newBuilder().setV("Test varchar")))) + .addColumns(Column + .newBuilder() + .setName("col_boolean") + .setData(Value.newBuilder().setBool(ValueBool.newBuilder().setV(true)))) + .addColumns(Column + .newBuilder() + .setName("col_bytea") + .setData(Value.newBuilder().setBinary(ValueBinary.newBuilder().setV(ByteString.fromHex("A1B2C3D4"))))) + .addColumns( + Column + .newBuilder() + .setName("col_json") + .setData( + Value + .newBuilder() + .setRecord( + ValueRecord + .newBuilder() + .addAtts(ValueRecordAttr + .newBuilder() + .setName("foo") + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("bar"))))))) + .addColumns( + Column + .newBuilder() + .setName("col_jsonb") + .setData( + Value + .newBuilder() + .setRecord( + ValueRecord + .newBuilder() + .addAtts(ValueRecordAttr + .newBuilder() + .setName("foo") + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("bar"))))))) + .addColumns( + Column + .newBuilder() + .setName("col_hstore") + .setData( + Value + .newBuilder() + .setRecord( + ValueRecord + .newBuilder() + .addAtts(ValueRecordAttr + .newBuilder() + .setName("a") + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("1")))) + .addAtts(ValueRecordAttr + .newBuilder() + .setName("b") + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("2"))))))) + .addColumns( + Column + .newBuilder() + .setName("col_int_array") + .setData( + Value + .newBuilder() + .setList(ValueList + .newBuilder() + .addAllValues(Seq( + Value.newBuilder().setInt(ValueInt.newBuilder().setV(1)).build(), + Value.newBuilder().setInt(ValueInt.newBuilder().setV(2)).build(), + Value.newBuilder().setInt(ValueInt.newBuilder().setV(3)).build()).asJava)))) + .addColumns( + Column + .newBuilder() + .setName("col_text_array") + .setData( + Value + .newBuilder() + .setList(ValueList + .newBuilder() + .addAllValues(Seq( + Value.newBuilder().setString(ValueString.newBuilder().setV("alpha")).build(), + Value.newBuilder().setString(ValueString.newBuilder().setV("beta")).build()).asJava)))) + .addColumns( + Column + .newBuilder() + .setName("col_date_array") + .setData( + Value + .newBuilder() + .setList(ValueList + .newBuilder() + .addAllValues(Seq( + Value.newBuilder().setDate(ValueDate.newBuilder().setYear(2025).setMonth(1).setDay(1)).build(), + Value + .newBuilder() + .setDate(ValueDate.newBuilder().setYear(2025).setMonth(1).setDay(2)) + .build()).asJava)))) + .addColumns( + Column + .newBuilder() + .setName("col_json_array") + .setData( + Value + .newBuilder() + .setList(ValueList + .newBuilder() + .addAllValues(Seq( + Value + .newBuilder() + .setRecord(ValueRecord + .newBuilder() + .addAtts(ValueRecordAttr + .newBuilder() + .setName("k1") + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("v1"))))) + .build(), + Value + .newBuilder() + .setRecord(ValueRecord + .newBuilder() + .addAtts(ValueRecordAttr + .newBuilder() + .setName("k2") + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("v2"))))) + .build()).asJava)))) + .addColumns( + Column + .newBuilder() + .setName("col_jsonb_array") + .setData( + Value + .newBuilder() + .setList(ValueList + .newBuilder() + .addAllValues(Seq( + Value + .newBuilder() + .setRecord(ValueRecord + .newBuilder() + .addAtts(ValueRecordAttr + .newBuilder() + .setName("k1") + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("v1"))))) + .build(), + Value + .newBuilder() + .setRecord(ValueRecord + .newBuilder() + .addAtts(ValueRecordAttr + .newBuilder() + .setName("k2") + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("v2"))))) + .build()).asJava)))) + .addColumns( + Column + .newBuilder() + .setName("col_hstore_array") + .setData( + Value + .newBuilder() + .setList(ValueList + .newBuilder() + .addAllValues(Seq( + Value + .newBuilder() + .setRecord( + ValueRecord + .newBuilder() + .addAtts(ValueRecordAttr + .newBuilder() + .setName("x") + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("10")))) + .addAtts(ValueRecordAttr + .newBuilder() + .setName("y") + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("20"))))) + .build(), + Value + .newBuilder() + .setRecord( + ValueRecord + .newBuilder() + .addAtts(ValueRecordAttr + .newBuilder() + .setName("a") + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("1")))) + .addAtts(ValueRecordAttr + .newBuilder() + .setName("b") + .setValue(Value.newBuilder().setString(ValueString.newBuilder().setV("2"))))) + .build()).asJava)))) + .build() + } + +}