SQL Query API + Job Updates (#104)

* Add new job tasks, add support for run_if, change to 2.2 API

* docs and tests

* adjusting job tests, docs for new job tasks, data security mode added.

* fixing issue with SQL tasks, minor edits to vignette

* Added SQL queries API coverage + tests, SQL task fixes.

* Updating NEWS.md and pkgdown.yml

---------

Co-authored-by: Zac Davies <[email protected]>

Author: zacdav-db

NAMESPACE

@@ -9,6 +9,7 @@ export(azure_attributes)
 export(close_workspace)
 export(cluster_autoscale)
 export(cluster_log_conf)
+export(condition_task)
 export(cron_schedule)
 export(db_cluster_create)
 export(db_cluster_delete)
@@ -76,6 +77,11 @@ export(db_mlflow_model_version_comment)
 export(db_mlflow_model_version_comment_delete)
 export(db_mlflow_model_version_comment_edit)
 export(db_mlflow_registered_model_details)
+export(db_query_create)
+export(db_query_delete)
+export(db_query_get)
+export(db_query_list)
+export(db_query_update)
 export(db_read_netrc)
 export(db_repl)
 export(db_repo_create)
@@ -160,6 +166,7 @@ export(email_notifications)
 export(embedding_source_column)
 export(embedding_vector_column)
 export(file_storage_info)
+export(for_each_task)
 export(gcp_attributes)
 export(get_and_start_cluster)
 export(get_and_start_warehouse)
@@ -174,6 +181,7 @@ export(is.aws_attributes)
 export(is.azure_attributes)
 export(is.cluster_autoscale)
 export(is.cluster_log_conf)
+export(is.condition_task)
 export(is.cron_schedule)
 export(is.dbfs_storage_info)
 export(is.delta_sync_index)
@@ -183,6 +191,7 @@ export(is.email_notifications)
 export(is.embedding_source_column)
 export(is.embedding_vector_column)
 export(is.file_storage_info)
+export(is.for_each_task)
 export(is.gcp_attributes)
 export(is.git_source)
 export(is.init_script_info)
@@ -199,10 +208,13 @@ export(is.new_cluster)
 export(is.notebook_task)
 export(is.pipeline_task)
 export(is.python_wheel_task)
+export(is.run_job_task)
 export(is.s3_storage_info)
 export(is.spark_jar_task)
 export(is.spark_python_task)
 export(is.spark_submit_task)
+export(is.sql_file_task)
+export(is.sql_query_task)
 export(is.valid_task_type)
 export(is.vector_search_index_spec)
 export(job_task)
@@ -220,10 +232,13 @@ export(open_workspace)
 export(pipeline_task)
 export(python_wheel_task)
 export(remove_lib_path)
+export(run_job_task)
 export(s3_storage_info)
 export(spark_jar_task)
 export(spark_python_task)
 export(spark_submit_task)
+export(sql_file_task)
+export(sql_query_task)
 export(wait_for_lib_installs)
 import(cli)
 import(httr2)

NEWS.md

@@ -1,5 +1,8 @@
 # brickster 0.2.8
 
+* Added SQL Queries API coverage
+* Updated Jobs to 2.2
+* Added additional tasks for jobs: `for_each_task`, `condition_task`, `sql_query_task`, `sql_file_task`, `run_job_task`
 * Removing the Python SQL connector as `db_sql_query` supersedes it.
 * Added `db_sql_query` to simplify execution of SQL
 * Adjusted `db_repl` to handle mulit-line expressions (R only)

R/clusters.R

@@ -68,6 +68,9 @@
 #' encryption of disks locally attached to the cluster is enabled.
 #' @param docker_image Instance of [docker_image()].
 #' @param policy_id String, ID of a cluster policy.
+#' @param kind The kind of compute described by this compute specification.
+#' @param data_security_mode Data security mode decides what data governance
+#' model to use when accessing data from a cluster.
 #' @inheritParams auth_params
 #' @inheritParams db_sql_warehouse_create
 #'
@@ -114,6 +117,19 @@ db_cluster_create <- function(
   enable_local_disk_encryption = TRUE,
   docker_image = NULL,
   policy_id = NULL,
+  kind = c("CLASSIC_PREVIEW"),
+  data_security_mode = c(
+    "NONE",
+    "SINGLE_USER",
+    "USER_ISOLATION",
+    "LEGACY_TABLE_ACL",
+    "LEGACY_PASSTHROUGH",
+    "LEGACY_SINGLE_USER",
+    "LEGACY_SINGLE_USER_STANDARD",
+    "DATA_SECURITY_MODE_STANDARD",
+    "DATA_SECURITY_MODE_DEDICATED",
+    "DATA_SECURITY_MODE_AUTO"
+  ),
   host = db_host(),
   token = db_token(),
   perform_request = TRUE
@@ -124,6 +140,9 @@ db_cluster_create <- function(
   # - all values in init_scripts must be from init_script_info() (class: InitScriptInfo)
   # - if specified, docker_image must be of class DockerImage
 
+  kind <- match.arg(kind)
+  data_security_mode <- match.arg(data_security_mode)
+
   body <- list(
     cluster_name = name,
     spark_version = spark_version,
@@ -143,7 +162,9 @@ db_cluster_create <- function(
     apply_policy_default_values = apply_policy_default_values,
     enable_local_disk_encryption = enable_local_disk_encryption,
     docker_image = docker_image,
-    policy_id = policy_id
+    policy_id = policy_id,
+    data_security_mode = data_security_mode,
+    kind = kind
   )
 
   if (is.null(num_workers)) {
@@ -229,6 +250,19 @@ db_cluster_edit <- function(
   enable_local_disk_encryption = NULL,
   docker_image = NULL,
   policy_id = NULL,
+  kind = c("CLASSIC_PREVIEW"),
+  data_security_mode = c(
+    "NONE",
+    "SINGLE_USER",
+    "USER_ISOLATION",
+    "LEGACY_TABLE_ACL",
+    "LEGACY_PASSTHROUGH",
+    "LEGACY_SINGLE_USER",
+    "LEGACY_SINGLE_USER_STANDARD",
+    "DATA_SECURITY_MODE_STANDARD",
+    "DATA_SECURITY_MODE_DEDICATED",
+    "DATA_SECURITY_MODE_AUTO"
+  ),
   host = db_host(),
   token = db_token(),
   perform_request = TRUE
@@ -242,6 +276,8 @@ db_cluster_edit <- function(
   # - if specified, log_conf must be of class ClusterLogConf
   # - all values in init_scripts must be from init_script_info() (class: InitScriptInfo)
   # - if specified, docker_image must be of class DockerImage
+  kind <- match.arg(kind)
+  data_security_mode <- match.arg(data_security_mode)
 
   body <- list(
     cluster_id = cluster_id,
@@ -263,7 +299,9 @@ db_cluster_edit <- function(
     apply_policy_default_values = apply_policy_default_values,
     enable_local_disk_encryption = enable_local_disk_encryption,
     docker_image = docker_image,
-    policy_id = policy_id
+    policy_id = policy_id,
+    data_security_mode = data_security_mode,
+    kind = kind
   )
 
   if (!(is.null(num_workers) && is.null(autoscale))) {