From 7d1eb0536daf38ec7b6930350229395aff6d2f26 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 12 Mar 2026 16:54:26 +0000 Subject: [PATCH 01/15] bulk_load happy path --- .../Redshift/Redshift_Connection.enso | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index c3829fbbb622..7e314b0a501f 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -1,14 +1,21 @@ from Standard.Base import all +import Standard.Base.Errors.Illegal_Argument.Illegal_Argument +import Standard.Base.Runtime.Context import Standard.Base.Metadata.File_Type import Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data from Standard.Base.Metadata.Choice import Option from Standard.Base.Metadata.Widget import File_Browse, Single_Choice, Text_Input import Standard.Table.Rows_To_Read.Rows_To_Read -from Standard.Table import Table +import Standard.Table.Internal.In_Memory_Helpers +from Standard.Table import Table, Value_Type +from Standard.Table.Internal.Storage import from_value_type +import Standard.Table.In_Memory_Table.In_Memory_Table +import Standard.Database.Bulk_Load_Exists.Bulk_Load_Exists import Standard.Database.Column_Description.Column_Description import Standard.Database.Connection.Connection.Connection +import Standard.Database.Connection.Credentials.Credentials import Standard.Database.DB_Table as DB_Table_Module import Standard.Database.DB_Table.DB_Table import Standard.Database.Dialects.Dialect.Dialect @@ -24,12 +31,16 @@ from Standard.Database.Errors import SQL_Error, Table_Already_Exists, Table_Not_ from Standard.Database.Internal.Postgres.Helpers import get_encoding_name, parse_postgres_encoding from Standard.Database.Internal.Upload.Helpers.Default_Arguments import first_column_name_in_structure from Standard.Database.Internal.Upload.Operations.Create import create_table_implementation +import project.S3.S3_File.S3_File +import project.AWS_Credential.AWS_Credential import project.Database.Redshift.Internal.Redshift_Dialect import project.Database.Redshift.Internal.Redshift_Type_Mapping.Redshift_Type_Mapping +polyglot java import java.lang.ArithmeticException polyglot java import org.enso.database.JDBCDriverTypes polyglot java import org.enso.database.JDBCProxy +polyglot java import org.enso.table.data.column.storage.type.BigDecimalType type Redshift_Connection ## --- @@ -360,6 +371,59 @@ type Redshift_Connection execute self query = self.connection.execute query + ## --- + icon: data_upload + --- + Upload a Table to a new table in the database in the current schema. + + ## Arguments: + - `table`: The input table to upload. + - `table_name`: The name of the table to create. + - `if_exists`: What action to take if the table already exists. Defaults to + raising a `Table_Already_Exists` error. + - `Raise_Error` - a `Table_Already_Exists` error will be raised. + - `Drop_Table` - the existing table will be dropped before creating the new one. + - `Truncate_Table` - the existing table will be truncated before loading data into it. + - `Append_To_Table` - data will be appended to the existing table. + - `temporary`: If set to `True`, the created table will be temporary. + Defaults to `False`. + + ## Returns + A `DB_Table` representing the created table. + bulk_load self table:Table=(Missing_Argument.throw "table") table_name:Text=(Missing_Argument.throw "table_name") staging_bucket:Text=(Missing_Argument.throw "staging_bucket") credentials:Credentials|AWS_Credential=..Profile iam_role:Text=(Missing_Argument.throw "iam_role") if_exists:Bulk_Load_Exists=..Raise_Error temporary:Boolean=False = case table of + _ : DB_Table -> + Error.throw (Illegal_Argument.Error "Cannot bulk load from a DB_Table, use `read` to materialize or use `select_into_database_table` if on same server.") + _ : In_Memory_Table -> + ## Check for illegal column types + removed = table.remove_columns [(..By_Type ..Null), (..By_Type ..Mixed), (..By_Type ..Unsupported_Data_Type), (..By_Type ..Binary)] + if removed.column_count != table.column_count then Error.throw (Illegal_Argument.Error "The table contains columns with unsupported types (Null, Mixed, Binary, or Unsupported_Data_Type) that cannot be uploaded to DuckDB. Please remove or convert these columns before uploading.") + + ## Check if table exists + exists = self.query (..Table_Name table_name) . is_error . not + if exists && if_exists==Bulk_Load_Exists.Raise_Error then Error.throw (Table_Already_Exists.Error table_name) + + Context.Output.with_enabled <| + ## Drop existing table if needed and then create new table + created_table = if exists.not then self.create_table table_name table primary_key=[] temporary=temporary else case if_exists of + Bulk_Load_Exists.Drop_Table -> + self.drop_table table_name if_exists=True + self.create_table table_name table primary_key=[] temporary=temporary + Bulk_Load_Exists.Truncate_Table -> + self.truncate_table table_name + self.query (..Table_Name table_name) + Bulk_Load_Exists.Append_To_Table -> + self.query (..Table_Name table_name) + + ## Assuming we managed to create the table, proceed with upload + created_table.if_not_error <| + s3_file_path = staging_bucket + "/" + table_name + "_" + Random.uuid + ".csv" + s3_file = S3_File.new s3_file_path credentials + table.write s3_file format=..Delimited + self.execute ("COPY " + table_name + " FROM '" + s3_file_path + "'" + " IAM_ROLE '" + iam_role + "'" + " FORMAT AS CSV" + " IGNOREHEADER 1;") + s3_file.delete + created_table + + ## --- private: true --- From e24a78b30c88ae5692bb7fc3b8a1bd3de9313e6e Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 12 Mar 2026 16:56:06 +0000 Subject: [PATCH 02/15] Update docs --- .../Redshift/Redshift_Connection.enso | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index 7e314b0a501f..fb4bcf184e9d 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -374,22 +374,41 @@ type Redshift_Connection ## --- icon: data_upload --- - Upload a Table to a new table in the database in the current schema. + Bulk load a Table into Redshift using the COPY command via S3 staging. + + This method uploads the table as a CSV file to S3, then uses Redshift's + COPY command to efficiently load the data into the database. The staging + file is automatically deleted after the load completes. ## Arguments: - - `table`: The input table to upload. - - `table_name`: The name of the table to create. + - `table`: The input table to upload. Must be an in-memory table; + DB_Tables are not supported. + - `table_name`: The name of the table to create in Redshift. + - `staging_bucket`: The S3 bucket path to use for staging the CSV file + (e.g., "s3://my-bucket/staging"). A unique filename will be generated + automatically. + - `credentials`: AWS credentials to use for S3 access. Defaults to + `AWS_Credential.Profile` which uses the default AWS profile. + - `iam_role`: The IAM role ARN that Redshift should assume to access the + S3 staging bucket (e.g., "arn:aws:iam::123456789012:role/MyRedshiftRole"). - `if_exists`: What action to take if the table already exists. Defaults to - raising a `Table_Already_Exists` error. + `Raise_Error`. - `Raise_Error` - a `Table_Already_Exists` error will be raised. - `Drop_Table` - the existing table will be dropped before creating the new one. - `Truncate_Table` - the existing table will be truncated before loading data into it. - `Append_To_Table` - data will be appended to the existing table. - `temporary`: If set to `True`, the created table will be temporary. - Defaults to `False`. + Defaults to `False`. ## Returns - A `DB_Table` representing the created table. + A `DB_Table` representing the created or updated table. + + ## Errors + - `Illegal_Argument` if the table contains unsupported column types (Null, + Mixed, Binary, or Unsupported_Data_Type), or if a `DB_Table` is provided + instead of an in-memory table. + - `Table_Already_Exists` if the table exists and `if_exists` is set to + `Raise_Error`. bulk_load self table:Table=(Missing_Argument.throw "table") table_name:Text=(Missing_Argument.throw "table_name") staging_bucket:Text=(Missing_Argument.throw "staging_bucket") credentials:Credentials|AWS_Credential=..Profile iam_role:Text=(Missing_Argument.throw "iam_role") if_exists:Bulk_Load_Exists=..Raise_Error temporary:Boolean=False = case table of _ : DB_Table -> Error.throw (Illegal_Argument.Error "Cannot bulk load from a DB_Table, use `read` to materialize or use `select_into_database_table` if on same server.") From 95140c35282b0c6ec55b14c10fb344fc5f0c96d1 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 13 Mar 2026 09:20:27 +0000 Subject: [PATCH 03/15] Check execution context --- .../0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso | 4 ++++ .../lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index fb4bcf184e9d..f77124a72653 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -421,6 +421,10 @@ type Redshift_Connection exists = self.query (..Table_Name table_name) . is_error . not if exists && if_exists==Bulk_Load_Exists.Raise_Error then Error.throw (Table_Already_Exists.Error table_name) + ## Check Execution Context + Execution_Context.Output.if_enabled disabled_message="As writing is disabled, cannot load data. Press the Write button ▶ to perform the operation." panic=False <| + Nothing + Context.Output.with_enabled <| ## Drop existing table if needed and then create new table created_table = if exists.not then self.create_table table_name table primary_key=[] temporary=temporary else case if_exists of diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso index 50c792d32cd7..e2b50e22bf00 100644 --- a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso @@ -571,7 +571,7 @@ type DuckDB_Connection if exists && if_exists==Bulk_Load_Exists.Raise_Error then Error.throw (Table_Already_Exists.Error table_name) ## Check Execution Context - if self.is_in_memory.not then Context.Output.if_enabled disabled_message="As writing is disabled, cannot create a new index. Press the Write button ▶ to perform the operation." panic=False <| + if self.is_in_memory.not then Context.Output.if_enabled disabled_message="As writing is disabled, cannot load data. Press the Write button ▶ to perform the operation." panic=False <| Nothing Context.Output.with_enabled <| From 5d2a1c34762571ee3c26e8adebaab9d93038f124 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 13 Mar 2026 13:26:08 +0000 Subject: [PATCH 04/15] staging_bucket widget --- .../Database/Redshift/Redshift_Connection.enso | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index f77124a72653..d0b8188fd3a5 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -33,6 +33,7 @@ from Standard.Database.Internal.Upload.Helpers.Default_Arguments import first_co from Standard.Database.Internal.Upload.Operations.Create import create_table_implementation import project.S3.S3_File.S3_File import project.AWS_Credential.AWS_Credential +import project.S3.S3 import project.Database.Redshift.Internal.Redshift_Dialect import project.Database.Redshift.Internal.Redshift_Type_Mapping.Redshift_Type_Mapping @@ -409,7 +410,8 @@ type Redshift_Connection instead of an in-memory table. - `Table_Already_Exists` if the table exists and `if_exists` is set to `Raise_Error`. - bulk_load self table:Table=(Missing_Argument.throw "table") table_name:Text=(Missing_Argument.throw "table_name") staging_bucket:Text=(Missing_Argument.throw "staging_bucket") credentials:Credentials|AWS_Credential=..Profile iam_role:Text=(Missing_Argument.throw "iam_role") if_exists:Bulk_Load_Exists=..Raise_Error temporary:Boolean=False = case table of + @staging_bucket _make_staging_bucket_widget + bulk_load self table:Table=(Missing_Argument.throw "table") table_name:Text=(Missing_Argument.throw "table_name") staging_bucket_credentials:Credentials|AWS_Credential=..Profile staging_bucket:Text=(Missing_Argument.throw "staging_bucket") iam_role:Text=(Missing_Argument.throw "iam_role") if_exists:Bulk_Load_Exists=..Raise_Error temporary:Boolean=False = case table of _ : DB_Table -> Error.throw (Illegal_Argument.Error "Cannot bulk load from a DB_Table, use `read` to materialize or use `select_into_database_table` if on same server.") _ : In_Memory_Table -> @@ -422,7 +424,7 @@ type Redshift_Connection if exists && if_exists==Bulk_Load_Exists.Raise_Error then Error.throw (Table_Already_Exists.Error table_name) ## Check Execution Context - Execution_Context.Output.if_enabled disabled_message="As writing is disabled, cannot load data. Press the Write button ▶ to perform the operation." panic=False <| + Context.Output.if_enabled disabled_message="As writing is disabled, cannot load data. Press the Write button ▶ to perform the operation." panic=False <| Nothing Context.Output.with_enabled <| @@ -440,13 +442,14 @@ type Redshift_Connection ## Assuming we managed to create the table, proceed with upload created_table.if_not_error <| s3_file_path = staging_bucket + "/" + table_name + "_" + Random.uuid + ".csv" - s3_file = S3_File.new s3_file_path credentials - table.write s3_file format=..Delimited + s3_file = S3_File.new s3_file_path staging_bucket_credentials + if s3_file.is_error then Error.throw s3_file + written_s3_file = table.write s3_file format=..Delimited + self.execute ("COPY " + table_name + " FROM '" + s3_file_path + "'" + " IAM_ROLE '" + iam_role + "'" + " FORMAT AS CSV" + " IGNOREHEADER 1;") s3_file.delete created_table - ## --- private: true --- @@ -529,6 +532,11 @@ type Redshift_Connection to_js_object self = JS_Object.from_pairs <| [["type", "Redshift_Connection"], ["links", self.connection.tables.at "Name" . to_vector]] +private _make_staging_bucket_widget connection cache=Nothing = + cached_credentials = cache.if_not_nothing <| cache "staging_bucket_credentials" + credentials = if cached_credentials.is_nothing then AWS_Credential.Default else cached_credentials + Single_Choice display=..Always values=((S3.list_buckets credentials).map c-> (Option c ("'"+c+"'"))) + ## --- private: true --- From 154a8ec2f806c1fbd088a26937624a0ebb04e175 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 13 Mar 2026 14:11:42 +0000 Subject: [PATCH 05/15] More error handling --- .../0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index d0b8188fd3a5..2a23f949345c 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -441,10 +441,13 @@ type Redshift_Connection ## Assuming we managed to create the table, proceed with upload created_table.if_not_error <| - s3_file_path = staging_bucket + "/" + table_name + "_" + Random.uuid + ".csv" + normalized_staging_bucket = if staging_bucket.starts_with "s3://" then staging_bucket else "s3://" + staging_bucket + "/" + normalized_staging_bucket2 = if normalized_staging_bucket.ends_with "/" then normalized_staging_bucket else normalized_staging_bucket + "/" + s3_file_path = normalized_staging_bucket2 + table_name + "_" + Random.uuid + ".csv" s3_file = S3_File.new s3_file_path staging_bucket_credentials if s3_file.is_error then Error.throw s3_file written_s3_file = table.write s3_file format=..Delimited + if written_s3_file.is_error then Error.throw written_s3_file self.execute ("COPY " + table_name + " FROM '" + s3_file_path + "'" + " IAM_ROLE '" + iam_role + "'" + " FORMAT AS CSV" + " IGNOREHEADER 1;") s3_file.delete From ce408e20a4099073a7c24cac72056d9effc75494 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 13 Mar 2026 14:18:46 +0000 Subject: [PATCH 06/15] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b38495c48dd..2c845b4caec5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ - [New `Profile` API for timing code execution.][14827] - [Expanded S3 API with versions and signed uri][14831] - [Support for reading JSON data from database connections.][14872] +- [Add Redshift bulk loading][14860] [14522]: https://github.com/enso-org/enso/pull/14522 [14476]: https://github.com/enso-org/enso/pull/14476 @@ -56,6 +57,7 @@ [14827]: https://github.com/enso-org/enso/pull/14827 [14831]: https://github.com/enso-org/enso/pull/14831 [14872]: https://github.com/enso-org/enso/pull/14872 +[14860]: https://github.com/enso-org/enso/pull/14860 #### Enso Language & Runtime From 3b6119b4e01cc42bda3063f5988473adfbb0dbbd Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 13 Mar 2026 14:25:45 +0000 Subject: [PATCH 07/15] import ordering --- .../src/Database/Redshift/Redshift_Connection.enso | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index 2a23f949345c..bc645b5c172a 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -1,17 +1,17 @@ from Standard.Base import all import Standard.Base.Errors.Illegal_Argument.Illegal_Argument -import Standard.Base.Runtime.Context import Standard.Base.Metadata.File_Type +import Standard.Base.Runtime.Context import Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data from Standard.Base.Metadata.Choice import Option from Standard.Base.Metadata.Widget import File_Browse, Single_Choice, Text_Input -import Standard.Table.Rows_To_Read.Rows_To_Read -import Standard.Table.Internal.In_Memory_Helpers from Standard.Table import Table, Value_Type from Standard.Table.Internal.Storage import from_value_type - import Standard.Table.In_Memory_Table.In_Memory_Table +import Standard.Table.Internal.In_Memory_Helpers +import Standard.Table.Rows_To_Read.Rows_To_Read + import Standard.Database.Bulk_Load_Exists.Bulk_Load_Exists import Standard.Database.Column_Description.Column_Description import Standard.Database.Connection.Connection.Connection @@ -31,12 +31,12 @@ from Standard.Database.Errors import SQL_Error, Table_Already_Exists, Table_Not_ from Standard.Database.Internal.Postgres.Helpers import get_encoding_name, parse_postgres_encoding from Standard.Database.Internal.Upload.Helpers.Default_Arguments import first_column_name_in_structure from Standard.Database.Internal.Upload.Operations.Create import create_table_implementation -import project.S3.S3_File.S3_File -import project.AWS_Credential.AWS_Credential -import project.S3.S3 import project.Database.Redshift.Internal.Redshift_Dialect import project.Database.Redshift.Internal.Redshift_Type_Mapping.Redshift_Type_Mapping +import project.AWS_Credential.AWS_Credential +import project.S3.S3 +import project.S3.S3_File.S3_File polyglot java import java.lang.ArithmeticException polyglot java import org.enso.database.JDBCDriverTypes From ed6966c8c3419380462a1208b65d2c86cce12ac7 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Mon, 16 Mar 2026 10:07:11 +0000 Subject: [PATCH 08/15] typo --- .../0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index bc645b5c172a..bc226a58673a 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -417,7 +417,7 @@ type Redshift_Connection _ : In_Memory_Table -> ## Check for illegal column types removed = table.remove_columns [(..By_Type ..Null), (..By_Type ..Mixed), (..By_Type ..Unsupported_Data_Type), (..By_Type ..Binary)] - if removed.column_count != table.column_count then Error.throw (Illegal_Argument.Error "The table contains columns with unsupported types (Null, Mixed, Binary, or Unsupported_Data_Type) that cannot be uploaded to DuckDB. Please remove or convert these columns before uploading.") + if removed.column_count != table.column_count then Error.throw (Illegal_Argument.Error "The table contains columns with unsupported types (Null, Mixed, Binary, or Unsupported_Data_Type) that cannot be uploaded to Redshift. Please remove or convert these columns before uploading.") ## Check if table exists exists = self.query (..Table_Name table_name) . is_error . not From 4206807cd905fc0f8fbab1f06bf585ff696ff048 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Mon, 16 Mar 2026 10:10:29 +0000 Subject: [PATCH 09/15] Remove duplicate check --- .../Redshift/Redshift_Connection.enso | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index bc226a58673a..642392c2dc34 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -427,31 +427,29 @@ type Redshift_Connection Context.Output.if_enabled disabled_message="As writing is disabled, cannot load data. Press the Write button ▶ to perform the operation." panic=False <| Nothing - Context.Output.with_enabled <| - ## Drop existing table if needed and then create new table - created_table = if exists.not then self.create_table table_name table primary_key=[] temporary=temporary else case if_exists of - Bulk_Load_Exists.Drop_Table -> - self.drop_table table_name if_exists=True - self.create_table table_name table primary_key=[] temporary=temporary - Bulk_Load_Exists.Truncate_Table -> - self.truncate_table table_name - self.query (..Table_Name table_name) - Bulk_Load_Exists.Append_To_Table -> - self.query (..Table_Name table_name) - - ## Assuming we managed to create the table, proceed with upload - created_table.if_not_error <| - normalized_staging_bucket = if staging_bucket.starts_with "s3://" then staging_bucket else "s3://" + staging_bucket + "/" - normalized_staging_bucket2 = if normalized_staging_bucket.ends_with "/" then normalized_staging_bucket else normalized_staging_bucket + "/" - s3_file_path = normalized_staging_bucket2 + table_name + "_" + Random.uuid + ".csv" - s3_file = S3_File.new s3_file_path staging_bucket_credentials - if s3_file.is_error then Error.throw s3_file - written_s3_file = table.write s3_file format=..Delimited - if written_s3_file.is_error then Error.throw written_s3_file - - self.execute ("COPY " + table_name + " FROM '" + s3_file_path + "'" + " IAM_ROLE '" + iam_role + "'" + " FORMAT AS CSV" + " IGNOREHEADER 1;") - s3_file.delete - created_table + created_table = if exists.not then self.create_table table_name table primary_key=[] temporary=temporary else case if_exists of + Bulk_Load_Exists.Drop_Table -> + self.drop_table table_name if_exists=True + self.create_table table_name table primary_key=[] temporary=temporary + Bulk_Load_Exists.Truncate_Table -> + self.truncate_table table_name + self.query (..Table_Name table_name) + Bulk_Load_Exists.Append_To_Table -> + self.query (..Table_Name table_name) + + ## Assuming we managed to create the table, proceed with upload + created_table.if_not_error <| + normalized_staging_bucket = if staging_bucket.starts_with "s3://" then staging_bucket else "s3://" + staging_bucket + "/" + normalized_staging_bucket2 = if normalized_staging_bucket.ends_with "/" then normalized_staging_bucket else normalized_staging_bucket + "/" + s3_file_path = normalized_staging_bucket2 + table_name + "_" + Random.uuid + ".csv" + s3_file = S3_File.new s3_file_path staging_bucket_credentials + if s3_file.is_error then Error.throw s3_file + written_s3_file = table.write s3_file format=..Delimited + if written_s3_file.is_error then Error.throw written_s3_file + + self.execute ("COPY " + table_name + " FROM '" + s3_file_path + "'" + " IAM_ROLE '" + iam_role + "'" + " FORMAT AS CSV" + " IGNOREHEADER 1;") + s3_file.delete + created_table ## --- private: true From f6a6b0a23e0c37d0ccff13aa324ba51433ddc4a9 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Mon, 16 Mar 2026 10:13:08 +0000 Subject: [PATCH 10/15] Refactor --- .../Redshift/Redshift_Connection.enso | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index 642392c2dc34..022798c1e4d0 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -427,6 +427,14 @@ type Redshift_Connection Context.Output.if_enabled disabled_message="As writing is disabled, cannot load data. Press the Write button ▶ to perform the operation." panic=False <| Nothing + normalized_staging_bucket = if staging_bucket.starts_with "s3://" then staging_bucket else "s3://" + staging_bucket + "/" + normalized_staging_bucket2 = if normalized_staging_bucket.ends_with "/" then normalized_staging_bucket else normalized_staging_bucket + "/" + s3_file_path = normalized_staging_bucket2 + table_name + "_" + Random.uuid + ".csv" + s3_file = S3_File.new s3_file_path staging_bucket_credentials + if s3_file.is_error then Error.throw s3_file + written_s3_file = table.write s3_file format=..Delimited + if written_s3_file.is_error then Error.throw written_s3_file + created_table = if exists.not then self.create_table table_name table primary_key=[] temporary=temporary else case if_exists of Bulk_Load_Exists.Drop_Table -> self.drop_table table_name if_exists=True @@ -437,19 +445,12 @@ type Redshift_Connection Bulk_Load_Exists.Append_To_Table -> self.query (..Table_Name table_name) - ## Assuming we managed to create the table, proceed with upload + ## Assuming we managed to create the table, proceed with copy created_table.if_not_error <| - normalized_staging_bucket = if staging_bucket.starts_with "s3://" then staging_bucket else "s3://" + staging_bucket + "/" - normalized_staging_bucket2 = if normalized_staging_bucket.ends_with "/" then normalized_staging_bucket else normalized_staging_bucket + "/" - s3_file_path = normalized_staging_bucket2 + table_name + "_" + Random.uuid + ".csv" - s3_file = S3_File.new s3_file_path staging_bucket_credentials - if s3_file.is_error then Error.throw s3_file - written_s3_file = table.write s3_file format=..Delimited - if written_s3_file.is_error then Error.throw written_s3_file - self.execute ("COPY " + table_name + " FROM '" + s3_file_path + "'" + " IAM_ROLE '" + iam_role + "'" + " FORMAT AS CSV" + " IGNOREHEADER 1;") - s3_file.delete - created_table + + s3_file.delete + created_table ## --- private: true @@ -534,6 +535,7 @@ type Redshift_Connection JS_Object.from_pairs <| [["type", "Redshift_Connection"], ["links", self.connection.tables.at "Name" . to_vector]] private _make_staging_bucket_widget connection cache=Nothing = + _ = connection cached_credentials = cache.if_not_nothing <| cache "staging_bucket_credentials" credentials = if cached_credentials.is_nothing then AWS_Credential.Default else cached_credentials Single_Choice display=..Always values=((S3.list_buckets credentials).map c-> (Option c ("'"+c+"'"))) From 4a84dde97e140be1ec2a8236db96e2fb09774b1a Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Mon, 16 Mar 2026 10:16:17 +0000 Subject: [PATCH 11/15] Lint fixes --- .../0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index 022798c1e4d0..29a1765f4c7e 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -6,8 +6,7 @@ import Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data from Standard.Base.Metadata.Choice import Option from Standard.Base.Metadata.Widget import File_Browse, Single_Choice, Text_Input -from Standard.Table import Table, Value_Type -from Standard.Table.Internal.Storage import from_value_type +from Standard.Table import Table import Standard.Table.In_Memory_Table.In_Memory_Table import Standard.Table.Internal.In_Memory_Helpers import Standard.Table.Rows_To_Read.Rows_To_Read From 2a6f4ebee02a76b07ecee96c066dd07190863dc7 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Tue, 17 Mar 2026 14:11:29 +0000 Subject: [PATCH 12/15] datetimefix --- .../0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index 29a1765f4c7e..c28e56a2ffd5 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -431,7 +431,8 @@ type Redshift_Connection s3_file_path = normalized_staging_bucket2 + table_name + "_" + Random.uuid + ".csv" s3_file = S3_File.new s3_file_path staging_bucket_credentials if s3_file.is_error then Error.throw s3_file - written_s3_file = table.write s3_file format=..Delimited + normalized_datetime_table = table.format [(..By_Type ..Date_Time)] 'yyyy-MM-dd HH:mm:ssz' + written_s3_file = normalized_datetime_table.write s3_file format=..Delimited if written_s3_file.is_error then Error.throw written_s3_file created_table = if exists.not then self.create_table table_name table primary_key=[] temporary=temporary else case if_exists of From 0fa60fa4368ba74d94b06f8a3a05421c6b599fb2 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Tue, 17 Mar 2026 15:19:42 +0000 Subject: [PATCH 13/15] API --- .../0.0.0-dev/docs/api/Database/Redshift/Redshift_Connection.md | 1 + 1 file changed, 1 insertion(+) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/docs/api/Database/Redshift/Redshift_Connection.md b/distribution/lib/Standard/AWS/0.0.0-dev/docs/api/Database/Redshift/Redshift_Connection.md index 0f5956cde9e9..916147e83d86 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/docs/api/Database/Redshift/Redshift_Connection.md +++ b/distribution/lib/Standard/AWS/0.0.0-dev/docs/api/Database/Redshift/Redshift_Connection.md @@ -2,6 +2,7 @@ ## module Standard.AWS.Database.Redshift.Redshift_Connection - type Redshift_Connection - base_connection self -> Standard.Base.Any.Any + - bulk_load self table:Standard.Table.Table.Table= table_name:Standard.Base.Data.Text.Text= staging_bucket_credentials:(Standard.Database.Connection.Credentials.Credentials|Standard.AWS.AWS_Credential.AWS_Credential)= staging_bucket:Standard.Base.Data.Text.Text= iam_role:Standard.Base.Data.Text.Text= if_exists:Standard.Database.Bulk_Load_Exists.Bulk_Load_Exists= temporary:Standard.Base.Data.Boolean.Boolean= -> Standard.Base.Any.Any - close self -> Standard.Base.Any.Any - create url:Standard.Base.Any.Any properties:Standard.Base.Any.Any make_new:Standard.Base.Any.Any -> Standard.Base.Any.Any - create_literal_table self source:Standard.Table.Table.Table alias:Standard.Base.Data.Text.Text -> (Standard.Table.Table.Table&Standard.Database.DB_Table.DB_Table&Standard.Base.Any.Any) From 91072983d43fec9fd80a8fe22d2583fd134b3b49 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Wed, 18 Mar 2026 20:12:57 +0000 Subject: [PATCH 14/15] Code review changes --- .../src/Database/Redshift/Redshift_Connection.enso | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index c28e56a2ffd5..d87531ff60f3 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -431,10 +431,9 @@ type Redshift_Connection s3_file_path = normalized_staging_bucket2 + table_name + "_" + Random.uuid + ".csv" s3_file = S3_File.new s3_file_path staging_bucket_credentials if s3_file.is_error then Error.throw s3_file - normalized_datetime_table = table.format [(..By_Type ..Date_Time)] 'yyyy-MM-dd HH:mm:ssz' - written_s3_file = normalized_datetime_table.write s3_file format=..Delimited - if written_s3_file.is_error then Error.throw written_s3_file - + written_s3_file = table.write s3_file format=(..Delimited value_formatter=(Data_Formatter.Value datetime_formats=['yyyy-MM-dd HH:mm:ssz'])) + Error.return_if_error written_s3_file + Error.return_if_error written_s3_file created_table = if exists.not then self.create_table table_name table primary_key=[] temporary=temporary else case if_exists of Bulk_Load_Exists.Drop_Table -> self.drop_table table_name if_exists=True From 8945dbebcd672735652e5cfbb8b59f6e2f1c6a6c Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 19 Mar 2026 10:41:23 +0000 Subject: [PATCH 15/15] Code review --- .../src/Database/Redshift/Redshift_Connection.enso | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso index d87531ff60f3..8e022d3357e6 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Redshift_Connection.enso @@ -6,7 +6,7 @@ import Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data from Standard.Base.Metadata.Choice import Option from Standard.Base.Metadata.Widget import File_Browse, Single_Choice, Text_Input -from Standard.Table import Table +from Standard.Table import Table, Data_Formatter import Standard.Table.In_Memory_Table.In_Memory_Table import Standard.Table.Internal.In_Memory_Helpers import Standard.Table.Rows_To_Read.Rows_To_Read @@ -446,7 +446,10 @@ type Redshift_Connection ## Assuming we managed to create the table, proceed with copy created_table.if_not_error <| - self.execute ("COPY " + table_name + " FROM '" + s3_file_path + "'" + " IAM_ROLE '" + iam_role + "'" + " FORMAT AS CSV" + " IGNOREHEADER 1;") + result = self.execute ("COPY " + table_name + " FROM '" + s3_file_path + "'" + " IAM_ROLE '" + iam_role + "'" + " FORMAT AS CSV" + " IGNOREHEADER 1;") + if result.is_error then + s3_file.delete + Error.throw result s3_file.delete created_table