Ingest v2: Snowflake connectors - add example SQL statements and more how-to instructions, and expand details for role and unique record ID (#450)

Author: Paul-Cornell

snippets/destination_connectors/snowflake.sh.mdx

@@ -6,8 +6,6 @@
 unstructured \
   local \
     --input-path $LOCAL_FILE_INPUT_DIR \
-    --partition-by-api \
-    --strategy hi_res \
     --chunking-strategy by_title \
     --embedding-provider huggingface \
     --partition-by-api \
@@ -18,10 +16,13 @@ unstructured \
     --account $SNOWFLAKE_ACCOUNT \
     --user $SNOWFLAKE_USER \
     --password $SNOWFLAKE_PASSWORD \
+    --role $SNOWFLAKE_ROLE \
     --host $SNOWFLAKE_HOST \
     --port $SNOWFLAKE_PORT \
     --database $SNOWFLAKE_DATABASE \
-    --schema PUBLIC \
-    --table-name elements \
-    --batch-size 50
+    --schema $SNOWFLAKE_SCHEMA \
+    --batch-size 50 \
+    --table-name $SNOWFLAKE_TABLE \
+    --record-id-key $SNOWFLAKE_RECORD_ID_KEY \
+
 ```

snippets/destination_connectors/snowflake.v2.py.mdx

@@ -40,18 +40,22 @@ if __name__ == "__main__":
         chunker_config=ChunkerConfig(chunking_strategy="by_title"),
         embedder_config=EmbedderConfig(embedding_provider="huggingface"),
         destination_connection_config=SnowflakeConnectionConfig(
-            access_config=SnowflakeAccessConfig(password=os.getenv("SNOWFLAKE_PASSWORD")),
+            access_config=SnowflakeAccessConfig(
+                password=os.getenv("SNOWFLAKE_PASSWORD")
+            ),
             account=os.getenv("SNOWFLAKE_ACCOUNT"),
             user=os.getenv("SNOWFLAKE_USER"),
             host=os.getenv("SNOWFLAKE_HOST"),
             port=os.getenv("SNOWFLAKE_PORT"),
             database=os.getenv("SNOWFLAKE_DATABASE"),
-            schema="PUBLIC"
+            schema=os.getenv("SNOWFLAKE_SCHEMA"),
+            role=os.getenv("SNOWFLAKE_ROLE")
         ),
         stager_config=SnowflakeUploadStagerConfig(),
         uploader_config=SnowflakeUploaderConfig(
             batch_size=50,
-            table_name="elements"
+            table_name=os.getenv("SNOWFLAKE_TABLE"),
+            record_id_key=os.getenv("SNOWFLAKE_RECORD_ID_KEY")
         )
     ).run()
 ```

snippets/general-shared-text/snowflake-cli-api.mdx

@@ -10,9 +10,16 @@ import AdditionalIngestDependencies from '/snippets/general-shared-text/ingest-d
 
 These environment variables:
 
-- `SNOWFLAKE_ACCOUNT` - The ID of the Snowflake account, represented by `--account` (CLI) or `account` (Python).
-- `SNOWFLAKE_USER`  - The name of the Snowflake user, represented by `--user` (CLI) or `user` (Python).
+- `SNOWFLAKE_ACCOUNT` - The ID of the target Snowflake account, represented by `--account` (CLI) or `account` (Python).
+- `SNOWFLAKE_USER`  - The name of the target Snowflake user, represented by `--user` (CLI) or `user` (Python).
 - `SNOWFLAKE_PASSWORD`  - The user's password, represented by `--password` (CLI) or `password` (Python).
-- `SNOWFLAKE_HOST`  - The hostname for the Snowflake account, represented by `--host` (CLI) or `host` (Python).
-- `SNOWFLAKE_PORT`  - The host's port number, represented by `--port` (CLI) or `port` (Python).
-- `SNOWFLAKE_DATABASE`  - The name of the Snowflake database, represented by `--database` (CLI) or `database` (Python).
+- `SNOWFLAKE_ROLE` - The target role for the user, represented by `--role` (CLI) or `role` (Python).
+- `SNOWFLAKE_HOST`  - The hostname for the target Snowflake warehouse, represented by `--host` (CLI) or `host` (Python).
+- `SNOWFLAKE_PORT`  - The warehouse's port number, represented by `--port` (CLI) or `port` (Python).  The default is `443` if not otherwise specified.
+- `SNOWFLAKE_DATABASE`  - The name of the target Snowflake database, represented by `--database` (CLI) or `database` (Python).
+- `SNOWFLAKE_SCHEMA` - The name of the target schema in the database, represented by `--schema` (CLI) or `schema` (Python).
+- `SNOWFLAKE_TABLE` - The name of the target table in the schema, represented by `--table-name` (CLI) or `table_name` (Python). For the destination connector, the default is `elements` if not otherwise specified.
+- `SNOWFLAKE_RECORD_ID_KEY` - The name of the column in the table that uniquely identifies each record, represented by:
+
+  - For the source connector, `--id-column` (CLI) or `id_column` (Python).
+  - For the destination connector, `--record-id-key` (CLI) or `record_id_key` (Python). For the destination connector, the default is `record_id` if not otherwise specified.

snippets/general-shared-text/snowflake.mdx

@@ -1,4 +1,4 @@
-- A Snowflake [account](https://signup.snowflake.com/) and its [identifier](https://docs.snowflake.com/user-guide/admin-account-identifier).
+- A Snowflake [account](https://signup.snowflake.com/) and its account identifier.
 
   <iframe
   width="560"
@@ -10,7 +10,19 @@
   allowfullscreen
   ></iframe>
 
-- The Snowflake [username and its password](https://docs.snowflake.com/user-guide/admin-user-management#creating-users) in the account.
+  To get the identifier for the current Snowflake account:
+
+  1. Log in to [Snowsight](https://docs.snowflake.com/user-guide/ui-snowsight-homepage) with your Snowflake account.
+  2. In Snowsight, on the navigation menu, click your username, and then click **Account > View account details**.
+  3. On the **Account** tab, note the value of the **Account Identifier** field.
+
+  Alternatively, the following Snowflake query returns the current account's identifier:
+
+  ```text
+  SELECT CURRENT_ORGANIZATION_NAME() || '-' || CURRENT_ACCOUNT_NAME() AS "Account Identifier"
+  ```
+
+- The Snowflake [user's login name (not its username) and its password](https://docs.snowflake.com/user-guide/admin-user-management#creating-users) in the account.
 
   <iframe
   width="560"
@@ -22,7 +34,35 @@
   allowfullscreen
   ></iframe>
 
-- The Snowflake [hostname and its port number](https://docs.snowflake.com/sql-reference/functions/system_allowlist) in the account.
+  To view the login name for a user:
+
+  1. Log in to [Snowsight](https://docs.snowflake.com/user-guide/ui-snowsight-homepage) with your Snowflake account.
+  2. In Snowsight, on the navigation menu, click **Admin > Users & Roles**.
+  3. On the **Users** tab, in the list of available users, click the name of the target user.
+  4. In the **About** tile, note the **Login Name** for the user.
+
+  Alternatively, the following Snowflake query returns information about the user with the username of `<my-user>`, including their `login_name` value representing their login name:
+
+  ```text
+  SHOW USERS LIKE '<my-user>';
+  ```
+
+- The Snowflake warehouse's [hostname and its port number](https://docs.snowflake.com/sql-reference/functions/system_allowlist) in the account. 
+
+  To view a list of available warehouses in the current Snowflake account:
+
+  1. Log in to [Snowsight](https://docs.snowflake.com/user-guide/ui-snowsight-homepage) with your Snowflake account.
+  2. In Snowsight, on the navigation menu, click **Admin > Warehouses**. This view does not provide access to the warehouses' hostnames or port numbers. To get this information, you must run a Snowflake query.
+
+  The following Snowflake query returns a list of available warehouse types, hostnames, and port numbers in the current account. Look for the row with a `type` of `SNOWFLAKE_DEPLOYMENT`:
+
+  ```text
+  SELECT t.VALUE:type::VARCHAR as type,
+         t.VALUE:host::VARCHAR as host,
+         t.VALUE:port as port
+  FROM TABLE(FLATTEN(input => PARSE_JSON(SYSTEM$ALLOWLIST()))) AS t;
+  ```
+
 - The name of the Snowflake [database](https://docs.snowflake.com/sql-reference/sql/create-database) in the account.
 
   <iframe
@@ -35,6 +75,17 @@
   allowfullscreen
   ></iframe>
 
+  To view a list of available databases in the current Snowflake account:
+
+  1. Log in to [Snowsight](https://docs.snowflake.com/user-guide/ui-snowsight-homepage) with your Snowflake account.
+  2. In Snowsight, on the navigation menu, click **Data > Databases**.
+
+  Alternatively, the following Snowflake query returns a list of available databases in the current account:
+
+  ```text
+  SHOW DATABASES;
+  ```
+
 - The name of the [schema](https://docs.snowflake.com/sql-reference/sql/create-schema) in the database.
 
   <iframe
@@ -47,6 +98,24 @@
   allowfullscreen
   ></iframe>
 
+  To view a list of available schemas for a database in the current Snowflake account:
+
+  1. Log in to [Snowsight](https://docs.snowflake.com/user-guide/ui-snowsight-homepage) with your Snowflake account.
+  2. In Snowsight, on the navigation menu, click **Data > Databases**.
+  3. Expand the name of the target database.
+
+  Alternatively, the following Snowflake query returns a list of available schemas in the current account:
+
+  ```text
+  SHOW SCHEMAS;
+  ```
+
+  The following Snowflake query returns a list of available schemas for the database named `<database-name>` in the current account:
+
+  ```text
+  SHOW SCHEMAS IN DATABASE <database-name>;
+  ```
+
 - The name of the [table](https://docs.snowflake.com/sql-reference/sql/create-table) in the schema.
 
   <iframe
@@ -59,6 +128,21 @@
   allowfullscreen
   ></iframe>
 
+  To view a list of available tables for a schema in a database in the current Snowflake account:
+
+  1. Log in to [Snowsight](https://docs.snowflake.com/user-guide/ui-snowsight-homepage) with your Snowflake account.
+  2. In Snowsight, on the navigation menu, click **Data > Databases**.
+  3. Expand the name of the database that contains the target schema.
+  4. Expand the name of the target schema. 
+  5. Expand **Tables**.
+
+  Alternatively, the following Snowflake query returns a list of available tables for the schema named `<schema-name>` in the datbase named 
+  `<database-name>` in the current account:
+
+  ```text
+  SHOW TABLES IN SCHEMA <database-name>.<schema-name>;
+  ```
+
   Snowflake requires the target table to have a defined schema before Unstructured can write to the table. The recommended table 
   schema for Unstructured is as follows:
 
@@ -107,3 +191,26 @@
     PRIMARY KEY (ID)
   );
   ```
+
+- The name of the column in the table that uniquely identifies each record (for example, `RECORD_ID`).
+- The name of the Snowflake [role](https://docs.snowflake.com/sql-reference/sql/create-role) that the user belongs to and that also has sufficient access to the Snowflake database, schema, table, and host.
+
+  - To create a database in Snowflake, the role needs to be granted `CREATE DATABASE` privilege at the current account level; and `USAGE` privilege on the warehouse that is used to create the database.
+  - To create a schema in a database in Snowflake, the role needs to be granted `USAGE` privilege on the database and the warehouse that is used to create the schema; and `CREATE SCHEMA` on the database.
+  - To create a table in a schema in Snowflake, the role needs to be granted `USAGE` privilege on the database and schema and the warehouse that is used to create the table; and `CREATE TABLE` on the schema.
+  - To write to a table in Snowflake, the role needs to be granted `USAGE` privilege on the database and schema and the warehouse that is used to write to the table; and `INSERT` on the table.
+  - To read from a table in Snowflake, the role needs to be granted `USAGE` privilege on the database and schema and the warehouse that is used to write to the table; and `SELECT` on the table.
+
+  To view a list of available roles in the current Snowflake account:
+
+  1. Log in to [Snowsight](https://docs.snowflake.com/user-guide/ui-snowsight-homepage) with your Snowflake account.
+  2. In Snowsight, on the navigation menu, click **Admin > Users & Roles**.
+  3. Click the **Roles** tab.
+
+  Alternatively, the following Snowflake query returns a list of available roles in the current account:
+
+  ```text
+  SHOW ROLES;
+  ```
+
+  [Grant privileges to a role](https://docs.snowflake.com/sql-reference/sql/grant-privilege). [Learn more](https://docs.snowflake.com/user-guide/security-access-control-privileges). 

snippets/source_connectors/snowflake.sh.mdx

@@ -6,13 +6,14 @@ unstructured \
     --account $SNOWFLAKE_ACCOUNT \
     --user $SNOWFLAKE_USER \
     --password $SNOWFLAKE_PASSWORD \
+    --role $SNOWFLAKE_ROLE \
     --host $SNOWFLAKE_HOST \
     --port $SNOWFLAKE_PORT \
     --database $SNOWFLAKE_DATABASE \
-    --schema PUBLIC \
-    --table-name elements\
-    --id-column id \
-    --batch-size 100 \
+    --schema $SNOWFLAKE_SCHEMA \
+    --batch-size 50 \
+    --table-name $SNOWFLAKE_TABLE \
+    --id-column $SNOWFLAKE_RECORD_ID_KEY \
     --download-dir $LOCAL_FILE_DOWNLOAD_DIR\
     --partition-by-api \
     --api-key $UNSTRUCTURED_API_KEY \

snippets/source_connectors/snowflake.v2.py.mdx

@@ -21,19 +21,22 @@ if __name__ == "__main__":
     Pipeline.from_configs(
         context=ProcessorConfig(),
         indexer_config=SnowflakeIndexerConfig(
-            table_name="elements",
-            id_column="id",
+            table_name=os.getenv("SNOWFLAKE_TABLE"),
+            id_column=os.getenv("SNOWFLAKE_RECORD_ID_KEY"),
             batch_size=100
         ),
         downloader_config=SnowflakeDownloaderConfig(download_dir=os.getenv("LOCAL_FILE_DOWNLOAD_DIR")),
         source_connection_config=SnowflakeConnectionConfig(
-            access_config=SnowflakeAccessConfig(password=os.getenv("SNOWFLAKE_PASSWORD")),
+            access_config=SnowflakeAccessConfig(
+                password=os.getenv("SNOWFLAKE_PASSWORD")
+            ),
             account=os.getenv("SNOWFLAKE_ACCOUNT"),
             user=os.getenv("SNOWFLAKE_USER"),
             host=os.getenv("SNOWFLAKE_HOST"),
             port=os.getenv("SNOWFLAKE_PORT"),
             database=os.getenv("SNOWFLAKE_DATABASE"),
-            schema="PUBLIC"
+            schema=os.getenv("SNOWFLAKE_SCHEMA"),
+            role=os.getenv("SNOWFLAKE_ROLE")
         ),
         partitioner_config=PartitionerConfig(
             partition_by_api=True,