From e42ec2337f998708701835344265af716083d751 Mon Sep 17 00:00:00 2001 From: AWS Gopher Date: Wed, 20 Aug 2025 15:28:35 -0400 Subject: [PATCH] Add integration tests for source and destination connectors - Introduced `TestDestinationPermutations` and `TestSourcePermutations` to validate various connector configurations with the live API server. - Each test checks the creation of sources and destinations using the unstructured SDK. - Added conditional skipping for tests if the `UNSTRUCTURED_API_KEY` environment variable is not set. --- test/destination_test.go | 214 +++++++++++++++++++++++++++++++++++++++ test/main_test.go | 13 ++- test/source_test.go | 194 +++++++++++++++++++++++++++++++++++ 3 files changed, 416 insertions(+), 5 deletions(-) create mode 100644 test/destination_test.go create mode 100644 test/source_test.go diff --git a/test/destination_test.go b/test/destination_test.go new file mode 100644 index 0000000..190cc17 --- /dev/null +++ b/test/destination_test.go @@ -0,0 +1,214 @@ +//go:build integration + +package test + +import ( + "context" + "crypto/rand" + "fmt" + "os" + "testing" + + "github.com/aws-gopher/unstructured-sdk-go" +) + +func TestDestinationPermutations(t *testing.T) { + t.Parallel() + + if os.Getenv("UNSTRUCTURED_API_KEY") == "" { + t.Skip("skipping because UNSTRUCTURED_API_KEY is not set") + } + + client, err := unstructured.New() + if err != nil { + t.Fatalf("failed to create client: %v", err) + } + + for name, src := range map[string]unstructured.DestinationConfigInput{ + "astra-db": unstructured.AstraDBConnectorConfigInput{ + CollectionName: "foo", + APIEndpoint: "https://foo.apps.astra.datastax.com", + Token: "foo", + }, + + "azure-ai-search": unstructured.AzureAISearchConnectorConfigInput{ + Endpoint: "https://foo.search.windows.net", + Index: "foo", + Key: "foo", + }, + + "couchbase": unstructured.CouchbaseDestinationConnectorConfigInput{ + Bucket: "foo", + ConnectionString: "couchbase://foo", + Username: "foo", + Password: "foo", + BatchSize: 100, + }, + + // server responds 500 + // "databricks-volume-delta-table": unstructured.DatabricksVDTDestinationConnectorConfigInput{ + // ServerHostname: "foo.cloud.databricks.com", + // HTTPPath: "/sql/1.0/warehouses/foo", + // Token: S("foo"), + // Catalog: "foo", + // Volume: "foo", + // }, + + "delta-table": unstructured.DeltaTableConnectorConfigInput{ + AwsAccessKeyID: "foo", + AwsSecretAccessKey: "foo", + AwsRegion: "us-east-1", + TableURI: "s3://foo/table", + }, + + "elasticsearch": unstructured.ElasticsearchConnectorConfigInput{ + Hosts: []string{"https://foo.elastic-cloud.com"}, + IndexName: "foo", + ESAPIKey: "foo", + }, + + "gcs": unstructured.GCSDestinationConnectorConfigInput{ + RemoteURL: "gs://foo", + ServiceAccountKey: "foo", + }, + + // server responds 412 asking for `bootstrap_server` instead of `bootstrap_servers` + // "kafka-cloud": unstructured.KafkaCloudDestinationConnectorConfigInput{ + // BootstrapServers: "foo.cloud.confluent.io", + // Topic: "foo", + // KafkaAPIKey: "foo", + // Secret: "foo", + // }, + + "milvus-token": unstructured.MilvusDestinationConnectorConfigInput{ + URI: "https://foo.zilliz.com", + CollectionName: "foo", + RecordIDKey: "foo", + Token: S("foo"), + }, + "milvus-password": unstructured.MilvusDestinationConnectorConfigInput{ + URI: "https://foo.zilliz.com", + CollectionName: "foo", + RecordIDKey: "foo", + User: S("foo"), + Password: S("foo"), + }, + + "mongo-db": unstructured.MongoDBConnectorConfigInput{ + Database: "foo", + Collection: "foo", + URI: "mongodb://foo:27017/foo", + }, + + // server responds 422: Destination Connector type motherduck not supported + // "mother-duck": unstructured.MotherduckDestinationConnectorConfigInput{ + // Account: "foo", + // Role: "foo", + // User: "foo", + // Password: "foo", + // Host: "foo.duckdb.io", + // Database: "foo", + // }, + + "neo4j": unstructured.Neo4jDestinationConnectorConfigInput{ + URI: "bolt://foo:7687", + Database: "foo", + Username: "foo", + Password: "foo", + }, + + "one-drive": unstructured.OneDriveDestinationConnectorConfigInput{ + ClientID: "foo", + UserPName: "foo", + Tenant: "foo", + AuthorityURL: "https://login.microsoftonline.com/foo", + ClientCred: "foo", + RemoteURL: "onedrive://foo", + }, + + "pinecone": unstructured.PineconeDestinationConnectorConfigInput{ + IndexName: "foo", + APIKey: "foo", + Namespace: "foo", + }, + + "postgres": unstructured.PostgresDestinationConnectorConfigInput{ + Host: "foo.com", + Database: "foo", + Port: 5432, + Username: "foo", + Password: "foo", + TableName: "foo", + BatchSize: 100, + }, + + "redis": unstructured.RedisDestinationConnectorConfigInput{ + Host: "foo.com", + Username: S("foo"), + Password: S("foo"), + }, + + "qdrant-cloud": unstructured.QdrantCloudDestinationConnectorConfigInput{ + URL: "https://foo.qdrant.io", + APIKey: "foo", + CollectionName: "foo", + }, + + "s3": unstructured.S3DestinationConnectorConfigInput{ + RemoteURL: "s3://foo", + Key: S("foo"), + Secret: S("foo"), + }, + + // server responds 500 + // "snowflake": unstructured.SnowflakeDestinationConnectorConfigInput{ + // Account: "foo", + // Role: "foo", + // User: "foo", + // Password: "foo", + // Host: "foo.snowflakecomputing.com", + // Database: "foo", + // }, + + "weaviate-cloud": unstructured.WeaviateDestinationConnectorConfigInput{ + ClusterURL: "https://foo.weaviate.network", + APIKey: "foo", + }, + + "ibm-watsonx-s3": unstructured.IBMWatsonxS3DestinationConnectorConfigInput{ + IAMApiKey: "foo", + AccessKeyID: "foo", + SecretAccessKey: "foo", + IcebergEndpoint: "https://foo.iceberg.cloud.ibm.com", + ObjectStorageEndpoint: "https://foo.s3.cloud.ibm.com", + ObjectStorageRegion: "us-east", + Catalog: "foo", + Namespace: "foo", + Table: "foo", + }, + + // server responds 500 + // "databricks-volumes": unstructured.DatabricksVolumesConnectorConfigInput{ + // Host: "foo.cloud.databricks.com", + // Catalog: "foo", + // Volume: "foo", + // VolumePath: "/foo", + // ClientSecret: "foo", + // ClientID: "foo", + // }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + destination, err := client.CreateDestination(t.Context(), unstructured.CreateDestinationRequest{ + Name: fmt.Sprintf("test-%s-%s", name, rand.Text()), + Config: src, + }) + if err != nil { + t.Fatalf("failed to create destination: %v", err) + } + + t.Cleanup(func() { _ = client.DeleteDestination(context.Background(), destination.ID) }) + }) + } +} diff --git a/test/main_test.go b/test/main_test.go index 2a33bc9..9b23979 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -15,9 +15,14 @@ import ( "github.com/aws-gopher/unstructured-sdk-go" ) +var S = unstructured.String +var I = unstructured.Int +var B = unstructured.Bool + func TestWorkflow(t *testing.T) { - key := os.Getenv("UNSTRUCTURED_API_KEY") - if key == "" { + t.Parallel() + + if os.Getenv("UNSTRUCTURED_API_KEY") == "" { t.Skip("skipping because UNSTRUCTURED_API_KEY is not set") } @@ -30,9 +35,7 @@ func TestWorkflow(t *testing.T) { return string(data) } - client, err := unstructured.New( - unstructured.WithKey(key), - ) + client, err := unstructured.New() if err != nil { t.Fatalf("failed to create client: %v", err) } diff --git a/test/source_test.go b/test/source_test.go new file mode 100644 index 0000000..92d2614 --- /dev/null +++ b/test/source_test.go @@ -0,0 +1,194 @@ +//go:build integration + +package test + +import ( + "context" + "crypto/rand" + "fmt" + "os" + "testing" + + "github.com/aws-gopher/unstructured-sdk-go" +) + +func TestSourcePermutations(t *testing.T) { + t.Parallel() + + if os.Getenv("UNSTRUCTURED_API_KEY") == "" { + t.Skip("skipping because UNSTRUCTURED_API_KEY is not set") + } + + client, err := unstructured.New() + if err != nil { + t.Fatalf("failed to create client: %v", err) + } + + for name, src := range map[string]unstructured.SourceConfigInput{ + "azure-account-key": unstructured.AzureSourceConnectorConfigInput{ + RemoteURL: "az://foo", + AccountName: S("foo"), + AccountKey: S("foo"), + }, + "azure-connection-string": unstructured.AzureSourceConnectorConfigInput{ + RemoteURL: "az://foo", + ConnectionString: S("foo"), + }, + "azure-sas-token": unstructured.AzureSourceConnectorConfigInput{ + RemoteURL: "az://foo", + AccountName: S("foo"), + SASToken: S("foo"), + }, + + "box": unstructured.BoxSourceConnectorConfigInput{ + BoxAppConfig: "foo", + RemoteURL: "box://foo", + }, + + // server responds 500 + // "confluence": unstructured.ConfluenceSourceConnectorConfigInput{ + // URL: "https://foo.atlassian.net", + // Username: "foo", + // Password: S("foo"), + // }, + + "couchbase": unstructured.CouchbaseSourceConnectorConfigInput{ + Bucket: "foo", + ConnectionString: "couchbase://foo", + Username: "foo", + Password: "foo", + CollectionID: "foo", + BatchSize: 100, + }, + + // server responds 500 + // "databricks-volumes": unstructured.DatabricksVolumesConnectorConfigInput{ + // Host: "foo.cloud.databricks.com", + // Catalog: "foo", + // Volume: "foo", + // VolumePath: "/foo", + // ClientSecret: "foo", + // ClientID: "foo", + // }, + + "dropbox": unstructured.DropboxSourceConnectorConfigInput{ + Token: "foo", + RemoteURL: "dropbox://foo", + }, + + "elasticsearch": unstructured.ElasticsearchConnectorConfigInput{ + Hosts: []string{"https://foo.elastic-cloud.com"}, + IndexName: "foo", + ESAPIKey: "foo", + }, + + "gcs": unstructured.GCSSourceConnectorConfigInput{ + RemoteURL: "gs://foo", + ServiceAccountKey: "foo", + }, + + "google-drive": unstructured.GoogleDriveSourceConnectorConfigInput{ + DriveID: "foo", + ServiceAccountKey: S("foo"), + }, + + "jira": unstructured.JiraSourceConnectorConfigInput{ + URL: "https://foo.atlassian.net", + Username: "foo", + Password: S("foo"), + }, + + // server responds 412 asking for `bootstrap_server` instead of `bootstrap_servers` + // "kafka-cloud": unstructured.KafkaCloudSourceConnectorConfigInput{ + // BootstrapServers: "foo.cloud.confluent.io", + // Topic: "foo", + // KafkaAPIKey: "foo", + // Secret: "foo", + // }, + + "mongodb": unstructured.MongoDBConnectorConfigInput{ + Database: "foo", + Collection: "foo", + URI: "mongodb://foo", + }, + + "onedrive": unstructured.OneDriveSourceConnectorConfigInput{ + ClientID: "foo", + UserPName: "foo", + Tenant: "foo", + AuthorityURL: "https://login.microsoftonline.com/foo", + ClientCred: "foo", + Path: "/foo", + }, + + "outlook": unstructured.OutlookSourceConnectorConfigInput{ + ClientID: "foo", + ClientCred: "foo", + UserEmail: "foo@example.com", + OutlookFolders: []string{"Inbox"}, + }, + + "postgres": unstructured.PostgresSourceConnectorConfigInput{ + Host: "foo.com", + Database: "foo", + Port: 5432, + Username: "foo", + Password: "foo", + TableName: "foo", + BatchSize: 100, + }, + + "s3": unstructured.S3SourceConnectorConfigInput{ + RemoteURL: "s3://foo", + Key: S("foo"), + Secret: S("foo"), + }, + + "salesforce": unstructured.SalesforceSourceConnectorConfigInput{ + Username: "foo", + ConsumerKey: "foo", + PrivateKey: "foo", + Categories: []string{"foo"}, + }, + + "sharepoint": unstructured.SharePointSourceConnectorConfigInput{ + Site: "https://foo.sharepoint.com/sites/foo", + Tenant: "foo", + UserPName: "foo", + ClientID: "foo", + ClientCred: "foo", + }, + + // server responds 500 + // "snowflake": unstructured.SnowflakeSourceConnectorConfigInput{ + // Account: "foo", + // Role: "foo", + // User: "foo", + // Password: "foo", + // Host: "foo.snowflakecomputing.com", + // Database: "foo", + // TableName: S("foo"), + // IDColumn: S("foo"), + // }, + + "zendesk": unstructured.ZendeskSourceConnectorConfigInput{ + Subdomain: "foo", + Email: "foo@example.com", + APIToken: "foo", + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + source, err := client.CreateSource(t.Context(), unstructured.CreateSourceRequest{ + Name: fmt.Sprintf("test-%s-%s", name, rand.Text()), + Config: src, + }) + if err != nil { + t.Fatalf("failed to create source: %v", err) + } + + t.Cleanup(func() { _ = client.DeleteSource(context.Background(), source.ID) }) + }) + } +}