feat: add V2 feed support

.gitignore

@@ -33,3 +33,6 @@ website/vendor
 !command/test-fixtures/**/.terraform/
 
 dist/
+
+# Compiled provider binary
+terraform-provider-chronicle

.golangci.yaml

@@ -60,3 +60,4 @@ issues:
       linters:
         - stylecheck
         - unused
+        - unparam

GNUmakefile

@@ -18,8 +18,17 @@ test: fmtcheck
 	go test -v . ./chronicle
 
 testacc: fmtcheck
+	@echo "WARNING: Running both V1 and V2 feed tests together. This may fail if your Chronicle SIEM"
+	@echo "         has both V1 and V2 feeds active. Consider using 'make testacc-v1' or 'make testacc-v2' instead."
+	@echo ""
 	TF_ACC=1  go test -v ./chronicle -timeout 120m  -parallel 1
 
+testacc-v1: fmtcheck
+	TF_ACC=1  go test -v ./chronicle -run='TestAcc' -skip='V2|EventDriven' -timeout 120m  -parallel 1
+
+testacc-v2: fmtcheck
+	TF_ACC=1  go test -v ./chronicle -run='TestAcc.*(V2|EventDriven)' -timeout 120m  -parallel 1	
+
 build:
 	@go build -mod=vendor -o $(PROJECT_NAME)
 	@echo "Build succeeded"
@@ -55,4 +64,4 @@ docs:
 vendor:
 	@go mod tidy && go mod vendor && go mod verify
 
-.PHONY: build install lint test clean testacc vet fmt fmtcheck docs vendor
+.PHONY: build install lint test clean testacc testacc-v1 testacc-v2 vet fmt fmtcheck docs vendor

README.md

@@ -8,7 +8,7 @@
 [![CI](https://github.com/form3tech-oss/terraform-provider-chronicle/actions/workflows/ci.yaml/badge.svg)](https://github.com/form3tech-oss/terraform-provider-chronicle/actions/workflows/ci.yaml)
 [![release](https://github.com/form3tech-oss/terraform-provider-chronicle/actions/workflows/release.yaml/badge.svg)](https://github.com/form3tech-oss/terraform-provider-chronicle/actions/workflows/release.yaml)
 
-Terraform provider for Chronicle
+Terraform provider for Chronicle (now known as Google Security Operations SIEM)
 
 # Documentation
 
@@ -52,7 +52,32 @@ In order to test the provider, you can simply run `make test`.
 make test
 ```
 
-In order to run the full suite of Acceptance tests, set the environment variables listed below and run `make testacc`.
+### Running Acceptance Tests
+
+Set the required environment variables and use one of these make targets:
+
+- `make testacc-v1` - Run V1 feed tests only
+- `make testacc-v2` - Run V2 feed tests only
+- `make testacc` - Run all tests (see limitations below)
+
+#### V1 vs V2 Feed Testing Constraints
+
+**Important:** Google Chronicle SIEM instances can only create **one feed version at a time** (either V1 or V2), which affects how you run acceptance tests.
+
+**How feed versions work:**
+- When you switch a SIEM to V2 feeds, you can no longer **create** new V1 feeds
+- However, any **existing** V1 feeds remain active and continue running (at time of writing)
+- This means you cannot run the full `make testacc` suite against a single SIEM instance
+
+**Recommended approach:**
+
+Match your test target to your SIEM configuration:
+- **V1-configured SIEM** → use `make testacc-v1`
+- **V2-configured SIEM** → use `make testacc-v2`
+
+Running the wrong test suite will fail because the SIEM won't allow creating feeds of the non-configured version.
+
+**Note:** You could theoretically run `make testacc` with multiple SIEM instances (one V1-configured, one V2-configured), but this setup is untested.
 
 The order of precedence for chronicle's API configuration is the following: `Credential file through TF > Access Token through TF > Environment Variable`.
 Environment variables always take the lowest precedence

chronicle/provider.go

@@ -195,14 +195,19 @@ func Provider() *schema.Provider {
 			"chronicle_rule":                                          resourceRule(),
 			"chronicle_reference_list":                                resourceReferenceList(),
 			"chronicle_feed_amazon_s3":                                NewResourceFeedAmazonS3().TerraformResource,
+			"chronicle_feed_amazon_s3_v2":                             NewResourceFeedAmazonS3V2().TerraformResource,
 			"chronicle_feed_amazon_sqs":                               NewResourceFeedAmazonSQS().TerraformResource,
+			"chronicle_feed_amazon_sqs_v2":                            NewResourceFeedAmazonSQSV2().TerraformResource,
 			"chronicle_feed_qualys_vm":                                NewResourceFeedQualysVM().TerraformResource,
 			"chronicle_feed_microsoft_office_365_management_activity": NewResourceFeedMicrosoftOffice365ManagementActivity().TerraformResource,
 			"chronicle_feed_okta_system_log":                          NewResourceFeedOktaSystemLog().TerraformResource,
 			"chronicle_feed_okta_users":                               NewResourceFeedOktaUsers().TerraformResource,
 			"chronicle_feed_proofpoint_siem":                          NewResourceFeedProofpointSIEM().TerraformResource,
 			"chronicle_feed_google_cloud_storage_bucket":              NewResourceFeedGoogleCloudStorageBucket().TerraformResource,
+			"chronicle_feed_google_cloud_storage_v2":                  NewResourceFeedGoogleCloudStorageV2().TerraformResource,
+			"chronicle_feed_google_cloud_storage_event_driven":        NewResourceFeedGoogleCloudStorageEventDriven().TerraformResource,
 			"chronicle_feed_azure_blobstore":                          NewResourceFeedAzureBlobStore().TerraformResource,
+			"chronicle_feed_azure_blobstore_v2":                       NewResourceFeedAzureBlobStoreV2().TerraformResource,
 			"chronicle_feed_thinkst_canary":                           NewResourceFeedThinkstCanary().TerraformResource,
 		},
 	}
@@ -272,6 +277,9 @@ func providerConfigure(ctx context.Context, d *schema.ResourceData, p *schema.Pr
 	if endpoint, isCustom := customEndpoint(d, "rule_custom_endpoint"); isCustom {
 		client.WithRuleBasePath(endpoint)
 	}
+	if endpoint, isCustom := customEndpoint(d, "feed_custom_endpoint"); isCustom {
+		client.WithFeedManagementBasePath(endpoint)
+	}
 	if endpoint, isCustom := customEndpoint(d, "subjects_custom_endpoint"); isCustom {
 		client.WithSubjectsBasePath(endpoint)
 	}
@@ -295,7 +303,7 @@ func getAPIAuthOpts(d *schema.ResourceData) []chronicle.Option {
 
 	if v, ok := d.GetOk("backstoryapi_credentials"); ok {
 		opts = append(opts, chronicle.WithBackstoryAPICredentials(v.(string)))
-	} else if v, ok := d.GetOk("backstoryapi_credentials"); ok {
+	} else if v, ok := d.GetOk("backstoryapi_access_token"); ok {
 		opts = append(opts, chronicle.WithBackstoryAPIAccessToken(v.(string)))
 	} else {
 		env := envSearch(chronicle.BackstoryAPIEnvVar)
@@ -306,7 +314,7 @@ func getAPIAuthOpts(d *schema.ResourceData) []chronicle.Option {
 
 	if v, ok := d.GetOk("ingestionapi_credentials"); ok {
 		opts = append(opts, chronicle.WithIngestionAPICredentials(v.(string)))
-	} else if v, ok := d.GetOk("ingestionapi_credentials"); ok {
+	} else if v, ok := d.GetOk("ingestionapi_access_token"); ok {
 		opts = append(opts, chronicle.WithIngestionAPIAccessToken(v.(string)))
 	} else {
 		env := envSearch(chronicle.IngestionAPIEnvVar)
@@ -322,7 +330,7 @@ func getAPIAuthOpts(d *schema.ResourceData) []chronicle.Option {
 	} else {
 		env := envSearch(chronicle.ForwarderAPIEnvVar)
 		if env != "" {
-			opts = append(opts, chronicle.WithBigQueryAPIEnvVar())
+			opts = append(opts, chronicle.WithForwarderAPIEnvVar())
 		}
 	}
 

chronicle/resource_feed_amazon_s3_test.go

@@ -200,7 +200,6 @@ func TestAccChronicleFeedAmazonS3_UpdateLogType(t *testing.T) {
 	})
 }
 
-//nolint:unparam
 func testAccCheckChronicleFeedAmazonS3AuthUpdated(t *testing.T, n, region, accessKeyID, secretAccessKey string) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		rs, ok := s.RootModule().Resources[n]
@@ -223,7 +222,6 @@ func testAccCheckChronicleFeedAmazonS3AuthUpdated(t *testing.T, n, region, acces
 	}
 }
 
-//nolint:unparam
 func testAccCheckChronicleFeedAmazonS3(displayName, logType, enabled, namespace, labels, s3Uri, s3SourceType,
 	sourceDeleteOptions, region, accesKeyID, secretAccessKey string) string {
 	return fmt.Sprintf(
@@ -265,7 +263,7 @@ func testAccCheckChronicleFeedAmazonS3Exists(n string) resource.TestCheckFunc {
 
 func testAccCheckChronicleFeedAmazonS3Destroy(s *terraform.State) error {
 	for _, rs := range s.RootModule().Resources {
-		if rs.Type != "chronicle_feed_amazon_s3.test" {
+		if rs.Type != "chronicle_feed_amazon_s3" {
 			continue
 		}
 
@@ -277,7 +275,6 @@ func testAccCheckChronicleFeedAmazonS3Destroy(s *terraform.State) error {
 	return nil
 }
 
-//nolint:unparam
 func feedAmazonS3Ref(name string) string {
 	return fmt.Sprintf("chronicle_feed_amazon_s3.%v", name)
 }

chronicle/resource_feed_amazon_s3_v2.go

@@ -0,0 +1,164 @@
+package chronicle
+
+import (
+	chronicle "github.com/form3tech-oss/terraform-provider-chronicle/client"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+type ResourceFeedAmazonS3V2 struct {
+	TerraformResource *schema.Resource
+}
+
+func NewResourceFeedAmazonS3V2() *ResourceFeedAmazonS3V2 {
+	details := &schema.Resource{
+		Schema: map[string]*schema.Schema{
+			"s3_uri": {
+				Type:        schema.TypeString,
+				Required:    true,
+				Description: `The S3 bucket URI in the format s3://bucket-name/path/.`,
+			},
+			"source_delete_options": {
+				Type:             schema.TypeString,
+				ValidateDiagFunc: validateFeedV2SourceDeleteOption,
+				Required:         true,
+				Description: `Whether to delete source files after they have been transferred to Chronicle. Valid values are:
+
+- NEVER: Never delete files from the source.
+- ON_SUCCESS: Delete files and empty directories from the source after successful ingestion.`,
+			},
+			"max_lookback_days": {
+				Type:             schema.TypeInt,
+				Optional:         true,
+				Default:          180,
+				ValidateDiagFunc: validateMaxLookbackDays,
+				Description:      `The maximum number of days in the past to look for files. Must be between 1 and 180. Default is 180 days.`,
+			},
+			"authentication": {
+				Type:        schema.TypeList,
+				Required:    true,
+				MaxItems:    1,
+				Description: `AWS authentication details. Use either access key credentials or IAM role ARN.`,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"access_key_id": {
+							Type:             schema.TypeString,
+							Optional:         true,
+							ValidateDiagFunc: validateAWSAccessKeyID,
+							RequiredWith:     []string{"details.0.authentication.0.secret_access_key"},
+							ConflictsWith:    []string{"details.0.authentication.0.aws_iam_role_arn"},
+							AtLeastOneOf: []string{
+								"details.0.authentication.0.access_key_id",
+								"details.0.authentication.0.aws_iam_role_arn",
+							},
+							Description: `The 20-character access key ID associated with your Amazon IAM account. Required if not using aws_iam_role_arn.`,
+						},
+						"secret_access_key": {
+							Type:             schema.TypeString,
+							Optional:         true,
+							Sensitive:        true,
+							ValidateDiagFunc: validateAWSSecretAccessKey,
+							RequiredWith:     []string{"details.0.authentication.0.access_key_id"},
+							ConflictsWith:    []string{"details.0.authentication.0.aws_iam_role_arn"},
+							Description:      `The 40-character secret access key associated with your Amazon IAM account. Required if not using aws_iam_role_arn.`,
+						},
+						"aws_iam_role_arn": {
+							Type:          schema.TypeString,
+							Optional:      true,
+							ConflictsWith: []string{"details.0.authentication.0.access_key_id", "details.0.authentication.0.secret_access_key"},
+							AtLeastOneOf: []string{
+								"details.0.authentication.0.access_key_id",
+								"details.0.authentication.0.aws_iam_role_arn",
+							},
+							Description: `ARN of the AWS IAM role configured to access S3 bucket. Use this for federated authentication instead of access keys.`,
+						},
+					},
+				},
+			},
+		},
+	}
+	description := "Creates a V2 feed from Amazon Simple Storage Service (S3). " +
+		"This feed type uses the Google Cloud Storage Transfer Service for improved ingestion."
+	resource := &ResourceFeedAmazonS3V2{}
+	resource.TerraformResource = newFeedResourceSchema(details, resource, description, true)
+
+	return resource
+}
+
+func (f *ResourceFeedAmazonS3V2) getLogType() string {
+	return ""
+}
+
+func (f *ResourceFeedAmazonS3V2) expandConcreteFeedConfiguration(d *schema.ResourceData) chronicle.ConcreteFeedConfiguration {
+	resourceDetailsInterface := readSliceFromResource(d, "details")
+	if resourceDetailsInterface == nil {
+		return nil
+	}
+
+	resourceDetails := resourceDetailsInterface[0].(map[string]interface{})
+	authenticationDetails := resourceDetails["authentication"].([]interface{})[0].(map[string]interface{})
+
+	config := &chronicle.S3V2FeedConfiguration{
+		S3URI:               resourceDetails["s3_uri"].(string),
+		SourceDeleteOptions: resourceDetails["source_delete_options"].(string),
+		MaxLookbackDays:     resourceDetails["max_lookback_days"].(int),
+		Authentication:      chronicle.S3V2FeedAuthentication{},
+	}
+
+	// Check which authentication method is used
+	if iamRoleArn, ok := authenticationDetails["aws_iam_role_arn"].(string); ok && iamRoleArn != "" {
+		config.Authentication.AWSIAMRoleAuth = &chronicle.S3V2AWSIAMRoleAuth{
+			AWSIAMRoleArn: iamRoleArn,
+		}
+	} else {
+		config.Authentication.AccessKeySecretAuth = &chronicle.S3V2AccessKeySecretAuth{
+			AccessKeyID:     authenticationDetails["access_key_id"].(string),
+			SecretAccessKey: authenticationDetails["secret_access_key"].(string),
+		}
+	}
+
+	return config
+}
+
+//nolint:all
+func (f *ResourceFeedAmazonS3V2) flattenDetailsFromReadOperation(originalConf chronicle.ConcreteFeedConfiguration, readConf chronicle.ConcreteFeedConfiguration) []map[string]interface{} {
+
+	readS3Conf := readConf.(*chronicle.S3V2FeedConfiguration)
+
+	// Import Case
+	if originalConf == nil {
+		authMap := make(map[string]interface{})
+		// Only populate non-secret auth fields during import
+		if readS3Conf.Authentication.AWSIAMRoleAuth != nil && readS3Conf.Authentication.AWSIAMRoleAuth.AWSIAMRoleArn != "" {
+			authMap["aws_iam_role_arn"] = readS3Conf.Authentication.AWSIAMRoleAuth.AWSIAMRoleArn
+		}
+
+		// Note: access_key_id and secret_access_key are not returned by the API
+		// and will remain empty in state after import until explicitly set by user
+
+		return []map[string]interface{}{{
+			"s3_uri":                readS3Conf.S3URI,
+			"source_delete_options": readS3Conf.SourceDeleteOptions,
+			"max_lookback_days":     readS3Conf.MaxLookbackDays,
+			"authentication":        []map[string]interface{}{authMap},
+		}}
+	}
+
+	originalS3Conf := originalConf.(*chronicle.S3V2FeedConfiguration)
+	// Default Case
+	authMap := make(map[string]interface{})
+	if originalS3Conf.Authentication.AWSIAMRoleAuth != nil && originalS3Conf.Authentication.AWSIAMRoleAuth.AWSIAMRoleArn != "" {
+		authMap["aws_iam_role_arn"] = originalS3Conf.Authentication.AWSIAMRoleAuth.AWSIAMRoleArn
+	}
+	if originalS3Conf.Authentication.AccessKeySecretAuth != nil {
+		authMap["access_key_id"] = originalS3Conf.Authentication.AccessKeySecretAuth.AccessKeyID
+		authMap["secret_access_key"] = originalS3Conf.Authentication.AccessKeySecretAuth.SecretAccessKey
+	}
+
+	return []map[string]interface{}{{
+		"s3_uri":                readS3Conf.S3URI,
+		"source_delete_options": originalS3Conf.SourceDeleteOptions, // not returned
+		"max_lookback_days":     readS3Conf.MaxLookbackDays,
+		// replace authentication block with original values because they are not returned within a read request
+		"authentication": []map[string]interface{}{authMap},
+	}}
+}