feat(CIS): Logpush to cloud logs and generic destination (IBM-Cloud#5872)

* feat(PDNS): update custom resolver profile and views

* fix minor warnigns

* add example

* add documentation

* change variable names

* fix styling

* add validator for profile

* modify example

* improve error message

* modified docs

* fix error message

* incorporate review comments for documentation

* add order in views and fix minor bugs

* change the variable instance

* changing update logic for views

* modified test for linked zone

* add test for custom resolver profile

* add test for forwarding rule views

* fix acceptance test FR

* change message format

* change fields to computed in data source file

* feat(CIS): Logpush to cloud logs and generic destination

* add support for get job api

* modify create resource

* modify update resource logic

* add remaining computed fields

* add examples

* add documentation

* fix key

* incorporated doc review changes

* fix go mod issues

* add go dependency

* incorporated doc review changes

* syncing with go sdk

* add use_daily_subfolder attribute in document.

Author: arjunchauhanibm

examples/ibm-cis/main.tf

@@ -509,6 +509,7 @@ data "ibm_cis_bot_analytics" "tests" {
 }
 
 # CIS Logpush Job
+# logdna
 resource "ibm_cis_logpush_job" "test" {
     cis_id          = data.ibm_cis.cis.id
     domain_id       = data.ibm_cis_domain.cis_domain.domain_id
@@ -519,16 +520,65 @@ resource "ibm_cis_logpush_job" "test" {
     frequency       = "high"
     logdna =<<LOG
         {
-                        "hostname": "cistest-load.com",
+            "hostname": "cistest-load.com",
             "ingress_key": "e2f7xxxxx73a251caxxxxxxxxxxxx",
             "region": "in-che"
         }
         LOG
 }
+
+# IBM Cloud Logs
+resource "ibm_cis_logpush_job" "test" {
+    cis_id          = "crn:v1:staging:public:internet-svcs-ci:global:a/01652b251c3ae2787110a995d8db0135:1a9174b6-0106-417a-844b-c8eb43a72f63::"
+    domain_id       = "601b728b86e630c744c81740f72570c3"
+    name            = "MylogpushJob"
+    enabled         = false
+    logpull_options = "timestamps=rfc3339&timestamps=rfc3339"
+    dataset         = "http_requests"
+    frequency       = "high"
+    ibmcl {
+        instance_id ="604a309c-585c-4a42-955d-76239ccc1905"
+        api_key = "zxzeNQI22dxxxxxxxxxxxxxtn1EVK"
+        region = "us-south"
+    }
+}
+
+# COS
+resource "ibm_cis_logpush_job" "test" {
+    cis_id              = "crn:v1:staging:public:internet-svcs-ci:global:a/01652b251c3ae2787110a995d8db0135:1a9174b6-0106-417a-844b-c8eb43a72f63::"
+    domain_id           = "601b728b86e630c744c81740f72570c3"
+    name                = "MylogpushJob"
+    enabled             = false
+    logpull_options     = "timestamps=rfc3339&timestamps=rfc3339"
+    dataset             = "http_requests"
+    frequency           = "high"
+    ownership_challenge = "xxx"
+    cos =<<COS
+        {
+          "bucket_name": "examplse.cistest-load.com",
+          "id": "e2f72cxxxxxxxxxxxxa0b87859e",
+          "region": "in-che"
+    }
+    COS
+}
+
+# Genral destination
+resource "ibm_cis_logpush_job" "test" {
+    cis_id          = "crn:v1:staging:public:internet-svcs-ci:global:a/01652b251c3ae2787110a995d8db0135:1a9174b6-0106-417a-844b-c8eb43a72f63::"
+    domain_id       = "601b728b86e630c744c81740f72570c3"
+    name            = "MylogpushJob"
+    enabled         = false
+    logpull_options = "timestamps=rfc3339&timestamps=rfc3339"
+    dataset         = "http_requests"
+    frequency       = "high"
+    destination_conf = "s3://mybucket/logs?region=us-west-2"
+}
 # CIS Logpush Job Data source
+
 data "ibm_cis_logpush_jobs" "test" {
     cis_id          = data.ibm_cis.cis.id
     domain_id       = data.ibm_cis_domain.cis_domain.domain_id
+    job_id          = data.ibm_cis_domain.job.job_id
 }
 
 #CIS MTLS instance

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/IBM/logs-go-sdk v0.4.0
 	github.com/IBM/logs-router-go-sdk v1.0.5
 	github.com/IBM/mqcloud-go-sdk v0.2.0
-	github.com/IBM/networking-go-sdk v0.50.0
+	github.com/IBM/networking-go-sdk v0.51.1
 	github.com/IBM/platform-services-go-sdk v0.72.0
 	github.com/IBM/project-go-sdk v0.3.5
 	github.com/IBM/push-notifications-go-sdk v0.0.0-20210310100607-5790b96c47f5

go.sum

@@ -154,8 +154,8 @@ github.com/IBM/logs-router-go-sdk v1.0.5 h1:r0kC1+HfmSeQCD6zQTUp4PDI/zp4Ueo1Zo19
 github.com/IBM/logs-router-go-sdk v1.0.5/go.mod h1:tCN2vFgu5xG0ob9iJcxi5M4bJ6mWmu3nhmRPnvlwev0=
 github.com/IBM/mqcloud-go-sdk v0.2.0 h1:QOWk8ZGk0QfIL0MOGTKzNdM3Qe0Hk+ifAFtNSFQo5HU=
 github.com/IBM/mqcloud-go-sdk v0.2.0/go.mod h1:VZQKMtqmcdXKhmLhLiPuS/UHMs/5yo2tA/nD83cQt9E=
-github.com/IBM/networking-go-sdk v0.50.0 h1:3BHO1GrFnkkZ4mYyn2nQc+Djw48/g3znJe3k4a4p1Yk=
-github.com/IBM/networking-go-sdk v0.50.0/go.mod h1:aDKmbdfJG+JnoV9RvxOPg7nmgTKReyBgX8aAZHolcm8=
+github.com/IBM/networking-go-sdk v0.51.1 h1:xRlDFxSMejMmJ7JPFZ9cflL86uJpbqUzuL/1D/pP7/g=
+github.com/IBM/networking-go-sdk v0.51.1/go.mod h1:5aBaPKkm47LHECEOOZ4ltbwypOhapVo5aSCP3MTxmgQ=
 github.com/IBM/platform-services-go-sdk v0.72.0 h1:AfJe6bgqmTQU4ff/2URu3wkRLZD0XIzojn7SLf2yIns=
 github.com/IBM/platform-services-go-sdk v0.72.0/go.mod h1:ApFkvqw7NaluWJ5Uq+afdM/2jQqo5ILc0SzKSVobYNw=
 github.com/IBM/project-go-sdk v0.3.5 h1:L+YClFUa14foS0B/hOOY9n7sIdsT5/XQicnXOyJSpyM=

ibm/service/cis/datasource_ibm_cis_logpush_jobs.go

@@ -5,6 +5,7 @@ package cis
 
 import (
 	"log"
+	"strconv"
 	"time"
 
 	"github.com/IBM-Cloud/terraform-provider-ibm/ibm/conns"
@@ -36,6 +37,11 @@ func DataSourceIBMCISLogPushJobs() *schema.Resource {
 				Required:         true,
 				DiffSuppressFunc: suppressDomainIDDiff,
 			},
+			cisLogpushJobID: {
+				Type:        schema.TypeInt,
+				Description: "ID of the Job",
+				Optional:    true,
+			},
 			cisLogpushJobs: {
 				Type:        schema.TypeList,
 				Computed:    true,
@@ -77,6 +83,21 @@ func DataSourceIBMCISLogPushJobs() *schema.Resource {
 							Computed:    true,
 							Description: "The frequency at which CIS sends batches of logs to your destination",
 						},
+						cisLogpushLastComplete: {
+							Type:        schema.TypeString,
+							Computed:    true,
+							Description: "Records the last time for which logs have been successfully pushed.",
+						},
+						cisLogpushLastError: {
+							Type:        schema.TypeString,
+							Computed:    true,
+							Description: "Records the last time the job failed.",
+						},
+						cisLogpushErrorMessage: {
+							Type:        schema.TypeString,
+							Computed:    true,
+							Description: "The last failure message.",
+						},
 					},
 				},
 			},
@@ -110,30 +131,72 @@ func ResourceIBMCISLogpushJobsRead(d *schema.ResourceData, meta interface{}) err
 	sess.Crn = core.StringPtr(crn)
 	zoneID, _, _ := flex.ConvertTftoCisTwoVar(d.Get(cisDomainID).(string))
 	sess.ZoneID = core.StringPtr(zoneID)
-	opt := sess.NewGetLogpushJobsV2Options()
-	result, resp, err := sess.GetLogpushJobsV2(opt)
-	if err != nil {
-		log.Printf("[WARN] List all Logpush jobs failed: %v\n", resp)
-		return err
-	}
+	jobId := d.Get(cisLogpushJobID).(int)
 	logPushList := make([]map[string]interface{}, 0)
-	for _, logpushObj := range result.Result {
+
+	if jobId != 0 {
+		opt := sess.NewGetLogpushJobV2Options(strconv.Itoa(jobId))
+		result, resp, err := sess.GetLogpushJobV2(opt)
+		if err != nil {
+			log.Printf("[WARN] Get Logpush job failed: %v\n", resp)
+			return err
+		}
 		logPushOpt := map[string]interface{}{}
-		logPushOpt[cisLogpushJobID] = int64(*logpushObj.ID)
-		logPushOpt[cisLogpushName] = *logpushObj.Name
-		logPushOpt[cisLogpullOpt] = *logpushObj.LogpullOptions
-		logPushOpt[cisLogpushEnabled] = *logpushObj.Enabled
-		logPushOpt[cisLogpushDataset] = *logpushObj.Dataset
-		logPushOpt[cisLogpushFreq] = *logpushObj.Frequency
-		logPushOpt[cisLogpushDestConf] = *logpushObj.DestinationConf
+		logPushOpt[cisLogpushJobID] = int64(*result.Result.ID)
+		logPushOpt[cisLogpushName] = *result.Result.Name
+		logPushOpt[cisLogpullOpt] = *result.Result.LogpullOptions
+		logPushOpt[cisLogpushEnabled] = *result.Result.Enabled
+		logPushOpt[cisLogpushDataset] = *result.Result.Dataset
+		logPushOpt[cisLogpushFreq] = *result.Result.Frequency
+		logPushOpt[cisLogpushDestConf] = *result.Result.DestinationConf
+		if result.Result.LastComplete != nil {
+			logPushOpt[cisLogpushLastComplete] = *result.Result.LastComplete
+		}
+		if result.Result.LastError != nil {
+			logPushOpt[cisLogpushLastError] = *result.Result.LastError
+		}
+		if result.Result.ErrorMessage != nil {
+			logPushOpt[cisLogpushErrorMessage] = *result.Result.ErrorMessage
+		}
 		logPushList = append(logPushList, logPushOpt)
+
+	} else {
+
+		opt := sess.NewGetLogpushJobsV2Options()
+		result, resp, err := sess.GetLogpushJobsV2(opt)
+		if err != nil {
+			log.Printf("[WARN] List all Logpush jobs failed: %v\n", resp)
+			return err
+		}
+
+		for _, logpushObj := range result.Result {
+			logPushOpt := map[string]interface{}{}
+			logPushOpt[cisLogpushJobID] = int64(*logpushObj.ID)
+			logPushOpt[cisLogpushName] = *logpushObj.Name
+			logPushOpt[cisLogpullOpt] = *logpushObj.LogpullOptions
+			logPushOpt[cisLogpushEnabled] = *logpushObj.Enabled
+			logPushOpt[cisLogpushDataset] = *logpushObj.Dataset
+			logPushOpt[cisLogpushFreq] = *logpushObj.Frequency
+			logPushOpt[cisLogpushDestConf] = *logpushObj.DestinationConf
+			if logpushObj.LastComplete != nil {
+				logPushOpt[cisLogpushLastComplete] = *logpushObj.LastComplete
+			}
+			if logpushObj.LastError != nil {
+				logPushOpt[cisLogpushLastError] = *logpushObj.LastError
+			}
+			if logpushObj.ErrorMessage != nil {
+				logPushOpt[cisLogpushErrorMessage] = *logpushObj.ErrorMessage
+			}
+
+			logPushList = append(logPushList, logPushOpt)
+		}
 	}
-	d.SetId(dataSourceCISLogpushJobsCheckID(d))
+	d.SetId(dataSourceCISLogpushJobsCheckID())
 	d.Set(cisID, crn)
 	d.Set(cisDomainID, zoneID)
 	d.Set(cisLogpushJobs, logPushList)
 	return nil
 }
-func dataSourceCISLogpushJobsCheckID(d *schema.ResourceData) string {
+func dataSourceCISLogpushJobsCheckID() string {
 	return time.Now().UTC().String()
 }