Merge pull request #130 from Cyb3r-Jak3/add-r2-metrics

feat: Add R2 metrics

Author: haad

README.md

@@ -32,6 +32,8 @@ Required authentication scopes:
 
 To authenticate this way, only set `CF_API_TOKEN` (omit `CF_API_EMAIL` and `CF_API_KEY`)
 
+[Shortcut to create the API token](https://dash.cloudflare.com/profile/api-tokens?permissionGroupKeys=%5B%7B%22key%22%3A%22account_analytics%22%2C%22type%22%3A%22read%22%7D%2C%7B%22key%22%3A%22account_settings%22%2C%22type%22%3A%22read%22%7D%2C%7B%22key%22%3A%22analytics%22%2C%22type%22%3A%22read%22%7D%2C%7B%22key%22%3A%22firewall_services%22%2C%22type%22%3A%22read%22%7D%5D&name=Cloudflare+Exporter&accountId=*&zoneId=all)
+
 ### User email + API key
 To authenticate with user email + API key, use the `Global API Key` from the Cloudflare dashboard.
 Beware that this key authenticates with write access to every Cloudflare resource.
@@ -104,6 +106,9 @@ Note: `ZONE_<name>` configuration is not supported as flag.
 # HELP cloudflare_zone_pool_requests_total Requests per pool
 # HELP cloudflare_logpush_failed_jobs_account_count Number of failed logpush jobs on the account level
 # HELP cloudflare_logpush_failed_jobs_zone_count Number of failed logpush jobs on the zone level
+# HELP cloudflare_r2_operation_count Number of operations performed by R2
+# HELP cloudflare_r2_storage_bytes Storage used by R2
+# HELP cloudflare_r2_storage_total_bytes Total storage used by R2
 ```
 
 ## Helm chart repository

cloudflare.go

@@ -5,13 +5,13 @@ import (
 	"strings"
 	"time"
 
-	cloudflare "github.com/cloudflare/cloudflare-go"
+	"github.com/cloudflare/cloudflare-go"
 	"github.com/machinebox/graphql"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 )
 
-var (
+const (
 	cfGraphQLEndpoint = "https://api.cloudflare.com/client/v4/graphql/"
 )
 
@@ -45,6 +45,35 @@ type cloudflareResponseLogpushAccount struct {
 	} `json:"viewer"`
 }
 
+type r2AccountResp struct {
+	R2StorageGroups []struct {
+		Dimensions struct {
+			BucketName string `json:"bucketName"`
+		} `json:"dimensions"`
+		Max struct {
+			MetadataSize uint64 `json:"metadataSize"`
+			PayloadSize  uint64 `json:"payloadSize"`
+			ObjectCount  uint64 `json:"objectCount"`
+		} `json:"max"`
+	} `json:"r2StorageAdaptiveGroups"`
+
+	R2StorageOperations []struct {
+		Dimensions struct {
+			Action     string `json:"actionType"`
+			BucketName string `json:"bucketName"`
+		} `json:"dimensions"`
+		Sum struct {
+			Requests uint64 `json:"requests"`
+		} `json:"sum"`
+	} `json:"r2OperationsAdaptiveGroups"`
+}
+
+type cloudflareResponseR2Account struct {
+	Viewer struct {
+		Accounts []r2AccountResp `json:"accounts"`
+	}
+}
+
 type cloudflareResponseLogpushZone struct {
 	Viewer struct {
 		Zones []logpushResponse `json:"zones"`
@@ -251,60 +280,38 @@ type lbResp struct {
 }
 
 func fetchZones() []cloudflare.Zone {
-	var api *cloudflare.API
-	var err error
-	if len(viper.GetString("cf_api_token")) > 0 {
-		api, err = cloudflare.NewWithAPIToken(viper.GetString("cf_api_token"))
-	} else {
-		api, err = cloudflare.New(viper.GetString("cf_api_key"), viper.GetString("cf_api_email"))
-	}
-	if err != nil {
-		log.Fatal("failed to make client to fetch zones: ", err)
-	}
-
 	ctx := context.Background()
-	z, err := api.ListZones(ctx)
+	z, err := cloudflareAPI.ListZones(ctx)
 	if err != nil {
-		log.Fatal("failed to fetch zones: ", err)
+		log.Fatalf("Error fetching zones: %s", err)
 	}
 
 	return z
 }
 
 func fetchFirewallRules(zoneID string) map[string]string {
-	var api *cloudflare.API
-	var err error
-	if len(viper.GetString("cf_api_token")) > 0 {
-		api, err = cloudflare.NewWithAPIToken(viper.GetString("cf_api_token"))
-	} else {
-		api, err = cloudflare.New(viper.GetString("cf_api_key"), viper.GetString("cf_api_email"))
-	}
-	if err != nil {
-		log.Fatal("failed to make client to fetch firewall rules: ", err)
-	}
-
 	ctx := context.Background()
-	listOfRules, _, err := api.FirewallRules(ctx,
+	listOfRules, _, err := cloudflareAPI.FirewallRules(ctx,
 		cloudflare.ZoneIdentifier(zoneID),
 		cloudflare.FirewallRuleListParams{})
 	if err != nil {
-		log.Fatal("failed to fetch firewall rules: ", err)
+		log.Fatalf("Error fetching firewall rules: %s", err)
 	}
 	firewallRulesMap := make(map[string]string)
 
 	for _, rule := range listOfRules {
 		firewallRulesMap[rule.ID] = rule.Description
 	}
 
-	listOfRulesets, err := api.ListRulesets(ctx, cloudflare.ZoneIdentifier(zoneID), cloudflare.ListRulesetsParams{})
+	listOfRulesets, err := cloudflareAPI.ListRulesets(ctx, cloudflare.ZoneIdentifier(zoneID), cloudflare.ListRulesetsParams{})
 	if err != nil {
-		log.Fatal("failed to fetch list of rulesets for firewall rules: ", err)
+		log.Fatalf("Error listing rulesets: %s", err)
 	}
 	for _, rulesetDesc := range listOfRulesets {
 		if rulesetDesc.Phase == "http_request_firewall_managed" {
-			ruleset, err := api.GetRuleset(ctx, cloudflare.ZoneIdentifier(zoneID), rulesetDesc.ID)
+			ruleset, err := cloudflareAPI.GetRuleset(ctx, cloudflare.ZoneIdentifier(zoneID), rulesetDesc.ID)
 			if err != nil {
-				log.Fatal("failed to fetch ruleset for firewall rules: ", err)
+        log.Fatalf("Error fetching ruleset for firewall rules: %s", err)
 			}
 			for _, rule := range ruleset.Rules {
 				firewallRulesMap[rule.ID] = rule.Description
@@ -314,7 +321,7 @@ func fetchFirewallRules(zoneID string) map[string]string {
 		if rulesetDesc.Phase == "http_request_firewall_custom" {
 			ruleset, err := api.GetRuleset(ctx, cloudflare.ZoneIdentifier(zoneID), rulesetDesc.ID)
 			if err != nil {
-				log.Fatal(err)
+        log.Fatal("Error fetching custom firewall rulesets: %s" err)
 			}
 			for _, rule := range ruleset.Rules {
 				firewallRulesMap[rule.ID] = rule.Description
@@ -326,21 +333,10 @@ func fetchFirewallRules(zoneID string) map[string]string {
 }
 
 func fetchAccounts() []cloudflare.Account {
-	var api *cloudflare.API
-	var err error
-	if len(viper.GetString("cf_api_token")) > 0 {
-		api, err = cloudflare.NewWithAPIToken(viper.GetString("cf_api_token"))
-	} else {
-		api, err = cloudflare.New(viper.GetString("cf_api_key"), viper.GetString("cf_api_email"))
-	}
-	if err != nil {
-		log.Fatal("failed to make client to fetch accounts: ", err)
-	}
-
 	ctx := context.Background()
-	a, _, err := api.Accounts(ctx, cloudflare.AccountsListParams{PaginationOptions: cloudflare.PaginationOptions{PerPage: 100}})
+	a, _, err := cloudflareAPI.Accounts(ctx, cloudflare.AccountsListParams{PaginationOptions: cloudflare.PaginationOptions{PerPage: 100}})
 	if err != nil {
-		log.Fatal("failed to fetch accounts: ", err)
+		log.Fatalf("Error fetching accounts: %s", err)
 	}
 
 	return a
@@ -580,7 +576,7 @@ func fetchWorkerTotals(accountID string) (*cloudflareResponseAccts, error) {
 	graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
 	var resp cloudflareResponseAccts
 	if err := graphqlClient.Run(ctx, request, &resp); err != nil {
-		log.Error("failed to fetch worker totals: ", err)
+		log.Errorf("Error fetching worker totals: %s", err)
 		return nil, err
 	}
 
@@ -657,7 +653,7 @@ func fetchLoadBalancerTotals(zoneIDs []string) (*cloudflareResponseLb, error) {
 	graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
 	var resp cloudflareResponseLb
 	if err := graphqlClient.Run(ctx, request, &resp); err != nil {
-		log.Error("failed to fetch load balancer totals: ", err)
+		log.Errorf("Error fetching load balancer totals: %s", err)
 		return nil, err
 	}
 	return &resp, nil
@@ -709,7 +705,7 @@ func fetchLogpushAccount(accountID string) (*cloudflareResponseLogpushAccount, e
 	graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
 	var resp cloudflareResponseLogpushAccount
 	if err := graphqlClient.Run(ctx, request, &resp); err != nil {
-		log.Error("failed to logpush account info: ", err)
+		log.Errorf("Error fetching logpush account totals: %s", err)
 		return nil, err
 	}
 	return &resp, nil
@@ -761,13 +757,71 @@ func fetchLogpushZone(zoneIDs []string) (*cloudflareResponseLogpushZone, error)
 	graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
 	var resp cloudflareResponseLogpushZone
 	if err := graphqlClient.Run(ctx, request, &resp); err != nil {
-		log.Error("failed to fetch logpush zone: ", err)
+		log.Errorf("Error fetching logpush zone totals: %s", err)
 		return nil, err
 	}
 
 	return &resp, nil
 }
 
+func fetchR2Account(accountID string) (*cloudflareResponseR2Account, error) {
+	now := time.Now().Add(-time.Duration(viper.GetInt("scrape_delay")) * time.Second).UTC()
+	s := 60 * time.Second
+	now = now.Truncate(s)
+
+	request := graphql.NewRequest(`query($accountID: String!, $limit: Int!, $date: String!) {
+		viewer {
+		  accounts(filter: {accountTag : $accountID }) {
+			r2StorageAdaptiveGroups(
+			  filter: {
+				date: $date
+			  },
+			  limit: $limit
+			) {
+			  dimensions {
+          		bucketName
+			  }
+        	  max {
+				metadataSize
+          		payloadSize
+				objectCount
+			  }
+      		}
+			r2OperationsAdaptiveGroups(filter: { date: $date }, limit: $limit) {
+				dimensions {
+					actionType
+					bucketName
+				}
+				sum {
+					requests
+				}
+			}
+			}
+		  }
+	  }`)
+
+	if len(viper.GetString("cf_api_token")) > 0 {
+		request.Header.Set("Authorization", "Bearer "+viper.GetString("cf_api_token"))
+	} else {
+		request.Header.Set("X-AUTH-EMAIL", viper.GetString("cf_api_email"))
+		request.Header.Set("X-AUTH-KEY", viper.GetString("cf_api_key"))
+	}
+
+	request.Var("accountID", accountID)
+	request.Var("limit", 9999)
+	request.Var("date", now.Format("2006-01-02"))
+
+	ctx := context.Background()
+	graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
+	graphqlClient.Log = func(s string) { log.Debug(s) }
+	var resp cloudflareResponseR2Account
+	if err := graphqlClient.Run(ctx, request, &resp); err != nil {
+		log.Errorf("Error fetching R2 account: %s", err)
+		return nil, err
+	}
+	return &resp, nil
+}
+
 func findZoneAccountName(zones []cloudflare.Zone, ID string) (string, string) {
 	for _, z := range zones {
 		if z.ID == ID {