Unicron add api logs in dynamodb

cla-backend-go/api_logs/models.go

@@ -0,0 +1,30 @@
+// Copyright The Linux Foundation and each contributor to CommunityBridge.
+// SPDX-License-Identifier: MIT
+
+package api_logs
+
+import (
+	"fmt"
+	"time"
+)
+
+// APILog data model for DynamoDB table cla-{stage}-api-log
+type APILog struct {
+	URL    string `dynamodbav:"url" json:"url"`
+	DT     int64  `dynamodbav:"dt" json:"dt"`
+	Bucket string `dynamodbav:"bucket" json:"bucket"`
+}
+
+// String returns a string representation of the APILog
+func (a *APILog) String() string {
+	return fmt.Sprintf("APILog{URL: %s, DT: %d, Bucket: %s}", a.URL, a.DT, a.Bucket)
+}
+
+// NewAPILog creates a new APILog entry with current timestamp
+func NewAPILog(url, bucket string) *APILog {
+	return &APILog{
+		URL:    url,
+		DT:     time.Now().UnixMilli(), // Unix timestamp in milliseconds
+		Bucket: bucket,
+	}
+}

cla-backend-go/api_logs/repository.go

@@ -0,0 +1,94 @@
+// Copyright The Linux Foundation and each contributor to CommunityBridge.
+// SPDX-License-Identifier: MIT
+
+package api_logs
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/dynamodb"
+	"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
+)
+
+const (
+	// APILogTableName is the DynamoDB table name for API logs
+	APILogTableName = "cla-%s-api-log"
+)
+
+// Repository interface for API logs
+type Repository interface {
+	LogAPIRequest(ctx context.Context, url string) error
+}
+
+// repository implements the Repository interface
+type repository struct {
+	stage          string
+	dynamoDBClient *dynamodb.DynamoDB
+}
+
+// NewRepository creates a new API logs repository
+func NewRepository(stage string, dynamoDBClient *dynamodb.DynamoDB) Repository {
+	return &repository{
+		stage:          stage,
+		dynamoDBClient: dynamoDBClient,
+	}
+}
+
+// LogAPIRequest logs an API request to the DynamoDB table
+// Creates three entries: ALL bucket, daily bucket (YYYY-MM-DD), and monthly bucket (YYYY-MM)
+// IMPORTANT: table key is (url, dt). To avoid overwrites, dt is shifted by -1/0/+1 ms per bucket.
+func (r *repository) LogAPIRequest(ctx context.Context, url string) error {
+	// 200% fail-safe: never panic on nil ctx/client
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if r == nil || r.dynamoDBClient == nil {
+		return fmt.Errorf("dynamodb client is nil")
+	}
+
+	now := time.Now().UTC()
+	timestamp := now.UnixMilli()
+
+	// Generate bucket names
+	dailyBucket := now.Format("2006-01-02") // YYYY-MM-DD
+	monthlyBucket := now.Format("2006-01")  // YYYY-MM
+
+	entries := []*APILog{
+		{URL: url, DT: timestamp - 1, Bucket: "ALL"},
+		{URL: url, DT: timestamp, Bucket: dailyBucket},
+		{URL: url, DT: timestamp + 1, Bucket: monthlyBucket},
+	}
+	tableName := fmt.Sprintf(APILogTableName, r.stage)
+
+	var errs []string
+	for _, logEntry := range entries {
+		// Convert to DynamoDB attribute value
+		av, err := dynamodbattribute.MarshalMap(logEntry)
+		if err != nil {
+			errs = append(errs, fmt.Sprintf("bucket=%s marshal=%v", logEntry.Bucket, err))
+			continue
+		}
+
+		// Put item to DynamoDB
+		input := &dynamodb.PutItemInput{
+			TableName: aws.String(tableName),
+			Item:      av,
+		}
+
+		_, err = r.dynamoDBClient.PutItemWithContext(ctx, input)
+		if err != nil {
+			errs = append(errs, fmt.Sprintf("bucket=%s put=%v", logEntry.Bucket, err))
+			continue
+		}
+	}
+
+	// Return error so middleware can emit a single LG:* line.
+	if len(errs) > 0 {
+		return fmt.Errorf("%s", strings.Join(errs, "; "))
+	}
+	return nil
+}

cla-backend-go/cmd/server.go

@@ -15,6 +15,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/linuxfoundation/easycla/cla-backend-go/project/repository"
 	"github.com/linuxfoundation/easycla/cla-backend-go/project/service"
@@ -78,6 +79,7 @@ import (
 
 	"github.com/linuxfoundation/easycla/cla-backend-go/users"
 
+	"github.com/linuxfoundation/easycla/cla-backend-go/api_logs"
 	"github.com/linuxfoundation/easycla/cla-backend-go/signatures"
 	v2Signatures "github.com/linuxfoundation/easycla/cla-backend-go/v2/signatures"
 
@@ -144,12 +146,34 @@ type combinedRepo struct {
 	projects_cla_groups.Repository
 }
 
-// in cmd/server.go (top-level imports already use logrus)
-func apiPathLogger(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		log.Infof("LG:api-request-path:%s", r.URL.Path)
-		next.ServeHTTP(w, r)
-	})
+// apiPathLoggerWithDB creates a middleware that logs API requests to DynamoDB
+func apiPathLoggerWithDB(apiLogsRepo api_logs.Repository) func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			log.Infof("LG:api-request-path:%s", r.URL.Path)
+
+			// Log to DynamoDB table (fire-and-forget, never fail request)
+			if apiLogsRepo != nil {
+				path := r.URL.Path
+				go func() {
+					defer func() {
+						if rec := recover(); rec != nil {
+							log.Infof("LG:api-log-dynamo-failed:%s err=panic:%v", path, rec)
+						}
+					}()
+
+					ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+					defer cancel()
+
+					if err := apiLogsRepo.LogAPIRequest(ctx, path); err != nil {
+						log.Infof("LG:api-log-dynamo-failed:%s err=%v", path, err)
+					}
+				}()
+			}
+
+			next.ServeHTTP(w, r)
+		})
+	}
 }
 
 // server function called by environment specific server functions
@@ -281,6 +305,7 @@ func server(localMode bool) http.Handler {
 	approvalListRepo := approval_list.NewRepository(awsSession, stage)
 	v1CompanyRepo := v1Company.NewRepository(awsSession, stage)
 	eventsRepo := events.NewRepository(awsSession, stage)
+	apiLogsRepo := api_logs.NewRepository(stage, dynamodb.New(awsSession))
 	v1ProjectClaGroupRepo := projects_cla_groups.NewRepository(awsSession, stage)
 	v1CLAGroupRepo := repository.NewRepository(awsSession, stage, gitV1Repository, gerritRepo, v1ProjectClaGroupRepo)
 	metricsRepo := metrics.NewRepository(awsSession, stage, configFile.APIGatewayURL, v1ProjectClaGroupRepo)
@@ -406,7 +431,7 @@ func server(localMode bool) http.Handler {
 	// The middleware configuration is for the handler executors. These do not apply to the swagger.json document.
 	// The middleware executes after routing but before authentication, binding and validation
 	middlewareSetupfunc := func(handler http.Handler) http.Handler {
-		return apiPathLogger(setRequestIDHandler(responseLoggingMiddleware(userCreaterMiddleware(handler))))
+		return apiPathLoggerWithDB(apiLogsRepo)(setRequestIDHandler(responseLoggingMiddleware(userCreaterMiddleware(handler))))
 	}
 
 	v2API.CsvProducer = openapi_runtime.ProducerFunc(func(w io.Writer, data interface{}) error {

cla-backend-go/serverless.yml

@@ -122,6 +122,7 @@ provider:
             - dynamodb:DescribeStream
             - dynamodb:ListStreams
           Resource:
+            - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-api-log"
             - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-ccla-whitelist-requests"
             - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-cla-manager-requests"
             - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-companies"
@@ -144,6 +145,7 @@ provider:
           Action:
             - dynamodb:Query
           Resource:
+            - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-api-log/index/bucket-dt-index"
             - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-ccla-whitelist-requests/index/company-id-project-id-index"
             - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-ccla-whitelist-requests/index/ccla-approval-list-request-project-id-index"
             - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-users/index/github-id-index"

cla-backend/cla/models/dynamo_models.py

@@ -58,6 +58,7 @@ def create_database():
         GerritModel,
         EventModel,
         CCLAAllowlistRequestModel,
+        APILogModel,
 
     ]
     # Create all required tables.
@@ -83,6 +84,7 @@ def delete_database():
         GitHubOrgModel,
         GerritModel,
         CCLAAllowlistRequestModel,
+        APILogModel,
     ]
     # Delete all existing tables.
     for table in tables:
@@ -5384,6 +5386,136 @@ def create_event(
             return {"errors": {"event_id": str(err)}}
 
 
+class APILogBucketDTIndex(GlobalSecondaryIndex):
+    """
+    This class represents a global secondary index for querying API logs by bucket and time range.
+    """
+
+    class Meta:
+        """Meta class for API Log bucket-dt index."""
+
+        index_name = "bucket-dt-index"
+        write_capacity_units = int(cla.conf.get("DYNAMO_WRITE_UNITS", 10))
+        read_capacity_units = int(cla.conf.get("DYNAMO_READ_UNITS", 10))
+        # All attributes are projected - not sure if this is necessary.
+        projection = AllProjection()
+
+    # This attribute is the hash key for the index.
+    bucket = UnicodeAttribute(hash_key=True)
+    # This attribute is the range key for the index.
+    dt = NumberAttribute(range_key=True)
+
+
+class APILogModel(BaseModel):
+    """
+    Represents an API log entry in the database
+    """
+
+    class Meta:
+        """Meta class for APILog."""
+
+        table_name = "cla-{}-api-log".format(stage)
+        if stage == "local":
+            host = "http://localhost:8000"
+
+    url = UnicodeAttribute(hash_key=True)
+    dt = NumberAttribute(range_key=True)
+    bucket = UnicodeAttribute(null=False)
+
+    # GSI for querying by bucket and time range
+    bucket_dt_index = APILogBucketDTIndex()
+
+
+class APILog(model_interfaces.APILog):
+    """
+    ORM-agnostic wrapper for the DynamoDB APILog model.
+    """
+
+    def __init__(self, url=None, dt=None, bucket=None):
+        super().__init__()
+        self.model = APILogModel()
+        self.model.url = url
+        self.model.dt = dt
+        self.model.bucket = bucket
+
+    def __str__(self):
+        return f"url:{self.model.url}, dt:{self.model.dt}, bucket:{self.model.bucket}"
+
+    def to_dict(self):
+        return dict(self.model)
+
+    def save(self) -> None:
+        # self.model.date_modified = datetime.datetime.utcnow()
+        self.model.save()
+
+    def load(self, url, dt):
+        try:
+            api_log = self.model.get(str(url), int(dt))
+        except APILogModel.DoesNotExist:
+            raise cla.models.DoesNotExist("API Log entry not found")
+        self.model = api_log
+
+    def delete(self):
+        self.model.delete()
+
+    def get_url(self):
+        return self.model.url
+
+    def get_dt(self):
+        return self.model.dt
+
+    def get_bucket(self):
+        return self.model.bucket
+
+    def set_url(self, url):
+        self.model.url = url
+
+    def set_dt(self, dt):
+        self.model.dt = dt
+
+    def set_bucket(self, bucket):
+        self.model.bucket = bucket
+
+    @classmethod
+    def log_api_request(cls, url: str):
+        """
+        Log an API request with the given URL.
+        Creates three entries: ALL bucket, daily bucket, and monthly bucket.
+        Never raises exceptions - logs errors instead.
+        """
+        try:
+            # Base timestamp in milliseconds
+            base_dt = int(time.time() * 1000)
+            dt_obj = datetime.datetime.utcnow()
+
+            # Buckets
+            daily_bucket = dt_obj.strftime('%Y-%m-%d')
+            monthly_bucket = dt_obj.strftime('%Y-%m')
+
+            # IMPORTANT: table key is (url, dt). To avoid overwrites we shift dt by -1/0/+1 ms.
+            entries = [
+                ("ALL", base_dt - 1),
+                (daily_bucket, base_dt),
+                (monthly_bucket, base_dt + 1),
+            ]
+
+            errors = []
+            for bucket, dt_value in entries:
+                try:
+                    api_log = cls(url=url, dt=dt_value, bucket=bucket)
+                    api_log.save()
+                except Exception as e:
+                    errors.append(f"bucket={bucket} err={e}")
+
+            if errors:
+                # Only AWS logs entry (LG-style), never fail request flow
+                cla.log.info(f"LG:api-log-dynamo-failed:{url} " + "; ".join(errors))
+
+        except Exception as e:
+            # Never let API logging failure break the request flow
+            cla.log.info(f"LG:api-log-dynamo-failed:{url} err={e}")
+
+
 class CCLAAllowlistRequestModel(BaseModel):
     """
     Represents a CCLAAllowlistRequest in the database