diff --git a/cla-backend-go/api_logs/models.go b/cla-backend-go/api_logs/models.go new file mode 100644 index 000000000..0f8b64ff9 --- /dev/null +++ b/cla-backend-go/api_logs/models.go @@ -0,0 +1,30 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package api_logs + +import ( + "fmt" + "time" +) + +// APILog data model for DynamoDB table cla-{stage}-api-log +type APILog struct { + URL string `dynamodbav:"url" json:"url"` + DT int64 `dynamodbav:"dt" json:"dt"` + Bucket string `dynamodbav:"bucket" json:"bucket"` +} + +// String returns a string representation of the APILog +func (a *APILog) String() string { + return fmt.Sprintf("APILog{URL: %s, DT: %d, Bucket: %s}", a.URL, a.DT, a.Bucket) +} + +// NewAPILog creates a new APILog entry with current timestamp +func NewAPILog(url, bucket string) *APILog { + return &APILog{ + URL: url, + DT: time.Now().UnixMilli(), // Unix timestamp in milliseconds + Bucket: bucket, + } +} diff --git a/cla-backend-go/api_logs/repository.go b/cla-backend-go/api_logs/repository.go new file mode 100644 index 000000000..ba2cb97ef --- /dev/null +++ b/cla-backend-go/api_logs/repository.go @@ -0,0 +1,94 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package api_logs + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" +) + +const ( + // APILogTableName is the DynamoDB table name for API logs + APILogTableName = "cla-%s-api-log" +) + +// Repository interface for API logs +type Repository interface { + LogAPIRequest(ctx context.Context, url string) error +} + +// repository implements the Repository interface +type repository struct { + stage string + dynamoDBClient *dynamodb.DynamoDB +} + +// NewRepository creates a new API logs repository +func NewRepository(stage string, dynamoDBClient *dynamodb.DynamoDB) Repository { + return &repository{ + stage: stage, + dynamoDBClient: dynamoDBClient, + } +} + +// LogAPIRequest logs an API request to the DynamoDB table +// Creates three entries: ALL bucket, daily bucket (YYYY-MM-DD), and monthly bucket (YYYY-MM) +// IMPORTANT: table key is (url, dt). To avoid overwrites, dt is shifted by -1/0/+1 ms per bucket. +func (r *repository) LogAPIRequest(ctx context.Context, url string) error { + // 200% fail-safe: never panic on nil ctx/client + if ctx == nil { + ctx = context.Background() + } + if r == nil || r.dynamoDBClient == nil { + return fmt.Errorf("dynamodb client is nil") + } + + now := time.Now().UTC() + timestamp := now.UnixMilli() + + // Generate bucket names + dailyBucket := now.Format("2006-01-02") // YYYY-MM-DD + monthlyBucket := now.Format("2006-01") // YYYY-MM + + entries := []*APILog{ + {URL: url, DT: timestamp - 1, Bucket: "ALL"}, + {URL: url, DT: timestamp, Bucket: dailyBucket}, + {URL: url, DT: timestamp + 1, Bucket: monthlyBucket}, + } + tableName := fmt.Sprintf(APILogTableName, r.stage) + + var errs []string + for _, logEntry := range entries { + // Convert to DynamoDB attribute value + av, err := dynamodbattribute.MarshalMap(logEntry) + if err != nil { + errs = append(errs, fmt.Sprintf("bucket=%s marshal=%v", logEntry.Bucket, err)) + continue + } + + // Put item to DynamoDB + input := &dynamodb.PutItemInput{ + TableName: aws.String(tableName), + Item: av, + } + + _, err = r.dynamoDBClient.PutItemWithContext(ctx, input) + if err != nil { + errs = append(errs, fmt.Sprintf("bucket=%s put=%v", logEntry.Bucket, err)) + continue + } + } + + // Return error so middleware can emit a single LG:* line. + if len(errs) > 0 { + return fmt.Errorf("%s", strings.Join(errs, "; ")) + } + return nil +} diff --git a/cla-backend-go/cmd/server.go b/cla-backend-go/cmd/server.go index 21aee4d56..94400a08d 100644 --- a/cla-backend-go/cmd/server.go +++ b/cla-backend-go/cmd/server.go @@ -15,6 +15,7 @@ import ( "runtime" "strconv" "strings" + "time" "github.com/linuxfoundation/easycla/cla-backend-go/project/repository" "github.com/linuxfoundation/easycla/cla-backend-go/project/service" @@ -78,6 +79,7 @@ import ( "github.com/linuxfoundation/easycla/cla-backend-go/users" + "github.com/linuxfoundation/easycla/cla-backend-go/api_logs" "github.com/linuxfoundation/easycla/cla-backend-go/signatures" v2Signatures "github.com/linuxfoundation/easycla/cla-backend-go/v2/signatures" @@ -144,12 +146,34 @@ type combinedRepo struct { projects_cla_groups.Repository } -// in cmd/server.go (top-level imports already use logrus) -func apiPathLogger(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - log.Infof("LG:api-request-path:%s", r.URL.Path) - next.ServeHTTP(w, r) - }) +// apiPathLoggerWithDB creates a middleware that logs API requests to DynamoDB +func apiPathLoggerWithDB(apiLogsRepo api_logs.Repository) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Infof("LG:api-request-path:%s", r.URL.Path) + + // Log to DynamoDB table (fire-and-forget, never fail request) + if apiLogsRepo != nil { + path := r.URL.Path + go func() { + defer func() { + if rec := recover(); rec != nil { + log.Infof("LG:api-log-dynamo-failed:%s err=panic:%v", path, rec) + } + }() + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + if err := apiLogsRepo.LogAPIRequest(ctx, path); err != nil { + log.Infof("LG:api-log-dynamo-failed:%s err=%v", path, err) + } + }() + } + + next.ServeHTTP(w, r) + }) + } } // server function called by environment specific server functions @@ -281,6 +305,7 @@ func server(localMode bool) http.Handler { approvalListRepo := approval_list.NewRepository(awsSession, stage) v1CompanyRepo := v1Company.NewRepository(awsSession, stage) eventsRepo := events.NewRepository(awsSession, stage) + apiLogsRepo := api_logs.NewRepository(stage, dynamodb.New(awsSession)) v1ProjectClaGroupRepo := projects_cla_groups.NewRepository(awsSession, stage) v1CLAGroupRepo := repository.NewRepository(awsSession, stage, gitV1Repository, gerritRepo, v1ProjectClaGroupRepo) metricsRepo := metrics.NewRepository(awsSession, stage, configFile.APIGatewayURL, v1ProjectClaGroupRepo) @@ -406,7 +431,7 @@ func server(localMode bool) http.Handler { // The middleware configuration is for the handler executors. These do not apply to the swagger.json document. // The middleware executes after routing but before authentication, binding and validation middlewareSetupfunc := func(handler http.Handler) http.Handler { - return apiPathLogger(setRequestIDHandler(responseLoggingMiddleware(userCreaterMiddleware(handler)))) + return apiPathLoggerWithDB(apiLogsRepo)(setRequestIDHandler(responseLoggingMiddleware(userCreaterMiddleware(handler)))) } v2API.CsvProducer = openapi_runtime.ProducerFunc(func(w io.Writer, data interface{}) error { diff --git a/cla-backend-go/serverless.yml b/cla-backend-go/serverless.yml index 97210ebf7..76bc17f37 100644 --- a/cla-backend-go/serverless.yml +++ b/cla-backend-go/serverless.yml @@ -122,6 +122,7 @@ provider: - dynamodb:DescribeStream - dynamodb:ListStreams Resource: + - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-api-log" - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-ccla-whitelist-requests" - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-cla-manager-requests" - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-companies" @@ -144,6 +145,7 @@ provider: Action: - dynamodb:Query Resource: + - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-api-log/index/bucket-dt-index" - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-ccla-whitelist-requests/index/company-id-project-id-index" - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-ccla-whitelist-requests/index/ccla-approval-list-request-project-id-index" - "arn:aws:dynamodb:${self:custom.dynamodb.region}:${aws:accountId}:table/cla-${opt:stage}-users/index/github-id-index" diff --git a/cla-backend/cla/models/dynamo_models.py b/cla-backend/cla/models/dynamo_models.py index 12406e64a..b518c3b3d 100644 --- a/cla-backend/cla/models/dynamo_models.py +++ b/cla-backend/cla/models/dynamo_models.py @@ -58,6 +58,7 @@ def create_database(): GerritModel, EventModel, CCLAAllowlistRequestModel, + APILogModel, ] # Create all required tables. @@ -83,6 +84,7 @@ def delete_database(): GitHubOrgModel, GerritModel, CCLAAllowlistRequestModel, + APILogModel, ] # Delete all existing tables. for table in tables: @@ -5384,6 +5386,136 @@ def create_event( return {"errors": {"event_id": str(err)}} +class APILogBucketDTIndex(GlobalSecondaryIndex): + """ + This class represents a global secondary index for querying API logs by bucket and time range. + """ + + class Meta: + """Meta class for API Log bucket-dt index.""" + + index_name = "bucket-dt-index" + write_capacity_units = int(cla.conf.get("DYNAMO_WRITE_UNITS", 10)) + read_capacity_units = int(cla.conf.get("DYNAMO_READ_UNITS", 10)) + # All attributes are projected - not sure if this is necessary. + projection = AllProjection() + + # This attribute is the hash key for the index. + bucket = UnicodeAttribute(hash_key=True) + # This attribute is the range key for the index. + dt = NumberAttribute(range_key=True) + + +class APILogModel(BaseModel): + """ + Represents an API log entry in the database + """ + + class Meta: + """Meta class for APILog.""" + + table_name = "cla-{}-api-log".format(stage) + if stage == "local": + host = "http://localhost:8000" + + url = UnicodeAttribute(hash_key=True) + dt = NumberAttribute(range_key=True) + bucket = UnicodeAttribute(null=False) + + # GSI for querying by bucket and time range + bucket_dt_index = APILogBucketDTIndex() + + +class APILog(model_interfaces.APILog): + """ + ORM-agnostic wrapper for the DynamoDB APILog model. + """ + + def __init__(self, url=None, dt=None, bucket=None): + super().__init__() + self.model = APILogModel() + self.model.url = url + self.model.dt = dt + self.model.bucket = bucket + + def __str__(self): + return f"url:{self.model.url}, dt:{self.model.dt}, bucket:{self.model.bucket}" + + def to_dict(self): + return dict(self.model) + + def save(self) -> None: + # self.model.date_modified = datetime.datetime.utcnow() + self.model.save() + + def load(self, url, dt): + try: + api_log = self.model.get(str(url), int(dt)) + except APILogModel.DoesNotExist: + raise cla.models.DoesNotExist("API Log entry not found") + self.model = api_log + + def delete(self): + self.model.delete() + + def get_url(self): + return self.model.url + + def get_dt(self): + return self.model.dt + + def get_bucket(self): + return self.model.bucket + + def set_url(self, url): + self.model.url = url + + def set_dt(self, dt): + self.model.dt = dt + + def set_bucket(self, bucket): + self.model.bucket = bucket + + @classmethod + def log_api_request(cls, url: str): + """ + Log an API request with the given URL. + Creates three entries: ALL bucket, daily bucket, and monthly bucket. + Never raises exceptions - logs errors instead. + """ + try: + # Base timestamp in milliseconds + base_dt = int(time.time() * 1000) + dt_obj = datetime.datetime.utcnow() + + # Buckets + daily_bucket = dt_obj.strftime('%Y-%m-%d') + monthly_bucket = dt_obj.strftime('%Y-%m') + + # IMPORTANT: table key is (url, dt). To avoid overwrites we shift dt by -1/0/+1 ms. + entries = [ + ("ALL", base_dt - 1), + (daily_bucket, base_dt), + (monthly_bucket, base_dt + 1), + ] + + errors = [] + for bucket, dt_value in entries: + try: + api_log = cls(url=url, dt=dt_value, bucket=bucket) + api_log.save() + except Exception as e: + errors.append(f"bucket={bucket} err={e}") + + if errors: + # Only AWS logs entry (LG-style), never fail request flow + cla.log.info(f"LG:api-log-dynamo-failed:{url} " + "; ".join(errors)) + + except Exception as e: + # Never let API logging failure break the request flow + cla.log.info(f"LG:api-log-dynamo-failed:{url} err={e}") + + class CCLAAllowlistRequestModel(BaseModel): """ Represents a CCLAAllowlistRequest in the database diff --git a/cla-backend/cla/models/model_interfaces.py b/cla-backend/cla/models/model_interfaces.py index 49510b992..dc27c9c36 100644 --- a/cla-backend/cla/models/model_interfaces.py +++ b/cla-backend/cla/models/model_interfaces.py @@ -2383,3 +2383,80 @@ def all(self): :rtype: [cla.models.model_interfaces.ProjectCLAGroup] """ raise NotImplementedError() + + +class APILog(object): + """ + Interface to the APILog Model for logging API requests + """ + + def to_dict(self): + """ + Converts models to dictionaries for JSON serialization. + + :return: A dict representation of the model. + :rtype: dict + """ + raise NotImplementedError() + + def save(self): + """ + Simple abstraction around the supported ORMs to save a model + """ + raise NotImplementedError() + + def delete(self): + """ + Simple abstraction around the supported ORMs to delete a model + """ + raise NotImplementedError() + + def load(self, url, dt): + """ + Simple abstraction around the supported ORMs to load a model + Populates the current object. + + :param url: The URL of the API call + :type url: string + :param dt: The timestamp of the API call + :type dt: int + """ + raise NotImplementedError() + + def get_url(self): + """ + Returns the URL of the API call + + :return: The URL string + :rtype: string + """ + raise NotImplementedError() + + def get_dt(self): + """ + Returns the timestamp of the API call + + :return: The timestamp + :rtype: int + """ + raise NotImplementedError() + + def get_bucket(self): + """ + Returns the bucket of the API call (ALL, YYYY-MM-DD, or YYYY-MM) + + :return: The bucket string + :rtype: string + """ + raise NotImplementedError() + + @classmethod + def log_api_request(cls, url): + """ + Log an API request with the given URL. + Creates three entries: ALL bucket, daily bucket, and monthly bucket. + + :param url: The API endpoint URL + :type url: string + """ + raise NotImplementedError() diff --git a/cla-backend/cla/routes.py b/cla-backend/cla/routes.py index b8ec153b8..f3e4a2386 100755 --- a/cla-backend/cla/routes.py +++ b/cla-backend/cla/routes.py @@ -38,19 +38,52 @@ get_log_middleware ) +_APILOG_CLS = None +_APILOG_IMPORT_ERROR = None + +def _get_apilog_cls(): + """ + Lazy, cached import to avoid per-request imports while staying safe + against circular-import/startup ordering issues. + """ + global _APILOG_CLS, _APILOG_IMPORT_ERROR + if _APILOG_CLS is not None: + return _APILOG_CLS + if _APILOG_IMPORT_ERROR is not None: + return None + try: + from cla.models.dynamo_models import APILog as _APILog + _APILOG_CLS = _APILog + return _APILOG_CLS + except Exception as e: + _APILOG_IMPORT_ERROR = e + cla.log.info(f"LG:api-log-import-failed err={e}") + return None # # Middleware # @hug.request_middleware() -def process_data(request, response): +def process_data_api_logs(request, response): """ - This middleware is needed here to copy the stream so we can re-read it - later on in the other handlers, currently only active on /github/activity - endpoint because it's an expensive operation. + Request middleware that logs API requests and, for the GitHub activity + endpoint, copies the request body stream so it can be re-read by other + handlers. The stream-copy behavior is currently only applied to the + /github/activity endpoint because it is an expensive operation, while + API request metadata is logged to the DynamoDB-backed APILog model + for all requests. """ cla.log.info('LG:api-request-path:' + request.path) + + # Log API request to DynamoDB table + apilog_cls = _get_apilog_cls() + if apilog_cls is not None: + try: + apilog_cls.log_api_request(request.path) + except Exception as e: + cla.log.info(f"LG:api-log-dynamo-failed:{request.path} err={e}") + if "/github/activity" in request.path: body = request.bounded_stream.read() request.bounded_stream.read = lambda: body diff --git a/cla-backend/serverless.yml b/cla-backend/serverless.yml index 2f6c4b5f1..68cc9bfd5 100644 --- a/cla-backend/serverless.yml +++ b/cla-backend/serverless.yml @@ -207,6 +207,7 @@ provider: - dynamodb:DescribeStream - dynamodb:ListStreams Resource: + - "arn:aws:dynamodb:${aws:region}:${aws:accountId}:table/cla-${sls:stage}-api-log" - "arn:aws:dynamodb:${aws:region}:${aws:accountId}:table/cla-${sls:stage}-ccla-whitelist-requests" - "arn:aws:dynamodb:${aws:region}:${aws:accountId}:table/cla-${sls:stage}-cla-manager-requests" - "arn:aws:dynamodb:${aws:region}:${aws:accountId}:table/cla-${sls:stage}-companies" @@ -229,6 +230,7 @@ provider: Action: - dynamodb:Query Resource: + - "arn:aws:dynamodb:${aws:region}:${aws:accountId}:table/cla-${sls:stage}-api-log/index/bucket-dt-index" - "arn:aws:dynamodb:${aws:region}:${aws:accountId}:table/cla-${sls:stage}-ccla-whitelist-requests/index/company-id-project-id-index" - "arn:aws:dynamodb:${aws:region}:${aws:accountId}:table/cla-${sls:stage}-ccla-whitelist-requests/index/ccla-approval-list-request-project-id-index" - "arn:aws:dynamodb:${aws:region}:${aws:accountId}:table/cla-${sls:stage}-users/index/github-id-index" diff --git a/infra/backup-dynamodb-tables.sh b/infra/backup-dynamodb-tables.sh index b07d51d9e..b30c42cdf 100755 --- a/infra/backup-dynamodb-tables.sh +++ b/infra/backup-dynamodb-tables.sh @@ -60,7 +60,8 @@ declare -r region="us-east-1" declare -r profile="lfproduct-${env}" declare -r current_date="$(date +%Y%m%dT%H%M%S)" declare -r dry_run="false" -declare -a tables=( "cla-${env}-ccla-whitelist-requests" +declare -a tables=( "cla-${env}-api-log" + "cla-${env}-ccla-whitelist-requests" "cla-${env}-companies" "cla-${env}-company-invites" "cla-${env}-events" diff --git a/utils/aws_create_api_logs_table.sh b/utils/aws_create_api_logs_table.sh new file mode 100755 index 000000000..48c6c36b6 --- /dev/null +++ b/utils/aws_create_api_logs_table.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${AWS_REGION:?AWS_REGION is not set. Example: us-east-1}" +: "${STAGE:?STAGE is not set. Example: dev}" + +PROFILE="lfproduct-${STAGE}" +TABLE_NAME="cla-${STAGE}-api-log" + +echo "Creating table: ${TABLE_NAME} in ${AWS_REGION} using profile ${PROFILE}" + +# 1) Create table (ONLY define attrs used by the table key schema) +aws --profile "${PROFILE}" dynamodb create-table \ + --table-name "${TABLE_NAME}" \ + --attribute-definitions \ + AttributeName=url,AttributeType=S \ + AttributeName=dt,AttributeType=N \ + --key-schema \ + AttributeName=url,KeyType=HASH \ + AttributeName=dt,KeyType=RANGE \ + --billing-mode PAY_PER_REQUEST \ + --region "${AWS_REGION}" + +aws --profile "${PROFILE}" dynamodb wait table-exists \ + --table-name "${TABLE_NAME}" \ + --region "${AWS_REGION}" + +echo "Creating GSI bucket-dt-index (supports time-range queries across all URLs)" + +# 2) Add GSI: bucket (HASH) + dt (RANGE) +aws --profile "${PROFILE}" dynamodb update-table \ + --table-name "${TABLE_NAME}" \ + --attribute-definitions \ + AttributeName=bucket,AttributeType=S \ + AttributeName=dt,AttributeType=N \ + --global-secondary-index-updates '[ + { + "Create": { + "IndexName": "bucket-dt-index", + "KeySchema": [ + { "AttributeName": "bucket", "KeyType": "HASH" }, + { "AttributeName": "dt", "KeyType": "RANGE" } + ], + "Projection": { "ProjectionType": "ALL" } + } + } + ]' \ + --region "${AWS_REGION}" + +echo "Waiting for GSI to become ACTIVE..." +# Wait until the index becomes ACTIVE (polling) +while true; do + STATUS=$(aws --profile "${PROFILE}" dynamodb describe-table \ + --table-name "${TABLE_NAME}" \ + --region "${AWS_REGION}" \ + --query "Table.GlobalSecondaryIndexes[?IndexName=='bucket-dt-index'].IndexStatus | [0]" \ + --output text) + echo "bucket-dt-index status: ${STATUS}" + if [[ "${STATUS}" == "ACTIVE" ]]; then + break + fi + sleep 5 +done + +aws --profile ${PROFILE} dynamodb describe-table --table-name cla-${STAGE}-api-log --region ${AWS_REGION} + +echo "Done." diff --git a/utils/aws_edit_table_access.sh b/utils/aws_edit_table_access.sh new file mode 100755 index 000000000..9b6a5c2ed --- /dev/null +++ b/utils/aws_edit_table_access.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +if [ -z "$AWS_REGION" ]; then + echo "AWS_REGION is not set. Please set it before running the script." + exit 1 +fi +if [ -z "$STAGE" ]; then + echo "STAGE is not set. Please set it before running the script." + exit 1 +fi + +# aws --profile lfproduct-dev iam get-role-policy --role-name cla-backend-go-dev-us-east-2-lambdaRole --policy-name cla-backend-go-dev-lambda --query PolicyDocument --output json + +aws --profile lfproduct-${STAGE} --region ${AWS_REGION} \ + iam get-role-policy \ + --role-name cla-backend-${STAGE}-${AWS_REGION}-lambdaRole \ + --policy-name cla-backend-${STAGE}-lambda \ + --query PolicyDocument \ + --output json > /tmp/cla-backend-${STAGE}-${AWS_REGION}-lambda.json + +aws --profile lfproduct-${STAGE} --region ${AWS_REGION} \ + iam get-role-policy \ + --role-name cla-backend-go-${STAGE}-us-east-2-lambdaRole \ + --policy-name cla-backend-go-${STAGE}-lambda \ + --query PolicyDocument \ + --output json > /tmp/cla-backend-go-${STAGE}-us-east-2-lambda.json + +vim /tmp/cla-backend-${STAGE}-${AWS_REGION}-lambda.json /tmp/cla-backend-go-${STAGE}-us-east-2-lambda.json + +aws --profile lfproduct-${STAGE} --region ${AWS_REGION} \ + iam put-role-policy \ + --role-name cla-backend-${STAGE}-${AWS_REGION}-lambdaRole \ + --policy-name cla-backend-${STAGE}-lambda \ + --policy-document file:///tmp/cla-backend-${STAGE}-${AWS_REGION}-lambda.json + +aws --profile lfproduct-${STAGE} --region ${AWS_REGION} \ + iam put-role-policy \ + --role-name cla-backend-go-${STAGE}-us-east-2-lambdaRole \ + --policy-name cla-backend-go-${STAGE}-lambda \ + --policy-document file:///tmp/cla-backend-go-${STAGE}-us-east-2-lambda.json diff --git a/utils/aws_example_api_logs_usage.sh b/utils/aws_example_api_logs_usage.sh new file mode 100644 index 000000000..a0274f83d --- /dev/null +++ b/utils/aws_example_api_logs_usage.sh @@ -0,0 +1,63 @@ +aws --profile lfproduct-dev dynamodb put-item \ + --table-name cla-dev-api-log \ + --region us-east-1 \ + --item '{ + "url": {"S": "/health"}, + "dt": {"N": "1715086845123"}, + "bucket": {"S": "ALL"} + }' + +aws --profile lfproduct-dev dynamodb query \ + --table-name cla-dev-api-log \ + --region us-east-1 \ + --key-condition-expression "#u = :u" \ + --expression-attribute-names '{"#u":"url"}' \ + --expression-attribute-values '{":u":{"S":"/health"}}' +{ + "Items": [ + { + "url": { + "S": "/health" + }, + "bucket": { + "S": "ALL" + }, + "dt": { + "N": "1715086845123" + } + } + ], + "Count": 1, + "ScannedCount": 1, + "ConsumedCapacity": null +} + +aws --profile lfproduct-dev dynamodb query \ + --table-name cla-dev-api-log \ + --index-name bucket-dt-index \ + --region us-east-1 \ + --key-condition-expression "#b = :b AND #dt BETWEEN :f AND :t" \ + --expression-attribute-names '{"#b":"bucket","#dt":"dt"}' \ + --expression-attribute-values '{ + ":b":{"S":"ALL"}, + ":f":{"N":"0"}, + ":t":{"N":"9999999999999"} + }' +{ + "Items": [ + { + "dt": { + "N": "1715086845123" + }, + "url": { + "S": "/health" + }, + "bucket": { + "S": "ALL" + } + } + ], + "Count": 1, + "ScannedCount": 1, + "ConsumedCapacity": null +} diff --git a/utils/example_scan_api_log.sh b/utils/example_scan_api_log.sh new file mode 100755 index 000000000..785970df5 --- /dev/null +++ b/utils/example_scan_api_log.sh @@ -0,0 +1,2 @@ +#!/bin/bash +aws --region us-east-1 --profile lfproduct-dev dynamodb scan --table-name cla-dev-api-log --max-items 100 | jq -r '.Items'