updates

Author: ford-at-aws

aws_doc_sdk_examples_tools/stats/.gitignore

@@ -0,0 +1,8 @@
+*.js
+!jest.config.js
+*.d.ts
+node_modules
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

aws_doc_sdk_examples_tools/stats/.npmignore

@@ -0,0 +1,6 @@
+*.ts
+!*.d.ts
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

aws_doc_sdk_examples_tools/stats/README.md

@@ -0,0 +1,14 @@
+# Welcome to your CDK TypeScript project
+
+This is a blank project for CDK development with TypeScript.
+
+The `cdk.json` file tells the CDK Toolkit how to execute your app.
+
+## Useful commands
+
+* `npm run build`   compile typescript to js
+* `npm run watch`   watch for changes and compile
+* `npm run test`    perform the jest unit tests
+* `npx cdk deploy`  deploy this stack to your default AWS account/region
+* `npx cdk diff`    compare deployed stack with current state
+* `npx cdk synth`   emits the synthesized CloudFormation template

aws_doc_sdk_examples_tools/stats/app.ts

@@ -0,0 +1,102 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import * as cdk from "aws-cdk-lib";
+import * as lambda from "aws-cdk-lib/aws-lambda";
+import * as iam from "aws-cdk-lib/aws-iam";
+import * as events from "aws-cdk-lib/aws-events";
+import * as targets from "aws-cdk-lib/aws-events-targets";
+import { Construct } from "constructs";
+import * as path from "path";
+
+const repoName = "AWSDocsSdkExamplesPublic";
+const awsRegion = "us-west-2";
+
+class CodeCommitCloneStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
+    super(scope, id, props);
+
+    // Initialize Lambda function
+    const cloneLambda = this.initCloneLambda();
+
+    // Set up EventBridge rule to trigger Lambda on CodeCommit repository changes
+    this.initCodeCommitTrigger(cloneLambda);
+  }
+
+  private initCloneLambda(): lambda.Function {
+    // IAM Role and Policy for Lambda to access CodeCommit
+    const lambdaExecutionRole = new iam.Role(this, "CloneLambdaExecutionRole", {
+      assumedBy: new iam.ServicePrincipal("lambda.amazonaws.com"),
+      description: "Execution role for Lambda function to clone CodeCommit repo",
+      managedPolicies: [
+        iam.ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaBasicExecutionRole"),
+      ],
+    });
+
+    // Grant necessary permissions to CodeCommit and S3
+    lambdaExecutionRole.addToPolicy(
+      new iam.PolicyStatement({
+        actions: [
+          "codecommit:GetRepository",
+          "codecommit:GitPull",
+          "codecommit:GetBranch",
+          "codecommit:GetDifferences",
+          "codecommit:GetFile"
+        ],
+        resources: [`arn:aws:codecommit:${awsRegion}:${this.account}:${repoName}`],
+      })
+    );
+
+    // Grant necessary permissions to S3 bucket "codeexamplestats" for Get and Put
+    lambdaExecutionRole.addToPolicy(
+      new iam.PolicyStatement({
+        actions: ["s3:GetObject", "s3:PutObject"],
+        resources: [`arn:aws:s3:::codeexamplestats/*`],  // Allow access to all objects in the bucket
+      })
+    );
+
+    // Define the Lambda function, pointing directly to the source code directory
+    const cloneLambda = new lambda.Function(this, "CodeCommitCloneLambda", {
+      runtime: lambda.Runtime.PYTHON_3_9,
+      handler: "index.lambda_handler",
+      code: lambda.Code.fromAsset(path.join(__dirname, "lambda")), // Pointing to the directory of the lambda function code
+      environment: {
+        REPO_NAME: repoName,
+      },
+      timeout: cdk.Duration.minutes(5),
+      role: lambdaExecutionRole,
+    });
+
+    return cloneLambda;
+  }
+
+  private initCodeCommitTrigger(cloneLambda: lambda.Function): void {
+    // Create EventBridge rule for CodeCommit repository updates
+    const codeCommitRule = new events.Rule(this, "CodeCommitUpdateRule", {
+      eventPattern: {
+        source: ["aws.codecommit"],
+        detailType: ["CodeCommit Repository State Change"],
+        resources: [`arn:aws:codecommit:${awsRegion}:${this.account}:${repoName}`],
+        detail: {
+          event: [
+            "referenceCreated",
+            "referenceUpdated",
+            "referenceDeleted"
+          ]
+        }
+      }
+    });
+
+    // Add Lambda function as the target of the EventBridge rule
+    codeCommitRule.addTarget(new targets.LambdaFunction(cloneLambda));
+  }
+}
+
+const app = new cdk.App();
+new CodeCommitCloneStack(app, "CodeCommitCloneStack", {
+  env: {
+    account: process.env.CDK_DEFAULT_ACCOUNT,
+    region: "us-west-2",  // Where codecommit is stored
+  },
+});
+app.synth();

aws_doc_sdk_examples_tools/stats/build.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Define paths
+LAMBDA_DIR="lambda"  # Directory where your Lambda function code is
+BUILD_DIR="build"    # Temporary directory to install dependencies
+OUTPUT_ZIP="lambda_package.zip"  # Final zip package
+
+# Clean up any previous builds
+rm -rf $BUILD_DIR $OUTPUT_ZIP
+mkdir -p $BUILD_DIR/python
+
+# Install dependencies to the build directory
+pip install -r $LAMBDA_DIR/requirements.txt -t $BUILD_DIR/python
+
+# Copy Lambda code to the build directory
+cp -r $LAMBDA_DIR/* $BUILD_DIR/python
+
+# Zip everything in the build directory into the output zip
+cd $BUILD_DIR/python && zip -r ../../$OUTPUT_ZIP . && cd ../../
+
+# Clean up build directory
+rm -rf $BUILD_DIR
+

aws_doc_sdk_examples_tools/stats/cdk.json

@@ -0,0 +1,3 @@
+{
+  "app": "npx ts-node app.ts"
+}

aws_doc_sdk_examples_tools/stats/dataset/manifest.json

@@ -0,0 +1,16 @@
+{
+    "fileLocations": [
+        {
+            "URIs": [
+                "s3://codeexamplestats/stats.csv"
+            ]
+        }
+    ],
+    "globalUploadSettings": {
+        "format": "CSV",
+        "delimiter": ",",
+        "textqualifier": "\"",
+        "containsHeader": "true"
+    }
+}
+

aws_doc_sdk_examples_tools/stats/jest.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  testEnvironment: 'node',
+  roots: ['<rootDir>/test'],
+  testMatch: ['**/*.test.ts'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest'
+  }
+};

aws_doc_sdk_examples_tools/stats/lambda/index.py

@@ -0,0 +1,74 @@
+import boto3
+import os
+import tempfile
+from io import BytesIO, StringIO
+import json
+import csv
+from datetime import datetime  # Import datetime module
+
+def lambda_handler(event, context):
+    # Initialize boto3 clients
+    codecommit = boto3.client("codecommit", region_name=os.environ["AWS_REGION"])
+    s3 = boto3.client("s3")
+    
+    # Environment variables
+    repo_name = os.environ.get("REPO_NAME")
+    branch_name = os.environ.get("BRANCH_NAME", "mainline")  # Default to "mainline"
+    bucket_name = "codeexamplestats"
+    csv_file_key = "stats.csv"
+
+    try:
+        # Step 1: Retrieve the .stats file content directly from CodeCommit
+        file_response = codecommit.get_file(
+            repositoryName=repo_name,
+            filePath='.stats',  # Specify the path to the .stats file
+            commitSpecifier=branch_name
+        )
+        
+        # Convert .stats content to valid JSON if necessary
+        file_content = file_response['fileContent'].decode("utf-8")
+        try:
+            stats_data = json.loads(file_content)  # Valid JSON parsing
+        except json.JSONDecodeError:
+            file_content = file_content.replace("'", "\"")  # Replace single quotes
+            stats_data = json.loads(file_content)
+        
+        # Step 2: Fetch the current stats.csv file from S3
+        existing_rows = []
+        try:
+            csv_obj = s3.get_object(Bucket=bucket_name, Key=csv_file_key)
+            csv_data = csv_obj['Body'].read().decode('utf-8')
+            csv_reader = csv.DictReader(StringIO(csv_data))
+            existing_rows = list(csv_reader)
+        except s3.exceptions.NoSuchKey:
+            existing_rows = []
+
+        # Step 3: Append the new data from .stats file with a formatted timestamp
+        new_row = {
+            'sdks': stats_data['sdks'],
+            'services': stats_data['services'],
+            'examples': stats_data['examples'],
+            'versions': stats_data['versions'],
+            'snippets': stats_data['snippets'],
+            'genai_none': stats_data['genai']['none'],
+            'genai_some': stats_data['genai']['some'],
+            'genai_most': stats_data['genai']['most'],
+            'timestamp': datetime.now().strftime("%d/%m/%Y %H:%M:%S")  # Formatted timestamp
+        }
+        existing_rows.append(new_row)
+
+        # Step 4: Save the updated data back to CSV and upload it to S3
+        with StringIO() as output:
+            # Include 'timestamp' in fieldnames to write the timestamp in the CSV
+            writer = csv.DictWriter(output, fieldnames=new_row.keys())
+            writer.writeheader()  # Ensure we have headers
+            writer.writerows(existing_rows)
+            s3.put_object(Bucket=bucket_name, Key=csv_file_key, Body=output.getvalue().encode('utf-8'))
+        
+        print("Successfully updated stats.csv in S3 with new .stats data.")
+    
+    except Exception as e:
+        print(f"Error occurred: {e}")
+        return {"statusCode": 500, "body": "Failed to update stats.csv."}
+
+    return {"statusCode": 200, "body": "stats.csv updated successfully in S3."}

aws_doc_sdk_examples_tools/stats/lambda/requirements.txt

@@ -0,0 +1,2 @@
+pandas
+numpy