Merge pull request #257 from circleci/parallel-uploader

Replace slow Python+BASH tooling with Go

Author: pete-woods

.circleci/config.yml

@@ -8,16 +8,17 @@ parameters:
 orbs:
   aws-cli: circleci/[email protected]
   vale: circleci/[email protected]
+  go: circleci/[email protected]
 
 executors:
   node_executor:
     docker:
       - image: cimg/node:22.15.1
     working_directory: ~/project
 
-  python_executor:
+  go_executor:
     docker:
-      - image: cimg/python:3.12.11
+      - image: cimg/go:1.24
     working_directory: ~/project
 
   ruby_executor:
@@ -107,11 +108,6 @@ jobs:
           paths:
             - build
             - build.zip
-            - scripts
-      - persist_to_workspace:
-          root: .
-          paths:
-            - build
       - notify_error:
           message: "Build job failed for branch ${CIRCLE_BRANCH}"
 
@@ -190,7 +186,7 @@ jobs:
           message: "Deploy preview job failed for branch ${CIRCLE_BRANCH}"
 
   deploy-production:
-    executor: python_executor
+    executor: go_executor
     parameters:
       bucket_name:
         description: The name of the s3 bucket where static assets are stored.
@@ -200,39 +196,27 @@ jobs:
         description: The path to the docs build directory
         type: string
     steps:
+      - checkout
       - attach_workspace:
           at: .
       - aws-setup
-      - run:
-          name: Deploy Production Site to S3
-          command: |
-            AWS_S3_BUCKET=<< parameters.bucket_name >>
-            BUILD_DIRECTORY=<< parameters.build_dir >>
-
-            set -e
-            echo "[INFO] Deploying production site..."
-            aws s3 sync "$BUILD_DIRECTORY" "s3://$AWS_S3_BUCKET/"
-      - run:
-          name: install pyyaml requests
-          command: |
-            set -e
-            echo "[INFO] Installing pyyaml requests..."
-            pip install pyyaml requests
-      - run:
-          name: Deploy Redirects to S3
-          command: |
-            AWS_S3_BUCKET=<< parameters.bucket_name >>
-
-            set -e
-            echo "[INFO] Deploying redirects..."
-            #REMEMBER TO UPDATE THE START/END PARAMETER IN THE VALIDATE JOB TO MATCH
-            python scripts/create-redirects.py $AWS_S3_BUCKET
-      - run:
-          name: Validate Redirects
-          command: |
-            set -e
-            echo "[INFO] Validating redirects..."
-            python scripts/validate-redirects.py https://circleci.com/docs-preview
+      - go/with-cache:
+          steps:
+            - go/mod-download
+            - run:
+                name: Deploy Production Site to S3
+                command: |
+                  AWS_S3_BUCKET=<< parameters.bucket_name >>
+                  BUILD_DIRECTORY=<< parameters.build_dir >>
+      
+                  echo "[INFO] Deploying production site..."
+                  aws s3 sync "$BUILD_DIRECTORY" "s3://$AWS_S3_BUCKET/"
+            - run:
+                name: Deploy Redirects to S3
+                command: go run ./cmd/create-redirects "<< parameters.bucket_name >>"
+            - run:
+                name: Validate Redirects
+                command: go run ./cmd/validate-redirects https://circleci.com
       - notify_error:
           message: "Production deployment job failed for branch ${CIRCLE_BRANCH}"
 

cmd/create-redirects/main.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log/slog"
+	"os"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/lmittmann/tint"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/sync/semaphore"
+
+	"github.com/circleci/circleci-docs-static/internal/redirects"
+)
+
+type options struct {
+	Bucket string
+	Start  int
+	End    int
+}
+
+const (
+	concurrency = 20
+	docsPath    = "docs-preview"
+)
+
+func main() {
+	ctx := context.Background()
+
+	slog.SetDefault(slog.New(
+		tint.NewHandler(os.Stderr, &tint.Options{
+			Level: slog.LevelDebug,
+		}),
+	))
+
+	opts := options{}
+	flag.IntVar(&opts.Start, "start", 0, "Start index (inclusive)")
+	flag.IntVar(&opts.End, "end", -1, "Start index (inclusive)")
+	flag.Parse()
+	opts.Bucket = flag.Arg(0)
+
+	if opts.Bucket == "" {
+		slog.Error("Bucket is required")
+		os.Exit(1)
+	}
+
+	err := run(ctx, opts)
+	if err != nil {
+		slog.Error("Unexpected error", "error", err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context, opts options) error {
+	awsConfig, err := config.LoadDefaultConfig(ctx)
+	if err != nil {
+		return err
+	}
+
+	s3Client := s3.NewFromConfig(awsConfig)
+
+	redirs, err := redirects.Load(redirects.DefaultFile)
+	if err != nil {
+		return err
+	}
+
+	if opts.End < 0 {
+		opts.End = len(redirs)
+	}
+
+	sliceRedirects := redirs[opts.Start:opts.End]
+	slog.Info("Redirects",
+		"start", opts.Start,
+		"end", opts.End,
+		"total", len(sliceRedirects),
+	)
+
+	u := newUploader(opts.Bucket, s3Client, concurrency)
+	g, ctx := errgroup.WithContext(ctx)
+	for _, r := range sliceRedirects {
+		g.Go(func() (err error) {
+			return u.Upload(ctx, r)
+		})
+	}
+	err = g.Wait()
+	if err != nil {
+		return err
+	}
+
+	slog.Info("Successfully processed redirects")
+	return nil
+}
+
+type uploader struct {
+	bucket string
+	client *s3.Client
+	sem    *semaphore.Weighted
+}
+
+func newUploader(bucket string, client *s3.Client, max int64) *uploader {
+	return &uploader{
+		bucket: bucket,
+		client: client,
+		sem:    semaphore.NewWeighted(max),
+	}
+}
+
+func (u *uploader) Upload(ctx context.Context, r redirects.Redirect) error {
+	err := u.sem.Acquire(ctx, 1)
+	if err != nil {
+		return err
+	}
+
+	defer u.sem.Release(1)
+
+	key := fmt.Sprintf("%s%sindex.html", docsPath, r.Old)
+	redir := fmt.Sprintf("/%s%s", docsPath, r.New)
+	slog.Info("Creating redirect",
+		"old", r.Old,
+		"new", r.New,
+		"key", key,
+		"redirect", redir,
+	)
+	_, err = u.client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket:                  aws.String(u.bucket),
+		Key:                     aws.String(key),
+		WebsiteRedirectLocation: aws.String(redir),
+		ContentType:             aws.String("text/html"),
+		ContentLength:           aws.Int64(0),
+	})
+	return err
+}