1+ #! /bin/bash
2+
3+ set -e
4+
5+ # Check required parameters
6+ if [ -z " $1 " || [ -z " $2 " ; then
7+ echo " Usage: $0 <bucket_name> <redirects_file>"
8+ echo " Example: $0 circleci-docs-platform-assets/docs-preview scripts/redirects_v2.yml"
9+ exit 1
10+ fi
11+
12+ BUCKET_NAME=" $1 "
13+ REDIRECTS_FILE=" $2 "
14+ TEMP_DIR=$( mktemp -d)
15+ BATCH_SIZE=50
16+
17+ echo " [INFO] Processing redirects from $REDIRECTS_FILE to bucket s3://$BUCKET_NAME "
18+ echo " [INFO] Using temporary directory: $TEMP_DIR "
19+
20+ # Cleanup function
21+ cleanup () {
22+ echo " [INFO] Cleaning up temporary files..."
23+ rm -rf " $TEMP_DIR "
24+ }
25+ trap cleanup EXIT
26+
27+ # Check if redirects file exists
28+ if [ ! -f " $REDIRECTS_FILE " ; then
29+ echo " [ERROR] Redirects file not found: $REDIRECTS_FILE "
30+ exit 1
31+ fi
32+
33+ # Parse YAML and create redirect objects in batches
34+ python3 << 'EOF '
35+ import yaml
36+ import subprocess
37+ import sys
38+ import os
39+ import tempfile
40+ import json
41+ from concurrent.futures import ThreadPoolExecutor, as_completed
42+ import threading
43+
44+ bucket_name = sys.argv[1]
45+ redirects_file = sys.argv[2]
46+ temp_dir = sys.argv[3]
47+ batch_size = int(sys.argv[4])
48+
49+ print(f"[INFO] Loading redirects from {redirects_file}")
50+
51+ def create_redirect_object(bucket, key, redirect_location):
52+ """Create a single redirect object using aws s3api put-object"""
53+ try:
54+ cmd = [
55+ 'aws', 's3api', 'put-object',
56+ '--bucket', bucket,
57+ '--key', key,
58+ '--website-redirect-location', redirect_location,
59+ '--content-type', 'text/html',
60+ '--content-length', '0'
61+ ]
62+
63+ result = subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=30)
64+ return True, key, None
65+ except subprocess.CalledProcessError as e:
66+ return False, key, f"S3 API error: {e.stderr.strip()}"
67+ except subprocess.TimeoutExpired:
68+ return False, key, "Request timeout"
69+ except Exception as e:
70+ return False, key, f"Unexpected error: {str(e)}"
71+
72+ def process_batch(batch_redirects):
73+ """Process a batch of redirects with thread pool"""
74+ success_count = 0
75+ error_count = 0
76+ errors = []
77+
78+ # Use ThreadPoolExecutor for concurrent uploads
79+ with ThreadPoolExecutor(max_workers=10) as executor:
80+ future_to_redirect = {}
81+
82+ for redirect in batch_redirects:
83+ old_path = redirect['old'].rstrip('/')
84+ new_path = redirect['new']
85+
86+ # Ensure paths start with /
87+ if not old_path.startswith('/'):
88+ old_path = '/' + old_path
89+ if not new_path.startswith('/'):
90+ new_path = '/' + new_path
91+
92+ # S3 object key (remove leading slash for S3)
93+ s3_key = old_path.lstrip('/')
94+
95+ # If the old path doesn't end with a file extension, add index.html
96+ if not s3_key.endswith('.html') and not s3_key.endswith('/'):
97+ s3_key += '/index.html'
98+ elif s3_key.endswith('/'):
99+ s3_key += 'index.html'
100+
101+ future = executor.submit(create_redirect_object, bucket_name, s3_key, new_path)
102+ future_to_redirect[future] = (old_path, new_path, s3_key)
103+
104+ # Collect results
105+ for future in as_completed(future_to_redirect):
106+ old_path, new_path, s3_key = future_to_redirect[future]
107+ success, key, error = future.result()
108+
109+ if success:
110+ success_count += 1
111+ print(f"[SUCCESS] {old_path} -> {new_path}")
112+ else:
113+ error_count += 1
114+ error_msg = f"{old_path} -> {new_path}: {error}"
115+ errors.append(error_msg)
116+ print(f"[ERROR] {error_msg}")
117+
118+ return success_count, error_count, errors
119+
120+ try:
121+ with open(redirects_file, 'r') as f:
122+ redirects = yaml.safe_load(f)
123+
124+ if not redirects:
125+ print("[WARN] No redirects found in file")
126+ sys.exit(0)
127+
128+ total_redirects = len(redirects)
129+ print(f"[INFO] Found {total_redirects} redirects to process")
130+ print(f"[INFO] Processing in batches of {batch_size} with concurrent uploads")
131+
132+ total_success = 0
133+ total_errors = 0
134+ all_errors = []
135+
136+ # Process redirects in batches
137+ for i in range(0, total_redirects, batch_size):
138+ batch = redirects[i:i + batch_size]
139+ batch_num = (i // batch_size) + 1
140+ total_batches = (total_redirects + batch_size - 1) // batch_size
141+
142+ print(f"[INFO] Processing batch {batch_num}/{total_batches} ({len(batch)} redirects)")
143+
144+ success_count, error_count, errors = process_batch(batch)
145+ total_success += success_count
146+ total_errors += error_count
147+ all_errors.extend(errors)
148+
149+ print(f"[INFO] Batch {batch_num} complete: {success_count} success, {error_count} errors")
150+
151+ print(f"[INFO] Redirect processing complete:")
152+ print(f"[INFO] - Total redirects processed: {total_redirects}")
153+ print(f"[INFO] - Successfully created: {total_success}")
154+ print(f"[INFO] - Errors: {total_errors}")
155+
156+ if all_errors:
157+ print(f"[ERROR] Failed redirects:")
158+ for error in all_errors[:10]: # Show first 10 errors
159+ print(f"[ERROR] - {error}")
160+ if len(all_errors) > 10:
161+ print(f"[ERROR] ... and {len(all_errors) - 10} more errors")
162+
163+ # Exit with error if too many failed
164+ if total_errors > total_redirects * 0.1: # More than 10% failed
165+ print(f"[ERROR] Too many redirects failed ({total_errors}/{total_redirects})")
166+ sys.exit(1)
167+
168+ if total_errors > 0:
169+ print(f"[WARN] {total_errors} redirects failed, but continuing since error rate is acceptable")
170+
171+ except Exception as e:
172+ print(f"[ERROR] Failed to process redirects: {e}")
173+ import traceback
174+ traceback.print_exc()
175+ sys.exit(1)
176+
177+ EOF " $BUCKET_NAME " " $REDIRECTS_FILE " " $TEMP_DIR " " $BATCH_SIZE "
178+
179+ echo " [INFO] Redirects deployment completed successfully"
0 commit comments