Skip to content

Commit 57453a5

Browse files
keackeac
authored
Update sync-tf-mirror workflow for strict verification
1 parent bdbc218 commit 57453a5

1 file changed

Lines changed: 109 additions & 81 deletions

File tree

‎.github/workflows/sync-tf-mirror.yml‎

Lines changed: 109 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ name: Sync TF Mirror
22

33
on:
44
schedule:
5-
- cron: '0 4 * * 0' # 每周日运行
6-
workflow_dispatch:
5+
- cron: '0 4 * * 0' # 每周日凌晨 4 点自动同步
6+
workflow_dispatch: # 支持手动触发
77

88
permissions:
99
contents: write
@@ -20,7 +20,7 @@ jobs:
2020
with:
2121
python-version: '3.11'
2222

23-
- name: Force Sync & Verify
23+
- name: Secure Sync & Verify
2424
shell: python
2525
run: |
2626
import json
@@ -32,16 +32,25 @@ jobs:
3232
import datetime
3333
import textwrap
3434
import hashlib
35+
import binascii
3536
import base64
3637
from pathlib import Path
3738
from concurrent.futures import ThreadPoolExecutor, as_completed
3839
39-
# ================= 配置区域 =================
40+
# ==============================================================================
41+
# 1. 全局配置 (Configuration)
42+
# ==============================================================================
43+
44+
# [必须修改] 您的镜像站地址，必须以 / 结尾
45+
MIRROR_URL = 'https://redc.wgpsec.org/tf-mirror/'
46+
47+
# 需要同步的 Provider 列表
4048
PROVIDERS = [
4149
'random', 'null', 'time', 'local',
4250
'tls', 'http', 'archive', 'external'
4351
]
4452
53+
# 需要同步的操作系统和架构
4554
TARGET_PLATFORMS = [
4655
('linux', 'amd64'), ('linux', 'arm64'),
4756
('windows', 'amd64'),
@@ -50,30 +59,54 @@ jobs:
5059
5160
MIRROR_ROOT = Path('public/tf-mirror')
5261
REGISTRY_API = 'https://registry.terraform.io/v1/providers'
53-
MIRROR_URL = 'https://redc.wgpsec.org/tf-mirror/'
54-
MAX_WORKERS = 4
55-
# ===========================================
62+
MAX_WORKERS = 4 # 并发下载数
5663
57-
# 1. 暴力清理旧目录
58-
if MIRROR_ROOT.exists():
59-
print("🧹 Cleaning up old mirror directory...")
60-
shutil.rmtree(MIRROR_ROOT)
61-
MIRROR_ROOT.mkdir(parents=True, exist_ok=True)
64+
# ==============================================================================
65+
# 2. 核心逻辑 (Core Logic)
66+
# ==============================================================================
6267
6368
class MirrorSync:
6469
def __init__(self, base_dir: Path):
6570
self.base_dir = base_dir
6671
67-
def fetch_text(self, url):
72+
def fetch_json(self, url):
73+
"""从 Terraform Registry API 获取元数据"""
6874
req = urllib.request.Request(url, headers={'User-Agent': 'TF-Mirror-Bot'})
6975
with urllib.request.urlopen(req, timeout=15) as r:
70-
return r.read().decode('utf-8')
76+
return json.loads(r.read().decode())
7177
72-
def fetch_json(self, url):
73-
content = self.fetch_text(url)
74-
return json.loads(content)
78+
def verify_and_convert_hash(self, file_path: Path, expected_hex: str):
79+
"""
80+
[核心安全逻辑]
81+
1. 读取已下载文件的内容，计算本地 SHA256 (Hex)。
82+
2. 与官方 API 返回的 expected_hex 进行比对。
83+
3. 如果一致，将 Hex 转换为 Terraform 需要的 h1:Base64 格式。
84+
"""
85+
print(f" 🛡️ Verifying checksum for {file_path.name}...")
86+
sha256 = hashlib.sha256()
87+
88+
# 分块读取文件，计算 Hash (避免大文件撑爆内存)
89+
with open(file_path, 'rb') as f:
90+
for block in iter(lambda: f.read(4096), b""):
91+
sha256.update(block)
92+
93+
local_hex = sha256.hexdigest()
94+
95+
# === 步骤 A: 安全比对 ===
96+
if local_hex != expected_hex:
97+
print(f" ❌ CRITICAL: Hash Mismatch!")
98+
print(f" Expected: {expected_hex}")
99+
print(f" Actual: {local_hex}")
100+
return None # 校验失败
101+
102+
# === 步骤 B: 格式转换 ===
103+
# Terraform Network Mirror 协议要求: "h1:" + Base64编码的二进制Hash
104+
binary_hash = sha256.digest() # 获取二进制摘要
105+
b64_hash = base64.b64encode(binary_hash).decode('utf-8')
106+
return f"h1:{b64_hash}"
75107
76108
def download_file(self, url, dest_path: Path):
109+
"""下载文件，如果已存在则跳过"""
77110
if dest_path.exists() and dest_path.stat().st_size > 0:
78111
return 'SKIP'
79112
try:
@@ -83,67 +116,47 @@ jobs:
83116
shutil.copyfileobj(r, f)
84117
return 'OK'
85118
except Exception as e:
86-
if dest_path.exists(): dest_path.unlink()
119+
if dest_path.exists(): dest_path.unlink() # 下载失败清理垃圾
87120
raise e
88121
89-
def verify_integrity(self, file_path: Path, filename: str, official_sums: str):
90-
"""
91-
计算本地文件的 SHA256，并检查是否在官方 SHA256SUMS 文件中
92-
"""
93-
print(f" 🛡️ Verifying integrity for {filename}...")
94-
sha256_hash = hashlib.sha256()
95-
with open(file_path, "rb") as f:
96-
for byte_block in iter(lambda: f.read(4096), b""):
97-
sha256_hash.update(byte_block)
98-
99-
# 1. 获取 Hex 格式 Hash (用于比对)
100-
local_hex = sha256_hash.hexdigest()
101-
102-
# 2. 构造官方校验行特征 (格式: hash filename)
103-
expected_entry = f"{local_hex} {filename}"
104-
105-
if expected_entry not in official_sums:
106-
print(f" ❌ Hash Mismatch! Local: {local_hex}")
107-
return None # 校验失败
108-
109-
# 3. 生成 Terraform 需要的 h1:Base64 格式
110-
digest = sha256_hash.digest()
111-
b64_hash = base64.b64encode(digest).decode('utf-8')
112-
return f"h1:{b64_hash}"
113-
114-
def process_single_arch(self, namespace, type_name, version, os_name, arch, provider_dir, official_sums):
122+
def process_single_arch(self, namespace, type_name, version, os_name, arch, provider_dir):
123+
"""处理单个架构：下载 -> 校验 -> 返回元数据"""
115124
try:
125+
# 1. 调用 API 获取下载链接和官方 Hash
116126
dl_api = f'{REGISTRY_API}/{namespace}/{type_name}/{version}/download/{os_name}/{arch}'
117127
dl_info = self.fetch_json(dl_api)
128+
118129
filename = dl_info['filename']
119130
file_url = dl_info['download_url']
131+
official_hex = dl_info['shasum'] # 官方宣称的 Hash (Hex格式)
120132
121133
dest = provider_dir / filename
122134
123-
# 1. 下载
135+
# 2. 下载文件
124136
self.download_file(file_url, dest)
125137
126-
# 2. 校验 (对比官方指纹)
127-
tf_hash = self.verify_integrity(dest, filename, official_sums)
138+
# 3. [关键] 本地计算 Hash 并与官方比对
139+
tf_hash = self.verify_and_convert_hash(dest, official_hex)
128140
129141
if not tf_hash:
130-
# 校验失败，删除文件，抛出异常
142+
# 校验失败，立即删除文件，防止污染镜像
131143
if dest.exists(): dest.unlink()
132-
return {"status": "failed", "error": "Checksum verification failed"}
144+
raise ValueError("Checksum verification failed (Local vs Official)")
133145
134146
return {
135147
"status": "success",
136148
"os": os_name,
137149
"arch": arch,
138150
"filename": filename,
139-
"hash": tf_hash
151+
"hash": tf_hash # 验证通过后的 h1 Hash
140152
}
141153
except urllib.error.HTTPError:
142-
return {"status": "skipped"}
154+
return {"status": "skipped"} # 架构不存在
143155
except Exception as e:
144156
return {"status": "failed", "error": str(e)}
145157
146158
def process_provider(self, full_name):
159+
"""处理单个 Provider 全流程"""
147160
if '/' in full_name:
148161
namespace, type_name = full_name.split('/')
149162
else:
@@ -153,57 +166,46 @@ jobs:
153166
print(f'🔍 Analyzing {namespace}/{type_name}...')
154167
155168
try:
156-
# 1. 获取版本信息
169+
# 1. 获取版本
157170
meta_url = f'{REGISTRY_API}/{namespace}/{type_name}'
158171
meta = self.fetch_json(meta_url)
159172
version = meta['version']
160173
161-
# 2. 获取该版本的详细元数据 (为了拿到 shasums_url)
162-
version_meta_url = f'{REGISTRY_API}/{namespace}/{type_name}/{version}'
163-
version_meta = self.fetch_json(version_meta_url)
164-
shasums_url = version_meta.get('shasums_url')
165-
shasums_sig_url = version_meta.get('shasums_signature_url')
166-
167-
if not shasums_url:
168-
print(f" ⚠️ No SHA256SUMS url found for {type_name}, skipping verification.")
169-
return {'status': 'failed', 'name': type_name}
170-
171-
# 3. 下载官方 SHA256SUMS 内容 (用于内存校验)
172-
print(f" ⬇️ Fetching official SHA256SUMS for v{version}...")
173-
official_sums_content = self.fetch_text(shasums_url)
174-
174+
# 目录结构: public/tf-mirror/registry.terraform.io/NAMESPACE/TYPE/
175175
provider_dir = self.base_dir / 'registry.terraform.io' / namespace / type_name
176176
provider_dir.mkdir(parents=True, exist_ok=True)
177177
178178
archives_data = {}
179179
180-
# 4. 并发下载并校验
180+
# 2. 并发处理所有架构
181181
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
182182
futures = []
183183
for os_name, arch in TARGET_PLATFORMS:
184184
futures.append(executor.submit(
185185
self.process_single_arch,
186-
namespace, type_name, version, os_name, arch, provider_dir, official_sums_content
186+
namespace, type_name, version, os_name, arch, provider_dir
187187
))
188188
189189
for future in as_completed(futures):
190190
res = future.result()
191191
if res["status"] == "success":
192192
key = f"{res['os']}_{res['arch']}"
193-
print(f" ✅ Verified: {res['filename']}")
193+
print(f" ✅ Verified & Ready: {res['filename']}")
194194
archives_data[key] = {
195195
"url": res['filename'],
196196
"hashes": [res['hash']]
197197
}
198198
elif res["status"] == "failed":
199199
print(f" ❌ Failed: {res.get('error')}")
200200
201-
# 5. 生成 JSON 文件
201+
# 3. 生成索引文件
202202
if archives_data:
203+
# index.json
203204
index_data = {"versions": {version: {}}}
204205
with open(provider_dir / 'index.json', 'w') as f:
205206
json.dump(index_data, f, indent=2)
206207
208+
# VERSION.json (如 3.8.1.json)
207209
version_file_data = {"archives": archives_data}
208210
with open(provider_dir / f'{version}.json', 'w') as f:
209211
json.dump(version_file_data, f, indent=2)
@@ -213,10 +215,19 @@ jobs:
213215
return {'name': type_name, 'namespace': namespace, 'version': version, 'status': 'empty'}
214216
215217
except Exception as e:
216-
print(f'🚫 Error: {namespace}/{type_name} - {e}')
218+
print(f'🚫 Error processing {namespace}/{type_name}: {e}')
217219
return {'name': type_name, 'namespace': namespace, 'version': 'Error', 'status': 'failed'}
218220
219-
# --- Main ---
221+
# ==============================================================================
222+
# 3. 主程序执行 (Main Execution)
223+
# ==============================================================================
224+
225+
# 清理旧目录
226+
if MIRROR_ROOT.exists():
227+
print("🧹 Cleaning up old mirror directory...")
228+
shutil.rmtree(MIRROR_ROOT)
229+
MIRROR_ROOT.mkdir(parents=True, exist_ok=True)
230+
220231
syncer = MirrorSync(MIRROR_ROOT)
221232
results = []
222233
@@ -225,21 +236,33 @@ jobs:
225236
for f in as_completed(futures):
226237
res = f.result()
227238
results.append(res)
228-
print(f"🎉 Completed {res['namespace']}/{res['name']}")
239+
240+
ns = res.get('namespace', 'unknown')
241+
nm = res.get('name', 'unknown')
242+
if res.get('status') == 'success':
243+
print(f"🎉 Completed {ns}/{nm}")
244+
else:
245+
print(f"⚠️ Skipped {ns}/{nm}")
229246
230-
# --- Generate HTML ---
247+
# ==============================================================================
248+
# 4. 生成 HTML 索引页面
249+
# ==============================================================================
231250
print('📝 Generating index.html...')
232251
timestamp = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')
233252
253+
sorted_results = sorted(results, key=lambda x: (x.get('namespace', ''), x.get('name', '')))
254+
255+
# 生成 Terraform include 配置块
234256
config_lines = []
235-
for r in sorted(results, key=lambda x: x['name']):
236-
if r['status'] == 'success':
257+
for r in sorted_results:
258+
if r.get('status') == 'success':
237259
config_lines.append(f'"registry.terraform.io/{r["namespace"]}/{r["name"]}"')
238260
config_str = ",\n ".join(config_lines)
239261
262+
# 生成卡片 HTML
240263
cards_html = ""
241-
for r in sorted(results, key=lambda x: x['name']):
242-
if r['status'] == 'success':
264+
for r in sorted_results:
265+
if r.get('status') == 'success':
243266
cards_html += f"""
244267
<div class="card">
245268
<div class="name">{r['namespace']}/{r['name']}</div>
@@ -271,8 +294,9 @@ jobs:
271294
<body>
272295
<h1>Terraform Provider Mirror</h1>
273296
<div class="status">
274-
<p><strong>Protocol:</strong> Network Mirror (Integrity Verified)</p>
297+
<p><strong>Protocol:</strong> Network Mirror (Strict Integrity Check)</p>
275298
<p><strong>Last Updated:</strong> {{TIMESTAMP}}</p>
299+
<p><strong>Mirror URL:</strong> <code>{{MIRROR_URL}}</code></p>
276300
</div>
277301
278302
<h2>Usage Configuration</h2>
@@ -296,14 +320,18 @@ jobs:
296320
<div class="grid">
297321
{{CARDS_HTML}}
298322
</div>
323+
324+
<footer>
325+
wgpsec terraform mirror | <a href="https://github.com/wgpsec/redc-template">Source Repository</a>
326+
</footer>
299327
</body>
300328
</html>
301329
""")
302330
303331
final_html = html_template.replace("{{TIMESTAMP}}", timestamp) \
304332
.replace("{{MIRROR_URL}}", MIRROR_URL) \
305333
.replace("{{CONFIG_STR}}", config_str) \
306-
.replace("{{COUNT}}", str(len(results))) \
334+
.replace("{{COUNT}}", str(len(config_lines))) \
307335
.replace("{{CARDS_HTML}}", cards_html)
308336
309337
with open(MIRROR_ROOT / 'index.html', 'w', encoding='utf-8') as f:
@@ -317,4 +345,4 @@ jobs:
317345
publish_dir: ./public
318346
publish_branch: gh-pages
319347
keep_files: true
320-
commit_message: "Mirror: Sync with Official Integrity Verification"
348+
commit_message: "Mirror: Sync providers with Strict Local Hash Verification"

0 commit comments

Comments
 (0)