diff --git a/.github/workflows/auto_rules_version.yml b/.github/workflows/auto_rules_version.yml new file mode 100644 index 0000000..9e63e26 --- /dev/null +++ b/.github/workflows/auto_rules_version.yml @@ -0,0 +1,156 @@ +name: Auto Rules Version + +on: + push: + branches: [main] + paths: + - 'rostran/rules/**' + - '!rostran/rules/VERSIONS.json' + pull_request: + branches: [main] + paths: + - 'rostran/rules/**' + - '!rostran/rules/VERSIONS.json' + +permissions: + contents: write + +jobs: + auto-rules-version: + runs-on: ubuntu-22.04 + if: "!contains(github.event.head_commit.message, 'chore: bump rules version')" + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + # For same-repo PRs, check out the head branch so we can push back; + # for fork PRs, fall back to the default merge ref (branch doesn't exist in base repo) + ref: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.head.ref || '' }} + + - name: Check if rule files actually changed + id: check_changes + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + else + CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD) + fi + + RULES_CHANGED=false + while IFS= read -r file; do + [ -z "$file" ] && continue + case "$file" in + rostran/rules/VERSIONS.json) continue ;; + rostran/rules/*) + RULES_CHANGED=true + echo "Rule file changed: $file" + ;; + esac + done <<< "$CHANGED_FILES" + + echo "rules_changed=$RULES_CHANGED" >> "$GITHUB_OUTPUT" + + - name: Set up Python + if: steps.check_changes.outputs.rules_changed == 'true' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Bump VERSIONS.json + if: steps.check_changes.outputs.rules_changed == 'true' + id: bump + run: | + VERSIONS_JSON="rostran/rules/VERSIONS.json" + + CURRENT_VERSION=$(python3 -c "import json; print(json.load(open('$VERSIONS_JSON'))['latest'])") + echo "Current rules version: $CURRENT_VERSION" + + # Bump patch version + NEW_VERSION=$(python3 -c " + parts = '$CURRENT_VERSION'.split('.') + parts[-1] = str(int(parts[-1]) + 1) + print('.'.join(parts)) + ") + echo "New rules version: $NEW_VERSION" + + # Get commit info (use current HEAD; this commit will be + # amended after we add our changes, but the SHA recorded + # here will be updated by the commit step below) + COMMIT_SHA=$(git rev-parse HEAD) + TODAY=$(date -u +%Y-%m-%d) + + # Update VERSIONS.json + python3 << PYEOF + import json + + version = "$NEW_VERSION" + commit_sha = "$COMMIT_SHA" + today = "$TODAY" + + with open("$VERSIONS_JSON", "r") as f: + data = json.load(f) + + data["latest"] = version + if "versions" not in data: + data["versions"] = {} + + new_versions = { + version: { + "commit": commit_sha, + "date": today, + "description": "" + } + } + new_versions.update(data["versions"]) + data["versions"] = new_versions + + with open("$VERSIONS_JSON", "w") as f: + json.dump(data, f, indent=2) + f.write("\n") + PYEOF + + echo "old_version=$CURRENT_VERSION" >> "$GITHUB_OUTPUT" + echo "new_version=$NEW_VERSION" >> "$GITHUB_OUTPUT" + + - name: Commit version bump + if: >- + steps.check_changes.outputs.rules_changed == 'true' && + (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add rostran/rules/VERSIONS.json + git diff --cached --quiet && echo "No changes to commit" && exit 0 + git commit -m "chore: bump rules version ${{ steps.bump.outputs.old_version }} -> ${{ steps.bump.outputs.new_version }}" + git push + + - name: Update VERSIONS.json with final commit SHA + if: >- + steps.check_changes.outputs.rules_changed == 'true' && + (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + run: | + VERSIONS_JSON="rostran/rules/VERSIONS.json" + NEW_VERSION="${{ steps.bump.outputs.new_version }}" + FINAL_SHA=$(git rev-parse HEAD) + TODAY=$(date -u +%Y-%m-%d) + + python3 << PYEOF + import json + + with open("$VERSIONS_JSON", "r") as f: + data = json.load(f) + + data["versions"]["$NEW_VERSION"]["commit"] = "$FINAL_SHA" + data["versions"]["$NEW_VERSION"]["date"] = "$TODAY" + + with open("$VERSIONS_JSON", "w") as f: + json.dump(data, f, indent=2) + f.write("\n") + PYEOF + + git add rostran/rules/VERSIONS.json + git diff --cached --quiet && echo "SHA already correct, skipping." && exit 0 + git commit -m "chore: update VERSIONS.json commit SHA for rules version $NEW_VERSION" + git push diff --git a/docs/usage.md b/docs/usage.md index c1908e0..a47a771 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -110,6 +110,65 @@ The following options are supported: - `--no-markdown`: [Default] Show rules in normal way. - `--with-link`: Append a link when showing rules in markdown format. - `--no-with-link`: [Default] No link is attached when showing rules in markdown format. +- `--version`/`-V`: Show the version and source of the currently active rules. + +### Show Rules Version + +```bash +rostran rules --version +``` + +When using locally cached rules (after running `rostran rules update`): + +``` +Rules source : local cache (~/.rostran/rules/) +Rules version: 1.2.0 +Built-in ver : 1.0.0 +``` + +When using built-in rules (default): + +``` +Rules source : built-in (shipped with package) +Rules version: 1.0.0 +``` + +## Update Transform Rules + +Update transform rules from the remote repository without upgrading the package. +Downloaded rules are cached locally at `~/.rostran/rules/` and take precedence over the built-in rules shipped with the package. + +### Command + +```bash +rostran rules update [OPTIONS] +rostran rules list +rostran rules reset +``` + +#### `rules update` options + +- `--version`/`-v`: Specific rules version to install (e.g. `1.2.0`). Defaults to the latest version on the main branch. +- `--force`: Force re-download even if the local rules are already up-to-date. + +#### Examples + +```bash +# Update to the latest rules version +rostran rules update + +# Install a specific rules version +rostran rules update --version 1.2.0 + +# List all available rules versions +rostran rules list + +# Force re-download +rostran rules update --force + +# Remove local cache and revert to built-in rules +rostran rules reset +``` ## View Help Information diff --git a/docs/zh-cn/usage.md b/docs/zh-cn/usage.md index 2317d59..8773408 100644 --- a/docs/zh-cn/usage.md +++ b/docs/zh-cn/usage.md @@ -95,6 +95,65 @@ rostran rules [OPTIONS] - `--no-markdown`: 【默认】以普通方式展示规则。 - `--with-link`: 以 markdown 格式展示规则时附加链接。 - `--no-with-link`: 【默认】以 markdown 格式展示规则时不附加链接。 +- `--version`/`-V`: 显示当前使用的规则版本和来源。 + +### 查看规则版本 + +```bash +rostran rules --version +``` + +使用本地缓存规则时（执行过 `rostran rules update` 后）： + +``` +Rules source : local cache (~/.rostran/rules/) +Rules version: 1.2.0 +Built-in ver : 1.0.0 +``` + +使用内置规则时（默认）： + +``` +Rules source : built-in (shipped with package) +Rules version: 1.0.0 +``` + +## 更新转换规则 + +从远程仓库更新转换规则，无需升级整个软件包。 +下载的规则会缓存在本地 `~/.rostran/rules/` 目录中，优先于软件包内置的规则使用。 + +### 命令 + +```bash +rostran rules update [OPTIONS] +rostran rules list +rostran rules reset +``` + +#### `rules update` 可选项 + +- `--version`/`-v`：指定要安装的规则版本（如 `1.2.0`）。默认安装 main 分支上的最新版本。 +- `--force`：即使本地规则已是最新版本，也强制重新下载。 + +#### 示例 + +```bash +# 更新到最新规则版本 +rostran rules update + +# 安装指定版本的规则 +rostran rules update --version 1.2.0 + +# 列出所有可用的规则版本 +rostran rules list + +# 强制重新下载 +rostran rules update --force + +# 清除本地缓存，恢复使用内置规则 +rostran rules reset +``` ## 查看帮助信息 diff --git a/requirements.txt b/requirements.txt index f73c825..943d51f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -typer>=0.6.0,<1.0.0 +typer>=0.12.3,<1.0.0 colorama>=0.4.0,<1.0.0 openpyxl==3.* ruamel.yaml >=0.17.0 diff --git a/rostran/__init__.py b/rostran/__init__.py index 5963297..8b301a7 100644 --- a/rostran/__init__.py +++ b/rostran/__init__.py @@ -1 +1 @@ -__version__ = "0.22.0" +__version__ = "0.23.0" diff --git a/rostran/cli/__main__.py b/rostran/cli/__main__.py index c34d90b..e0c647f 100644 --- a/rostran/cli/__main__.py +++ b/rostran/cli/__main__.py @@ -326,8 +326,40 @@ def _format_directory( return formatted_paths -@app.command() +rules_app = typer.Typer( + help="Show transform rules of Terraform and CloudFormation, or manage rules via sub-commands.", + invoke_without_command=True, +) +app.add_typer(rules_app, name="rules") + + +def _show_rules_version(): + from rostran.core.rules_updater import ( + get_local_rules_version, + get_builtin_rules_version, + has_user_rules, + ) + + local_ver = get_local_rules_version() + builtin_ver = get_builtin_rules_version() + + if has_user_rules() and local_ver: + typer.secho("Rules source : local cache (~/.rostran/rules/)", fg=typer.colors.BLUE) + typer.secho(f"Rules version: {local_ver}", fg=typer.colors.GREEN) + if builtin_ver: + typer.secho(f"Built-in ver : {builtin_ver}", fg=typer.colors.BRIGHT_BLACK) + else: + typer.secho("Rules source : built-in (shipped with package)", fg=typer.colors.BLUE) + if builtin_ver: + typer.secho(f"Rules version: {builtin_ver}", fg=typer.colors.GREEN) + else: + from rostran import __version__ + typer.secho(f"Package ver : {__version__}", fg=typer.colors.GREEN) + + +@rules_app.callback() def rules( + ctx: typer.Context, terraform: bool = typer.Option( True, help="Whether to show Terraform transform rules.", @@ -344,10 +376,23 @@ def rules( False, help="Whether to include links when showing rules in markdown format.", ), + version: bool = typer.Option( + False, + "--version", + "-V", + help="Show the version of the currently active rules.", + ), ): """ Show transform rules of Terraform and CloudFormation. """ + if ctx.invoked_subcommand is not None: + return + + if version: + _show_rules_version() + return + newline = False if terraform: rule_manager = RuleManager.initialize(RuleClassifier.TerraformAliCloud) @@ -360,6 +405,96 @@ def rules( rule_manager.show(markdown, with_link) +@rules_app.command("list") +def rules_list(): + """List all available rules versions from remote.""" + from rostran.core.rules_updater import ( + get_local_rules_version, + fetch_available_versions, + RulesUpdateError, + ) + + typer.secho("Fetching available rules versions...", fg=typer.colors.BLUE) + try: + versions = fetch_available_versions() + except RulesUpdateError as e: + typer.secho(f"Failed: {e}", fg=typer.colors.RED) + raise typer.Exit(1) + if not versions: + typer.secho("No rules versions found on remote.", fg=typer.colors.YELLOW) + else: + local_ver = get_local_rules_version() + for entry in versions: + v = entry["version"] + date = entry.get("date", "") + desc = entry.get("description", "") + marker = " <-- installed" if v == local_ver else "" + info = f" ({date})" if date else "" + typer.echo(f" {v}{info}{marker}") + if desc: + typer.secho(f" {desc}", fg=typer.colors.BRIGHT_BLACK) + + +@rules_app.command("reset") +def rules_reset(): + """Remove the local rules cache and revert to built-in rules.""" + from rostran.core.rules_updater import clean_user_rules + + msg = clean_user_rules() + typer.secho(msg, fg=typer.colors.GREEN) + + +@rules_app.command() +def update( + version: str = typer.Option( + None, + "--version", + "-v", + help="Specific rules version to install (e.g. 1.2.0). " + "Defaults to the latest version on the main branch.", + ), + force: bool = typer.Option( + False, + help="Force re-download even if the local rules are already up-to-date.", + ), +): + """ + Update transform rules from the remote repository without upgrading the package. + + Downloads rules and caches them locally at ~/.rostran/rules/. + The cached rules take precedence over the built-in rules shipped with the package. + + \b + Examples: + rostran rules update # update to latest + rostran rules update -v 1.2.0 # install specific version + rostran rules list # list available versions + rostran rules reset # revert to built-in rules + """ + from rostran.core.rules_updater import ( + update_rules as do_update, + get_local_rules_version, + RulesUpdateError, + ) + + local_ver = get_local_rules_version() + if local_ver: + typer.secho(f"Current rules version: {local_ver}", fg=typer.colors.BLUE) + else: + typer.secho( + "No local rules cache found, using built-in rules.", + fg=typer.colors.BLUE, + ) + + typer.secho("Checking for rules updates...", fg=typer.colors.BLUE) + try: + msg = do_update(version=version, force=force) + except RulesUpdateError as e: + typer.secho(f"Update failed: {e}", fg=typer.colors.RED) + raise typer.Exit(1) + typer.secho(msg, fg=typer.colors.GREEN) + + def main(): logging.basicConfig(level=logging.INFO, format="%(message)s") try: diff --git a/rostran/core/rule_manager.py b/rostran/core/rule_manager.py index 584d286..13cc187 100644 --- a/rostran/core/rule_manager.py +++ b/rostran/core/rule_manager.py @@ -1,3 +1,4 @@ +import logging import os from typing import Dict @@ -9,10 +10,11 @@ RuleTypeNotSupport, RuleAlreadyExist, ) -from rostran.core.settings import RULES_DIR +from rostran.core.settings import RULES_DIR, USER_RULES_DIR import rostran.handlers.resource as resource_handler_module yaml = YAML() +logger = logging.getLogger(__name__) class RuleClassifier: @@ -45,11 +47,32 @@ def initialize(cls, rule_classifier): rule_manager.load() return rule_manager + def _get_rules_dirs(self): + """Return a list of rules directories in loading order. + + If a user-level rules cache exists (``~/.rostran/rules/``), it takes + precedence over the built-in rules shipped with the package. Only one + source is used — the first existing directory wins — so there is no + partial-merge complexity. + """ + user_dir = os.path.join(USER_RULES_DIR, self.rule_classifier) + builtin_dir = os.path.join(RULES_DIR, self.rule_classifier) + + if os.path.isdir(user_dir): + logger.debug("Using user-level rules from %s", user_dir) + return [user_dir] + + if os.path.isdir(builtin_dir): + return [builtin_dir] + + return [] + def load(self): - rules_dir = os.path.join(RULES_DIR, self.rule_classifier) - if not os.path.exists(rules_dir): - return + rules_dirs = self._get_rules_dirs() + for rules_dir in rules_dirs: + self._load_from_dir(rules_dir) + def _load_from_dir(self, rules_dir): for root, dirs, files in os.walk(rules_dir): for filename in files: if not filename.endswith(".yml"): diff --git a/rostran/core/rules_updater.py b/rostran/core/rules_updater.py new file mode 100644 index 0000000..e34c0b6 --- /dev/null +++ b/rostran/core/rules_updater.py @@ -0,0 +1,349 @@ +""" +Dynamic rules updater: downloads rules from the GitHub repository and +caches them locally under ``~/.rostran/rules/``. + +Version convention +------------------ +- ``rostran/rules/VERSIONS.json`` holds the current ``latest`` semantic version and + maps each version to a commit SHA:: + + { + "latest": "1.2.0", + "versions": { + "1.2.0": {"commit": "abc123...", "date": "2024-06-01", "description": "..."}, + "1.1.0": {"commit": "def456...", "date": "2024-05-01", "description": "..."} + } + } + +- Maintainers update the yml files, bump ``latest`` / ``versions`` in + ``VERSIONS.json``, and push to main. No git tags needed. +- ``rostran rules update`` pulls the latest version. +- ``rostran rules update -v 1.1.0`` pulls a specific version by its + recorded commit SHA. +- ``rostran rules list`` lists available versions from remote. +- ``rostran rules reset`` removes the local cache and reverts to built-in rules. +""" + +import errno +import json +import logging +import os +import shutil +import socket +import tarfile +import tempfile +import time +from datetime import datetime, timezone +from io import BytesIO +from typing import Dict, List, Optional +from urllib import request, error + +from rostran.core.settings import ( + USER_DATA_DIR, + USER_RULES_DIR, + USER_RULES_META_FILE, + RULES_VERSIONS_JSON_FILE, + RULES_REPO_OWNER, + RULES_REPO_NAME, + RULES_REPO_BRANCH, + RULES_RAW_URL_TEMPLATE, + RULES_ARCHIVE_URL_TEMPLATE, + RULES_ARCHIVE_INNER_PATH, +) + +logger = logging.getLogger(__name__) + +# Single HTTP attempt timeout (seconds). Large archives may need a long read. +_REQUEST_TIMEOUT = 180 +# After a network timeout, retry this many additional times (1 + N total attempts). +_HTTP_TIMEOUT_RETRIES = 5 + + +class RulesUpdateError(Exception): + pass + + +# --------------------------------------------------------------------------- +# Local metadata / version helpers +# --------------------------------------------------------------------------- + +def _read_meta() -> dict: + if os.path.isfile(USER_RULES_META_FILE): + with open(USER_RULES_META_FILE) as f: + try: + return json.load(f) + except json.JSONDecodeError: + return {} + return {} + + +def _write_meta(meta: dict) -> None: + os.makedirs(USER_DATA_DIR, exist_ok=True) + with open(USER_RULES_META_FILE, "w") as f: + json.dump(meta, f, indent=2) + + +def get_local_rules_version() -> Optional[str]: + """Return the semantic version of locally cached rules, or None.""" + meta = _read_meta() + return meta.get("version") + + +def _read_latest_from_versions_json(path: str) -> Optional[str]: + """Return the ``latest`` field from a VERSIONS.json file, or None.""" + if not os.path.isfile(path): + return None + try: + with open(path) as f: + data = json.load(f) + latest = data.get("latest") + return str(latest).strip() if latest is not None else None + except (json.JSONDecodeError, OSError, TypeError): + return None + + +def get_builtin_rules_version() -> Optional[str]: + """Read ``latest`` from the VERSIONS.json shipped inside the package.""" + return _read_latest_from_versions_json(RULES_VERSIONS_JSON_FILE) + + +# --------------------------------------------------------------------------- +# HTTP helpers +# --------------------------------------------------------------------------- + +def _urlerror_is_timeout(exc: error.URLError) -> bool: + """True if this error is worth retrying (connection timed out, read timeout).""" + r = exc.reason + if isinstance(r, socket.timeout): + return True + if isinstance(r, TimeoutError): + return True + if isinstance(r, OSError) and getattr(r, "errno", None) == errno.ETIMEDOUT: + return True + return False + + +def _http_get(url: str, headers: dict = None, decode: bool = True): + hdrs = {"User-Agent": "rostran-rules-updater"} + if headers: + hdrs.update(headers) + req = request.Request(url, headers=hdrs) + max_attempts = 1 + _HTTP_TIMEOUT_RETRIES + + for attempt in range(max_attempts): + try: + with request.urlopen(req, timeout=_REQUEST_TIMEOUT) as resp: + data = resp.read() + return data.decode() if decode else data + except error.HTTPError as exc: + raise RulesUpdateError( + f"HTTP request failed ({exc.code}): {url}" + ) from exc + except error.URLError as exc: + if _urlerror_is_timeout(exc) and attempt < max_attempts - 1: + logger.warning( + "HTTP timeout (%s), retry %s/%s: %s", + exc.reason, + attempt + 1, + max_attempts, + url, + ) + time.sleep(min(2**attempt, 8)) + continue + raise RulesUpdateError( + f"Cannot reach remote: {exc.reason}" + ) from exc + + +# --------------------------------------------------------------------------- +# Remote version discovery +# --------------------------------------------------------------------------- + +def _fetch_remote_versions_json() -> dict: + """Fetch the VERSIONS.json from the remote main branch.""" + url = RULES_RAW_URL_TEMPLATE.format( + owner=RULES_REPO_OWNER, + repo=RULES_REPO_NAME, + ref=RULES_REPO_BRANCH, + file="VERSIONS.json", + ) + body = _http_get(url) + try: + return json.loads(body) + except json.JSONDecodeError as exc: + raise RulesUpdateError("Invalid VERSIONS.json from remote") from exc + + +def fetch_remote_latest_version() -> str: + """Get the latest rules version string from the remote.""" + versions_data = _fetch_remote_versions_json() + latest = versions_data.get("latest") + if not latest: + raise RulesUpdateError("No 'latest' field in remote VERSIONS.json") + return latest + + +def fetch_available_versions() -> List[Dict]: + """List all available rules versions with their metadata. + + Returns a list of dicts: ``[{"version": "1.2.0", "date": "...", "description": "..."}, ...]`` + sorted by version key (newest first based on insertion order in JSON). + """ + versions_data = _fetch_remote_versions_json() + result = [] + for ver, info in versions_data.get("versions", {}).items(): + result.append({ + "version": ver, + "commit": info.get("commit", ""), + "date": info.get("date", ""), + "description": info.get("description", ""), + }) + return result + + +def _resolve_commit_for_version(version: str) -> str: + """Look up the commit SHA for a given version string.""" + versions_data = _fetch_remote_versions_json() + versions = versions_data.get("versions", {}) + if version not in versions: + available = ", ".join(versions.keys()) or "none" + raise RulesUpdateError( + f"Version '{version}' not found. Available versions: {available}" + ) + commit = versions[version].get("commit") + if not commit: + raise RulesUpdateError( + f"Version '{version}' has no commit SHA in VERSIONS.json" + ) + return commit + + +# --------------------------------------------------------------------------- +# Download & extraction +# --------------------------------------------------------------------------- + +def _download_and_extract_rules(ref: str) -> str: + """Download the repo archive for *ref* (branch or commit SHA) and + extract the ``rostran/rules/`` subtree into a temp directory.""" + archive_url = RULES_ARCHIVE_URL_TEMPLATE.format( + owner=RULES_REPO_OWNER, + repo=RULES_REPO_NAME, + ref=ref, + ) + logger.info("Downloading rules from %s ...", archive_url) + raw = _http_get(archive_url, decode=False) + + tmpdir = tempfile.mkdtemp(prefix="rostran_rules_") + try: + with tarfile.open(fileobj=BytesIO(raw), mode="r:gz") as tf: + top_dirs = {m.name.split("/")[0] for m in tf.getmembers()} + archive_prefix = top_dirs.pop() if len(top_dirs) == 1 else "" + + rules_prefix = ( + f"{archive_prefix}/{RULES_ARCHIVE_INNER_PATH}" + if archive_prefix + else RULES_ARCHIVE_INNER_PATH + ) + rules_prefix_slash = rules_prefix + "/" + + for member in tf.getmembers(): + if not (member.name == rules_prefix or member.name.startswith(rules_prefix_slash)): + continue + rel = os.path.relpath(member.name, archive_prefix) if archive_prefix else member.name + inner_rel = os.path.relpath(rel, RULES_ARCHIVE_INNER_PATH) + if inner_rel == ".": + continue + if member.isdir(): + os.makedirs(os.path.join(tmpdir, inner_rel), exist_ok=True) + else: + fobj = tf.extractfile(member) + if fobj is None: + continue + dest = os.path.join(tmpdir, inner_rel) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, "wb") as out: + out.write(fobj.read()) + except (tarfile.TarError, OSError) as exc: + shutil.rmtree(tmpdir, ignore_errors=True) + raise RulesUpdateError(f"Failed to extract rules archive: {exc}") from exc + return tmpdir + + +def _read_version_from_dir(rules_dir: str) -> Optional[str]: + """Read ``latest`` from VERSIONS.json in an extracted rules directory.""" + vj = os.path.join(rules_dir, "VERSIONS.json") + return _read_latest_from_versions_json(vj) + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def update_rules(version: Optional[str] = None, force: bool = False) -> str: + """Download rules for a given version (or the latest) and cache locally. + + Parameters + ---------- + version: + Semantic version to install (e.g. ``"1.2.0"``). + ``None`` means "fetch the latest from the main branch". + force: + Re-download even if local version matches. + + Returns + ------- + str + Human-readable status message. + """ + if version is None: + target_version = fetch_remote_latest_version() + ref = RULES_REPO_BRANCH + else: + target_version = version + ref = _resolve_commit_for_version(version) + + local_version = get_local_rules_version() + + if not force and local_version == target_version: + return f"Rules are already up-to-date (version: {target_version})." + + tmpdir = _download_and_extract_rules(ref) + try: + downloaded_version = _read_version_from_dir(tmpdir) + if downloaded_version and downloaded_version != target_version: + logger.warning( + "VERSIONS.json latest mismatch: expected %s, got %s", + target_version, + downloaded_version, + ) + + if os.path.exists(USER_RULES_DIR): + shutil.rmtree(USER_RULES_DIR) + shutil.copytree(tmpdir, USER_RULES_DIR) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + actual_version = downloaded_version or target_version + _write_meta( + { + "version": actual_version, + "updated_at": datetime.now(timezone.utc).isoformat(), + "ref": ref, + } + ) + old_info = f" (was: {local_version})" if local_version else "" + return f"Rules updated to version {actual_version}{old_info}." + + +def has_user_rules() -> bool: + """Return True if user-level cached rules exist.""" + return os.path.isdir(USER_RULES_DIR) + + +def clean_user_rules() -> str: + """Remove the local rules cache, reverting to built-in rules.""" + if os.path.isdir(USER_RULES_DIR): + shutil.rmtree(USER_RULES_DIR) + if os.path.isfile(USER_RULES_META_FILE): + os.remove(USER_RULES_META_FILE) + return "Local rules cache cleared. Built-in rules will be used." diff --git a/rostran/core/settings.py b/rostran/core/settings.py index eb47987..51082e1 100644 --- a/rostran/core/settings.py +++ b/rostran/core/settings.py @@ -5,3 +5,29 @@ ROOT = os.path.dirname(BASE_DIR) RULES_DIR = os.path.join(BASE_DIR, "rules") BIN_DIR = os.path.join(BASE_DIR, "bin") + +# User-level rules directory for dynamic updates (overrides built-in rules) +USER_DATA_DIR = os.path.join(os.path.expanduser("~"), ".rostran") +USER_RULES_DIR = os.path.join(USER_DATA_DIR, "rules") +USER_RULES_META_FILE = os.path.join(USER_DATA_DIR, "rules_meta.json") + +# Remote rules source (GitHub repo) +RULES_REPO_OWNER = os.environ.get("ROSTRAN_RULES_REPO_OWNER", "aliyun") +RULES_REPO_NAME = os.environ.get( + "ROSTRAN_RULES_REPO_NAME", "alibabacloud-ros-tool-transformer" +) +RULES_REPO_BRANCH = os.environ.get("ROSTRAN_RULES_BRANCH", "main") + +# Built-in rules version index (``latest`` is the current semantic version) +RULES_VERSIONS_JSON_FILE = os.path.join(RULES_DIR, "VERSIONS.json") + +# GitHub raw content URL templates +RULES_RAW_URL_TEMPLATE = ( + "https://raw.githubusercontent.com/{owner}/{repo}/{ref}/rostran/rules/{file}" +) +# GitHub archive URL pattern (supports branch names and commit SHAs) +RULES_ARCHIVE_URL_TEMPLATE = ( + "https://github.com/{owner}/{repo}/archive/{ref}.tar.gz" +) +# Path inside the archive where rules live +RULES_ARCHIVE_INNER_PATH = "rostran/rules" diff --git a/rostran/rules/VERSIONS.json b/rostran/rules/VERSIONS.json new file mode 100644 index 0000000..8999292 --- /dev/null +++ b/rostran/rules/VERSIONS.json @@ -0,0 +1,10 @@ +{ + "latest": "1.0.0", + "versions": { + "1.0.0": { + "commit": "b7935bb63ce23206ff5c10ff5b2759ffacc4f8de", + "date": "2026-03-20", + "description": "Initial versioned rules release" + } + } +} diff --git a/tools/tf_ali_ros_generate_mappings.json b/rostran/rules/tf_ali_ros_generate_mappings.json similarity index 100% rename from tools/tf_ali_ros_generate_mappings.json rename to rostran/rules/tf_ali_ros_generate_mappings.json diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/core/test_rules_updater.py b/tests/core/test_rules_updater.py new file mode 100644 index 0000000..dd009e7 --- /dev/null +++ b/tests/core/test_rules_updater.py @@ -0,0 +1,327 @@ +import errno +import io +import json +import os +import shutil +import socket +import tarfile +import tempfile +from unittest import mock +from urllib import error + +import pytest + +from rostran.core.rules_updater import ( + _read_meta, + _write_meta, + get_local_rules_version, + get_builtin_rules_version, + fetch_remote_latest_version, + fetch_available_versions, + update_rules, + has_user_rules, + clean_user_rules, + RulesUpdateError, +) + + +@pytest.fixture +def temp_user_dir(tmp_path, monkeypatch): + """Redirect all user-level paths to a temp directory.""" + user_data = tmp_path / ".rostran" + user_data.mkdir() + rules_dir = user_data / "rules" + meta_file = user_data / "rules_meta.json" + + monkeypatch.setattr("rostran.core.rules_updater.USER_DATA_DIR", str(user_data)) + monkeypatch.setattr("rostran.core.rules_updater.USER_RULES_DIR", str(rules_dir)) + monkeypatch.setattr("rostran.core.rules_updater.USER_RULES_META_FILE", str(meta_file)) + + return { + "user_data": user_data, + "rules_dir": rules_dir, + "meta_file": meta_file, + } + + +FAKE_VERSIONS_JSON = { + "latest": "2.0.0", + "versions": { + "2.0.0": { + "commit": "abc123def456abc123def456abc123def456abc1", + "date": "2026-03-20", + "description": "Added new DTS resources", + }, + "1.0.0": { + "commit": "b7935bb63ce23206ff5c10ff5b2759ffacc4f8de", + "date": "2026-03-01", + "description": "Initial versioned release", + }, + }, +} + + +class TestMeta: + def test_read_meta_missing(self, temp_user_dir): + assert _read_meta() == {} + + def test_write_and_read_meta(self, temp_user_dir): + _write_meta({"version": "1.0.0"}) + assert _read_meta() == {"version": "1.0.0"} + + def test_get_local_version_none(self, temp_user_dir): + assert get_local_rules_version() is None + + def test_get_local_version(self, temp_user_dir): + _write_meta({"version": "1.2.3"}) + assert get_local_rules_version() == "1.2.3" + + +class TestBuiltinVersion: + def test_reads_versions_json_latest(self): + version = get_builtin_rules_version() + assert version is not None + assert version == "1.0.0" + + +class TestHasUserRules: + def test_no_rules(self, temp_user_dir): + assert has_user_rules() is False + + def test_with_rules(self, temp_user_dir): + rules_dir = temp_user_dir["rules_dir"] + rules_dir.mkdir(parents=True) + assert has_user_rules() is True + + +class TestCleanUserRules: + def test_clean_removes_cache(self, temp_user_dir): + rules_dir = temp_user_dir["rules_dir"] + rules_dir.mkdir(parents=True) + (rules_dir / "test.yml").write_text("data") + _write_meta({"version": "1.0.0"}) + + msg = clean_user_rules() + assert "cleared" in msg.lower() + assert not rules_dir.exists() + assert not temp_user_dir["meta_file"].exists() + + +def _make_rules_tarball(rules_files: dict, prefix: str = "repo-main") -> bytes: + """Create an in-memory tar.gz archive mimicking the GitHub archive layout.""" + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tf: + for path, content in rules_files.items(): + full_path = f"{prefix}/rostran/rules/{path}" + data = content.encode() if isinstance(content, str) else content + info = tarfile.TarInfo(name=full_path) + info.size = len(data) + tf.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +class TestFetchRemoteLatestVersion: + def test_fetches_version(self): + with mock.patch( + "rostran.core.rules_updater._http_get", + return_value=json.dumps(FAKE_VERSIONS_JSON), + ): + ver = fetch_remote_latest_version() + assert ver == "2.0.0" + + +class TestHttpGetRetry: + def test_retries_on_socket_timeout_then_succeeds(self): + from rostran.core import rules_updater as ru + + resp_ok = mock.MagicMock() + resp_ok.__enter__ = mock.Mock(return_value=resp_ok) + resp_ok.__exit__ = mock.Mock(return_value=False) + resp_ok.read.return_value = b'{"latest": "1.0.0", "versions": {}}' + + with mock.patch( + "rostran.core.rules_updater.request.urlopen", + side_effect=[ + error.URLError(socket.timeout("timed out")), + error.URLError(socket.timeout("timed out")), + resp_ok, + ], + ) as mock_urlopen, mock.patch( + "rostran.core.rules_updater.time.sleep" + ) as mock_sleep: + out = ru._http_get("https://example.com/x") + assert '{"latest":' in out + assert mock_urlopen.call_count == 3 + assert mock_sleep.call_count == 2 + + def test_fails_after_six_timeout_attempts(self): + from rostran.core import rules_updater as ru + + with mock.patch( + "rostran.core.rules_updater.request.urlopen", + side_effect=error.URLError(socket.timeout()), + ) as mock_urlopen, mock.patch( + "rostran.core.rules_updater.time.sleep" + ): + with pytest.raises(RulesUpdateError, match="Cannot reach remote"): + ru._http_get("https://example.com/x") + assert mock_urlopen.call_count == 6 + + def test_no_retry_on_non_timeout_urlerror(self): + from rostran.core import rules_updater as ru + + with mock.patch( + "rostran.core.rules_updater.request.urlopen", + side_effect=error.URLError( + OSError(errno.ECONNREFUSED, "Connection refused") + ), + ) as mock_urlopen, mock.patch( + "rostran.core.rules_updater.time.sleep" + ) as mock_sleep: + with pytest.raises(RulesUpdateError, match="Cannot reach remote"): + ru._http_get("https://example.com/x") + assert mock_urlopen.call_count == 1 + assert mock_sleep.call_count == 0 + + +class TestFetchAvailableVersions: + def test_lists_versions_with_metadata(self): + with mock.patch( + "rostran.core.rules_updater._http_get", + return_value=json.dumps(FAKE_VERSIONS_JSON), + ): + versions = fetch_available_versions() + assert len(versions) == 2 + assert versions[0]["version"] == "2.0.0" + assert versions[0]["commit"].startswith("abc123") + assert versions[1]["version"] == "1.0.0" + + def test_empty_when_no_versions(self): + empty_data = {"latest": "1.0.0", "versions": {}} + with mock.patch( + "rostran.core.rules_updater._http_get", + return_value=json.dumps(empty_data), + ): + versions = fetch_available_versions() + assert versions == [] + + +class TestUpdateRules: + def _mock_http_get(self, tarball_data, versions_json=None): + """Returns a side_effect function for _http_get.""" + vj = versions_json or FAKE_VERSIONS_JSON + + def side_effect(url, headers=None, decode=True): + if "VERSIONS.json" in url: + return json.dumps(vj) + if url.endswith(".tar.gz"): + return tarball_data + return "" + return side_effect + + def test_update_to_latest(self, temp_user_dir): + tarball = _make_rules_tarball({ + "VERSIONS.json": json.dumps( + {"latest": "2.0.0", "versions": FAKE_VERSIONS_JSON["versions"]} + ), + "terraform/alicloud/vpc.yml": "Version: '2020-06-01'\nType: Resource\n", + }) + + with mock.patch( + "rostran.core.rules_updater._http_get", + side_effect=self._mock_http_get(tarball), + ): + msg = update_rules() + assert "2.0.0" in msg + assert has_user_rules() + assert get_local_rules_version() == "2.0.0" + assert (temp_user_dir["rules_dir"] / "terraform" / "alicloud" / "vpc.yml").exists() + + def test_update_to_specific_version(self, temp_user_dir): + tarball = _make_rules_tarball({ + "VERSIONS.json": json.dumps( + {"latest": "1.0.0", "versions": FAKE_VERSIONS_JSON["versions"]} + ), + "terraform/alicloud/vpc.yml": "data", + }, prefix="repo-b7935bb") + + with mock.patch( + "rostran.core.rules_updater._http_get", + side_effect=self._mock_http_get(tarball), + ): + msg = update_rules(version="1.0.0") + assert "1.0.0" in msg + assert get_local_rules_version() == "1.0.0" + + def test_update_specific_version_not_found(self, temp_user_dir): + with mock.patch( + "rostran.core.rules_updater._http_get", + return_value=json.dumps(FAKE_VERSIONS_JSON), + ): + with pytest.raises(RulesUpdateError, match="not found"): + update_rules(version="9.9.9") + + def test_update_skip_when_up_to_date(self, temp_user_dir): + _write_meta({"version": "2.0.0"}) + + with mock.patch( + "rostran.core.rules_updater._http_get", + return_value=json.dumps(FAKE_VERSIONS_JSON), + ): + msg = update_rules() + assert "up-to-date" in msg.lower() + + def test_update_force_redownloads(self, temp_user_dir): + _write_meta({"version": "2.0.0"}) + + tarball = _make_rules_tarball({ + "VERSIONS.json": json.dumps( + {"latest": "2.0.0", "versions": FAKE_VERSIONS_JSON["versions"]} + ), + }) + + with mock.patch( + "rostran.core.rules_updater._http_get", + side_effect=self._mock_http_get(tarball), + ): + msg = update_rules(force=True) + assert "2.0.0" in msg + + +class TestRuleManagerUserRules: + """Test that RuleManager prefers user-level rules when available.""" + + def test_uses_user_rules_dir(self, temp_user_dir, monkeypatch): + from rostran.core.rule_manager import RuleManager, RuleClassifier + + monkeypatch.setattr( + "rostran.core.rule_manager.USER_RULES_DIR", + str(temp_user_dir["rules_dir"]), + ) + + rules_dir = temp_user_dir["rules_dir"] / "terraform" / "alicloud" + rules_dir.mkdir(parents=True) + (rules_dir / "vpc.yml").write_text( + "Version: '2020-06-01'\n" + "Type: Resource\n" + "ResourceType:\n" + " From: alicloud_vpc\n" + " To: ALIYUN::ECS::VPC\n" + "Properties: {}\n" + "Attributes: {}\n" + ) + + rm = RuleManager.initialize(RuleClassifier.TerraformAliCloud) + assert "alicloud_vpc" in rm.resource_rules + assert len(rm.resource_rules) == 1 + + def test_falls_back_to_builtin(self, temp_user_dir, monkeypatch): + from rostran.core.rule_manager import RuleManager, RuleClassifier + + monkeypatch.setattr( + "rostran.core.rule_manager.USER_RULES_DIR", + str(temp_user_dir["rules_dir"]), + ) + + rm = RuleManager.initialize(RuleClassifier.TerraformAliCloud) + assert len(rm.resource_rules) > 0 diff --git a/tools/settings.py b/tools/settings.py index 359e376..14c41f6 100644 --- a/tools/settings.py +++ b/tools/settings.py @@ -55,8 +55,7 @@ "alicloud_slb_acl": {"entry_list": "AclEntries"}, } -current_dir = os.path.dirname(os.path.abspath(__file__)) -with open(os.path.join(current_dir, 'tf_ali_ros_generate_mappings.json'), 'r') as f: +with open(os.path.join(RULES_DIR, 'tf_ali_ros_generate_mappings.json'), 'r') as f: tf_ali_ros_generate_mappings = json.load(f) TF_ALI_ROS_GENERATE_MAPPINGS = tf_ali_ros_generate_mappings or {