diff --git a/.gitattributes b/.gitattributes index 809378b..0b5241c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,4 +33,4 @@ Makefile text eol=lf .pre-commit-config.yaml export-ignore tests/ export-ignore docs/ export-ignore -*.md export-ignore \ No newline at end of file +*.md export-ignore diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 341dabf..8a18000 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -2,9 +2,9 @@ name: OVMobileBench CI on: push: - branches: [ main, develop ] + branches: [main, develop] pull_request: - branches: [ main ] + branches: [main] workflow_dispatch: inputs: device_serial: @@ -17,7 +17,14 @@ concurrency: cancel-in-progress: true jobs: + # Run pre-commit only once on ubuntu-latest + pre-commit: + uses: ./.github/workflows/stage-pre-commit.yml + secrets: inherit + + # Run CI matrix for all OS after pre-commit passes ci-matrix: + needs: pre-commit strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] diff --git a/.github/workflows/ci-orchestrator.yml b/.github/workflows/ci-orchestrator.yml index e4cda61..b34e365 100644 --- a/.github/workflows/ci-orchestrator.yml +++ b/.github/workflows/ci-orchestrator.yml @@ -12,16 +12,16 @@ on: default: 'emulator-5554' jobs: - # Stage 1: Lint and Test - lint-test: - uses: ./.github/workflows/stage-lint-test.yml + # Stage 1: Test + test: + uses: ./.github/workflows/stage-test.yml with: os: ${{ inputs.os }} secrets: inherit # Stage 2: Build Package (runs in parallel with validation) build: - needs: lint-test + needs: test uses: ./.github/workflows/stage-build.yml with: os: ${{ inputs.os }} @@ -29,7 +29,7 @@ jobs: # Stage 3: Validation (runs in parallel with build) validation: - needs: lint-test + needs: test uses: ./.github/workflows/stage-validation.yml with: os: ${{ inputs.os }} diff --git a/.github/workflows/stage-device-tests.yml b/.github/workflows/stage-device-tests.yml index 4ba2ee8..ae8277a 100644 --- a/.github/workflows/stage-device-tests.yml +++ b/.github/workflows/stage-device-tests.yml @@ -57,8 +57,8 @@ jobs: device-test-adb: if: | - (github.event_name == 'workflow_dispatch' || - contains(github.event.head_commit.message, '[device-test-adb]')) && + (github.event_name == 'workflow_dispatch' || + contains(github.event.head_commit.message, '[device-test-adb]')) && inputs.os == 'ubuntu-latest' runs-on: self-hosted steps: diff --git a/.github/workflows/stage-pre-commit.yml b/.github/workflows/stage-pre-commit.yml new file mode 100644 index 0000000..5e8b2dc --- /dev/null +++ b/.github/workflows/stage-pre-commit.yml @@ -0,0 +1,29 @@ +name: Pre-commit + +on: + workflow_call: + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 # Fetch all history for all branches + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: | + requirements.txt + setup.py + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install pre-commit + + - name: Run pre-commit + run: pre-commit run --all-files --show-diff-on-failure diff --git a/.github/workflows/stage-lint-test.yml b/.github/workflows/stage-test.yml similarity index 78% rename from .github/workflows/stage-lint-test.yml rename to .github/workflows/stage-test.yml index a0b9eab..163fee2 100644 --- a/.github/workflows/stage-lint-test.yml +++ b/.github/workflows/stage-test.yml @@ -1,4 +1,4 @@ -name: Lint and Test +name: Test on: workflow_call: @@ -8,7 +8,7 @@ on: type: string jobs: - lint-and-test: + test: runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v5 @@ -30,15 +30,6 @@ jobs: pip install -r requirements.txt pip install -e . - - name: Run Black - run: black --check ovmobilebench tests - - - name: Run Ruff - run: ruff check ovmobilebench tests - - - name: Run MyPy - run: mypy ovmobilebench --ignore-missing-imports - - name: Run tests run: pytest tests/ -v --cov=ovmobilebench --cov=scripts --junitxml=junit.xml -o junit_family=legacy diff --git a/.github/workflows/stage-validation.yml b/.github/workflows/stage-validation.yml index fe33866..cc7ca71 100644 --- a/.github/workflows/stage-validation.yml +++ b/.github/workflows/stage-validation.yml @@ -30,7 +30,8 @@ jobs: - name: Validate example config run: | - python -c "from ovmobilebench.config.loader import load_experiment; load_experiment('experiments/android_example.yaml')" + python -c "from ovmobilebench.config.loader import load_experiment; \ + load_experiment('experiments/android_example.yaml')" - name: CLI help test run: | diff --git a/.gitignore b/.gitignore index c6eddf7..f6cf879 100644 --- a/.gitignore +++ b/.gitignore @@ -136,3 +136,5 @@ experiments/results/ ovmb_cache artifacts junit.xml + +*.DS_Store* diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 0000000..6c36e6a --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,18 @@ +{ + "default": true, + "MD013": false, + "MD033": { + "allowed_elements": ["br", "sup", "sub", "details", "summary", "img", "API", "arch", "target"] + }, + "MD041": false, + "MD024": { + "siblings_only": true + }, + "MD040": false, + "MD025": false, + "MD036": false, + "MD028": false, + "MD051": false, + "MD046": false, + "MD001": false +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07fa03c..e851244 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,12 @@ repos: - id: name-tests-test args: ['--pytest-test-first'] + - repo: https://github.com/asottile/pyupgrade + rev: v3.19.1 + hooks: + - id: pyupgrade + args: ['--py311-plus'] + - repo: https://github.com/psf/black rev: 24.4.2 hooks: @@ -35,9 +41,9 @@ repos: rev: v1.10.0 hooks: - id: mypy - additional_dependencies: [types-all] + additional_dependencies: [types-PyYAML, types-paramiko] args: ['--ignore-missing-imports'] - exclude: '^tests/' + files: '^ovmobilebench/(?!android/installer/).*\.py$' - repo: https://github.com/pycqa/isort rev: 5.13.2 @@ -45,18 +51,33 @@ repos: - id: isort args: ['--profile', 'black', '--line-length', '100'] - - repo: https://github.com/pycqa/docformatter - rev: v1.7.5 - hooks: - - id: docformatter - args: ['--in-place', '--wrap-summaries', '100', '--wrap-descriptions', '100'] - - repo: https://github.com/commitizen-tools/commitizen rev: v3.27.0 hooks: - id: commitizen stages: [commit-msg] + - repo: https://github.com/adrienverge/yamllint + rev: v1.35.1 + hooks: + - id: yamllint + args: ['--strict'] + + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.43.0 + hooks: + - id: markdownlint + args: ['--fix'] + + - repo: https://github.com/codespell-project/codespell + rev: v2.3.0 + hooks: + - id: codespell + args: ['--skip=*.json,*.yaml,*.yml,*.txt,*.csv,*.lock', '--ignore-words-list=nd,teh,hist'] + ci: autofix_prs: false - autoupdate_schedule: monthly \ No newline at end of file + autoupdate_schedule: weekly + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + skip: [] + submodules: false diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..df4c4c0 --- /dev/null +++ b/.yamllint @@ -0,0 +1,12 @@ +extends: default + +rules: + line-length: + max: 120 + level: warning + comments: + min-spaces-from-content: 1 + comments-indentation: disable + document-start: disable + truthy: + allowed-values: ['true', 'false', 'on', 'off'] diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a882b33..7d70957 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -35,6 +35,7 @@ OVMobileBench is an end-to-end benchmarking pipeline for OpenVINO on mobile devi **Purpose**: Define and validate experiment configurations. **Key Classes**: + - `Experiment`: Top-level configuration container - `BuildConfig`: OpenVINO build settings - `DeviceConfig`: Target device specifications @@ -48,6 +49,7 @@ OVMobileBench is an end-to-end benchmarking pipeline for OpenVINO on mobile devi **Purpose**: Command-line interface for user interaction. **Commands**: + - `build`: Build OpenVINO from source - `package`: Create deployment bundle - `deploy`: Push to device(s) @@ -62,6 +64,7 @@ OVMobileBench is an end-to-end benchmarking pipeline for OpenVINO on mobile devi **Purpose**: Coordinate execution of all pipeline stages. **Responsibilities**: + - Stage dependency management - Error handling and recovery - Progress tracking @@ -74,11 +77,13 @@ OVMobileBench is an end-to-end benchmarking pipeline for OpenVINO on mobile devi **Purpose**: Uniform interface for different device types. **Implementations**: + - `AndroidDevice`: ADB-based Android device control - `LinuxDevice`: SSH-based Linux device control (planned) - `iOSDevice`: iOS device control (stub) **Interface**: + ```python class Device(ABC): def push(local, remote) @@ -95,12 +100,14 @@ class Device(ABC): **Purpose**: Build OpenVINO runtime for target platforms. **Features**: + - CMake configuration generation - Cross-compilation support (Android NDK) - Build caching - Artifact collection **Supported Platforms**: + - Android (arm64-v8a) - Linux ARM (aarch64) @@ -109,6 +116,7 @@ class Device(ABC): **Purpose**: Bundle runtime, libraries, and models. **Bundle Structure**: + ``` ovbundle.tar.gz ├── bin/ @@ -127,6 +135,7 @@ ovbundle.tar.gz **Purpose**: Execute benchmark_app with various configurations. **Features**: + - Matrix expansion (device, threads, streams, precision) - Timeout handling - Cooldown between runs @@ -138,6 +147,7 @@ ovbundle.tar.gz **Purpose**: Extract metrics from benchmark_app output. **Metrics**: + - Throughput (FPS) - Latencies (avg, median, min, max) - Device utilization @@ -148,6 +158,7 @@ ovbundle.tar.gz **Purpose**: Generate structured reports from results. **Formats**: + - JSON: Machine-readable format - CSV: Spreadsheet-compatible - SQLite: Database format (planned) @@ -156,6 +167,7 @@ ovbundle.tar.gz ### 10. Core Utilities (`ovmobilebench/core/`) **Shared Components**: + - `shell.py`: Command execution with timeout - `fs.py`: File system operations - `artifacts.py`: Artifact management @@ -165,31 +177,37 @@ ovbundle.tar.gz ## Data Flow ### 1. Configuration Loading + ``` YAML File → Pydantic Validation → Experiment Object ``` ### 2. Build Flow + ``` Git Checkout → CMake Configure → Ninja Build → Artifact Collection ``` ### 3. Package Flow + ``` Build Artifacts + Models → Tar Archive → Checksum Generation ``` ### 4. Deployment Flow + ``` Bundle → ADB/SSH Push → Remote Extraction → Permission Setup ``` ### 5. Execution Flow + ``` Matrix Expansion → Device Preparation → Benchmark Execution → Output Collection ``` ### 6. Reporting Flow + ``` Raw Output → Parsing → Aggregation → Format Conversion → Sink Writing ``` @@ -197,29 +215,30 @@ Raw Output → Parsing → Aggregation → Format Conversion → Sink Writing ## Configuration Schema ### Experiment Configuration + ```yaml project: name: string run_id: string - + build: openvino_repo: path toolchain: android_ndk: path - + device: kind: android|linux_ssh serials: [string] - + models: - name: string path: path - + run: matrix: threads: [int] nstreams: [string] - + report: sinks: - type: json|csv @@ -229,16 +248,19 @@ report: ## Security Considerations ### Input Validation + - All user inputs validated via Pydantic - Shell commands parameterized to prevent injection - Path traversal prevention ### Secrets Management + - No hardcoded credentials - Environment variables for sensitive data - SSH key-based authentication ### Device Security + - ADB authorization required - Limited command set execution - Temporary file cleanup @@ -246,16 +268,19 @@ report: ## Performance Optimizations ### Build Caching + - CMake build cache - ccache integration (planned) - Incremental builds ### Parallel Execution + - Multiple device support - Concurrent stage execution (where safe) - Async I/O for file operations ### Resource Management + - Automatic cleanup of temporary files - Connection pooling for SSH - Memory-mapped file I/O for large files @@ -263,16 +288,19 @@ report: ## Extensibility Points ### Adding New Device Types + 1. Inherit from `Device` base class 2. Implement required methods 3. Register in `pipeline.py` ### Adding New Report Formats + 1. Inherit from `ReportSink` 2. Implement `write()` method 3. Register in configuration schema ### Adding New Benchmark Tools + 1. Create runner in `runners/` 2. Create parser in `parsers/` 3. Update configuration schema @@ -280,16 +308,19 @@ report: ## Testing Strategy ### Unit Tests + - Configuration validation - Parser accuracy - Device command generation ### Integration Tests + - Pipeline stage transitions - File operations - Mock device operations ### System Tests + - End-to-end pipeline execution - Real device testing (CI) - Performance regression tests @@ -297,20 +328,24 @@ report: ## CI/CD Pipeline ### Build Stage + - Lint (Black, Ruff) - Type check (MyPy) - Unit tests (pytest) - Coverage report ### Package Stage + - Build distribution - Generate artifacts ### Test Stage + - Integration tests - Dry-run validation ### Deploy Stage (manual) + - PyPI publishing - Docker image creation - Documentation update @@ -318,12 +353,14 @@ report: ## Future Enhancements ### Near Term + - SQLite report sink - Linux SSH device support - HTML report generation - Docker development environment ### Long Term + - Web UI dashboard - Real-time monitoring - Cloud device farm integration @@ -334,6 +371,7 @@ report: ## Dependencies ### Runtime + - Python 3.11+ - typer: CLI framework - pydantic: Data validation @@ -343,11 +381,13 @@ report: - rich: Terminal formatting ### Build + - Android NDK r26d+ - CMake 3.24+ - Ninja 1.11+ ### Development + - pip: Dependency management - pytest: Testing framework - black: Code formatting @@ -356,4 +396,4 @@ report: ## License -Apache License 2.0 \ No newline at end of file +Apache License 2.0 diff --git a/CODEOWNERS b/CODEOWNERS index 124d20b..b8311ec 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -40,4 +40,4 @@ /tests/ @ovmobilebench/qa-team # Configuration examples -/experiments/ @ovmobilebench/maintainers \ No newline at end of file +/experiments/ @ovmobilebench/maintainers diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 7af8d7d..9c612d9 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -5,7 +5,7 @@ We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, +identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. @@ -43,7 +43,7 @@ an individual is officially representing the community in public spaces. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at -conduct@ovmobilebench.example.com. +. All complaints will be reviewed and investigated promptly and fairly. @@ -89,4 +89,4 @@ version 2.1, available at [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. [homepage]: https://www.contributor-covenant.org -[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html \ No newline at end of file +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d9ad4b..ea5067f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,12 +18,14 @@ Please read and follow our [Code of Conduct](CODE_OF_CONDUCT.md). ### Submitting Pull Requests 1. **Fork and Clone** + ```bash git clone https://github.com/embedded-dev-research/OVMobileBench cd OVMobileBench ``` 2. **Create Branch** + ```bash git checkout -b feature/your-feature-name # or @@ -31,6 +33,7 @@ Please read and follow our [Code of Conduct](CODE_OF_CONDUCT.md). ``` 3. **Set Up Development Environment** + ```bash pip install -r requirements.txt pip install -e . @@ -43,26 +46,32 @@ Please read and follow our [Code of Conduct](CODE_OF_CONDUCT.md). - Update documentation if needed 5. **Run Quality Checks** + ```bash - # Format code - black ovmobilebench tests - - # Lint - ruff ovmobilebench tests - - # Type check + # Run all pre-commit checks + pre-commit run --all-files + + # Or run individual checks: + black --check ovmobilebench tests + ruff check ovmobilebench tests mypy ovmobilebench --ignore-missing-imports - + yamllint . + markdownlint docs/ *.md + + # Auto-fix issues + pre-commit run --all-files --hook-stage manual + # Run tests pytest tests/ ``` 6. **Commit Changes** + ```bash git add . git commit -m "feat: add new feature" ``` - + Follow [Conventional Commits](https://www.conventionalcommits.org/): - `feat:` new feature - `fix:` bug fix @@ -73,10 +82,11 @@ Please read and follow our [Code of Conduct](CODE_OF_CONDUCT.md). - `ci:` CI/CD changes 7. **Push and Create PR** + ```bash git push origin feature/your-feature-name ``` - + Then create a Pull Request on GitHub. ## Development Guidelines diff --git a/Makefile b/Makefile index 51b0cb4..4a8d993 100644 --- a/Makefile +++ b/Makefile @@ -59,4 +59,4 @@ clean: rm -rf dist/ rm -rf *.egg-info find . -type d -name __pycache__ -exec rm -rf {} + - find . -type f -name "*.pyc" -delete \ No newline at end of file + find . -type f -name "*.pyc" -delete diff --git a/README.md b/README.md index 9061d7c..8ce55f0 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,18 @@ Apache License 2.0 - See [LICENSE](LICENSE) for details. We welcome contributions! Please see our [Contributing Guide](CONTRIBUTING.md) for details. +### CI/CD Pipeline + +Our CI runs on every push and pull request with the following stages: + +1. **Pre-commit checks** (Ubuntu only) - Code formatting, linting, and style checks +2. **Tests** - Unit tests on Ubuntu, macOS, and Windows +3. **Build** - Package building verification +4. **Validation** - Configuration and CLI validation +5. **Device tests** - Integration tests with Android devices (when available) + ## 💬 Support - 📝 [GitHub Issues](https://github.com/embedded-dev-research/OVMobileBench/issues) - Bug reports and feature requests - 💡 [Discussions](https://github.com/embedded-dev-research/OVMobileBench/discussions) - Questions and ideas -- 📧 Contact: nesterov.alexander@outlook.com +- 📧 Contact: diff --git a/SECURITY.md b/SECURITY.md index 56725d8..bd27abe 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,7 +12,7 @@ We take security seriously. If you discover a security vulnerability, please follow these steps: 1. **DO NOT** create a public GitHub issue -2. Email security report to: nesterov.alexander@outlook.com +2. Email security report to: 3. Include: - Description of the vulnerability - Steps to reproduce @@ -65,16 +65,19 @@ We take security seriously. If you discover a security vulnerability, please fol ## Known Security Considerations ### ADB Security + - ADB runs with elevated privileges - Ensure devices are trusted - Use ADB authorization ### SSH Security + - Use key-based authentication - Verify host keys - Limit SSH access scope ### Model Execution + - Models run with benchmark_app privileges - Potential for malicious models - Validate model sources @@ -82,11 +85,13 @@ We take security seriously. If you discover a security vulnerability, please fol ## Security Features ### Current + - Input validation in configuration - Parameterized shell commands - Secrets excluded from logs ### Planned + - Model signature verification - Encrypted credential storage - Audit logging @@ -94,20 +99,22 @@ We take security seriously. If you discover a security vulnerability, please fol ## Vulnerability Disclosure After a security issue is resolved: + 1. Security advisory will be published 2. CVE will be requested if applicable 3. Users will be notified via GitHub ## Contact -Security Team: nesterov.alexander@outlook.com +Security Team: PGP Key: [Link to public key] ## Acknowledgments We appreciate responsible disclosure and will acknowledge security researchers who: + - Follow responsible disclosure practices - Allow reasonable time for fixes - Don't exploit vulnerabilities -Thank you for helping keep OVMobileBench secure! \ No newline at end of file +Thank you for helping keep OVMobileBench secure! diff --git a/docs/android-setup.md b/docs/android-setup.md index 92df0d8..7887f07 100644 --- a/docs/android-setup.md +++ b/docs/android-setup.md @@ -115,6 +115,7 @@ ovmobilebench-android-installer setup \ ### What Gets Installed **Full Installation (default):** + - Android SDK Command Line Tools - Android SDK Platform Tools (includes `adb`) - Android SDK Build Tools @@ -124,22 +125,26 @@ ovmobilebench-android-installer setup \ - Android Emulator (optional) **NDK-Only Installation:** + - Android SDK Command Line Tools (required) - Android NDK ### Platform-Specific Details #### Windows + - Downloads Windows-specific packages - Installs to `%USERPROFILE%\android-sdk` by default - Uses `.bat` scripts for SDK manager #### macOS + - Downloads macOS-specific packages (supports both Intel and Apple Silicon) - Installs to `~/android-sdk` by default - Handles DMG extraction for NDK #### Linux + - Downloads Linux-specific packages - Installs to `~/android-sdk` by default - Works on x86_64 and ARM64 architectures @@ -183,6 +188,7 @@ ovmobilebench-android-installer export-env ` ``` The module sets the following environment variables: + - `ANDROID_HOME` - Android SDK root directory - `ANDROID_SDK_ROOT` - Same as ANDROID_HOME - `ANDROID_NDK_HOME` - NDK installation directory @@ -235,6 +241,7 @@ chmod +x scripts/setup_android_tools.py #### Download Failures If downloads fail, you can: + 1. Try again (the script caches partial downloads) 2. Use a VPN if you're in a region with restricted access 3. Download files manually and place them in the installation directory @@ -242,6 +249,7 @@ If downloads fail, you can: #### SDK Manager Issues If `sdkmanager` fails to install packages: + 1. Make sure you have Java 11 or higher installed 2. Accept all licenses manually: `sdkmanager --licenses` 3. Check proxy settings if behind a corporate firewall @@ -249,9 +257,11 @@ If `sdkmanager` fails to install packages: #### ADB Connection Issues If `adb devices` doesn't show your device: + 1. Enable USB debugging on your Android device 2. Install device drivers (Windows) -3. Add udev rules (Linux): +3. Add udev rules (Linux): + ```bash echo 'SUBSYSTEM=="usb", ATTR{idVendor}=="18d1", MODE="0666", GROUP="plugdev"' | sudo tee /etc/udev/rules.d/51-android.rules sudo udevadm control --reload-rules @@ -262,12 +272,14 @@ If `adb devices` doesn't show your device: Once Android tools are installed, you can use them with OVMobileBench: 1. **List connected devices:** + ```bash ovmobilebench list-devices ``` 2. **Build OpenVINO for Android:** Configure your experiment YAML with the NDK path: + ```yaml build: toolchain: @@ -277,6 +289,7 @@ Once Android tools are installed, you can use them with OVMobileBench: ``` 3. **Deploy and run benchmarks:** + ```bash ovmobilebench all -c experiments/android_config.yaml ``` @@ -307,4 +320,4 @@ rm -rf ~/android-sdk rmdir /s %USERPROFILE%\android-sdk ``` -And remove the environment variables from your shell configuration. \ No newline at end of file +And remove the environment variables from your shell configuration. diff --git a/docs/android_installer.md b/docs/android_installer.md index 5f3cf7f..f66d9c9 100644 --- a/docs/android_installer.md +++ b/docs/android_installer.md @@ -194,7 +194,7 @@ avd.create( ) # List AVDs -avds = avd.list() +avds = avd.list_avds() # Get AVD info info = avd.get_info("test_avd") @@ -328,6 +328,7 @@ def ensure_android_tools( ``` **Parameters:** + - `sdk_root`: Android SDK installation directory - `api`: Android API level (e.g., 30, 31, 33) - `target`: System image target ("google_atd", "google_apis", "default") @@ -344,6 +345,7 @@ def ensure_android_tools( **Returns:** `InstallerResult` dictionary with: + - `sdk_root`: SDK installation path - `ndk_path`: NDK installation path (if installed) - `avd_created`: Whether AVD was created @@ -362,6 +364,7 @@ def export_android_env( ``` **Parameters:** + - `sdk_root`: Android SDK root directory - `ndk_path`: NDK installation path - `format`: Output format ("dict", "bash", "fish", "windows", "github") @@ -381,6 +384,7 @@ def verify_installation( ``` **Parameters:** + - `sdk_root`: Android SDK root directory - `verbose`: Print verification results @@ -489,26 +493,31 @@ print("Would install:", result["performed"]) ## Supported Platforms ### Host Operating Systems + - **Linux**: x86_64, arm64 (Ubuntu 20.04+, RHEL 8+) - **macOS**: x86_64, arm64 (macOS 11+) - **Windows**: x86_64 (Windows 10+) ### Android API Levels + - API 21-34 (Android 5.0 - 14) ### System Image Targets + - `default`: Basic Android system image - `google_apis`: Includes Google Play Services - `google_atd`: Automated Test Device (faster, for testing) - `google_apis_playstore`: Includes Play Store ### Architectures + - `arm64-v8a`: 64-bit ARM (recommended for M1/M2 Macs) - `armeabi-v7a`: 32-bit ARM - `x86_64`: 64-bit x86 (recommended for Intel with KVM) - `x86`: 32-bit x86 ### NDK Versions + - Aliases: r21e, r22b, r23c, r24, r25c, r26d - Direct versions: 21.4.7075529, 22.1.7171670, etc. @@ -553,12 +562,14 @@ except InstallerError as e: The module supports multiple logging modes: ### Console Logging + ```python # Verbose console output ensure_android_tools(..., verbose=True) ``` ### JSON Lines Logging + ```python # Structured logging to file ensure_android_tools(..., jsonl_path="/tmp/install.jsonl") @@ -572,6 +583,7 @@ with open("/tmp/install.jsonl") as f: ``` ### Custom Logger + ```python from ovmobilebench.android.installer.logging import StructuredLogger from ovmobilebench.android.installer import set_logger @@ -586,6 +598,7 @@ ensure_android_tools(...) ## Best Practices ### 1. Use Dry Run First + Always test with `dry_run=True` before actual installation: ```python @@ -597,6 +610,7 @@ if result["performed"]: ``` ### 2. Verify After Installation + Always verify the installation succeeded: ```python @@ -607,6 +621,7 @@ assert status["ndk"], "Missing NDK" ``` ### 3. Handle Errors Gracefully + Wrap installations in try-except blocks: ```python @@ -619,11 +634,13 @@ except DownloadError: ``` ### 4. Use Appropriate Targets + - Use `google_atd` for CI/testing (faster) - Use `google_apis` for development - Use `google_apis_playstore` for Play Store testing ### 5. Clean Up Temporary Files + Remove downloads after installation: ```python @@ -639,33 +656,43 @@ installer.cleanup(remove_downloads=True) ### Common Issues #### 1. SSL Certificate Errors + ``` DownloadError: certificate verify failed ``` + **Solution**: Update certificates or use corporate proxy settings #### 2. Permission Denied + ``` PermissionError: Permission denied: /opt/android-sdk ``` + **Solution**: Ensure write permissions or use user directory #### 3. Disk Space + ``` Warning: Low disk space detected (< 15GB free) ``` + **Solution**: Free up space or use different location #### 4. Java Not Found + ``` Warning: Java not detected ``` + **Solution**: Install JDK 11+ and ensure it's in PATH #### 5. KVM Not Available + ``` Info: KVM not available, using software acceleration ``` + **Solution**: Enable virtualization in BIOS or use ARM images on ARM hosts ### Debug Mode @@ -706,4 +733,4 @@ See [CONTRIBUTING.md](../CONTRIBUTING.md) for development guidelines. ## License -Apache License 2.0. See [LICENSE](../LICENSE) for details. \ No newline at end of file +Apache License 2.0. See [LICENSE](../LICENSE) for details. diff --git a/docs/api-reference.md b/docs/api-reference.md index e908570..552a64f 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -25,42 +25,42 @@ Main orchestration module for running benchmarks. ```python class Pipeline: """Main pipeline orchestrator""" - + def __init__(self, config: Union[str, Path, Experiment]): """ Initialize pipeline with configuration. - + Args: config: Path to YAML config file or Experiment object """ - + def run(self, stages: Optional[List[str]] = None) -> Dict[str, Any]: """ Run pipeline stages. - + Args: stages: List of stages to run. If None, runs all stages. Valid stages: ['build', 'package', 'deploy', 'run', 'report'] - + Returns: Dictionary with results from each stage - + Raises: PipelineError: If any stage fails """ - + def build(self) -> BuildResult: """Build OpenVINO from source""" - + def package(self) -> PackageResult: """Create deployment bundle""" - + def deploy(self) -> DeployResult: """Deploy to target devices""" - + def run_benchmarks(self) -> BenchmarkResult: """Execute benchmarks""" - + def report(self) -> ReportResult: """Generate reports""" ``` @@ -95,7 +95,7 @@ Pydantic models for configuration validation. ```python class Experiment(BaseModel): """Top-level experiment configuration""" - + project: ProjectConfig build: BuildConfig package: PackageConfig @@ -103,14 +103,14 @@ class Experiment(BaseModel): models: List[ModelItem] run: RunConfig report: ReportConfig - + @classmethod def from_yaml(cls, path: Union[str, Path]) -> "Experiment": """Load configuration from YAML file""" - + def to_yaml(self, path: Union[str, Path]) -> None: """Save configuration to YAML file""" - + def override(self, overrides: Dict[str, Any]) -> "Experiment": """Apply overrides to configuration""" ``` @@ -120,17 +120,17 @@ class Experiment(BaseModel): ```python class BuildConfig(BaseModel): """Build configuration""" - + enabled: bool = True openvino_repo: str openvino_commit: str = "HEAD" build_type: Literal["Release", "Debug", "RelWithDebInfo"] = "Release" toolchain: Toolchain options: Dict[str, str] = {} - + def validate_paths(self) -> None: """Validate that all paths exist""" - + def get_cmake_args(self) -> List[str]: """Generate CMake arguments""" ``` @@ -140,16 +140,16 @@ class BuildConfig(BaseModel): ```python class RunConfig(BaseModel): """Benchmark execution configuration""" - + repeats: int = 3 warmup_runs: int = 0 cooldown_sec: int = 0 timeout_sec: Optional[int] = None matrix: RunMatrix - + def expand_matrix(self) -> List[Dict[str, Any]]: """Expand matrix into individual configurations""" - + def estimate_duration(self) -> timedelta: """Estimate total execution time""" ``` @@ -195,39 +195,39 @@ Base class for device implementations. ```python class Device(ABC): """Abstract base class for devices""" - + @abstractmethod def connect(self) -> None: """Establish connection to device""" - + @abstractmethod def disconnect(self) -> None: """Close connection to device""" - + @abstractmethod def push(self, local: Path, remote: str) -> None: """Push file to device""" - + @abstractmethod def pull(self, remote: str, local: Path) -> None: """Pull file from device""" - + @abstractmethod def shell(self, command: str, timeout: Optional[int] = None) -> ShellResult: """Execute command on device""" - + @abstractmethod def exists(self, path: str) -> bool: """Check if path exists on device""" - + @abstractmethod def mkdir(self, path: str, parents: bool = True) -> None: """Create directory on device""" - + @abstractmethod def rm(self, path: str, recursive: bool = False) -> None: """Remove file/directory on device""" - + @abstractmethod def info(self) -> DeviceInfo: """Get device information""" @@ -242,25 +242,25 @@ Android device implementation using **adbutils** library for direct Python-based ```python class AndroidDevice(Device): """Android device via adbutils Python library""" - + def __init__(self, serial: str, use_root: bool = False): """ Initialize Android device. - + Args: serial: Device serial number use_root: Whether to use root access """ - + def get_temperature(self) -> float: """Get device temperature in Celsius""" - + def get_cpu_info(self) -> CPUInfo: """Get CPU information""" - + def set_cpu_governor(self, governor: str) -> None: """Set CPU frequency governor (requires root)""" - + def screenshot(self, path: Path) -> None: """Take screenshot and save to path""" ``` @@ -274,7 +274,7 @@ Linux device implementation using **paramiko** library for secure SSH connection ```python class LinuxSSHDevice(Device): """Linux device via SSH using paramiko""" - + def __init__( self, host: str, @@ -286,7 +286,7 @@ class LinuxSSHDevice(Device): ): """ Initialize SSH connection to Linux device. - + Args: host: Hostname or IP address username: SSH username @@ -295,7 +295,7 @@ class LinuxSSHDevice(Device): port: SSH port (default 22) push_dir: Remote directory for deployment """ - + def get_env(self) -> Dict[str, str]: """Get environment variables for benchmark execution""" ``` @@ -364,24 +364,24 @@ OpenVINO build management. ```python class OpenVINOBuilder: """Build OpenVINO from source""" - + def __init__(self, config: BuildConfig): """Initialize builder with configuration""" - + def configure(self) -> None: """Run CMake configuration""" - + def build(self, targets: List[str] = None) -> None: """ Build specified targets. - + Args: targets: List of CMake targets. If None, builds all. """ - + def clean(self) -> None: """Clean build directory""" - + def package(self, output_dir: Path) -> PackageInfo: """Package build artifacts""" ``` @@ -417,22 +417,22 @@ Benchmark execution management. ```python class BenchmarkRunner: """Execute benchmark_app""" - + def __init__(self, device: Device, config: RunConfig): """ Initialize runner. - + Args: device: Target device config: Run configuration """ - + def run_single(self, params: Dict[str, Any]) -> BenchmarkResult: """Run single benchmark with given parameters""" - + def run_matrix(self, model: ModelItem) -> List[BenchmarkResult]: """Run complete parameter matrix for a model""" - + def run_all(self, models: List[ModelItem]) -> List[BenchmarkResult]: """Run benchmarks for all models""" ``` @@ -443,7 +443,7 @@ class BenchmarkRunner: @dataclass class BenchmarkResult: """Result from single benchmark run""" - + model: str device: str parameters: Dict[str, Any] @@ -494,25 +494,25 @@ Parse benchmark_app output. ```python class BenchmarkParser: """Parse benchmark_app output""" - + def parse(self, stdout: str, stderr: str = "") -> ParsedMetrics: """ Parse benchmark output. - + Args: stdout: Standard output from benchmark_app stderr: Standard error from benchmark_app - + Returns: Parsed metrics - + Raises: ParserError: If output cannot be parsed """ - + def extract_throughput(self, text: str) -> float: """Extract throughput in FPS""" - + def extract_latencies(self, text: str) -> LatencyMetrics: """Extract latency metrics""" ``` @@ -552,7 +552,7 @@ Report generation and output. ```python class ReportSink(ABC): """Abstract base class for report sinks""" - + @abstractmethod def write(self, results: List[BenchmarkResult]) -> None: """Write results to sink""" @@ -563,10 +563,10 @@ class ReportSink(ABC): ```python class JSONSink(ReportSink): """Write results to JSON file""" - + def __init__(self, path: Path, indent: int = 2): """Initialize JSON sink""" - + def write(self, results: List[BenchmarkResult]) -> None: """Write results as JSON""" ``` @@ -576,10 +576,10 @@ class JSONSink(ReportSink): ```python class CSVSink(ReportSink): """Write results to CSV file""" - + def __init__(self, path: Path, columns: Optional[List[str]] = None): """Initialize CSV sink""" - + def write(self, results: List[BenchmarkResult]) -> None: """Write results as CSV""" ``` @@ -593,14 +593,14 @@ Statistical summarization of results. ```python class Summarizer: """Summarize benchmark results""" - + def summarize(self, results: List[BenchmarkResult]) -> Summary: """Generate statistical summary""" - - def group_by(self, results: List[BenchmarkResult], + + def group_by(self, results: List[BenchmarkResult], keys: List[str]) -> Dict[tuple, List[BenchmarkResult]]: """Group results by specified keys""" - + def compare(self, baseline: List[BenchmarkResult], current: List[BenchmarkResult]) -> Comparison: """Compare two sets of results""" @@ -657,7 +657,7 @@ def ensure_android_tools( ) -> InstallerResult: """ Install and configure Android SDK/NDK. - + Args: sdk_root: Android SDK installation directory api: Android API level (e.g., 30, 31, 33) @@ -672,10 +672,10 @@ def ensure_android_tools( dry_run: Preview without making changes verbose: Enable detailed logging jsonl_path: Path for JSON Lines log output - + Returns: Dictionary with installation results - + Raises: InstallerError: If installation fails """ @@ -691,12 +691,12 @@ def export_android_env( ) -> Union[Dict[str, str], str]: """ Export Android environment variables. - + Args: sdk_root: Android SDK root directory ndk_path: NDK installation path format: Output format (dict, bash, fish, windows, github) - + Returns: Environment variables as dictionary or formatted string """ @@ -711,11 +711,11 @@ def verify_installation( ) -> Dict[str, Any]: """ Verify Android tools installation. - + Args: sdk_root: Android SDK root directory verbose: Print verification results - + Returns: Dictionary with installation status """ @@ -728,18 +728,18 @@ def verify_installation( ```python class AndroidInstaller: """Main installer orchestrator""" - + def __init__(self, sdk_root: Path, logger: Optional[StructuredLogger] = None): """Initialize installer with SDK root""" - - def ensure(self, api: int, target: str, arch: str, + + def ensure(self, api: int, target: str, arch: str, ndk: Optional[NdkSpec] = None, **kwargs) -> InstallerResult: """Install Android tools with specified configuration""" - + def verify(self) -> Dict[str, Any]: """Verify installation status""" - - def cleanup(self, remove_downloads: bool = True, + + def cleanup(self, remove_downloads: bool = True, remove_temp: bool = True) -> None: """Clean up temporary files""" ``` @@ -814,14 +814,14 @@ def run_command( ) -> CommandResult: """ Execute shell command. - + Args: command: Command to execute cwd: Working directory env: Environment variables timeout: Timeout in seconds capture_output: Whether to capture stdout/stderr - + Returns: Command result with output and return code """ @@ -859,19 +859,19 @@ Artifact management utilities. ```python class ArtifactManager: """Manage build and benchmark artifacts""" - + def __init__(self, base_dir: Path): """Initialize artifact manager""" - + def create_artifact(self, name: str, content: Any) -> Artifact: """Create new artifact""" - + def get_artifact(self, artifact_id: str) -> Artifact: """Retrieve artifact by ID""" - + def list_artifacts(self, filter_type: Optional[str] = None) -> List[Artifact]: """List all artifacts""" - + def cleanup(self, older_than: timedelta) -> int: """Clean up old artifacts""" ``` @@ -903,18 +903,18 @@ from ovmobilebench.devices.base import Device, DeviceInfo class CustomDevice(Device): """Custom device implementation""" - + def __init__(self, config: Dict[str, Any]): self.config = config - + def connect(self) -> None: # Implement connection logic pass - + def shell(self, command: str, timeout: Optional[int] = None) -> ShellResult: # Implement command execution pass - + # Implement other required methods... ``` @@ -925,7 +925,7 @@ from ovmobilebench.parsers.base import Parser class CustomParser(Parser): """Custom output parser""" - + def parse(self, output: str) -> Dict[str, Any]: # Implement parsing logic metrics = {} @@ -940,10 +940,10 @@ from ovmobilebench.report.sink import ReportSink class HTMLSink(ReportSink): """Generate HTML reports""" - + def __init__(self, template_path: Path): self.template = load_template(template_path) - + def write(self, results: List[BenchmarkResult]) -> None: html = self.template.render(results=results) # Save HTML... @@ -996,4 +996,4 @@ except BenchmarkError as e: - [Troubleshooting](troubleshooting.md) - Common issues - [Architecture](architecture.md) - System design -- [Examples](https://github.com/embedded-dev-research/OVMobileBench/tree/main/examples) - Code examples \ No newline at end of file +- [Examples](https://github.com/embedded-dev-research/OVMobileBench/tree/main/examples) - Code examples diff --git a/docs/architecture.md b/docs/architecture.md index fa738f4..73664ec 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -35,6 +35,7 @@ OVMobileBench is an end-to-end benchmarking pipeline for OpenVINO on mobile devi **Purpose**: Define and validate experiment configurations. **Key Classes**: + - `Experiment`: Top-level configuration container - `BuildConfig`: OpenVINO build settings - `DeviceConfig`: Target device specifications @@ -48,6 +49,7 @@ OVMobileBench is an end-to-end benchmarking pipeline for OpenVINO on mobile devi **Purpose**: Command-line interface for user interaction. **Commands**: + - `build`: Build OpenVINO from source - `package`: Create deployment bundle - `deploy`: Push to device(s) @@ -62,6 +64,7 @@ OVMobileBench is an end-to-end benchmarking pipeline for OpenVINO on mobile devi **Purpose**: Coordinate execution of all pipeline stages. **Responsibilities**: + - Stage dependency management - Error handling and recovery - Progress tracking @@ -74,11 +77,13 @@ OVMobileBench is an end-to-end benchmarking pipeline for OpenVINO on mobile devi **Purpose**: Uniform interface for different device types. **Implementations**: + - `AndroidDevice`: Python adbutils-based Android device control (no external ADB binary needed) - `LinuxDevice`: SSH-based Linux device control (planned) - `iOSDevice`: iOS device control (stub) **Interface**: + ```python class Device(ABC): def push(local, remote) @@ -95,12 +100,14 @@ class Device(ABC): **Purpose**: Build OpenVINO runtime for target platforms. **Features**: + - CMake configuration generation - Cross-compilation support (Android NDK) - Build caching - Artifact collection **Supported Platforms**: + - Android (arm64-v8a) - Linux ARM (aarch64) @@ -109,6 +116,7 @@ class Device(ABC): **Purpose**: Bundle runtime, libraries, and models. **Bundle Structure**: + ``` ovbundle.tar.gz ├── bin/ @@ -127,6 +135,7 @@ ovbundle.tar.gz **Purpose**: Execute benchmark_app with various configurations. **Features**: + - Matrix expansion (device, threads, streams, precision) - Timeout handling - Cooldown between runs @@ -138,6 +147,7 @@ ovbundle.tar.gz **Purpose**: Extract metrics from benchmark_app output. **Metrics**: + - Throughput (FPS) - Latencies (avg, median, min, max) - Device utilization @@ -148,6 +158,7 @@ ovbundle.tar.gz **Purpose**: Generate structured reports from results. **Formats**: + - JSON: Machine-readable format - CSV: Spreadsheet-compatible - SQLite: Database format (planned) @@ -156,6 +167,7 @@ ovbundle.tar.gz ### 10. Core Utilities (`ovmobilebench/core/`) **Shared Components**: + - `shell.py`: Command execution with timeout - `fs.py`: File system operations - `artifacts.py`: Artifact management @@ -165,31 +177,37 @@ ovbundle.tar.gz ## Data Flow ### 1. Configuration Loading + ``` YAML File → Pydantic Validation → Experiment Object ``` ### 2. Build Flow + ``` Git Checkout → CMake Configure → Ninja Build → Artifact Collection ``` ### 3. Package Flow + ``` Build Artifacts + Models → Tar Archive → Checksum Generation ``` ### 4. Deployment Flow + ``` Bundle → Device Push → Remote Extraction → Permission Setup ``` ### 5. Execution Flow + ``` Matrix Expansion → Device Preparation → Benchmark Execution → Output Collection ``` ### 6. Reporting Flow + ``` Raw Output → Parsing → Aggregation → Format Conversion → Sink Writing ``` @@ -197,29 +215,30 @@ Raw Output → Parsing → Aggregation → Format Conversion → Sink Writing ## Configuration Schema ### Experiment Configuration + ```yaml project: name: string run_id: string - + build: openvino_repo: path toolchain: android_ndk: path - + device: kind: android|linux_ssh serials: [string] - + models: - name: string path: path - + run: matrix: threads: [int] nstreams: [string] - + report: sinks: - type: json|csv @@ -229,16 +248,19 @@ report: ## Security Considerations ### Input Validation + - All user inputs validated via Pydantic - Shell commands parameterized to prevent injection - Path traversal prevention ### Secrets Management + - No hardcoded credentials - Environment variables for sensitive data - SSH key-based authentication ### Device Security + - USB debugging authorization required - Limited command set execution - Temporary file cleanup @@ -246,16 +268,19 @@ report: ## Performance Optimizations ### Build Caching + - CMake build cache - ccache integration (planned) - Incremental builds ### Parallel Execution + - Multiple device support - Concurrent stage execution (where safe) - Async I/O for file operations ### Resource Management + - Automatic cleanup of temporary files - Connection pooling for SSH - Memory-mapped file I/O for large files @@ -263,16 +288,19 @@ report: ## Extensibility Points ### Adding New Device Types + 1. Inherit from `Device` base class 2. Implement required methods 3. Register in `pipeline.py` ### Adding New Report Formats + 1. Inherit from `ReportSink` 2. Implement `write()` method 3. Register in configuration schema ### Adding New Benchmark Tools + 1. Create runner in `runners/` 2. Create parser in `parsers/` 3. Update configuration schema @@ -280,16 +308,19 @@ report: ## Testing Strategy ### Unit Tests + - Configuration validation - Parser accuracy - Device command generation ### Integration Tests + - Pipeline stage transitions - File operations - Mock device operations ### System Tests + - End-to-end pipeline execution - Real device testing (CI) - Performance regression tests @@ -297,20 +328,24 @@ report: ## CI/CD Pipeline ### Build Stage + - Lint (Black, Ruff) - Type check (MyPy) - Unit tests (pytest) - Coverage report ### Package Stage + - Build distribution - Generate artifacts ### Test Stage + - Integration tests - Dry-run validation ### Deploy Stage (manual) + - PyPI publishing - Docker image creation - Documentation update @@ -318,12 +353,14 @@ report: ## Future Enhancements ### Near Term + - SQLite report sink - Linux SSH device support - HTML report generation - Docker development environment ### Long Term + - Web UI dashboard - Real-time monitoring - Cloud device farm integration @@ -334,6 +371,7 @@ report: ## Dependencies ### Runtime + - Python 3.11+ - typer: CLI framework - pydantic: Data validation @@ -343,11 +381,13 @@ report: - rich: Terminal formatting ### Build + - Android NDK r26d+ - CMake 3.24+ - Ninja 1.11+ ### Development + - pip: Dependency management - pytest: Testing framework - black: Code formatting @@ -356,4 +396,4 @@ report: ## License -Apache License 2.0 \ No newline at end of file +Apache License 2.0 diff --git a/docs/benchmarking.md b/docs/benchmarking.md index 3488c55..e2fe3ed 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -17,6 +17,7 @@ This guide covers running benchmarks, interpreting results, and optimizing perfo ### What is benchmark_app? `benchmark_app` is OpenVINO's official tool for measuring inference performance. It: + - Loads a model and runs inference - Measures throughput and latency - Supports various execution modes @@ -41,6 +42,7 @@ ovmobilebench run -c experiments/basic.yaml ``` Basic configuration: + ```yaml run: repeats: 3 @@ -119,14 +121,17 @@ run: ### Primary Metrics #### Throughput + ``` Throughput: 156.23 FPS ``` + - Total inferences per second - Higher is better - Best for batch processing #### Latency + ``` Latency: Median: 6.4 ms @@ -134,6 +139,7 @@ Latency: Min: 5.8 ms Max: 8.2 ms ``` + - Time per inference - Lower is better - Critical for real-time applications @@ -141,18 +147,21 @@ Latency: ### Secondary Metrics #### CPU Utilization + ```bash # Monitor during benchmark adb shell top -d 1 | grep benchmark_app ``` #### Memory Usage + ```bash # Check memory consumption adb shell dumpsys meminfo | grep benchmark_app ``` #### Power Consumption + ```bash # Battery stats (Android) adb shell dumpsys batterystats @@ -163,6 +172,7 @@ adb shell dumpsys batterystats ### Thread Optimization Find optimal thread count: + ```yaml run: matrix: @@ -170,6 +180,7 @@ run: ``` Analysis: + ```python import pandas as pd import matplotlib.pyplot as plt @@ -192,6 +203,7 @@ run: ``` Best practices: + - nstreams ≤ number of CPU cores - nireq ≥ nstreams for async mode - Use AUTO for automatic optimization @@ -210,6 +222,7 @@ run: ``` Trade-offs: + - Larger batch = better throughput - Larger batch = higher latency - Memory constraints limit max batch @@ -347,6 +360,7 @@ graph TD ### Configuration Examples #### Latency-Optimized + ```yaml run: matrix: @@ -357,6 +371,7 @@ run: ``` #### Throughput-Optimized + ```yaml run: matrix: @@ -367,6 +382,7 @@ run: ``` #### Power-Efficient + ```yaml run: matrix: @@ -409,7 +425,7 @@ def detect_throttling(fps_over_time): """Detect performance degradation over time""" first_quarter = np.mean(fps_over_time[:len(fps_over_time)//4]) last_quarter = np.mean(fps_over_time[-len(fps_over_time)//4:]) - + degradation = (first_quarter - last_quarter) / first_quarter if degradation > 0.1: # 10% drop return True, degradation @@ -432,7 +448,7 @@ run: adb shell ps -A | grep -v idle # Monitor CPU frequency -adb shell "while true; do +adb shell "while true; do cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_cur_freq sleep 1 done" @@ -466,7 +482,7 @@ models: run: matrix: - input_shape: + input_shape: - [1, 3, 224, 224] - [1, 3, 416, 416] - [1, 3, 640, 640] @@ -478,18 +494,18 @@ run: def calculate_efficiency_metrics(df): """Calculate custom efficiency metrics""" metrics = {} - + # FPS per thread metrics['fps_per_thread'] = df['throughput_fps'] / df['threads'] - + # FPS per watt (if power data available) if 'power_w' in df.columns: metrics['fps_per_watt'] = df['throughput_fps'] / df['power_w'] - + # Latency consistency if 'latency_std' in df.columns: metrics['latency_cv'] = df['latency_std'] / df['latency_avg'] - + return pd.DataFrame(metrics) ``` @@ -523,4 +539,4 @@ def calculate_efficiency_metrics(df): - [CI/CD Integration](ci-cd.md) - Automated benchmarking - [API Reference](api-reference.md) - Programming interface -- [Troubleshooting](troubleshooting.md) - Common issues \ No newline at end of file +- [Troubleshooting](troubleshooting.md) - Common issues diff --git a/docs/build-guide.md b/docs/build-guide.md index 199c181..dbfaa26 100644 --- a/docs/build-guide.md +++ b/docs/build-guide.md @@ -24,11 +24,13 @@ This guide covers building OpenVINO and benchmark_app for mobile platforms. ### Platform-Specific Requirements #### Android + - Android NDK r26d or later - Android SDK (for adb) - Java JDK 8+ (for Android SDK) #### Linux ARM + - Cross-compilation toolchain (for cross-compiling) - OR native build environment (when building on target) @@ -284,10 +286,10 @@ build: options: # For ARMv8.2-A with FP16 CMAKE_CXX_FLAGS: "-march=armv8.2-a+fp16+dotprod" - + # Enable ARM Compute Library ENABLE_ARM_COMPUTE: "ON" - + # Enable oneDNN for ARM ENABLE_ONEDNN_FOR_ARM: "ON" ``` @@ -323,7 +325,7 @@ build: ENABLE_INTEL_GPU: "OFF" ENABLE_INTEL_NPU: "OFF" ENABLE_INTEL_GNA: "OFF" - + # Enable specific features ENABLE_OPENCV: "OFF" # Disable if not needed ENABLE_PYTHON: "OFF" # Disable for smaller size @@ -483,17 +485,17 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - + - name: Setup NDK run: | wget https://dl.google.com/android/repository/android-ndk-r26d-linux.zip unzip -q android-ndk-r26d-linux.zip echo "ANDROID_NDK=$PWD/android-ndk-r26d" >> $GITHUB_ENV - + - name: Build run: | ovmobilebench build -c config.yaml - + - name: Upload artifacts uses: actions/upload-artifact@v4 with: @@ -532,4 +534,4 @@ WORKDIR /openvino - [Benchmarking Guide](benchmarking.md) - Running benchmarks - [Device Setup](device-setup.md) - Preparing target devices -- [Troubleshooting](troubleshooting.md) - Common issues and solutions \ No newline at end of file +- [Troubleshooting](troubleshooting.md) - Common issues and solutions diff --git a/docs/ci-cd.md b/docs/ci-cd.md index 48832a0..a7b61a4 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -40,6 +40,7 @@ graph LR ### Basic Workflow `.github/workflows/benchmark.yml`: + ```yaml name: Benchmark Pipeline @@ -56,20 +57,20 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - + - name: Setup Python uses: actions/setup-python@v5 with: python-version: '3.11' - + - name: Install OVMobileBench run: | pip install -e .[dev] - + - name: Build OpenVINO run: | ovmobilebench build -c ci/config.yaml - + - name: Upload artifacts uses: actions/upload-artifact@v4 with: @@ -82,19 +83,19 @@ jobs: runs-on: [self-hosted, android] steps: - uses: actions/checkout@v4 - + - name: Download artifacts uses: actions/download-artifact@v4 with: name: ovbundle path: artifacts/ - + - name: Run benchmarks run: | ovmobilebench deploy -c ci/config.yaml ovmobilebench run -c ci/config.yaml ovmobilebench report -c ci/config.yaml - + - name: Upload results uses: actions/upload-artifact@v4 with: @@ -114,9 +115,9 @@ jobs: device: [pixel6, galaxy-s21, oneplus9] precision: [FP32, FP16, INT8] threads: [1, 4, 8] - + runs-on: [self-hosted, "${{ matrix.device }}"] - + steps: - name: Run benchmark env: @@ -144,22 +145,22 @@ jobs: steps: - name: Checkout PR uses: actions/checkout@v4 - + - name: Checkout main uses: actions/checkout@v4 with: ref: main path: baseline - + - name: Run PR benchmark run: | ovmobilebench all -c ci/pr.yaml -o results/pr.json - + - name: Run baseline benchmark run: | cd baseline ovmobilebench all -c ci/pr.yaml -o results/baseline.json - + - name: Compare results id: compare run: | @@ -167,7 +168,7 @@ jobs: --baseline results/baseline.json \ --current results/pr.json \ --threshold -5.0 - + - name: Comment on PR uses: actions/github-script@v7 with: @@ -194,17 +195,17 @@ jobs: runs-on: [self-hosted, android] steps: - uses: actions/checkout@v4 - + - name: Full benchmark suite run: | ovmobilebench all -c ci/nightly.yaml - + - name: Upload to dashboard run: | python ci/upload_dashboard.py \ --results results/ \ --endpoint $PERF_DASHBOARD_URL - + - name: Check regressions run: | python ci/check_regression.py \ @@ -218,6 +219,7 @@ jobs: ### Basic Pipeline `.gitlab-ci.yml`: + ```yaml stages: - build @@ -291,14 +293,14 @@ benchmark:mr: - | # Run benchmarks ovmobilebench all -c ci/mr.yaml - + # Compare with target branch git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME ovmobilebench all -c ci/mr.yaml -o baseline.json - + # Generate comparison python ci/compare.py --format markdown > comparison.md - + # Post comment curl -X POST \ -H "PRIVATE-TOKEN: $CI_JOB_TOKEN" \ @@ -313,12 +315,12 @@ benchmark:mr: ```groovy pipeline { agent any - + environment { ANDROID_SERIAL = credentials('android-device-serial') OVBENCH_CONFIG = 'ci/jenkins.yaml' } - + stages { stage('Build') { agent { @@ -331,7 +333,7 @@ pipeline { stash includes: 'artifacts/**', name: 'build-artifacts' } } - + stage('Package') { steps { unstash 'build-artifacts' @@ -339,7 +341,7 @@ pipeline { stash includes: 'bundles/**', name: 'bundles' } } - + stage('Benchmark') { agent { label 'android-device' @@ -353,7 +355,7 @@ pipeline { ''' } } - + stage('Analysis') { steps { publishHTML([ @@ -361,7 +363,7 @@ pipeline { reportFiles: 'index.html', reportName: 'Performance Report' ]) - + perfReport( sourceDataFiles: 'results/*.json', compareBuildPrevious: true, @@ -370,7 +372,7 @@ pipeline { } } } - + post { always { archiveArtifacts artifacts: 'results/**', fingerprint: true @@ -395,7 +397,7 @@ pipeline { choice(name: 'PRECISION', choices: ['FP32', 'FP16', 'INT8'], description: 'Inference precision') string(name: 'THREADS', defaultValue: '4', description: 'Number of threads') } - + stages { stage('Benchmark') { steps { @@ -418,6 +420,7 @@ pipeline { ### GitHub Actions Runner Setup #### Installation + ```bash # Download runner mkdir actions-runner && cd actions-runner @@ -437,6 +440,7 @@ sudo ./svc.sh start ``` #### Runner Configuration + ```yaml # .github/workflows/device-test.yml jobs: @@ -452,6 +456,7 @@ jobs: ### Device Farm Management #### Device Registry + ```yaml # devices/inventory.yaml devices: @@ -460,7 +465,7 @@ devices: model: Pixel 6 soc: Google Tensor labels: [android, arm64, tensor] - + - id: galaxy-s21-01 serial: RF8N1234567 model: Galaxy S21 @@ -469,6 +474,7 @@ devices: ``` #### Health Monitoring + ```python # ci/device_health.py import subprocess @@ -485,7 +491,7 @@ def check_device_health(serial): ) if result.returncode != 0: return False, "ADB connection failed" - + # Check temperature temp = subprocess.run( f"adb -s {serial} shell cat /sys/class/thermal/thermal_zone0/temp", @@ -494,7 +500,7 @@ def check_device_health(serial): temp_c = int(temp.stdout) / 1000 if temp_c > 45: return False, f"Temperature too high: {temp_c}°C" - + # Check battery battery = subprocess.run( f"adb -s {serial} shell dumpsys battery | grep level", @@ -503,7 +509,7 @@ def check_device_health(serial): level = int(battery.stdout.split(':')[1]) if level < 20: return False, f"Battery too low: {level}%" - + return True, "Healthy" except Exception as e: return False, str(e) @@ -546,7 +552,7 @@ integration: - | # Wait for emulator adb wait-for-device - + # Run integration tests pytest tests/integration/ -v ``` @@ -563,7 +569,7 @@ def test_baseline_performance(): """Test that performance meets baseline""" config = load_config('tests/baseline.yaml') results = pipeline.run(config) - + assert results['throughput_fps'] > 50.0 assert results['latency_avg_ms'] < 20.0 @@ -572,10 +578,10 @@ def test_thread_scaling(threads): """Test thread scaling efficiency""" config = load_config('tests/scaling.yaml') config['run']['matrix']['threads'] = [threads] - + results = pipeline.run(config) efficiency = results['throughput_fps'] / (threads * baseline_fps) - + assert efficiency > 0.7 # At least 70% scaling efficiency ``` @@ -593,7 +599,7 @@ def collect_metrics(results_path, metadata): """Collect and format metrics for tracking""" with open(results_path) as f: results = json.load(f) - + metrics = { 'timestamp': datetime.utcnow().isoformat(), 'commit': metadata['commit'], @@ -606,15 +612,15 @@ def collect_metrics(results_path, metadata): 'ndk_version': metadata['ndk_version'] } } - + return metrics # Send to time-series database def send_to_influxdb(metrics): from influxdb import InfluxDBClient - + client = InfluxDBClient('127.0.0.1', 8086, database='ovmobilebench') - + points = [] for result in metrics['results']: points.append({ @@ -631,7 +637,7 @@ def send_to_influxdb(metrics): }, 'time': metrics['timestamp'] }) - + client.write_points(points) ``` @@ -644,11 +650,11 @@ dashboard: panels: - title: "Throughput Over Time" query: | - SELECT mean("throughput_fps") - FROM "benchmark" - WHERE $timeFilter + SELECT mean("throughput_fps") + FROM "benchmark" + WHERE $timeFilter GROUP BY time($interval), "model" - + - title: "Latency Distribution" query: | SELECT percentile("latency_avg_ms", 50) as median, @@ -669,20 +675,20 @@ from scipy import stats def detect_regression(baseline_data, current_data, threshold=0.05): """Detect performance regression using statistical test""" regressions = [] - + for metric in ['throughput_fps', 'latency_avg_ms']: baseline = [r[metric] for r in baseline_data] current = [r[metric] for r in current_data] - + # Perform t-test t_stat, p_value = stats.ttest_ind(baseline, current) - + # Check if significant difference if p_value < threshold: baseline_mean = np.mean(baseline) current_mean = np.mean(current) change = (current_mean - baseline_mean) / baseline_mean - + # For throughput, decrease is regression # For latency, increase is regression if (metric == 'throughput_fps' and change < -0.05) or \ @@ -694,7 +700,7 @@ def detect_regression(baseline_data, current_data, threshold=0.05): 'change': change, 'p_value': p_value }) - + return regressions ``` @@ -703,6 +709,7 @@ def detect_regression(baseline_data, current_data, threshold=0.05): ### CI Configuration 1. **Resource Management** + ```yaml jobs: benchmark: @@ -712,6 +719,7 @@ def detect_regression(baseline_data, current_data, threshold=0.05): ``` 2. **Caching** + ```yaml - name: Cache build uses: actions/cache@v3 @@ -723,6 +731,7 @@ def detect_regression(baseline_data, current_data, threshold=0.05): ``` 3. **Artifact Management** + ```yaml - uses: actions/upload-artifact@v4 with: @@ -735,6 +744,7 @@ def detect_regression(baseline_data, current_data, threshold=0.05): ### Security 1. **Secrets Management** + ```yaml env: ANDROID_KEYSTORE: ${{ secrets.ANDROID_KEYSTORE }} @@ -742,6 +752,7 @@ def detect_regression(baseline_data, current_data, threshold=0.05): ``` 2. **Access Control** + ```yaml jobs: deploy: @@ -752,6 +763,7 @@ def detect_regression(baseline_data, current_data, threshold=0.05): ### Monitoring 1. **Job Status Notification** + ```yaml - name: Notify Team if: failure() @@ -762,6 +774,7 @@ def detect_regression(baseline_data, current_data, threshold=0.05): ``` 2. **Performance Alerts** + ```python if regression_detected: send_alert( @@ -773,28 +786,30 @@ def detect_regression(baseline_data, current_data, threshold=0.05): ### Documentation 1. **CI Documentation** + ```markdown ## CI/CD Pipeline - + ### Triggers - Push to main: Full benchmark suite - Pull request: Quick validation - Nightly: Extended testing - + ### Required Secrets - ANDROID_SERIAL: Device identifier - PERF_API_KEY: Dashboard API key ``` 2. **Runbook** + ```markdown ## Troubleshooting CI Failures - + ### Device Offline 1. SSH to runner machine 2. Run `adb devices` 3. Restart ADB: `adb kill-server && adb start-server` - + ### Build Failure 1. Check NDK version 2. Clear cache @@ -805,4 +820,4 @@ def detect_regression(baseline_data, current_data, threshold=0.05): - [API Reference](api-reference.md) - Programming interface - [Troubleshooting](troubleshooting.md) - Common CI/CD issues -- [Architecture](architecture.md) - System design \ No newline at end of file +- [Architecture](architecture.md) - System design diff --git a/docs/configuration.md b/docs/configuration.md index 27a360e..0dff030 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,6 +31,7 @@ project: ``` **Example:** + ```yaml project: name: "mobile-benchmark" @@ -51,7 +52,7 @@ build: build_type: string # CMAKE_BUILD_TYPE (default: "Release") build_dir: path # Build directory (optional) clean_build: boolean # Clean before build (default: false) - + toolchain: android_ndk: path # Android NDK path (Android only) abi: string # Target ABI (default: "arm64-v8a") @@ -59,7 +60,7 @@ build: cmake: path # CMake executable (default: "cmake") ninja: path # Ninja executable (default: "ninja") compiler: string # Compiler choice (optional) - + options: # CMake options ENABLE_INTEL_CPU: ON|OFF ENABLE_INTEL_GPU: ON|OFF @@ -77,6 +78,7 @@ build: **Examples:** Android build: + ```yaml build: enabled: true @@ -93,6 +95,7 @@ build: ``` Linux ARM build: + ```yaml build: enabled: true @@ -107,6 +110,7 @@ build: ``` Using prebuilt: + ```yaml build: enabled: false @@ -129,6 +133,7 @@ package: ``` **Example:** + ```yaml package: include_symbols: false @@ -149,18 +154,18 @@ Defines target device(s) for deployment. ```yaml device: kind: android|linux_ssh|ios # Device type (required) - + # Android-specific serials: [string] # Device serials from 'adb devices' use_root: boolean # Use root access (default: false) - + # Linux SSH-specific host: string # Hostname or IP port: integer # SSH port (default: 22) user: string # SSH username password: string # SSH password (not recommended) key_path: path # SSH private key path - + # Common options push_dir: path # Remote directory (required) env_vars: {string: string} # Environment variables @@ -171,6 +176,7 @@ device: **Examples:** Android devices: + ```yaml device: kind: "android" @@ -182,6 +188,7 @@ device: ``` Linux SSH device: + ```yaml device: kind: "linux_ssh" @@ -210,6 +217,7 @@ models: ``` **Example:** + ```yaml models: - name: "resnet50" @@ -222,7 +230,7 @@ models: tags: dataset: "imagenet" accuracy_top1: 76.1 - + - name: "yolo_v5" path: "models/yolov5s.xml" precision: "INT8" @@ -243,7 +251,7 @@ run: warmup_runs: integer # Warmup iterations (default: 0) cooldown_sec: integer # Cooldown between runs (default: 0) timeout_sec: integer # Timeout per run (optional) - + matrix: # Parameter combinations to test niter: [integer] # Iterations per run api: [sync|async] # Inference API @@ -253,19 +261,20 @@ run: threads: [integer] # Number of threads infer_precision: [string] # Inference precision hint batch: [integer] # Batch size (optional) - + advanced: # Advanced options pin_threads: boolean # Pin threads to cores numa_node: integer # NUMA node affinity enable_profiling: boolean # Enable performance profiling cache_dir: path # Model cache directory - + custom_args: [string] # Additional benchmark_app arguments ``` **Examples:** Simple matrix: + ```yaml run: repeats: 3 @@ -276,6 +285,7 @@ run: ``` Complex matrix: + ```yaml run: repeats: 5 @@ -306,26 +316,26 @@ Configures output generation. ```yaml report: enabled: boolean # Enable reporting (default: true) - + sinks: # Output destinations - type: json|csv|sqlite|html # Sink type path: path # Output file path options: {} # Type-specific options - + aggregation: # Statistical aggregation metrics: [string] # Metrics to compute percentiles: [float] # Percentiles to calculate - + filters: # Result filtering min_throughput: float # Minimum FPS threshold max_latency: float # Maximum latency threshold - + comparison: # Baseline comparison baseline_path: path # Path to baseline results regression_threshold: float # Regression threshold (%) - + tags: {string: any} # Metadata tags - + artifacts: # Additional artifacts save_logs: boolean # Save raw logs save_stdout: boolean # Save benchmark stdout @@ -335,6 +345,7 @@ report: **Examples:** Multiple output formats: + ```yaml report: sinks: @@ -342,13 +353,13 @@ report: path: "results/output.json" options: indent: 2 - + - type: "csv" path: "results/output.csv" options: sep: "," index: false - + - type: "html" path: "results/report.html" options: @@ -356,6 +367,7 @@ report: ``` With aggregation and filtering: + ```yaml report: aggregation: @@ -414,7 +426,7 @@ models: precision: "FP16" tags: dataset: "imagenet" - + - name: "mobilenet_v2" path: "models/mobilenet_v2_int8.xml" precision: "INT8" @@ -489,4 +501,4 @@ Find example configurations in the `experiments/` directory: - `android_matrix.yaml` - Complex parameter matrix - `linux_ssh.yaml` - Linux device via SSH - `prebuilt.yaml` - Using prebuilt OpenVINO -- `ci_template.yaml` - CI/CD optimized configuration \ No newline at end of file +- `ci_template.yaml` - CI/CD optimized configuration diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..4f4e0a7 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,311 @@ +# Development Guide + +This guide covers the development setup, tools, and practices for OVMobileBench. + +## Development Setup + +### Prerequisites + +- Python 3.11+ +- Git +- Android SDK/NDK (for Android development) +- Docker (optional, for containerized testing) + +### Installation + +1. **Clone the repository** + + ```bash + git clone https://github.com/embedded-dev-research/OVMobileBench + cd OVMobileBench + ``` + +2. **Create virtual environment** + + ```bash + python -m venv .venv + source .venv/bin/activate # On Windows: .venv\Scripts\activate + ``` + +3. **Install dependencies** + + ```bash + pip install -r requirements.txt + pip install -e . # Install in editable mode + ``` + +4. **Install pre-commit hooks** + + ```bash + pre-commit install + ``` + +## Code Quality Tools + +### Pre-commit Hooks + +We use pre-commit to ensure code quality. It runs automatically on `git commit`. + +**Included checks:** + +- **pyupgrade** - Modernizes Python syntax to 3.11+ +- **black** - Code formatting (100 char line length) +- **ruff** - Fast Python linter (replaces flake8, isort, and more) +- **mypy** - Static type checking +- **isort** - Import sorting +- **yamllint** - YAML file linting +- **markdownlint** - Markdown file linting +- **codespell** - Spell checking +- **commitizen** - Commit message validation + +### Running Checks Manually + +```bash +# Run all checks +pre-commit run --all-files + +# Run specific check +pre-commit run black --all-files +pre-commit run ruff --all-files + +# Auto-fix issues +pre-commit run --all-files --hook-stage manual + +# Run without pre-commit +black ovmobilebench tests +ruff check ovmobilebench tests +mypy ovmobilebench --ignore-missing-imports +pytest tests/ +``` + +### Bypassing Checks (Emergency Only) + +```bash +git commit --no-verify -m "Emergency fix" +``` + +## Testing + +### Running Tests + +```bash +# Run all tests +pytest tests/ + +# Run with coverage +pytest tests/ --cov=ovmobilebench --cov-report=html + +# Run specific test file +pytest tests/test_config.py + +# Run specific test +pytest tests/test_config.py::TestConfigLoader::test_load_experiment + +# Run with verbose output +pytest tests/ -v + +# Run in parallel +pytest tests/ -n auto +``` + +### Test Structure + +``` +tests/ +├── android/ +│ └── installer/ # Android installer tests +├── test_*.py # Unit tests for each module +├── conftest.py # Shared fixtures +└── data/ # Test data files +``` + +### Writing Tests + +1. Use pytest fixtures for reusable test data +2. Mock external dependencies (ADB, SSH, file system) +3. Aim for >70% test coverage +4. Test both success and failure cases + +Example test: + +```python +import pytest +from unittest.mock import Mock, patch + +def test_feature(): + # Arrange + mock_device = Mock() + mock_device.shell.return_value = "output" + + # Act + result = process_device(mock_device) + + # Assert + assert result == "expected" + mock_device.shell.assert_called_once_with("command") +``` + +## CI/CD Pipeline + +### GitHub Actions Workflow + +The CI pipeline runs on every push and pull request: + +1. **Pre-commit Stage** (Ubuntu only) + - Runs all linting and formatting checks + - Fastest stage, fails early on style issues + +2. **Test Stage** (Ubuntu, macOS, Windows) + - Runs unit tests on all platforms + - Generates coverage reports + +3. **Build Stage** (All platforms) + - Verifies package building + - Tests installation process + +4. **Validation Stage** (All platforms) + - Validates configuration schemas + - Tests CLI commands + +5. **Device Tests** (When available) + - Integration tests with real/emulated devices + - Android emulator tests + +### Local CI Testing + +Test CI locally using act: + +```bash +# Install act +brew install act # macOS +# or download from https://github.com/nektos/act + +# Run CI locally +act push +``` + +## Code Style Guidelines + +### Python + +- Use type hints for all function parameters and returns +- Write docstrings for all public functions/classes +- Follow PEP 8 with exceptions defined in `.ruff.toml` +- Maximum line length: 100 characters +- Use f-strings for string formatting +- Prefer pathlib over os.path + +### Documentation + +- Use Markdown for all documentation +- Include code examples in docstrings +- Keep README.md updated with user-facing changes +- Update ARCHITECTURE.md for design changes + +### Commit Messages + +Follow [Conventional Commits](https://www.conventionalcommits.org/): + +``` +(): + + + +