diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml new file mode 100644 index 0000000..53d811c --- /dev/null +++ b/.github/workflows/security-scan.yml @@ -0,0 +1,112 @@ +name: Security Scan + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + # Run security scan weekly + - cron: '0 2 * * 1' + +jobs: + basic-checks: + runs-on: ubuntu-latest + name: Basic Checks (dev.sh) + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + # Install shellcheck + sudo apt-get update + sudo apt-get install -y shellcheck + + # Install yamllint + pip install yamllint + + - name: Run all checks + run: ./dev.sh check-all + + advanced-security: + runs-on: ubuntu-latest + name: Advanced Security Scans + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run TruffleHog OSS + uses: trufflesecurity/trufflehog@main + with: + path: ./ + base: main + head: HEAD + extra_args: --debug --only-verified + + - name: Find and scan Dockerfiles + run: | + # Find all Dockerfiles and run hadolint on each + dockerfiles=$(find . -name "Dockerfile*" -type f | grep -v node_modules | grep -v .git) + if [ -n "$dockerfiles" ]; then + echo "Found Dockerfiles:" + echo "$dockerfiles" + # Run hadolint on all found Dockerfiles + docker run --rm -i hadolint/hadolint:latest-debian hadolint --format sarif - < <(cat $dockerfiles) > hadolint-results.sarif || true + else + echo "No Dockerfiles found" + echo '{"version": "2.1.0", "runs": []}' > hadolint-results.sarif + fi + + - name: Upload Hadolint results + uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + sarif_file: hadolint-results.sarif + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: python + queries: security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:python" + + security-summary: + runs-on: ubuntu-latest + needs: [basic-checks, advanced-security] + if: always() + name: Security Summary + + steps: + - name: Security Scan Summary + run: | + echo "## Security Scan Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Check job results + basic_result="${{ needs.basic-checks.result }}" + advanced_result="${{ needs.advanced-security.result }}" + + echo "| Security Check | Status |" >> $GITHUB_STEP_SUMMARY + echo "|----------------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Basic Checks (dev.sh) | $basic_result |" >> $GITHUB_STEP_SUMMARY + echo "| Advanced Security | $advanced_result |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Overall status + if [[ "$basic_result $advanced_result" == *"failure"* ]]; then + echo "🔴 **Security issues detected!** Please review the scan results." >> $GITHUB_STEP_SUMMARY + else + echo "🟢 **All security scans passed successfully.**" >> $GITHUB_STEP_SUMMARY + fi \ No newline at end of file diff --git a/.github/workflows/validate-examples.yml b/.github/workflows/validate-examples.yml new file mode 100644 index 0000000..d328fb6 --- /dev/null +++ b/.github/workflows/validate-examples.yml @@ -0,0 +1,40 @@ +name: Validate Examples + +on: + push: + branches: [main, develop] + pull_request: + branches: [main, develop] + schedule: + # Run weekly to catch dependency issues + - cron: '0 0 * * 0' + +jobs: + check-all: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + # Install shellcheck + sudo apt-get update + sudo apt-get install -y shellcheck + + # Install yamllint + pip install yamllint + + - name: Run all checks + run: ./dev.sh check-all + + - name: Create summary + if: always() + run: | + echo "## Validation Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "✅ Ran comprehensive checks including:" >> $GITHUB_STEP_SUMMARY + echo "- Structure validation for all examples" >> $GITHUB_STEP_SUMMARY + echo "- Docker Compose syntax validation" >> $GITHUB_STEP_SUMMARY + echo "- Security scanning" >> $GITHUB_STEP_SUMMARY + echo "- Shell script linting" >> $GITHUB_STEP_SUMMARY + echo "- YAML linting" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.yamlfmt b/.yamlfmt new file mode 100644 index 0000000..e7cbe68 --- /dev/null +++ b/.yamlfmt @@ -0,0 +1,8 @@ +formatter: + type: basic + retain_line_breaks: true + trim_trailing_whitespace: true + scan_folded_as_literal: true + include_document_start: false + line_ending: lf + indent: 2 \ No newline at end of file diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..ad3bcba --- /dev/null +++ b/.yamllint @@ -0,0 +1,29 @@ +--- +# yamllint configuration for dstack-examples + +extends: default + +rules: + # Don't require document start marker (---) + document-start: disable + + # Reasonable line length for docker-compose files + line-length: + max: 160 + level: warning + + # Allow both styles of bracketed lists + brackets: + min-spaces-inside: 0 + max-spaces-inside: 1 + + # Be flexible with truthy values (yes, true, on, etc.) + truthy: + allowed-values: ['true', 'false', 'yes', 'no', 'on', 'off'] + + # Don't be too strict about comments + comments: + min-spaces-from-content: 1 + + # Allow empty values which are common in docker-compose + empty-values: enable diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..7c0d186 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +dstack@phala.network. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..aea4a3b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,176 @@ +# Contributing to dstack Examples + +Thank you for your interest in contributing to dstack Examples! This guide will help you contribute new examples or improve existing ones. + +## Types of Contributions + +We welcome: +- **New Examples** - Demonstrate new dstack features or use cases +- **Example Improvements** - Enhance existing examples with better practices +- **Documentation** - Improve READMEs and inline documentation +- **Bug Fixes** - Fix issues in existing examples + +## Creating a New Example + +### 1. Choose Your Example Type + +Consider what your example will demonstrate: +- **Security & Attestation** - TEE verification, remote attestation patterns +- **Networking & Domains** - Custom domains, port forwarding, gateway patterns +- **Development Tools** - Debugging, deployment utilities +- **Advanced Use Cases** - Complex integrations, blockchain, cryptography + +### 2. Set Up Your Example Directory + +```bash +# Create your example directory +mkdir my-example-name +cd my-example-name + +# Create required files +touch README.md +touch docker-compose.yaml +touch .env.example # If environment variables are needed +``` + +### 3. Create docker-compose.yaml + +Your `docker-compose.yaml` should follow these patterns: + +```yaml +services: + your-service: + image: your-image@sha256:1234abcd # Use specific image digest to pin the image (not tags) + restart: unless-stopped + environment: + - SOME_API_KEY=${SOME_API_KEY} # Use encrypted secrets when passing sensitive info + ports: + - "8080:80" # Only expose necessary ports + volumes: + - data:/app/data: # Use persistent storage + # Add health checks for production readiness + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost/health"] + interval: 30s + timeout: 10s + retries: 3 +volumes: + data: +``` + +### 4. Write a Comprehensive README.md + +Every example MUST include a README.md. Suggested format: + +```markdown +# Example Name + +## Description +Brief description of what this example demonstrates and why it's useful. + +## Prerequisites +- List any required knowledge +- Required dstack features +- External services needed + +## Quick Start +Step-by-step deployment instructions: +1. Copy docker-compose.yaml to your dstack deployment +2. Configure environment variables +3. Deploy through dstack interface + +## Configuration +### Environment Variables +| Variable | Description | Required | Default | +|----------|-------------|----------|---------| +| API_KEY | Your api key | Yes | - | + +## How It Works +Explain the technical implementation and what makes this example special. + +## Security Considerations +- TEE-specific security features used +- Any security best practices demonstrated +- Important security notes + +## Troubleshooting +Common issues and solutions. +``` + +## Improving Existing Examples + +When improving existing examples: + +1. **Maintain Compatibility** - Don't break existing deployments +2. **Update Documentation** - Reflect all changes in README +3. **Test Thoroughly** - Ensure improvements work correctly + +## Development Workflow + +### 1. Fork and Clone + +```bash +git clone https://github.com/YOUR-USERNAME/dstack-examples.git +cd dstack-examples +``` + +### 2. Create Feature Branch + +```bash +git checkout -b feat/add-awesome-example +``` + +### 3. Develop Your Example + +Follow the patterns above and use an existing example as a reference. + +### 4. Commit Your Changes + +Use conventional commits: + +```bash +# For new examples +git commit -m "feat: add blockchain verification example" + +# For improvements +git commit -m "fix: resolve port conflict in webshell example" +git commit -m "docs: improve attestation example README" +git commit -m "refactor: simplify launcher deployment" +``` + +### 5. Push and Create PR + +```bash +git push origin feat/add-awesome-example +``` + +Then create a Pull Request with: +- Clear description of what the example demonstrates +- Any special considerations for reviewers +- Testing steps followed + +## Commit Convention + +- `feat`: New example or major feature addition to existing example +- `fix`: Bug fixes in examples +- `docs`: Documentation improvements +- `refactor`: Code improvements without changing functionality +- `test`: Adding tests or test scripts +- `ci`: CI/CD related changes +- `chore`: Maintenance tasks +``` + +## Getting Help + +- **Technical Questions**: [GitHub Discussions](https://github.com/Dstack-TEE/dstack-examples/discussions) +- **Bug Reports**: [GitHub Issues](https://github.com/Dstack-TEE/dstack-examples/issues) +- **Real-time Chat**: [Telegram Community](https://t.me/+UO4bS4jflr45YmUx) + +## Recognition + +Contributors are recognized in: +- Repository contributors list +- Release notes +- Special mentions for significant contributions + +Thank you for contributing to dstack Examples! 🎉 diff --git a/LICENSE b/LICENSE index c8e6532..261eeb9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,201 @@ -MIT License - -Copyright (c) 2024 Dstack TEE - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index d31d00b..a869c1b 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,136 @@ -# Dstack Examples -This repository contains examples of Dstack applications. - -*Note on single-file example style:* Sometimes we use a style of packing the entire application into a single docker-compose.yml file. -But more commonly a dstack example would have Dockerfile and some other code. - -## Useful Utilities -These show useful patterns you may want to copy: -- [./lightclient](./lightclient) use a light client so that the dstack app can follow a blockchain -- [./custom-domain](./custom-domain) shows how to serve a secure website from a custom domain, by requesting a letsencrypt certificate from within the app -- [./ssh-over-tproxy](./ssh-over-tproxy) shows how to tunnel arbitrary sockets over https so it can work with tproxy -- [./webshell](./webshell) This is an alternative way to allow logging into a Dstack container (for debug only!) -## Showcases of porting existing tools -- [./tor-hidden-service](./tor-hidden-service) connects to the tor network and serves a website as a hidden service -## Illustrating Dstack Features -- [./prelaunch-script](./prelaunch-script) -- [./private-docker-image-deployment](./private-docker-image-deployment) -- [./attestation](./attestation) shows how to verify attestation for Dstack apps -## App examples -- [./timelock-nts](./timelock-nts) a timelock decryption example using secure NTP (NTS) from Cloudflare as a time oracle -## Tutorial (Coming soon) +# dstack Examples + +
+ +[![GitHub Stars](https://img.shields.io/github/stars/Dstack-TEE/dstack?style=flat-square)](https://github.com/Dstack-TEE/dstack-examples/stargazers) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg?style=flat-square)](LICENSE) +[![Telegram](https://img.shields.io/badge/Telegram-Community-blue?style=flat-square&logo=telegram)](https://t.me/+UO4bS4jflr45YmUx) +[![Documentation](https://img.shields.io/badge/Documentation-Phala%20Network-green?style=flat-square)](https://docs.phala.network/dstack) + +**Example applications for [dstack](https://github.com/Dstack-TEE/dstack) - Deploy containerized apps to TEEs with end-to-end security in minutes** + +[Getting Started](#getting-started) • [Examples](#examples) • [Contributing](CONTRIBUTING.md) • [Documentation](#documentation) • [Community](#community) + +
+ +--- + +## Overview + +This repository contains ready-to-deploy examples demonstrating how to build and run applications on [dstack](https://github.com/Dstack-TEE/dstack), the developer-friendly SDK for deploying containerized apps in Trusted Execution Environments (TEE). + +### What You'll Find Here + +- **Security Features** - Remote attestation, verification, and privacy-preserving apps +- **Secret Management** - Secure handling of credentials and sensitive data in TEE environments +- **Networking Patterns** - HTTPS termination, custom domains, port forwarding in the cloud +- **Best Practices** - Production-ready implementations following TEE security principles + +## Prerequisites + +Before you begin, ensure you have: + +- Access to a dstack environment +- Basic understanding of [TEE concepts](https://docs.phala.network/dstack) +- Basic familiarity with Docker Compose configuration files +- Git for cloning the repository + +You can deploy dstack on your own server, or use [Phala Cloud](https://cloud.phala.network). + +## Getting Started + +### Quick Start + +```bash +# Clone the repository +git clone https://github.com/Dstack-TEE/dstack-examples.git +cd dstack-examples + +# Choose an example +cd attestation/configid-based + +# Copy the docker-compose.yaml content to your dstack deployment +# Follow the example-specific README for deployment instructions +``` + +## Examples + +### Security & Attestation +| Example | Description | +|---------|-------------| +| [attestation/configid-based](./attestation/configid-based) | ConfigID-based remote attestation verification | +| [attestation/rtmr3-based](./attestation/rtmr3-based) | RTMR3-based attestation (legacy) | + +### Networking & Domains +| Example | Description | +|---------|-------------| +| [custom-domain](./custom-domain) | Set up custom domain with automatic TLS certificate management via zt-https | +| [ssh-over-gateway](./ssh-over-gateway) | SSH tunneling through dstack gateway | +| [tcp-port-forwarding](./tcp-port-forwarding) | Arbitrary TCP port forwarding | +| [tor-hidden-service](./tor-hidden-service) | Run Tor hidden services in TEEs | + +### Development Tools +| Example | Description | +|---------|-------------| +| [launcher](./launcher) | Generic launcher pattern for Docker Compose apps | +| [webshell](./webshell) | Web-based shell access for debugging | +| [prelaunch-script](./prelaunch-script) | Pre-launch script patterns used by Phala Cloud | + +### Advanced Use Cases +| Example | Description | +|---------|-------------| +| [lightclient](./lightclient) | Blockchain light client integration | +| [timelock-nts](./timelock-nts) | Timelock decryption with NTS | +| [private-docker-image-deployment](./private-docker-image-deployment) | Using private Docker registries | + +## Documentation + +- **[dstack Documentation](https://docs.phala.network/dstack)** - Official platform documentation +- **[Main Repository](https://github.com/Dstack-TEE/dstack)** - Core dstack framework +- **[Security Guide](SECURITY.md)** - Security best practices +- **[Contributing Guide](CONTRIBUTING.md)** - How to contribute + +## Development + +Use the `dev.sh` script for validation and development tasks: + +```bash +./dev.sh help # Show available commands +./dev.sh validate # Validate a specific example +./dev.sh validate-all # Validate all examples +./dev.sh security # Run security checks +./dev.sh lint # Run linting checks +./dev.sh check-all # Run all checks +``` ## Contributing -Pull requests are welcomed, curation plan to come soon + +We welcome contributions! Please see our [Contributing Guidelines](CONTRIBUTING.md) for details. + +## Community + +### Getting Help + +- **Telegram**: [Join our community](https://t.me/+UO4bS4jflr45YmUx) +- **Issues**: [GitHub Issues](https://github.com/Dstack-TEE/dstack-examples/issues) + +### Reporting Issues + +When reporting issues, please include: + +1. Example name and version +2. Steps to reproduce +3. Expected vs actual behavior +4. Relevant logs and error messages + +## License + +This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENSE) file for details. + +--- + +
+ +**[⬆ Back to top](#dstack-examples)** + +
diff --git a/attestation/configid-based/attest.sh b/attestation/configid-based/attest.sh index a1b4b85..b087b2c 100755 --- a/attestation/configid-based/attest.sh +++ b/attestation/configid-based/attest.sh @@ -91,7 +91,8 @@ download_command() { # Download and extract image echo "Downloading Dstack image..." local image_url="${IMAGE_DL_URL}" - local image_filename="$(basename "${image_url}")" + local image_filename + image_filename="$(basename "${image_url}")" local image_path="${IMAGES_DIR}/${image_filename}" echo "Downloading image from ${image_url}" @@ -186,7 +187,7 @@ calc_mrs_command() { # Run dstack-mr with metadata.json from the image echo "Running dstack-mr to calculate MRs..." - ${TOOLS_DIR}/dstack-mr -metadata "${IMAGES_DIR}/dstack-${IMAGE_VERSION}/metadata.json" \ + "${TOOLS_DIR}/dstack-mr" -metadata "${IMAGES_DIR}/dstack-${IMAGE_VERSION}/metadata.json" \ -cpu ${VCPU_COUNT} \ -memory "${MEM_SIZE}M" \ -json > "known_good_mrs.json.tmp" @@ -245,7 +246,7 @@ EOF if [ -n "$instance_id" ]; then echo "✅ App started successfully!" echo "Instance ID: $instance_id" - echo $instance_id > instance_id.txt + echo "$instance_id" > instance_id.txt break fi # Check for boot errors diff --git a/custom-domain/dstack-ingress/build-image.sh b/custom-domain/dstack-ingress/build-image.sh index 8b4405d..7c11151 100755 --- a/custom-domain/dstack-ingress/build-image.sh +++ b/custom-domain/dstack-ingress/build-image.sh @@ -10,6 +10,6 @@ if ! docker buildx inspect buildkit_20 &>/dev/null; then fi touch pinned-packages.txt git rev-parse HEAD > .GIT_REV -docker buildx build --builder buildkit_20 --no-cache --build-arg SOURCE_DATE_EPOCH="0" --output type=docker,name=$NAME,rewrite-timestamp=true . -docker run --rm --entrypoint bash $NAME -c "dpkg -l | grep '^ii' |awk '{print \$2\"=\"\$3}' | sort" > pinned-packages.txt +docker buildx build --builder buildkit_20 --no-cache --build-arg SOURCE_DATE_EPOCH="0" --output type=docker,name="$NAME",rewrite-timestamp=true . +docker run --rm --entrypoint bash "$NAME" -c "dpkg -l | grep '^ii' |awk '{print \$2\"=\"\$3}' | sort" > pinned-packages.txt rm .GIT_REV diff --git a/custom-domain/dstack-ingress/scripts/entrypoint.sh b/custom-domain/dstack-ingress/scripts/entrypoint.sh index 5a34793..972453b 100644 --- a/custom-domain/dstack-ingress/scripts/entrypoint.sh +++ b/custom-domain/dstack-ingress/scripts/entrypoint.sh @@ -80,9 +80,9 @@ obtain_certificate() { certbot certonly --dns-cloudflare \ --dns-cloudflare-credentials ~/.cloudflare/cloudflare.ini \ --dns-cloudflare-propagation-seconds 120 \ - --email $CERTBOT_EMAIL \ + --email "$CERTBOT_EMAIL" \ --agree-tos --no-eff-email --non-interactive \ - -d $DOMAIN + -d "$DOMAIN" } set_cname_record() { diff --git a/custom-domain/dstack-ingress/scripts/generate-evidences.sh b/custom-domain/dstack-ingress/scripts/generate-evidences.sh index 7b8d1fa..6db82ea 100644 --- a/custom-domain/dstack-ingress/scripts/generate-evidences.sh +++ b/custom-domain/dstack-ingress/scripts/generate-evidences.sh @@ -6,8 +6,8 @@ CERT_FILE=/etc/letsencrypt/live/${DOMAIN}/fullchain.pem mkdir -p /evidences cd /evidences -cp ${ACME_ACCOUNT_FILE} acme-account.json -cp ${CERT_FILE} cert.pem +cp "${ACME_ACCOUNT_FILE}" acme-account.json +cp "${CERT_FILE}" cert.pem sha256sum acme-account.json cert.pem > sha256sum.txt @@ -20,5 +20,5 @@ while [ ${#PADDED_HASH} -lt 128 ]; do done QUOTED_HASH="${PADDED_HASH}" -curl -s --unix-socket /var/run/tappd.sock http://localhost/prpc/Tappd.RawQuote?report_data=${QUOTED_HASH} > quote.json +curl -s --unix-socket /var/run/tappd.sock "http://localhost/prpc/Tappd.RawQuote?report_data=${QUOTED_HASH}" > quote.json echo "Generated evidences successfully" diff --git a/dev.sh b/dev.sh new file mode 100755 index 0000000..04bc081 --- /dev/null +++ b/dev.sh @@ -0,0 +1,268 @@ +#!/bin/bash + +# Dstack Examples - Development Helper Script + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXAMPLES_DIR="${SCRIPT_DIR}" + +# Helper functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Find all docker-compose files +find_compose_files() { + find "${EXAMPLES_DIR}" -name "docker-compose.y*ml" -not -path "./.github/*" -not -path "./.git/*" +} + +# Show help +show_help() { + cat << EOF +Dstack Examples Development Helper + +Usage: $0 [options] + +Commands: + list List all examples + validate Validate a specific example + validate-all Validate all examples + security Run security checks + lint Run linting (shellcheck, yamllint) + check-all Run all checks (validate, security, lint) + docs Show documentation status + help Show this help message + +Examples: + $0 validate attestation/configid-based + $0 check-all +EOF +} + +# List examples +list_examples() { + log_info "Available examples:" + find_compose_files | xargs dirname | sort | uniq | sed 's/^/ - /' +} + +# Validate structure +validate_structure() { + local example=$1 + + if [ ! -d "${example}" ]; then + log_error "Example directory not found: ${example}" + return 1 + fi + + if [ ! -f "${example}/README.md" ]; then + log_error "Missing README.md in: ${example}" + return 1 + fi + + if [ ! -f "${example}/docker-compose.yml" ] && [ ! -f "${example}/docker-compose.yaml" ]; then + log_error "Missing docker-compose file in: ${example}" + return 1 + fi + + log_success "Structure validation passed" + return 0 +} + +# Validate compose syntax +validate_compose() { + local example=$1 + + log_info "Validating Docker Compose syntax..." + if cd "${example}" && docker compose config --quiet > /dev/null 2>&1; then + log_success "Docker Compose syntax is valid" + return 0 + else + log_error "Docker Compose syntax is invalid" + return 1 + fi +} + +# Validate single example +validate_example() { + local example=$1 + + if [ -z "${example}" ]; then + log_error "Please specify an example: $0 validate " + exit 1 + fi + + log_info "Validating example: ${example}" + + local failed=0 + validate_structure "${example}" || failed=$((failed + 1)) + validate_compose "${example}" || failed=$((failed + 1)) + + if [ ${failed} -eq 0 ]; then + log_success "All validations passed for: ${example}" + return 0 + else + log_error "${failed} validation(s) failed for: ${example}" + return 1 + fi +} + +# Validate all examples +validate_all() { + log_info "Validating all examples..." + local failed=0 + + find_compose_files | while read -r compose_file; do + example_dir=$(dirname "${compose_file}") + echo "" + validate_example "${example_dir}" || failed=$((failed + 1)) + done + + if [ ${failed} -eq 0 ]; then + log_success "All examples validated successfully!" + else + log_error "${failed} example(s) failed validation" + fi +} + +# Security checks +security_checks() { + log_info "Running security checks..." + + # Check for hardcoded secrets + log_info "Scanning for potential secrets..." + if grep -r -i -E "(password|secret|key|token|api_key).*=.*['\"][^'\"]{8,}['\"]" "${EXAMPLES_DIR}" \ + --exclude-dir=.git --exclude-dir=.github --exclude="*.md" --exclude="Makefile" --exclude="dev.sh" 2>/dev/null; then + log_warning "Potential hardcoded secrets found!" + else + log_success "No obvious hardcoded secrets detected" + fi + + # Check Docker Compose security + log_info "Checking Docker Compose security..." + find_compose_files | while read -r compose_file; do + if grep -q "privileged.*true" "${compose_file}"; then + log_error "Privileged container found in: ${compose_file}" + fi + if grep -q "network_mode.*host" "${compose_file}"; then + log_warning "Host network mode found in: ${compose_file}" + fi + if grep -q "/var/run/docker.sock" "${compose_file}"; then + log_error "Docker socket mount found in: ${compose_file}" + fi + done + + log_success "Security checks completed" +} + +# Lint checks +lint_checks() { + log_info "Running lint checks..." + + # Shellcheck + if command -v shellcheck >/dev/null 2>&1; then + log_info "Linting shell scripts..." + find "${EXAMPLES_DIR}" -name "*.sh" -not -path "./.git/*" -exec shellcheck {} \; || log_warning "ShellCheck issues found" + else + log_warning "shellcheck not installed, skipping shell script linting" + fi + + # Yamllint + if command -v yamllint >/dev/null 2>&1; then + log_info "Linting YAML files..." + find "${EXAMPLES_DIR}" -name "*.yml" -o -name "*.yaml" | grep -v ".git" | xargs yamllint || log_warning "YAML linting issues found" + else + log_warning "yamllint not installed, skipping YAML linting" + fi + + log_success "Lint checks completed" +} + +# Documentation status +docs_status() { + log_info "Documentation status:" + echo "" + find_compose_files | while read -r compose_file; do + example_dir=$(dirname "${compose_file}") + if [ -f "${example_dir}/README.md" ]; then + echo -e " ${GREEN}✓${NC} ${example_dir}" + else + echo -e " ${RED}✗${NC} ${example_dir} (missing README.md)" + fi + done +} + +# Run all checks +check_all() { + log_info "Running comprehensive checks..." + echo "" + + validate_all + echo "" + + security_checks + echo "" + + lint_checks + echo "" + + log_success "All checks completed!" +} + +# Main script logic +main() { + case "${1:-help}" in + list) + list_examples + ;; + validate) + validate_example "${2:-}" + ;; + validate-all) + validate_all + ;; + security) + security_checks + ;; + lint) + lint_checks + ;; + check-all) + check_all + ;; + docs) + docs_status + ;; + help|--help|-h) + show_help + ;; + *) + log_error "Unknown command: $1" + show_help + exit 1 + ;; + esac +} + +# Run main function +main "$@" diff --git a/launcher/build-image.sh b/launcher/build-image.sh index c742a87..16e1d2f 100755 --- a/launcher/build-image.sh +++ b/launcher/build-image.sh @@ -10,6 +10,6 @@ if ! docker buildx inspect buildkit_20 &>/dev/null; then fi touch pinned-packages.txt git rev-parse HEAD > .GIT_REV -docker buildx build --platform linux/amd64 --builder buildkit_20 --no-cache --build-arg SOURCE_DATE_EPOCH="0" --output type=docker,name=$NAME,rewrite-timestamp=true . -docker run --rm --entrypoint bash $NAME -c "dpkg -l | grep '^ii' |awk '{print \$2\"=\"\$3}' | sort" > pinned-packages.txt -rm .GIT_REV \ No newline at end of file +docker buildx build --platform linux/amd64 --builder buildkit_20 --no-cache --build-arg SOURCE_DATE_EPOCH="0" --output "type=docker,name=$NAME,rewrite-timestamp=true" . +docker run --rm --entrypoint bash "$NAME" -c "dpkg -l | grep '^ii' |awk '{print \$2\"=\"\$3}' | sort" > pinned-packages.txt +rm .GIT_REV diff --git a/launcher/entrypoint.sh b/launcher/entrypoint.sh index daefad6..a079860 100644 --- a/launcher/entrypoint.sh +++ b/launcher/entrypoint.sh @@ -6,7 +6,7 @@ EXTERNAL_PORT=10080 SERVICE_NAME=server -cd $WORKDIR +cd $WORKDIR || exit check-update() { echo "Checking for updates..." diff --git a/launcher/get-latest.sh b/launcher/get-latest.sh index 6cd936c..bd1443f 100644 --- a/launcher/get-latest.sh +++ b/launcher/get-latest.sh @@ -29,8 +29,8 @@ MINUTE=$(date +%-M) # Use time-based selection instead of random to create more predictable upgrade patterns # This will switch images roughly every minute if [ $((MINUTE % 2)) -eq 0 ]; then - echo "nginx@sha256:d67fed8b03f1ed3d2a5e3cbc5ca268ad7a7528adfdd1220c420c8cf4e3802d9c" > $OUTPUT + echo "nginx@sha256:d67fed8b03f1ed3d2a5e3cbc5ca268ad7a7528adfdd1220c420c8cf4e3802d9c" > "$OUTPUT" else - echo "nginx@sha256:81aa342ba08035632898b78d46d0e11d79abeee63b3a6994a44ac34e102ef888" > $OUTPUT + echo "nginx@sha256:81aa342ba08035632898b78d46d0e11d79abeee63b3a6994a44ac34e102ef888" > "$OUTPUT" fi diff --git a/lightclient/docker-compose.yml b/lightclient/docker-compose.yml index a5fc219..716edd0 100644 --- a/lightclient/docker-compose.yml +++ b/lightclient/docker-compose.yml @@ -33,7 +33,7 @@ configs: # Let it sync #TODO do this smarter sleep 5 - + # Then run some queries. This would be a good place to run an api server. # Cast <-> Helios <-> Untrusted RPCs cast block --rpc-url=localhost:8545 | tee response.txt @@ -43,6 +43,6 @@ configs: PAYLOAD="{\"report_data\": \"$$(echo -n $$HASH | od -A n -t x1 | tr -d ' \n')\"}" ATTEST=$$(curl -X POST --unix-socket /var/run/tappd.sock -d "$$PAYLOAD" http://localhost/prpc/Tappd.TdxQuote?json) # TODO: Fallback to the dummy remote attestation - + echo ATTEST=$${ATTEST} >> response.txt cat response.txt diff --git a/phala-cloud-prelaunch-script/prelaunch.sh b/phala-cloud-prelaunch-script/prelaunch.sh index 08b95de..47fc75d 100644 --- a/phala-cloud-prelaunch-script/prelaunch.sh +++ b/phala-cloud-prelaunch-script/prelaunch.sh @@ -112,7 +112,7 @@ elif [[ -n "$DSTACK_AWS_ACCESS_KEY_ID" && -n "$DSTACK_AWS_SECRET_ACCESS_KEY" && fi echo "Logging in to AWS ECR..." - aws ecr get-login-password --region $DSTACK_AWS_REGION | docker login --username AWS --password-stdin "$DSTACK_AWS_ECR_REGISTRY" + aws ecr get-login-password --region "$DSTACK_AWS_REGION" | docker login --username AWS --password-stdin "$DSTACK_AWS_ECR_REGISTRY" if [ $? -eq 0 ]; then echo "AWS ECR login successful" notify_host_hoot_info "AWS ECR login successful" @@ -142,17 +142,21 @@ fi if [[ -e /var/run/dstack.sock ]]; then - export DSTACK_APP_ID=$(curl -s --unix-socket /var/run/dstack.sock http://dstack/Info | jq -j .app_id) + DSTACK_APP_ID=$(curl -s --unix-socket /var/run/dstack.sock http://dstack/Info | jq -j .app_id) + export DSTACK_APP_ID else - export DSTACK_APP_ID=$(curl -s --unix-socket /var/run/tappd.sock http://dstack/prpc/Tappd.Info | jq -j .app_id) + DSTACK_APP_ID=$(curl -s --unix-socket /var/run/tappd.sock http://dstack/prpc/Tappd.Info | jq -j .app_id) + export DSTACK_APP_ID fi # Check if app-compose.json has default_gateway_domain field and DSTACK_GATEWAY_DOMAIN is not set # If true, set DSTACK_GATEWAY_DOMAIN from app-compose.json if [[ $(jq 'has("default_gateway_domain")' app-compose.json) == "true" && -z "$DSTACK_GATEWAY_DOMAIN" ]]; then - export DSTACK_GATEWAY_DOMAIN=$(jq -j '.default_gateway_domain' app-compose.json) + DSTACK_GATEWAY_DOMAIN=$(jq -j '.default_gateway_domain' app-compose.json) + export DSTACK_GATEWAY_DOMAIN fi if [[ -n "$DSTACK_GATEWAY_DOMAIN" ]]; then - export DSTACK_APP_DOMAIN=$DSTACK_APP_ID"."$DSTACK_GATEWAY_DOMAIN + DSTACK_APP_DOMAIN=$DSTACK_APP_ID"."$DSTACK_GATEWAY_DOMAIN + export DSTACK_APP_DOMAIN fi echo "----------------------------------------------" diff --git a/timelock-nts/docker-compose.yml b/timelock-nts/docker-compose.yml index 62821ac..2da1997 100644 --- a/timelock-nts/docker-compose.yml +++ b/timelock-nts/docker-compose.yml @@ -16,11 +16,11 @@ services: configs: run.sh: - content: | + content: |- #!/bin/bash key=$$(openssl genpkey -algorithm Ed25519) echo "Public Key:"; echo "$$key" | openssl pkey -pubout - + # Get timestamp from cloudflare and add 5 minutes get_time() { ntpdate -4q time.cloudflare.com 2>/dev/null | head -1 | cut -d' ' -f1,2 | date +%s -f -; } deadline=$$(($$(get_time) + 300)) @@ -28,7 +28,7 @@ configs: echo "Release: $$deadline_str" # Fetch the quote - get_quote() { + get_quote() { PAYLOAD="{\"report_data\": \"$$(echo -n $$1 | od -A n -t x1 | tr -d ' \n')\"}" curl -X POST --unix-socket /var/run/tappd.sock -d "$$PAYLOAD" http://localhost/prpc/Tappd.TdxQuote?json }