Merge branch 'main' into RAG-deployment

raphaeltm · raphaeltm · commit fb0f82d68207 · 2024-08-28T15:41:23.000-07:00
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -59,18 +59,8 @@ jobs:
         with:
           go-version: "1.21"
 
-      - name: Generate CLI docs
-        run: go run main.go ./tmp
-        working-directory: defang/src/cmd/gendocs
-
-      - name: Copy CLI docs to current repository
-        run: cp -r ./defang/src/cmd/gendocs/tmp/* ./docs/cli
-
-      - name: Prep CLI docs
-        run: npm run prep-cli-docs
-
-      - name: Prep Samples
-        run: npm run prep-samples
+      - name: Pre-build
+        run: npm run prebuild
 
       - name: Build website
         run: npm run build
@@ -93,3 +83,12 @@ jobs:
           cname: docs.defang.io
 
 
+      - name: Notify Slack of Action Failures
+        uses: ravsamhq/notify-slack-action@2.5.0
+        if: ${{ always() && github.ref_name == 'main' }}
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "{workflow} is failing"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFIER_WEBHOOK_URL }}
diff --git a/.github/workflows/test-deploy.yml b/.github/workflows/test-deploy.yml
@@ -40,18 +40,8 @@ jobs:
         with:
           go-version: "1.21"
 
-      - name: Generate CLI docs
-        run: go run main.go ./tmp
-        working-directory: defang/src/cmd/gendocs
-
-      - name: Copy CLI docs to current repository
-        run: cp -r ./defang/src/cmd/gendocs/tmp/* ./docs/cli
-
-      - name: Prep CLI docs
-        run: npm run prep-cli-docs
-
-      - name: Prep Samples
-        run: npm run prep-samples
+      - name: Pre-build
+        run: npm run prebuild
 
       - name: Test build website
         run: npm run build
diff --git a/blog/2024-03-28-slackbot-sample.md b/blog/2024-03-28-slackbot-sample.md
@@ -5,7 +5,7 @@ tags: [Cloud, NoDevOps, BYOC, AWS ECS, defang compose up, Slack API]
 author: Raphaël Titsworth-Morin
 ---
 
-Hey folks! Today, I'm going to share one of our code samples that will show you how to deploy a simple Slack bot. If you're looking to connect a cloud service to Slack to publish status updates, or something else like that, this should help you get started. We'll walk through a step-by-step process of writing a Go program using the [github.com/slack-go/slack](github.com/slack-go/slack) library to interact with the Slack API and easily deploy it using Defang.
+Hey folks! Today, I'm going to share one of our code samples that will show you how to deploy a simple Slack bot. If you're looking to connect a cloud service to Slack to publish status updates, or something else like that, this should help you get started. We'll walk through a step-by-step process of writing a Go program using the [github.com/slack-go/slack](https://github.com/slack-go/slack) library to interact with the Slack API and easily deploy it using Defang.
 
 ## Prerequisites
 
@@ -57,4 +57,3 @@ curl 'https://raphaeltm-bot--8080.prod1.defang.dev/' \
 
 ## Takeaways
 Congratulations! You've successfully deployed a Slackbot using Defang. If you deployed this as an internal service, you could use it to send status updates, alerts, or other important messages to your team. The possibilities are endless!
-
diff --git a/blog/2024-05-01-may-product-updates.md b/blog/2024-05-01-may-product-updates.md
@@ -1,6 +1,6 @@
 ---
 slug: may-2024-product-update
-title: May 2024 Product Update
+title: April 2024 Product Update
 tags: [Cloud, NoDevOps, BYOC, AWS ECS, defang compose up]
 author: Defang Team
 ---
diff --git a/blog/2024-06-01-june-product-updates.md b/blog/2024-06-01-june-product-updates.md
@@ -1,6 +1,6 @@
 ---
 slug: june-2024-product-update
-title: June 2024 Product Update
+title: May 2024 Product Update
 tags: [Cloud, NoDevOps, BYOC, Playground, Load Testing, ACME, Managed Redis, Kaniko, Postgres, ECS]
 author: Defang Team
 ---
diff --git a/blog/2024-07-01-july-product-updates.md b/blog/2024-07-01-july-product-updates.md
@@ -1,6 +1,6 @@
 ---
 slug: july-2024-product-update
-title: July 2024 Product Update 
+title: June 2024 Product Update 
 tags: [Cloud, NoDevOps, BYOC, Windows, Managed Redis, CLI]
 author: Defang Team
 ---
diff --git a/blog/2024-07-31-july-product-updates-2.md b/blog/2024-07-31-july-product-updates-2.md
@@ -0,0 +1,31 @@
+---
+slug: 2024-07-31-product-update
+title: July 2024 Product Update 
+tags: [Cloud, NoDevOps, BYOC, CLI, AI, Debugging]
+author: Defang Team
+---
+
+![Defang Compose Update](/img/defang-compose-update.webp)
+
+Hey folks! We can’t believe a month has gone by already, time flies when you’re having fun! We continued to make progress on our vision of providing a radically simpler way to Develop, Deploy, and Debug cloud applications. In July, we spent time on a couple of key areas:
+
+1.  As our user-base grows, we wanted to make sure we’re able to scale our [Playground](https://docs.defang.io/docs/concepts/defang-playground) environment to be able to handle the load. This involved being able to shard the workload across multiple ALBs and being able to dynamically move some workloads  across shards where possible. With these changes, we are now able handle a large number of concurrent users comfortably. The only noticeable change in behavior you would see is that Defang will now ask you to “`compose down`” your previous project before you are able to deploy a new project on Playground.
+
+2.  The major news this month was the introduction of our “`debug`” functionality. The motivation for this feature was that while the Defang experience is amazing when everything goes smoothly, we saw users (including our own interns who are helping write all those [samples](https://docs.defang.io/docs/samples)) struggle when they hit an error. The underlying reason for the error could come from a variety of sources: an error in the developer’s application (including in their Dockerfile or Compose file), an issue in the way Defang is processing the application, or an issue in the underlying cloud platform (currently, AWS). To the developer, it is often not obvious what the issue is or how to fix it. That got us thinking how we could make this debugging experience “radically simpler” and thus the idea for `defang debug` was born.
+    
+    Now (with CLI v0.5.37 if your application encounters an error that leads to a failed deployment, a failed health-check, or a run-time error, Defang will automatically detect the issue. It will then offer to help you debug it by running the `defang debug`  command. If you choose to proceed, Defang will apply an LLM model to try to determine the precise cause of the error, with the context of your application source, logs, error code etc. And it will try to come up with one or more actionable insights on how to fix the error. For an example, see the case below:
+    
+    
+    
+    Behind the scenes, Defang is having a conversation on your behalf with the LLM to narrow down to the cause of the error.  We would love for you to try the `debug` feature and give us your feedback so we can improve it further. One future improvement already on our list is the ability to, with user consent, automatically apply a chosen fix and re-try. We are also looking for way to improve the range of failures we are able to diagnose successfully.
+
+## Townhall
+
+If you're excited about what's coming next and want to hear more about our vision for the future, join us for our Townhall on August 21st. We'll be sharing more about our roadmap and what we're working on next. We'll also be making sure to take time to answer any questions you have, hear your feedback, and learn more about what you want from Defang!
+
+**[Register here](https://lu.ma/rlj13eq5)**
+    
+---
+
+We’re excited to keep improving Defang to make it the easiest way for you to Develop, Deploy, and Debug cloud application. Stay tuned for more updates next month.
+
diff --git a/docs/concepts/configuration.md b/docs/concepts/configuration.md
@@ -14,7 +14,12 @@ You can find a sample of how to set environment variables with Pulumi [here](htt
 
 # Sensitive Config aka Secrets
 
-The Defang CLI allows you to securely store sensitive information such as API keys, passwords, and other credentials.
+The Defang CLI allows you to securely store sensitive information such as API keys, passwords, and other credentials. To do so just run:
+
+```bash
+# Set a configuration value called API_KEY
+defang config set API_KEY
+```
 
 You can use sensitive config by specifying them in the `environment` section of a service in a `compose.yaml` file without any value, or by specifying an environment key with a `null` value in your Pulumi code.
 
diff --git a/docs/concepts/debug.md b/docs/concepts/debug.md
@@ -0,0 +1,13 @@
+---
+title: Debug
+description: Defang uses AI to help you debug your cloud applications.
+sidebar_position: 650
+---
+
+# Debug
+
+Defang includes an AI-driven tool to help you debug your cloud applications. The AI assistant will use your service logs as well as the files in your project to help you identify and resolve issues.
+
+:::warning
+The AI debugging assistant is currently in preview and is currently limited in its capabilities. We plan to expand the capabilities of the AI assistant in the future.
+:::
diff --git a/docs/concepts/generate.md b/docs/concepts/generate.md
@@ -1,16 +1,15 @@
 ---
-title: AI
-description: How Defang uses AI to help you build and deploy your cloud applications.
+title: Generate
+description: Defang uses AI to help you generate a starting point for your cloud applications.
 sidebar_position: 100
 ---
 
-# AI
-
+# Generate
 
 Defang includes an AI-driven assistant that translates natural language prompts to an outline for your project that you can then refine. The AI assistant is available through the [Defang CLI](../getting-started/installing.md).
 
 :::info
-The AI assistant is currently in beta and only supports a limited set of prompts. We plan to expand the capabilities of the AI assistant in the future.
+The AI assistant is currently in preview and only supports a limited set of prompts, varying in complexity. We plan to expand the capabilities of the AI assistant in the future.
 :::
 
 We are working on expanding the range of supported prompts and improving the accuracy of the AI assistant. If you have any feedback or suggestions, please let us know by [opening an issue](https://github.com/DefangLabs/defang/issues/new).
@@ -38,3 +37,9 @@ A simple next.js app that serves a static page
 ```
 A simple api with a single route built on express
 ```
+
+**An example of a more complex prompt**
+
+```
+A service that uses bullmq and redis. The compose file should also include a redis service. The code should setup a queue. It should also expose three endpoints with express: one to add a job to the queue, another to check how many tasks are in the queue, and a last one that runs on / which is just a healthcheck. The code should also include a worker that processes the queue by logging what's in each task. Use environment variables to configure the redis connection. No password.
+```
diff --git a/docs/concepts/managed-storage.md b/docs/concepts/managed-storage.md
@@ -0,0 +1,43 @@
+---
+title: Managed Storage
+description: Defang can help you provision managed storage services.
+sidebar_position: 450
+---
+
+# Managed Storage
+
+Defang helps you provision the infrastructure you need to run your services. That infrastructure is designed to scale in and out without persistent storage, so you can build highly scalable services. But Defang can also help you provision managed services to store and persist your data, like [caches](#managed-redis), databases[<sup>\*</sup>](#managed-postgres) , and object storage[<sup>\*</sup>](#managed-object-storage).
+
+## Managed Redis
+
+Redis is an in-memory data structure store widely used for caching, real-time analytics, and session management due to its high performance, low latency, and support for various data types. Defang can help you provision a managed Redis instance. To read more about the specifics of how this works for each provider, see the following:
+  - [Playground](../providers/playground#managed-redis)
+  - [AWS](../providers/aws#managed-redis)
+
+### How to use Managed Redis
+
+To use managed Redis, in your `compose.yaml` file, use the `x-defang-redis` extension to define your Redis service. Adding the annotation will tell Defang to provision a managed instance, rather than running Redis as a service. Here's an example:
+
+```yaml
+  redisx:
+    image: redis:6.2
+    x-defang-redis: true
+    restart: unless-stopped
+    ports:
+      - mode: host
+        target: 6379
+```
+
+## Managed Postgres
+
+:::info
+As of July 22, 2024, managed Postgres is in development.
+:::
+
+Postgres, or PostgreSQL, is an advanced open-source relational database system known for its robustness, extensibility, and compliance with SQL standards, making it a popular choice for complex applications requiring reliable data integrity and sophisticated querying capabilities.
+
+## Managed Object Storage
+
+:::info
+As of July 22, 2024, managed object storage is planned for future development.
+:::
diff --git a/docs/concepts/projects.md b/docs/concepts/projects.md
@@ -0,0 +1,22 @@
+---
+title: Projects
+description: A _project_ refers to a cohesive collection of services which are defined and managed using a `compose.yml` file.
+sidebar_position: 200
+---
+
+# Projects
+
+A _project_ refers to a cohesive collection of services which are defined and managed using a `compose.yaml` file.
+
+## Project Names
+
+The _project name_ can be defined in the compose file with [the `name` property](https://docs.docker.com/compose/compose-file/04-version-and-name/#name-top-level-element), otherwise the base name of the project directory will be used.
+
+The project name may then be used when performing project-wide operations such as listing services, tailing logs, or deprovisioning. For example:
+```
+defang services --project-name <project-name>
+defang tail --project-name <project-name>
+defang compose down --project-name <project-name>
+```
+
+By default `defang` will use the project name from the `compose.yaml` file in the working directory. The [`--project-name` CLI flag](/docs/cli/defang) may be used when running `defang` from outside the working directory.
diff --git a/docs/intro.md b/docs/intro.md
@@ -7,7 +7,7 @@ description: Overview of Defang, a radically simpler way to build, deploy, and o
 
 ### What is Defang?
 
-Defang is a radically simpler way for developers to build, deploy their apps to the cloud. Defang enables you to easily author cloud application in any language, build and deploy to the cloud with a single command, and iterate quickly.
+Defang is a radically simpler way for developers to develop, deploy, and debug their cloud applications. Defang enables you to easily author cloud application in any language, build and deploy to the cloud with a single command, and iterate quickly with AI-assisted tooling.
 
 - The [Defang CLI](./getting-started/installing.md) includes an AI-driven assistant that translates natural language prompts to an outline for your project that you can then refine.
 - Defang can automatically build and deploy your project with a single command.
@@ -21,7 +21,8 @@ Defang provides a streamlined experience to develop, deploy, observe, and update
 
 - Support for [various types of applications](./use-cases/use-cases.md): Web services and APIs, mobile app backends, ML services, hosting LLMs, etc.
 - Support for your programming [language of choice](./samples.md): Node.js, Python, Golang, or anything else you can package in a Dockerfile.
-- Built-in [AI assistant](./concepts/ai.md) to go from natural language prompt to an outline project
+- Built-in AI assistant to go [from natural language prompt to an outline project](./concepts/generate.md)
+- Built-in AI assistant to help you [debug your cloud applications](./concepts/debug.md)
 - Automated [Dockerfile builds](./concepts/deployments.md)
 - Support for [pre-built Docker containers](./tutorials/deploy-container-using-the-cli.mdx), from public or private image registries
 - Ability to express your project configuration using a [Docker Compose YAML](./concepts/compose.md) file
diff --git a/docs/providers/aws.md b/docs/providers/aws.md
@@ -1,7 +1,7 @@
 ---
 title: AWS
 description: Defang allows you to easily create and manage full, scalable applications with AWS.
-sidebar_position: 000
+sidebar_position: 010
 ---
 
 # AWS
@@ -43,3 +43,7 @@ To deploy your services, the Defang CLI packages your code and uploads it to an
 ### Runtime
 
 The provider runs your workloads using ECS using Fargate. It provisions a VPC with public and private subnets, and deploys your services to the private subnets. It then provisions an Application Load Balancer (ALB) and routes traffic to your services.
+
+### Managed Redis
+
+When using [Managed Redis](../concepts/managed-storage.md#managed-redis), the Defang CLI provisions an ElastiCache Redis cluster in your account.
diff --git a/docs/providers/playground.md b/docs/providers/playground.md
@@ -0,0 +1,18 @@
+---
+title: Playground
+description: The Defang Playground is a free tier that allows you to experiment with Defang.
+sidebar_position: 000
+---
+
+The Defang Playground is a free tier that allows you to experiment with Defang. The Playground is a shared that cannot be used to run production workloads. The Playground is a great way to get started with Defang and to experiment with the Defang CLI.
+
+This page highlights architectural considerations when deploying to the playground and any differences you might encounter when deploying to the playground versus deploying to your own cloud account.
+
+## Overview
+
+Overall, the Defang Playground is very similar to deploying to your own cloud account. The Playground runs on a Defang-managed AWS account, so you can expect it to work similarly to deploying to [AWS](./aws.md).
+
+## Managed Redis
+
+The playground supports [Managed Redis](../concepts/managed-storage.md#managed-redis). Fundamentally, this works the same as it does in [AWS](./aws.md#managed-redis).
+
diff --git a/package.json b/package.json
@@ -5,16 +5,15 @@
   "scripts": {
     "docusaurus": "docusaurus",
     "start": "docusaurus start",
+    "prebuild": "./scripts/prebuild.sh",
     "build": "docusaurus build",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
     "clear": "docusaurus clear",
     "serve": "docusaurus serve",
     "write-translations": "docusaurus write-translations",
     "write-heading-ids": "docusaurus write-heading-ids",
-    "typecheck": "tsc",
-    "prep-cli-docs": "node scripts/prep-cli-docs.js",
-    "prep-samples": "node scripts/prep-samples.js"
+    "typecheck": "tsc"
   },
   "dependencies": {
     "@docusaurus/core": "3.0.0",
diff --git a/scripts/prebuild.sh b/scripts/prebuild.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+CWD=$(pwd)
+CLI_DOCS_PATH=$(readlink -f docs/cli)
+
+# In CI (github actions), the defang and samples repositories must be cloned
+# into the working directory of the `defang-docs` repository.
+# In local development, however, the defang and samples repositories are cloned
+# into the parent directory of the `defang-docs` repository.
+if [ -d "../defang" ]; then
+  DEFANG_PATH=$(readlink -f ../defang)
+else
+  DEFANG_PATH=$(readlink -f ./defang)
+fi
+if [ -d "../samples" ]; then
+  SAMPLES_PATH=$(readlink -f ../samples)
+else
+  SAMPLES_PATH=$(readlink -f ./samples)
+fi
+
+cd "$DEFANG_PATH/src/cmd/gendocs" && go run main.go "$CLI_DOCS_PATH"
+cd "$CWD"
+node scripts/prep-cli-docs.js
+node scripts/prep-samples.js "$SAMPLES_PATH/samples"
diff --git a/scripts/prep-cli-docs.js b/scripts/prep-cli-docs.js
@@ -47,4 +47,4 @@ fs.readdirSync(directoryPath).forEach(file => {
 
         fs.writeFileSync(filePath, fileContent);
     }
-});
+});
diff --git a/scripts/prep-samples.js b/scripts/prep-samples.js
@@ -2,7 +2,7 @@ const fs = require('fs');
 const path = require('path');
 const YAML = require('yaml');
 
-const samplesDir = path.join(__dirname, '..', 'samples', 'samples');
+const samplesDir = process.argv[2];
 
 // categories are directories in the current directory (i.e. we're running in samples/ and we might have a samples/ruby/ directory)
 const directories = fs.readdirSync(samplesDir).filter(file => fs.statSync(path.join(samplesDir, file)).isDirectory());
@@ -11,6 +11,7 @@ let jsonArray = [];
 
 directories.forEach((sample) => {
     const directoryName = sample;
+    console.log(`@@ Adding ${sample}`);
     let readme;
     try {
         readme = fs.readFileSync(path.join(samplesDir, sample, 'README.md'), 'utf8');
@@ -19,11 +20,11 @@ directories.forEach((sample) => {
     }
 
     // The readme should contain lines that start with the following:
-    // Title: 
-    // Short Description: 
-    // Tags: 
-    // Languages: 
-    // 
+    // Title:
+    // Short Description:
+    // Tags:
+    // Languages:
+    //
     // We want to extract the title, short description, tags, and languages from the readme. Tags and languages are comma separated lists.
     const title = readme.match(/Title: (.*)/)[1];
     const shortDescription = readme.match(/Short Description: (.*)/)[1];
diff --git a/src/components/Samples/index.tsx b/src/components/Samples/index.tsx
diff --git a/static/img/defang-compose-update.webp b/static/img/defang-compose-update.webp
