define sample apps in sample app json to be used by the console

.github/workflows/update-cloud-samples.yml

@@ -0,0 +1,149 @@
+name: Update on Vespa Cloud
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+defaults:
+  run:
+    # Specify to ensure "pipefail and errexit" are set.
+    # Ref: https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#defaultsrunshell
+    shell: bash
+
+permissions:
+  contents: read
+  id-token: write # Required for OIDC authentication
+  deployments: write
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+
+    outputs:
+      app-list-json: ${{ steps.read-json.outputs.APP_LIST }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Read JSON
+        id: read-json
+        run: |
+          delimiter="$(openssl rand -hex 8)"
+          {
+            echo "APP_LIST<<${delimiter}"
+            sed -e '$a\' console-sample-apps.json # Ensures that an empty line is always present.
+            echo "${delimiter}"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Upload artifact JSON
+        uses: actions/upload-artifact@v4
+        with:
+          name: console-json
+          path: console-sample-apps.json
+
+  create-and-validate:
+    name: Create Zip and Validate JSON
+    runs-on: ubuntu-latest
+
+    needs:
+      - setup
+
+    strategy:
+      fail-fast: false
+      matrix:
+        app: ${{ fromJSON(needs.setup.outputs.app-list-json) }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Create artifact
+        env:
+          DEBUG: ${{ runner.debug }} # Automatically set by GitHub Actions when running in debug mode.
+        run: |
+          ./scripts/build-sample-app.sh "${{ matrix.app.name }}"
+
+          # Ensure that both application.zip and documents.jsonl are created.
+          if [ ! -f "${{ matrix.app.name }}/dist/application.zip" ] ||
+             [ ! -f "${{ matrix.app.name }}/dist/documents.jsonl" ]; then
+            echo "::error:: Build failure: application.zip or documents.jsonl not found in ${GITHUB_WORKSPACE}/${{ matrix.app.name }}/dist/"
+            exit 1
+          fi
+
+      - name: Validate documents.jsonl
+        env:
+          DOCUMENTS_JSONL: ${{ matrix.app.name }}/dist/documents.jsonl
+        run: |
+          # Validate each line as a standalone JSON object/array
+          LINE_NUM=0
+          while IFS= read -r line || [ -n "$line" ]; do
+              LINE_NUM=$((LINE_NUM + 1))
+              echo "$line" | jq empty || (
+                  echo "::error:: Invalid JSON on line $LINE_NUM: $line"
+                  exit 1
+              )
+          done < "$DOCUMENTS_JSONL"
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.app.shortname }}
+          path: ${{ matrix.app.name }}/dist/
+
+  push:
+    name: Push to Vespa Cloud
+    runs-on: ubuntu-latest
+
+    environment:
+      # Append "CD" to the environment name if not on "push" event on default branch.
+      name: ${{ (github.event_name != 'push' || github.ref_name != 'master' ) && 'Vespa Cloud CD' || 'Vespa Cloud' }}
+      url: https://cloud.vespa.ai
+
+    needs:
+      - create-and-validate
+
+    env:
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_ROLE_SAMPLE_APP_DEPLOY: ${{ vars.AWS_ROLE_SAMPLE_APP_DEPLOY }}
+      AWS_CLOUDFRONT_DISTRIBUTION_ID: ${{ vars.AWS_CLOUDFRONT_DISTRIBUTION_ID }}
+      AWS_S3_BUCKET_NAME: ${{ vars.AWS_S3_BUCKET_NAME }}
+      S3_UPLOAD_PREFIX: console/one-click-sample-apps
+
+    steps:
+      - name: Download All Packages
+        uses: actions/download-artifact@v4
+        with:
+          path: apps
+
+      - name: List Downloaded
+        if: ${{ runner.debug }}
+        run: |
+          ls -lr apps
+
+      - name: Setup AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+          role-to-assume: ${{ env.AWS_ROLE_SAMPLE_APP_DEPLOY }}
+
+      - name: Push Zip and JSON to S3
+        env:
+          # Add `--dryrun` if not on "push" event on default branch.
+          AWS_S3_OPTIONS: --color=on --no-progress ${{ (github.event_name == 'push' && github.ref_name == 'master' ) &&  '' || ' --dryrun' }}
+        run: |
+          # Not an app, but artifact from the setup job.
+          mv ./apps/console-json ./console-json
+
+          aws s3 sync ${{ env.AWS_S3_OPTIONS }} ./apps/ "s3://${AWS_S3_BUCKET_NAME}/${S3_UPLOAD_PREFIX}/"
+          aws s3 cp ${{ env.AWS_S3_OPTIONS }} ./console-json/console-sample-apps.json "s3://${S3_BUCKET}/${S3_UPLOAD_PREFIX}/"
+
+      - name: Invalidate Cloudfront Cache
+        env:
+          ECHO: ${{ (github.event_name == 'push' && github.ref_name == 'master') && '' || 'echo' }}
+        run: |
+          $ECHO aws cloudfront create-invalidation --distribution-id "${AWS_CLOUDFRONT_DISTRIBUTION_ID}" --paths "${S3_UPLOAD_PREFIX}/*" --output text

README.md

@@ -19,6 +19,59 @@ First-time users should go through the [getting-started](https://docs.vespa.ai/e
 Explore the [examples](examples) for smaller applications helping you getting started with a particular feature,
 and see [operations](examples/operations) for operational examples.
 
+## Standard structure of a Sample Application
+
+We try to build each sample application with a standard structure, so you can
+ easily find your way around. This section gives you an overview of this structure.
+
+<details>
+<summary>Click to expand</summary>
+
+The standard structure is as follows:
+
+```plaintext
+one-sample-app/
+├─ README.md
+├─ dataset/
+│  ├─ a-document.json
+│  ├─ another-doc.json
+│  ├─ ...
+├─ app/
+│  ├─ security/
+│  ├─ schemas/
+│  │  ├─ my_vespa_schema.sd
+│  ├─ services.xml
+│  ├─ ...
+├─ script/
+│  ├─ build.sh             # when required
+│  ├─ ...
+```
+
+There might be other files and folders than the ones listed here, but this should be the bare minimum that you can expect.
+
+Let's analyze the structure:
+
+- `README.md` - self explainatory: the README specific for the sample application, we recommend reading it through.
+- `dataset/` - contains the dataset used for the sample application. This is usually a small set of documents, but it can be larger.
+
+  **Note:** Sometimes this might be missing, and will be generated by the build script (see below), this is usually because the dataset is too large to be included in the repository.
+-  `app/` - contains the [Vespa application package](https://docs.vespa.ai/en/application-packages.html), this is the actual definition of the Vespa deployment. The bare minimum includes:
+    - `security/` - this is where you will place your `clients.pem` certificate for data-plane authentication. _(Refer to our [Security Guide](https://cloud.vespa.ai/en/security/guide.html#application-configuration))_
+    - `schemas/` - contains [the schema](https://docs.vespa.ai/en/schemas.html) files used for the application. This is usually a single file, but it can be multiple files.
+    - `services.xml` - contains the services configuration for the application. This is usually a single file.
+    - Other files and folders that are required for the application to run.
+- `script/` - contains the optional scripts used to help building the application package. This is usually a single file, but it can be multiple files.
+  - `build.sh` - this is the main script, if multiple files are present, this will be the only one you'll need to worry about.
+
+    **Note:** We recomend refering to the README for the sample application for details on how to run it.
+
+
+> [!NOTE]
+> We are in the process of updating the sample applications to follow the
+> structure described here, but some applications may not be fully updated
+> yet, so please check the README files in each application for details.
+
+</details>
 
 
 ## Getting Started
@@ -139,7 +192,7 @@ with a 32x decrease in storage footprint.
 
 
 ## Retrieval Augmented Generation (RAG) and Generative AI
-[![logo](/assets/vespa-logomark-tiny.png) Retrieval Augmented Generation (RAG) in Vespa](retrieval-augmented-generation) 
+[![logo](/assets/vespa-logomark-tiny.png) Retrieval Augmented Generation (RAG) in Vespa](retrieval-augmented-generation)
 is an end-to-end RAG application where all the steps are run within Vespa.
 This application focuses on the generation part of RAG,
 with a simple text search using [BM25](https://docs.vespa.ai/en/reference/bm25.html).
@@ -213,11 +266,11 @@ feeds to Vespa and lets you query the videos in text using a Streamlit applicati
 It is a good start for creating a video search application using Vespa!
 
 [![logo](/assets/vespa-logomark-tiny.png) Video Search and Retrieval with Vespa and TwelveLabs](https://github.com/vespa-engine/pyvespa/blob/master/docs/sphinx/source/examples/video_search_twelvelabs_cloud.ipynb) is a notebook
-showcasing the use of [TwelveLabs](https://www.twelvelabs.io/) state-of-the-art generation and embedding models 
-for video processing. It demonstrates how to generate rich metadata (including summaries and keywords) for videos 
-using TwelveLabs' technology, and how to embed video chunks for efficient retrieval. The notebook processes three 
-sample videos, segments them into chunks, and stores their embeddings along with metadata in Vespa's multi-vector 
-tensors. You can perform hybrid searches to find specific video scenes based on natural language descriptions. 
+showcasing the use of [TwelveLabs](https://www.twelvelabs.io/) state-of-the-art generation and embedding models
+for video processing. It demonstrates how to generate rich metadata (including summaries and keywords) for videos
+using TwelveLabs' technology, and how to embed video chunks for efficient retrieval. The notebook processes three
+sample videos, segments them into chunks, and stores their embeddings along with metadata in Vespa's multi-vector
+tensors. You can perform hybrid searches to find specific video scenes based on natural language descriptions.
 This serves as an excellent starting point for implementing advanced video retrieval with Vespa!
 
 ## Ranking
@@ -342,12 +395,12 @@ lets users quickly create good queries.
 
 ### Search as you type and query suggestions
 [![logo](/assets/vespa-logomark-tiny.png) Incremental Search](incremental-search/) shows search-as-you-type functionality,
-where for each keystroke of the user, it retrieves matching documents. 
+where for each keystroke of the user, it retrieves matching documents.
 It also demonstrates search suggestions (query auto-completion).
 
 
 ### Vespa as ML inference server
-[![logo](/assets/vespa-logomark-tiny.png) Stateless model evaluation](model-inference/) demonstrates 
+[![logo](/assets/vespa-logomark-tiny.png) Stateless model evaluation](model-inference/) demonstrates
 using Vespa as a stateless ML model inference server
 where Vespa takes care of distributing ML models to multiple serving containers,
 offering horizontal scaling and safe deployment.

album-recommendation/.gitignore

@@ -1,2 +1 @@
-src/main/application/security/
-.idea/
+dist/*

colbert/README.md

@@ -27,7 +27,7 @@ through the <code>vespa deploy</code> step, cloning `colbert` instead of `album-
 
 Feed documents (this includes embed inference in Vespa):
 <pre data-test="exec">
-vespa feed ext/*.json
+vespa feed dataset/*.json
 </pre>
 
 Example queries:
@@ -53,8 +53,8 @@ vespa query 'yql=select * from doc where userQuery() or ({targetHits: 100}neares
  </pre>
 
 
-### Export ColBERT models from HF 
-See the [model2onnx.py](model2onnx.py) script for exporting the ColBERT model from Hugging Face to ONNX format. 
+### Export ColBERT models from HF
+See the [model2onnx.py](model2onnx.py) script for exporting the ColBERT model from Hugging Face to ONNX format.
 
 Notice that these three models use different embedding dimensionality.
 
@@ -65,7 +65,7 @@ Example usage:
 This is the recommended colbert model for this application as it is optimized for speed and accuracy. See [blog post](https://blog.vespa.ai/introducing-answerai-colbert-small/)
 
 ```bash
-python3 model2onnx.py --hf_model answerdotai/answerai-colbert-small-v1 --dims 96 
+python3 model2onnx.py --hf_model answerdotai/answerai-colbert-small-v1 --dims 96
 ```
 Can be used with:
 ```
@@ -75,22 +75,21 @@ field colbert type tensor<int8>(dt{}, x[12])
 
 #### https://huggingface.co/mixedbread-ai/mxbai-colbert-large-v1
 ```bash
-python3 model2onnx.py --hf_model mixedbread-ai/mxbai-colbert-large-v1 --dims 128 
+python3 model2onnx.py --hf_model mixedbread-ai/mxbai-colbert-large-v1 --dims 128
 ```
 
 ### https://huggingface.co/vespa-engine/col-minilm
 ```bash
-python3 model2onnx.py --hf_model vespa-engine/col-minilm --dims 32 
+python3 model2onnx.py --hf_model vespa-engine/col-minilm --dims 32
 ```
 Can be used with:
 ```
 field colbert type tensor<int8>(dt{}, x[4])
 ```
 
-### Terminate container 
+### Terminate container
 
 Remove the container after use:
 <pre data-test="exec">
 $ docker rm -f vespa
 </pre>
-

console-sample-apps.json

@@ -0,0 +1,60 @@
+[
+    {
+      "name": "album-recommendation",
+      "shortname": "album-recommendation",
+      "title": "Album Recommendation",
+      "description": "This application ranks music albums using a user profile: Albums with scores for a set of categories are matched with a user's preference.",
+      "features": [
+        "Search",
+        "Recommendation"
+      ],
+      "category": "Getting Started",
+      "repository": "https://github.com/vespa-engine/sample-apps/tree/master/album-recommendation#readme",
+      "exampleQuery": {
+        "yql": "select * from music where true",
+        "ranking": {
+            "profile": "rank_albums",
+            "features": {
+                "query(user_profile)": "{{cat:pop}:0.8,{cat:rock}:0.2,{cat:jazz}:0.1}"
+            }
+        }
+      }
+    },
+    {
+      "name": "text-search",
+      "shortname": "text-search",
+      "title": "Text Search",
+      "description": "The Text Search Tutorial demonstrates traditional text search using BM25/Vespa nativeRank, and is a good start to using the MS Marco dataset.",
+      "features": [
+        "BM25",
+        "nativeRank",
+        "MS Marco"
+      ],
+      "category": "Getting Started",
+      "repository": "https://github.com/vespa-engine/sample-apps/tree/master/text-search#readme",
+      "exampleQuery": {
+        "yql": "select title,url,id from msmarco where userQuery()",
+        "query": "what is dad bod"
+      }
+    },
+    {
+      "name": "colbert",
+      "shortname": "colbert",
+      "title": "Simple hybrid search with ColBERT",
+      "description": "This application uses a single vector embedding model for retrieval and ColBERT (multi-token vector representation) for re-ranking. This semantic search application demonstrates the colbert-embedder and the tensor expressions for ColBERT MaxSim.",
+      "features": [
+        "ColBERT",
+        "Re-ranking"
+      ],
+      "category": "Ranking",
+      "repository": "https://github.com/vespa-engine/sample-apps/tree/master/colbert#readme",
+      "exampleQuery": {
+        "query": "shipping stuff over the sea",
+        "yql": "select * from doc where userQuery() or ({targetHits: 100}nearestNeighbor(embedding, q))",
+        "input": {
+          "query(q)": "embed(e5, @query)",
+          "query(qt)": "embed(colbert, @query)"
+        }
+      }
+    }
+  ]