diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 31734f9cbd..4a8fd8a8ac 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -23,7 +23,7 @@ "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack", "nextflow.nextflow", - "codezombiech.gitignore", + "codezombiech.gitignore" ] } }, diff --git a/.github/gitpod.Dockerfile b/.github/gitpod.Dockerfile index c293de839f..1b565355f0 100644 --- a/.github/gitpod.Dockerfile +++ b/.github/gitpod.Dockerfile @@ -15,14 +15,17 @@ RUN apt-get update --quiet && \ curl \ tree \ graphviz \ - software-properties-common - + software-properties-common && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Taken from: https://github.com/nf-core/tools/blob/master/nf_core/gitpod/gitpod.Dockerfile # Install Apptainer (Singularity) RUN add-apt-repository -y ppa:apptainer/ppa && \ apt-get update --quiet && \ - apt install -y apptainer + apt install -y apptainer && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Install Conda RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ @@ -35,6 +38,11 @@ ENV PATH="/opt/conda/bin:$PATH" RUN mkdir -p /workspace/data \ && chown -R gitpod:gitpod /opt/conda /workspace/data +# Install Tower Agent +RUN curl -fSL https://github.com/seqeralabs/tower-agent/releases/latest/download/tw-agent-linux-x86_64 > tw-agent && \ + chmod +x tw-agent && \ + mv tw-agent /usr/local/bin/tw-agent + # Change user to gitpod USER gitpod diff --git a/docs/hello_nextflow/10_hello_seqera.md b/docs/hello_nextflow/10_hello_seqera.md index 1444085c40..9da4c1d2b4 100644 --- a/docs/hello_nextflow/10_hello_seqera.md +++ b/docs/hello_nextflow/10_hello_seqera.md @@ -1,7 +1,21 @@ +--- +title: "Part 9: Hello Seqera" +description: Get started with Seqera Platform +--- + # Part 9: Hello Seqera -TODO +So far we've been running Nextflow workflows on our local machine using the command line interface. +In this section, we'll introduce you to Seqera Platform, a powerful cloud-based platform for running, monitoring, and sharing Nextflow workflows. ---- +!!! tip + + [Sign up](https://cloud.seqera.io/) to try Seqera for free or request a [demo](https://seqera.io/demo/) for deployments in your own on-premise or cloud environment. + +You'll learn how to use Seqera Platform within Nextflow CLI, the Seqera Platform GUI, and the API. + +You can use Seqera Platform via either the **CLI**, through the **online GUI** or through the **API**. + +--8<-- "docs/hello_nextflow/seqera/01_run_with_cli.md" -TODO +--8<-- "docs/hello_nextflow/seqera/02_run_with_launchpad.md" diff --git a/docs/hello_nextflow/seqera/01_run_with_cli.md b/docs/hello_nextflow/seqera/01_run_with_cli.md new file mode 100644 index 0000000000..24108a5997 --- /dev/null +++ b/docs/hello_nextflow/seqera/01_run_with_cli.md @@ -0,0 +1,172 @@ +## 1. Use Seqera Platform to capture and monitor Nextflow jobs launched from the CLI + +We'll start by using the Nextflow CLI to launch a pipeline and monitor it in Seqera Platform. +Start by logging into the [Seqera Platform](https://cloud.seqera.io/). + +!!! info "Nextflow Tower" + + Seqera Platform was previously known as Nextflow Tower. + You'll still see references to the previous name in environment variables and CLI option names. + +### 1.1. Set up your Seqera Platform token by exporting it to your environment + +Follow these steps to set up your token: + +1. Create a new token by clicking on the **Settings** drop-down menu: + + ![Create a token](seqera/img/usage_create_token.png) + +2. Name your token: + + ![Name your token](seqera/img/usage_name_token.png) + +3. Save your token safely: + + ![Save token](seqera/img/usage_token.png) + + !!! note + + Leave the browser tab with the token open as we will need it once more to store it as a Nextflow secret. + +4. To make your token available to the Nextflow CLI, export it on the command line: + + Open a terminal and type: + + ```bash + export TOWER_ACCESS_TOKEN=eyxxxxxxxxxxxxxxxQ1ZTE= + ``` + + Where `eyxxxxxxxxxxxxxxxQ1ZTE=` is the token you have just created. + + !!! Warning "Security Note" + + Keep your token secure and do not share it with others. + You can add a ++space++ before the `export` command to prevent it from being saved in your shell history. + +### 1.2. Run Nextflow CLI with Seqera Platform visualizing and capturing logs + +Run a Nextflow workflow with the addition of the `-with-tower` command: + +```bash +nextflow run nextflow-io/hello -with-tower +``` + +You will see output similar to the following: + +```console title="Output" + N E X T F L O W ~ version 24.04.4 + +Launching `https://github.com/nextflow-io/hello` [evil_engelbart] DSL2 - revision: afff16a9b4 [master] + +Downloading plugin nf-tower@1.9.1 +Monitor the execution with Seqera Platform using this URL: https://cloud.seqera.io/user/kenbrewer/watch/5Gs0qqV9Y9rguE +executor > local (4) +[80/810411] process > sayHello (1) [100%] 4 of 4 ✔ +Ciao world! + +Bonjour world! + +Hola world! + +Hello world! +``` + +Hold ++ctrl++ or ++cmd++ and click on the link to open it in your browser. +You'll see the Seqera Platform interface with the job finished and the logs captured. + +![Seqera Platform](seqera/img/run_with_tower.png) + +You will see and be able to monitor your **Nextflow jobs** in Seqera Platform. + +### 1.3. Set up Seqera Platform in Nextflow configuration + +Doing that token setup regularly can get a bit tedious, so let's set this configuration for all our pipeline runs with the global Nextflow configuration file located at `$HOME/.nextflow/config`. + +Before we set the configuration, we need to permanently store the token in Nextflow using a [Nextflow secret](https://www.nextflow.io/docs/latest/secrets.html): + +```bash +nextflow secrets set tower_access_token "eyxxxxxxxxxxxxxxxQ1ZTE=" +``` + +Open the Nextflow configuration file located at `$HOME/.nextflow/config`: + +```bash +code $HOME/.nextflow/config +``` + +Then add the following block of configuration: + +```groovy title="$HOME/.nextflow/config" +tower { + enabled = true + endpoint = "https://api.cloud.seqera.io" + accessToken = secrets.tower_access_token + workspaceId = secrets.tower_workspace_id +} +``` + +!!! hint "Workspace ID and Endpoint" + + We haven't set `secrets.tower_workspace_id` yet, and so Nextflow will fill in an empty string for this value. + This will default to the user's workspace in Seqera Platform which is what we want for now. + + The `endpoint` is the URL of the Seqera Platform API. + If your institution is running a private instance of Seqera Platform, you will want to change this to the appropriate URL. + +Run your Nextflow workflows as before, but without the `-with-tower` command: + +```bash +nextflow run nextflow-io/hello +``` + +You will see the following output: + +```console title="Output" + N E X T F L O W ~ version 24.04.4 + +Launching `https://github.com/nextflow-io/hello` [fabulous_euclid] DSL2 - revision: afff16a9b4 [master] + +Monitor the execution with Seqera Platform using this URL: https://cloud.seqera.io/user/kenbrewer/watch/KYjRktIlOuxrh +executor > local (4) +[71/eaa915] process > sayHello (3) [100%] 4 of 4 ✔ +Ciao world! + +Bonjour world! + +Hola world! + +Hello world! +``` + +Note that we are logging to Seqera Platform even though we did not use the `-with-tower` command! + +### 1.4. Use Seqera Platform to explore the resolved configuration of a Nextflow pipeline + +Click on the link provided in the output to open the Seqera Platform for your run, then click on the `Configuration` tab. +If you ran your pipeline from the `hello_nextflow` directory, you'll see something like this: + +![Seqera Platform Configuration](seqera/img/resolved_configuration.png) + +Notice that configuration for our pipeline run is being run pulled from three separate files: + +- `/home/gitpod/.nextflow/config` - This is the global configuration file we just added. +- `/home/gitpod/.nextflow/assets/nextflow-io/hello/nextflow.config` - This is the `nextflow.config` file from the `nextflow-io/hello` repository. +- `/workspace/gitpod/nf-training/hello-nextflow/nextflow.config` - This is the `nextflow.config` file from our current working directory. + +Nextflow resolves these configurations at runtime with a [specific order of precedence](https://www.nextflow.io/docs/latest/config.html#configuration-file). +The general rule, however, is that more specific configurations override less specific ones, and config/params specified on the CLI will override defaults in the config files. + +Helpfully, Seqera Platform shows us the final output of this configuration resolution process which can be very useful for debugging! + +### Takeaway + +You have learned how to: + +- Set up your Seqera Platform token by exporting it to your environment. +- Run Nextflow CLI with Seqera Platform visualizing and capturing logs. +- Set up Seqera Platform logging by default. +- Use Seqera Platform to explore the resolved configuration of a Nextflow pipeline. + +### What's next? + +Learn how to launch Nextflow pipelines from Seqera Platform using the Launchpad feature. diff --git a/docs/hello_nextflow/seqera/02_run_with_launchpad.md b/docs/hello_nextflow/seqera/02_run_with_launchpad.md new file mode 100644 index 0000000000..45358d31e7 --- /dev/null +++ b/docs/hello_nextflow/seqera/02_run_with_launchpad.md @@ -0,0 +1,85 @@ +## 2 Using Seqera Platform Launchpad to run Nextflow workflows + +So far we've been running Nextflow workflows on our local machine using the command line interface but sending the logs to Seqera Platform for monitoring and visualization. +Next we want to start using Seqera Platform to launch Nextflow workflows on our behalf. + +!!! Note "Community Showcase" + + Having a compute environment capable of running Nextflow workflows configured in Seqera Platform is normally a prerequisite for this task. + But we want to see how it works before we put in that effort, so we'll start by launching a job in community/showcase workspace which has a compute environment already set up. + +!!! tip "Trainer Tip" + + Launch a test run of the nf-core/rnaseq pipeline in the community/showcase workspace prior to starting this session, so you'll have a recent run for participants to inspect. + +### 2.1. Navigate to the community/showcase workspace + +Seqera Platform has a concept of [**organizations**](https://docs.seqera.io/platform/latest/orgs-and-teams/organizations) and [**workspaces**](https://docs.seqera.io/platform/latest/orgs-and-teams/workspace-management) which are used to organize and share [pipelines](https://docs.seqera.io/platform/latest/launch/launchpad), [compute environments](https://docs.seqera.io/platform/latest/compute-envs/overview), [data](https://docs.seqera.io/platform/latest/data/data-explorer), [credentials](https://docs.seqera.io/platform/latest/credentials/overview), and more. +The `community/showcase` workspace is a public workspace where you can see some example pipelines and compute environments. +Each user has an alloted amount of free compute to use in this workspace. + +Click on your username in the top left corner of the screen to bring up the list of organizations and workspaces you have access to. +Select the `community/showcase` workspace. + +### 2.2. Launch a test run of nf-core/rnaseq pipeline + +In the `community/showcase` workspace, you will see a list of pipelines that have been set up by the workspace owner for you to run. +Follow these steps to launch a test run of a pipeline: + +![Launchpad](seqera/img/launchpad.gif) + +1. Find the `nf-core-rnaseq` pipeline in the list of pipelines. +2. Click on the `Launch` button to bring up the launch form. +3. Change the "Workflow run name" to "-rnaseq-test". +4. Click "Next" to bring up the parameters form. +5. Find the `trimmer` parameter and change it to `fastp`. +6. Click on "Next" to inspect the advanced configuration. +7. Click "Launch" to start the pipeline! + +!!! Tip + + In the advanced configuration, you'll see a section named "Pre-run script" with a script similar to the following: + + ```bash + export NXF_FILE_ROOT=s3://nf-tower-bucket/scratch/$TOWER_WORKFLOW_ID + ``` + + This is what ensures that everyone's pipeline will write to a unique location in cloud storage despite all having the `outdir` parameter set to `./results`. + +### 2.3. Monitor the pipeline run + +After launching the pipeline, you will be taken to the pipeline run page where you can monitor the progress of the pipeline. +It may take some time for the pipeline to start running while AWS Batch spins up the needed resources, so go to the "Runs" tab above the pipeline and open a recent completed (or failed) run by one of your "teammates" in the community. + +### 2.4. Inspect a pipeline run + +Scroll down to find the list of tasks that were executed in the pipeline run. +For example, by searching for `fastq` we can find the task `NFCORE_RNASEQ:RNASEQ:FASTQ_FASTQC_UMITOOLS_FASTP:FASTP (WT_REP2)` that was executed as part of the pipeline. + +Click on the task to see the task details: + +![Task details](seqera/img/task_details.png) + +1. Find the following details on the "About" page for the the task you're inspecting: + + - [ ] How long did the task script run (not including scheduling time)? + - [ ] How many CPUs were allocated to the task? + - [ ] What was the virtual machine type that the task ran on? + - [ ] What was the estimated cost of the task? + +2. Explore the Execution Log tab. What information is available here? + +3. Explore the Data Explorer tab. Note that the work directory structure we've seen during local runs is replicated here in cloud storage! + +### Takeaway + +You have learned how to: + +- Switch between organizations and workspaces in Seqera Platform. +- Launch a Nextflow pipeline that ran in the cloud using Seqera Platform. +- Monitor the progress of the pipeline run. +- Inspect the details of a task that was executed as part of the pipeline. + +### Next steps + +In the next section, we will learn how to set up a compute environment in Seqera Platform to run our own Nextflow workflows. diff --git a/docs/hello_nextflow/seqera/img/compute_env_platforms.png b/docs/hello_nextflow/seqera/img/compute_env_platforms.png new file mode 100644 index 0000000000..c76eb417b1 Binary files /dev/null and b/docs/hello_nextflow/seqera/img/compute_env_platforms.png differ diff --git a/docs/hello_nextflow/seqera/img/launchpad.gif b/docs/hello_nextflow/seqera/img/launchpad.gif new file mode 100644 index 0000000000..8dbe3bfd47 Binary files /dev/null and b/docs/hello_nextflow/seqera/img/launchpad.gif differ diff --git a/docs/hello_nextflow/seqera/img/resolved_configuration.png b/docs/hello_nextflow/seqera/img/resolved_configuration.png new file mode 100644 index 0000000000..01ae3e5467 Binary files /dev/null and b/docs/hello_nextflow/seqera/img/resolved_configuration.png differ diff --git a/docs/hello_nextflow/seqera/img/run_with_tower.png b/docs/hello_nextflow/seqera/img/run_with_tower.png new file mode 100644 index 0000000000..94b04172ae Binary files /dev/null and b/docs/hello_nextflow/seqera/img/run_with_tower.png differ diff --git a/docs/hello_nextflow/seqera/img/running_pipeline.png b/docs/hello_nextflow/seqera/img/running_pipeline.png new file mode 100644 index 0000000000..eef880448a Binary files /dev/null and b/docs/hello_nextflow/seqera/img/running_pipeline.png differ diff --git a/docs/hello_nextflow/seqera/img/task_details.png b/docs/hello_nextflow/seqera/img/task_details.png new file mode 100644 index 0000000000..88670186dc Binary files /dev/null and b/docs/hello_nextflow/seqera/img/task_details.png differ diff --git a/docs/hello_nextflow/seqera/img/usage_create_token.png b/docs/hello_nextflow/seqera/img/usage_create_token.png new file mode 100644 index 0000000000..d7149c2281 Binary files /dev/null and b/docs/hello_nextflow/seqera/img/usage_create_token.png differ diff --git a/docs/hello_nextflow/seqera/img/usage_name_token.png b/docs/hello_nextflow/seqera/img/usage_name_token.png new file mode 100644 index 0000000000..1a4abdf6ff Binary files /dev/null and b/docs/hello_nextflow/seqera/img/usage_name_token.png differ diff --git a/docs/hello_nextflow/seqera/img/usage_token.png b/docs/hello_nextflow/seqera/img/usage_token.png new file mode 100644 index 0000000000..6eab1a5f43 Binary files /dev/null and b/docs/hello_nextflow/seqera/img/usage_token.png differ diff --git a/mkdocs.yml b/mkdocs.yml index 3c3a5a54d6..548855b099 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -155,6 +155,9 @@ markdown_extensions: preserve_tabs: true - pymdownx.tabbed: alternate_style: true + - pymdownx.tasklist: + custom_checkbox: true + clickable_checkbox: true - tables - toc: title: On this page