Merge pull request #511 from rstudio/mm-verify-deployments

Verify deployments

Author: mmarchetti

.github/workflows/main.yml

@@ -63,7 +63,7 @@ jobs:
     - run: make fmt
     - run: make lint
     - run: rsconnect version
-    - run: make mock-test-3.8
+    - run: make test-3.8
 
   distributions:
     needs: test

CHANGELOG.md

@@ -32,6 +32,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added a new verbose logging level. Specifying `-v` on the command line uses this
   new level. Currently this will cause filenames to be logged as they are added to
   a bundle. To enable maximum verbosity (debug level), use `-vv`.
+- Added a verification step to the deployment process that accesses the deployed content.
+  This is a `GET` request to the content without parameters. The request is
+  considered successful if there isn't a 5xx code returned (errors like
+  400 Bad Request or 405 Method Not Allowed because not all apps support `GET /`).
+  For cases where this is not desired, use the `--no-verify` flag on the command line.
 - Added the `deploy flask` command.
 - Added the `write-manifest flask` command.
 

README.md

@@ -429,6 +429,14 @@ containing the API or application.
 When using `rsconnect deploy manifest`, the title is derived from the primary
 filename referenced in the manifest.
 
+#### Verification After Deployment
+After deploying your content, rsconnect accesses the deployed content
+to verify that the deployment is live. This is done with a `GET` request
+to the content, without parameters. The request is
+considered successful if there isn't a 5xx code returned. Errors like
+400 Bad Request or 405 Method Not Allowed because not all apps support `GET /`.
+For cases where this is not desired, use the `--no-verify` flag on the command line.
+
 ### Environment variables
 You can set environment variables during deployment. Their names and values will be
 passed to Posit Connect during deployment so you can use them in your code. Note that

rsconnect/api.py

@@ -3,7 +3,7 @@
 """
 import binascii
 import os
-from os.path import abspath
+from os.path import abspath, dirname
 import time
 from typing import IO, Callable
 import base64
@@ -195,6 +195,22 @@ def app_publish(self, app_id, access):
     def app_config(self, app_id):
         return self.get("applications/%s/config" % app_id)
 
+    def is_app_failed_response(self, response):
+        return isinstance(response, HTTPResponse) and response.status >= 500
+
+    def app_access(self, app_guid):
+        method = "GET"
+        base = dirname(self._url.path)  # remove __api__
+        path = f"{base}/content/{app_guid}/"
+        response = self._do_request(method, path, None, None, 3, {}, False)
+
+        if self.is_app_failed_response(response):
+            raise RSConnectException(
+                "Could not access the deployed content. "
+                + "The app might not have started successfully. "
+                + "Visit it in Connect to view the logs."
+            )
+
     def bundle_download(self, content_guid, bundle_id):
         response = self.get("v1/content/%s/bundles/%s/download" % (content_guid, bundle_id), decode_response=False)
         self._server.handle_bad_response(response)
@@ -300,7 +316,6 @@ def wait_for_task(
         poll_wait=0.5,
         raise_on_error=True,
     ):
-
         if log_callback is None:
             log_lines = []
             log_callback = log_lines.append
@@ -805,6 +820,13 @@ def save_deployed_info(self, *args, **kwargs):
 
         return self
 
+    @cls_logged("Verifying deployed content...")
+    def verify_deployment(self, *args, **kwargs):
+        if isinstance(self.remote_server, RSConnectServer):
+            deployed_info = self.get("deployed_info", *args, **kwargs)
+            app_guid = deployed_info["app_guid"]
+            self.client.app_access(app_guid)
+
     @cls_logged("Validating app mode...")
     def validate_app_mode(self, *args, **kwargs):
         path = (
@@ -1331,7 +1353,6 @@ def prepare_deploy(
         app_mode: AppMode,
         app_store_version: typing.Optional[int],
     ) -> PrepareDeployOutputResult:
-
         application_type = "static" if app_mode in [AppModes.STATIC, AppModes.STATIC_QUARTO] else "connect"
         logger.debug(f"application_type: {application_type}")
 

rsconnect/main.py

@@ -239,6 +239,11 @@ def content_args(func):
         "or just NAME to use the value from the local environment. "
         "May be specified multiple times. [v1.8.6+]",
     )
+    @click.option(
+        "--no-verify",
+        is_flag=True,
+        help="Don't access the deployed content to verify that it started correctly.",
+    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -851,6 +856,7 @@ def deploy_notebook(
     disable_env_management: bool,
     env_management_py: bool,
     env_management_r: bool,
+    no_verify: bool = False,
 ):
     kwargs = locals()
     set_verbosity(verbose)
@@ -893,6 +899,8 @@ def deploy_notebook(
             env_management_r=env_management_r,
         )
     ce.deploy_bundle().save_deployed_info().emit_task_log()
+    if not no_verify:
+        ce.verify_deployment()
 
 
 # noinspection SpellCheckingInspection,DuplicatedCode
@@ -971,6 +979,7 @@ def deploy_voila(
     cacert: typing.IO = None,
     connect_server: api.RSConnectServer = None,
     multi_notebook: bool = False,
+    no_verify: bool = False,
 ):
     kwargs = locals()
     set_verbosity(verbose)
@@ -994,6 +1003,8 @@ def deploy_voila(
         env_management_r=env_management_r,
         multi_notebook=multi_notebook,
     ).deploy_bundle().save_deployed_info().emit_task_log()
+    if not no_verify:
+        ce.verify_deployment()
 
 
 # noinspection SpellCheckingInspection,DuplicatedCode
@@ -1029,6 +1040,7 @@ def deploy_manifest(
     file: str,
     env_vars: typing.Dict[str, str],
     visibility: typing.Optional[str],
+    no_verify: bool = False,
 ):
     kwargs = locals()
     set_verbosity(verbose)
@@ -1049,6 +1061,8 @@ def deploy_manifest(
         .save_deployed_info()
         .emit_task_log()
     )
+    if not no_verify:
+        ce.verify_deployment()
 
 
 # noinspection SpellCheckingInspection,DuplicatedCode
@@ -1126,6 +1140,7 @@ def deploy_quarto(
     disable_env_management: bool,
     env_management_py: bool,
     env_management_r: bool,
+    no_verify: bool = False,
 ):
     kwargs = locals()
     set_verbosity(verbose)
@@ -1176,6 +1191,8 @@ def deploy_quarto(
         .save_deployed_info()
         .emit_task_log()
     )
+    if not no_verify:
+        ce.verify_deployment()
 
 
 # noinspection SpellCheckingInspection,DuplicatedCode
@@ -1229,6 +1246,7 @@ def deploy_html(
     account: str = None,
     token: str = None,
     secret: str = None,
+    no_verify: bool = False,
 ):
     kwargs = locals()
     set_verbosity(verbose)
@@ -1254,6 +1272,8 @@ def deploy_html(
         .save_deployed_info()
         .emit_task_log()
     )
+    if not no_verify:
+        ce.verify_deployment()
 
 
 def generate_deploy_python(app_mode: AppMode, alias: str, min_version: str, desc: Optional[str] = None):
@@ -1343,6 +1363,7 @@ def deploy_app(
         account: str = None,
         token: str = None,
         secret: str = None,
+        no_verify: bool = False,
     ):
         set_verbosity(verbose)
         kwargs = locals()
@@ -1374,6 +1395,8 @@ def deploy_app(
             .save_deployed_info()
             .emit_task_log()
         )
+        if not no_verify:
+            ce.verify_deployment()
 
     return deploy_app
 

tests/test_main.py

@@ -810,6 +810,22 @@ def test_deploy_api(self):
         result = runner.invoke(cli, args)
         assert result.exit_code == 0, result.output
 
+    def test_deploy_api_fail_verify(self):
+        target = optional_target(get_api_path("flask-bad"))
+        runner = CliRunner()
+        args = self.create_deploy_args("api", target)
+        args.extend(["-e", "badapp"])
+        result = runner.invoke(cli, args)
+        assert result.exit_code == 1, result.output
+
+    def test_deploy_api_fail_no_verify(self):
+        target = optional_target(get_api_path("flask-bad"))
+        runner = CliRunner()
+        args = self.create_deploy_args("api", target)
+        args.extend(["--no-verify", "-e", "badapp"])
+        result = runner.invoke(cli, args)
+        assert result.exit_code == 0, result.output
+
     def test_add_connect(self):
         connect_server = require_connect()
         api_key = require_api_key()
@@ -944,7 +960,6 @@ def setUp(self):
 
     def create_bootstrap_mock_callback(self, status, json_data):
         def request_callback(request, uri, response_headers):
-
             # verify auth header is sent correctly
             authorization = request.headers.get("Authorization")
             auth_split = authorization.split(" ")

tests/testdata/api/flask-bad/badapp.py

@@ -0,0 +1,19 @@
+import os
+from flask import Flask, jsonify, request, url_for
+
+app = Flask(__name__)
+
+
+@app.route("/ping")
+def ping():
+    return jsonify(
+        {
+            "headers": dict(request.headers),
+            "environ": dict(os.environ),
+            "link": url_for("ping"),
+            "external_link": url_for("ping", _external=True),
+        }
+    )
+
+
+raise Exception("this test app fails to start!")

tests/testdata/api/flask-bad/requirements.txt

@@ -0,0 +1,7 @@
+blinker==1.6.3
+click==8.1.7
+Flask==3.0.0
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.3
+Werkzeug==3.0.0

tests/testdata/api/flask/requirements.txt

@@ -1 +1,7 @@
-flask ~= 1.1.1
+blinker==1.6.3
+click==8.1.7
+Flask==3.0.0
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.3
+Werkzeug==3.0.0