add ol-concourse common components (#1894)

* add ol-concourse to requirements

* add common ol-concourse based classes for pipeline definitions

* fmt

* isort

* remove unused import

* add tests for add_error_handling and error if the Step object passed in does not extend StepModifierMixin

* Change add_error_handling type hint for step to StepModifierMixin

* remove unused `Step` class

* throw an error if add_error_handling is called with a value already set on on_failure, on_error or on_abort

* remove dev check from adding slack alert

* fix failure description for aborts

* make ClearCdnCacheStep a TaskStepWithErrorHandling

Author: gumaerc

content_sync/pipelines/definitions/__init__.py

@@ -0,0 +1,16 @@
+from ol_concourse.lib.models.pipeline import Identifier
+
+
+# Commonly used identifiers
+HTTP_RESOURCE_TYPE_IDENTIFIER = Identifier("http-resource")
+KEYVAL_RESOURCE_TYPE_IDENTIFIER = Identifier("keyval")
+S3_IAM_RESOURCE_TYPE_IDENTIFIER = Identifier("s3-resource-iam")
+OCW_STUDIO_WEBHOOK_RESOURCE_TYPE_IDENTIFIER = Identifier("ocw-studio-webhook")
+SLACK_ALERT_RESOURCE_IDENTIFIER = Identifier("slack-alert")
+OPEN_DISCUSSIONS_RESOURCE_IDENTIFIER = Identifier("open-discussions-webhook")
+WEBPACK_MANIFEST_S3_IDENTIFIER = Identifier("webpack-manifest-s3")
+WEBPACK_ARTIFACTS_IDENTIFIER = Identifier("webpack-artifacts")
+OCW_HUGO_THEMES_GIT_IDENTIFIER = Identifier("ocw-hugo-themes-git")
+OCW_HUGO_PROJECTS_GIT_IDENTIFIER = Identifier("ocw-hugo-projects-git")
+SITE_CONTENT_GIT_IDENTIFIER = Identifier("site-content-git")
+STATIC_RESOURCES_S3_IDENTIFIER = Identifier("static-resources-s3")

content_sync/pipelines/definitions/concourse/__init__.py

@@ -0,0 +1,21 @@
+from ol_concourse.lib.constants import REGISTRY_IMAGE
+from ol_concourse.lib.models.pipeline import AnonymousResource, RegistryImage
+
+
+"""
+Docker image for building OCW sites
+
+https://github.com/mitodl/ol-infrastructure/tree/main/dockerfiles/ocw/node-hugo
+"""
+OCW_COURSE_PUBLISHER_REGISTRY_IMAGE = AnonymousResource(
+    type=REGISTRY_IMAGE,
+    source=RegistryImage(repository="mitodl/ocw-course-publisher", tag="0.6"),
+)
+
+AWS_CLI_REGISTRY_IMAGE = AnonymousResource(
+    type=REGISTRY_IMAGE, source=RegistryImage(repository="amazon/aws-cli", tag="latest")
+)
+
+CURL_REGISTRY_IMAGE = AnonymousResource(
+    type=REGISTRY_IMAGE, source=RegistryImage(repository="curlimages/curl")
+)

content_sync/pipelines/definitions/concourse/common/__init__.py

@@ -0,0 +1,53 @@
+from ol_concourse.lib.constants import REGISTRY_IMAGE
+from ol_concourse.lib.models.pipeline import ResourceType
+
+from content_sync.pipelines.definitions.concourse.common.identifiers import (
+    HTTP_RESOURCE_TYPE_IDENTIFIER,
+    KEYVAL_RESOURCE_TYPE_IDENTIFIER,
+    S3_IAM_RESOURCE_TYPE_IDENTIFIER,
+)
+
+
+class HttpResourceType(ResourceType):
+    """
+    A Resource for making HTTP requests
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            name=HTTP_RESOURCE_TYPE_IDENTIFIER,
+            type=REGISTRY_IMAGE,
+            source={"repository": "jgriff/http-resource", "tag": "latest"},
+            **kwargs
+        )
+
+
+class KeyvalResourceType(ResourceType):
+    """
+    A resource for storing and recalling simple key / value pairs
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            name=KEYVAL_RESOURCE_TYPE_IDENTIFIER,
+            type=REGISTRY_IMAGE,
+            source={
+                "repository": "ghcr.io/cludden/concourse-keyval-resource",
+                "tag": "latest",
+            },
+            **kwargs
+        )
+
+
+class S3IamResourceType(ResourceType):
+    """
+    A resource for interacting with S3-compatible storage services that supports instance profiles
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            name=S3_IAM_RESOURCE_TYPE_IDENTIFIER,
+            type=REGISTRY_IMAGE,
+            source={"repository": "governmentpaas/s3-resource", "tag": "latest"},
+            **kwargs
+        )

content_sync/pipelines/definitions/concourse/common/identifiers.py

@@ -0,0 +1,95 @@
+from django.conf import settings
+from ol_concourse.lib.models.pipeline import Identifier, Resource
+from ol_concourse.lib.resource_types import slack_notification_resource
+
+from content_sync.pipelines.definitions.concourse.common.identifiers import (
+    HTTP_RESOURCE_TYPE_IDENTIFIER,
+    OCW_STUDIO_WEBHOOK_RESOURCE_TYPE_IDENTIFIER,
+    OPEN_DISCUSSIONS_RESOURCE_IDENTIFIER,
+    SLACK_ALERT_RESOURCE_IDENTIFIER,
+)
+
+
+class SlackAlertResource(Resource):
+    """
+    A Resource using the version of concourse-slack-notification specified by ol-concourse
+
+    It sends messages to a Slack channel
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            name=SLACK_ALERT_RESOURCE_IDENTIFIER,
+            icon="slack",
+            type=slack_notification_resource().name,
+            check_every="never",
+            source={"url": "((slack-url))", "disabled": "false"},
+            **kwargs,
+        )
+
+
+class OpenDiscussionsResource(Resource):
+    """
+    A Resource that uses the http-resource ResourceType to trigger API calls to open-discussions
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            name=OPEN_DISCUSSIONS_RESOURCE_IDENTIFIER,
+            icon="cloud-search",
+            type=HTTP_RESOURCE_TYPE_IDENTIFIER,
+            check_every="never",
+            source={
+                "url": f"{settings.OPEN_DISCUSSIONS_URL}/api/v0/ocw_next_webhook/",
+                "method": "POST",
+                "out_only": True,
+                "headers": {
+                    "Content-Type": "application/json",
+                },
+            },
+            **kwargs,
+        )
+
+
+class GitResource(Resource):
+    """
+    A Resource for interacting with git repositories
+    """
+
+    def __init__(self, name: Identifier, uri: str, branch: str, **kwagrs):
+        super().__init__(
+            name=name,
+            icon="git",
+            type="git",
+            source={"uri": uri, "branch": branch},
+            **kwagrs,
+        )
+
+
+class OcwStudioWebhookResource(Resource):
+    """
+    A Resource for making API calls ocw-studio to set a Website's status
+
+    args:
+        ocw_studio_url(str): The URL to the instance of ocw-studio to POST to
+        site_name(str): The name of the site the status is in reference to
+        api_token(str): The ocw-studio API token
+    """
+
+    def __init__(self, ocw_studio_url: str, site_name: str, api_token: str, **kwargs):
+        super().__init__(
+            name=OCW_STUDIO_WEBHOOK_RESOURCE_TYPE_IDENTIFIER,
+            icon="language-python",
+            type=HTTP_RESOURCE_TYPE_IDENTIFIER,
+            check_every="never",
+            source={
+                "url": f"{ocw_studio_url}/api/websites/{site_name}/pipeline_status/",
+                "method": "POST",
+                "out_only": True,
+                "headers": {
+                    "Content-Type": "application/json",
+                    "Authorization": f"Bearer {api_token}",
+                },
+            },
+            **kwargs,
+        )