⚠️ change catalog-specific URL from full path to API endpoint ref (#429)

* change catalog-specific URL from full path to API endpoint ref

solves #427 and implements phase 1 of the CatalogD expandable interface.
This phase just moves from `status.contentURL` to `status.baseURL`, and
will be used to provide reference to the catalog-specific API instead of
the full path to JSONLines-formatted content, plus tests and
documentation.

Signed-off-by: Jordan Keister <[email protected]>

* sticking to catalog-centric reference, and updated type

Signed-off-by: Jordan Keister <[email protected]>

* remove some magic numbers

Signed-off-by: Jordan Keister <[email protected]>

* review reconciliation

Signed-off-by: Jordan Keister <[email protected]>

---------

Signed-off-by: Jordan Keister <[email protected]>

Author: grokspawn

README.md

@@ -39,18 +39,18 @@ Procedure steps marked with an asterisk (`*`) are likely to change with future A
     *Example output*
     ```sh
     Name:         operatorhubio
-    Namespace:    
+    Namespace:
     Labels:       olm.operatorframework.io/metadata.name=operatorhubio
     Annotations:  <none>
     API Version:  olm.operatorframework.io/v1alpha1
     Kind:         ClusterCatalog
     Metadata:
-      Creation Timestamp:  2024-09-12T13:37:04Z
+      Creation Timestamp:  2024-10-17T13:48:46Z
       Finalizers:
         olm.operatorframework.io/delete-server-cache
       Generation:        1
-      Resource Version:  961
-      UID:               fa6bb9cf-1a36-4189-a7a0-83284c3f6f55
+      Resource Version:  7908
+      UID:               34eeaa91-9f8e-4254-9937-0ae9d25e92df
     Spec:
       Priority:  0
       Source:
@@ -60,38 +60,38 @@ Procedure steps marked with an asterisk (`*`) are likely to change with future A
         Type:             Image
     Status:
       Conditions:
-        Last Transition Time:  2024-09-12T13:37:53Z
-        Message:               Successfully unpacked and stored content from quay.io/operatorhubio/catalog:latest
+        Last Transition Time:  2024-10-17T13:48:59Z
+        Message:               Successfully unpacked and stored content from resolved source
+        Observed Generation:   1
         Reason:                Succeeded
         Status:                False
         Type:                  Progressing
-        Last Transition Time:  2024-09-12T13:37:53Z
-        Message:               Content from quay.io/operatorhubio/catalog:latest is being served
+        Last Transition Time:  2024-10-17T13:48:59Z
+        Message:               Serving desired content from resolved source
+        Observed Generation:   1
         Reason:                Available
         Status:                True
         Type:                  Serving
-      Content URL:             https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/all.json
-      Last Unpacked:           2024-09-12T13:37:52Z
-      Observed Generation:     1
+      Last Unpacked:           2024-10-17T13:48:58Z
       Resolved Source:
         Image:
-          Last Poll Attempt:  2024-09-12T13:37:52Z
-          Last Unpacked:      2024-09-12T13:37:52Z
-          Ref:                quay.io/operatorhubio/catalog:latest
-          Resolved Ref:       quay.io/operatorhubio/catalog@sha256:4453a361198d39d0390fd8c1a7f07b5a5a3ae1e8dac9979ef0c4eba46299df16
-        Type:                 Image
-   Events:                   <none>
+          Last Successful Poll Attempt:  2024-10-17T14:49:59Z
+          Ref:                           quay.io/operatorhubio/catalog@sha256:82be554b15ff246d8cc428f8d2f4cf5857c02ce3225d95d92a769ea3095e1fc7
+        Type:                            Image
+      Urls:
+        Base:  https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio
+    Events:    <none>
    ```
 
 1. Port forward the `catalogd-service` service in the `olmv1-system` namespace:
     ```sh
     $ kubectl -n olmv1-system port-forward svc/catalogd-service 8080:443
     ```
 
-1. Run the following command to get a list of packages:
+1. Access the `v1/all` service endpoint and filter the results to a list of packages by running the following command:
 
     ```sh
-    $ curl -k https://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.package") | .name'
+    $ curl https://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.package") | .name'
     ```
 
     *Example output*
@@ -118,7 +118,7 @@ Procedure steps marked with an asterisk (`*`) are likely to change with future A
 1. Run the following command to get a list of channels for the `ack-acm-controller` package:
 
     ```sh
-    $ curl -k https://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "ack-acm-controller") | .name'
+    $ curl https://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "ack-acm-controller") | .name'
     ```
 
     *Example output*
@@ -132,7 +132,7 @@ Procedure steps marked with an asterisk (`*`) are likely to change with future A
 1. Run the following command to get a list of bundles belonging to the `ack-acm-controller` package:
 
     ```sh
-    $ curl -k https://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "ack-acm-controller") | .name'
+    $ curl https://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "ack-acm-controller") | .name'
     ```
     
     *Example output*
@@ -153,7 +153,7 @@ Thanks for your interest in contributing to `catalogd`!
 
 `catalogd` is in the very early stages of development and a more in depth contributing guide will come in the near future.
 
-In the mean time, it is assumed you know how to make contributions to open source projects in general and this guide will only focus on how to manually test your changes (no automated testing yet).
+In the meantime, it is assumed you know how to make contributions to open source projects in general and this guide will only focus on how to manually test your changes (no automated testing yet).
 
 If you have any questions, feel free to reach out to us on the Kubernetes Slack channel [#olm-dev](https://kubernetes.slack.com/archives/C0181L6JYQ2) or [create an issue](https://github.com/operator-framework/catalogd/issues/new)
 ### Testing Local Changes

api/core/v1alpha1/clustercatalog_types.go

@@ -147,17 +147,24 @@ type ClusterCatalogStatus struct {
 	//  type: Image
 	// +optional
 	ResolvedSource *ResolvedCatalogSource `json:"resolvedSource,omitempty"`
-	// contentURL is a cluster-internal URL from which on-cluster components
-	// can read the content of a catalog
+	// urls contains the URLs that can be used to access the catalog.
 	// +optional
-	ContentURL string `json:"contentURL,omitempty"`
-
+	URLs *ClusterCatalogURLs `json:"urls,omitempty"`
 	// lastUnpacked represents the time when the
 	// ClusterCatalog object was last unpacked successfully.
 	// +optional
 	LastUnpacked metav1.Time `json:"lastUnpacked,omitempty"`
 }
 
+// ClusterCatalogURLs contains the URLs that can be used to access the catalog.
+type ClusterCatalogURLs struct {
+	// base is a required cluster-internal URL which provides API access for this ClusterCatalog.
+	// A suffix API access path can be added to retrieve catalog data for the ClusterCatalog.
+	// Currently, a 'v1' API access provides complete FBC retrival via the path "/api/v1/all", with the general form `{base}/api/v1/all`.
+	// +kubebuilder:validation:Required
+	Base string `json:"base"`
+}
+
 // CatalogSource is a discriminated union of possible sources for a Catalog.
 // CatalogSource contains the sourcing information for a Catalog
 // +union

api/core/v1alpha1/zz_generated.deepcopy.go

@@ -258,7 +258,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	localStorage = storage.LocalDir{RootDir: storeDir, BaseURL: baseStorageURL}
+	localStorage = storage.LocalDirV1{RootDir: storeDir, RootURL: baseStorageURL}
 
 	// Config for the the catalogd web server
 	catalogServerConfig := serverutil.CatalogServerConfig{

cmd/manager/main.go

@@ -221,11 +221,6 @@ spec:
                   - type
                   type: object
                 type: array
-              contentURL:
-                description: |-
-                  contentURL is a cluster-internal URL from which on-cluster components
-                  can read the content of a catalog
-                type: string
               lastUnpacked:
                 description: |-
                   lastUnpacked represents the time when the
@@ -278,6 +273,19 @@ spec:
                 x-kubernetes-validations:
                 - message: source type 'Image' requires image field
                   rule: self.type == 'Image' && has(self.image)
+              urls:
+                description: urls contains the URLs that can be used to access the
+                  catalog.
+                properties:
+                  base:
+                    description: |-
+                      base is a required cluster-internal URL which provides API access for this ClusterCatalog.
+                      A suffix API access path can be added to retrieve catalog data for the ClusterCatalog.
+                      Currently, a 'v1' API access provides complete FBC retrival via the path "/api/v1/all", with the general form `{base}/api/v1/all`.
+                    type: string
+                required:
+                - base
+                type: object
             type: object
         required:
         - metadata

config/base/crd/bases/olm.operatorframework.io_clustercatalogs.yaml

@@ -1,8 +1,9 @@
 # `ClusterCatalog` Interface
-`catalogd` serves catalog content via an HTTP(S) endpoint
+`catalogd` serves catalog content via a catalog-specific, versioned HTTP(S) endpoint. Clients access catalog information via this API endpoint and a versioned reference of the desired format. Current support includes only a complete catalog download, indicated by the path "api/v1/all", for example if `status.urls.base` is `https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio` then `https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/api/vi/all` would receive the complete FBC for the catalog `operatorhubio`.
+
 
 ## Response Format
-`catalogd` responses are encoded as a [JSON Lines](https://jsonlines.org/) stream of File-Based Catalog (FBC) [Meta](https://olm.operatorframework.io/docs/reference/file-based-catalogs/#schema) objects delimited by newlines.
+`catalogd` responses retrieved via the catalog-specific v1 API are encoded as a [JSON Lines](https://jsonlines.org/) stream of File-Based Catalog (FBC) [Meta](https://olm.operatorframework.io/docs/reference/file-based-catalogs/#schema) objects delimited by newlines.
 
 ### Example
 For an example JSON-encoded FBC snippet
@@ -79,30 +80,32 @@ For example purposes we make the following assumption:
 For local development, consider skipping TLS verification, such as `curl -k`, or reference external material
 on self-signed certificate verification.
 
-`ClusterCatalog` CRs have a status.contentURL field whose value is the location where the content 
-of a catalog can be read from:
+`ClusterCatalog` CRs have a `status.urls.base` field which identifies the catalog-specific API to access the catalog content:
 
 ```yaml
   status:
     .
     .
-    contentURL: https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/all.json
+    urls:
+        base: https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio
     resolvedSource:
       image:
         ref: quay.io/operatorhubio/catalog@sha256:e53267559addc85227c2a7901ca54b980bc900276fc24d3f4db0549cb38ecf76
       type: Image
 ```
 
-
 ## On cluster
 
-When making a request for the contents of the `operatorhubio` `ClusterCatalog` from within
-the cluster issue a HTTP `GET` request to 
-`https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/all.json`
+When making a request for the complete contents of the `operatorhubio` `ClusterCatalog` from within
+the cluster, clients would combine `status.urls.base` with the desired API service and issue an HTTP GET request for the URL.
+
+For example, to receive the complete catalog data for the `operatorhubio` catalog indicated above, the client would append the service point designator `api/v1/all`, like:
+
+`https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/api/v1/all`.
 
 An example command to run a `Pod` to `curl` the catalog contents:
 ```sh
-kubectl run fetcher --image=curlimages/curl:latest -- curl https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/all.json
+kubectl run fetcher --image=curlimages/curl:latest -- curl https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/api/v1/all
 ```
 
 ## Off cluster
@@ -115,11 +118,11 @@ kubectl -n olmv1-system port-forward svc/catalogd-service 8080:443
 ```
 
 Once the service has been successfully forwarded to a localhost port, issue a HTTP `GET`
-request to `https://localhost:8080/catalogs/operatorhubio/all.json`
+request to `https://localhost:8080/catalogs/operatorhubio/api/v1/all`
 
 An example `curl` request that assumes the port-forwarding is mapped to port 8080 on the local machine:
 ```sh
-curl http://localhost:8080/catalogs/operatorhubio/all.json
+curl http://localhost:8080/catalogs/operatorhubio/api/v1/all
 ```
 
 # Fetching `ClusterCatalog` contents from the `Catalogd` Service outside of the cluster
@@ -190,7 +193,7 @@ This section outlines a way of exposing the `Catalogd` Service's endpoints outsi
 1. Run the below example `curl` request to retrieve all of the catalog contents:
 
     ```sh
-      $ curl https://<address>/catalogs/operatorhubio/all.json
+      $ curl https://<address>/catalogs/operatorhubio/api/v1/all
     ```
 
     To obtain `address` of the ingress object, you can run the below command and look for the value in the `ADDRESS` field from output: 

docs/fetching-catalog-contents.md

@@ -11,6 +11,7 @@ sleep 10
 kubectl apply -f https://github.com/operator-framework/catalogd/releases/latest/download/catalogd.yaml
 kubectl wait --for=condition=Available -n olmv1-system deploy/catalogd-controller-manager --timeout=60s
 sleep 10
+
 # inspect crds (catalog)
 kubectl get crds -A
 
@@ -19,15 +20,15 @@ kubectl apply -f config/samples/core_v1alpha1_catalog.yaml
 # shows catalog-sample
 kubectl get catalog -A 
 # waiting for catalog to report ready status
-time kubectl wait --for=condition=Unpacked catalog/operatorhubio --timeout=1m
+time kubectl wait --for=condition=Serving catalog/operatorhubio --timeout=1m
 
 # port forward the catalogd-service service to interact with the HTTP server serving catalog contents
 (kubectl -n olmv1-system port-forward svc/catalogd-service 8080:80)&
 
 # check what 'packages' are available in this catalog
-curl http://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.package") | .name'
+curl http://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.package") | .name'
 # check what channels are included in the wavefront package
-curl http://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "wavefront") | .name'
+curl http://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "wavefront") | .name'
 # check what bundles are included in the wavefront package
-curl http://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "wavefront") | .name'
+curl http://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "wavefront") | .name'
 

hack/scripts/demo-script.sh

@@ -16,13 +16,13 @@ time kubectl wait --for=condition=Unpacked clustercatalog/operatorhubio --timeou
 sleep 5
 
 # retrieve catalog as plaintext JSONlines
-curl -k -vvv https://localhost:8080/catalogs/operatorhubio/all.json --output /tmp/cat-content.json
+curl -k -vvv https://localhost:8080/catalogs/operatorhubio/api/v1/all --output /tmp/cat-content.json
 
 # advertise handling of compressed content
-curl -vvv -k https://localhost:8080/catalogs/operatorhubio/all.json -H 'Accept-Encoding: gzip' --output /tmp/cat-content.gz
+curl -vvv -k https://localhost:8080/catalogs/operatorhubio/api/v1/all -H 'Accept-Encoding: gzip' --output /tmp/cat-content.gz
 
 # let curl handle the compress/decompress for us
-curl -vvv --compressed -k https://localhost:8080/catalogs/operatorhubio/all.json --output /tmp/cat-content-decompressed.txt
+curl -vvv --compressed -k https://localhost:8080/catalogs/operatorhubio/api/v1/all --output /tmp/cat-content-decompressed.txt
 
 # show that there's no content change with changed format
 diff /tmp/cat-content.json /tmp/cat-content-decompressed.txt

hack/scripts/gzip-demo-script.sh

@@ -159,7 +159,7 @@ func (r *ClusterCatalogReconciler) SetupWithManager(mgr ctrl.Manager) error {
 func (r *ClusterCatalogReconciler) reconcile(ctx context.Context, catalog *v1alpha1.ClusterCatalog) (ctrl.Result, error) {
 	l := log.FromContext(ctx)
 	// Check if the catalog availability is set to disabled, if true then
-	// unset content URL, delete it from the cache and set appropriate status
+	// unset base URL, delete it from the cache and set appropriate status
 	if catalog.Spec.Availability == v1alpha1.AvailabilityDisabled {
 		// Delete the catalog from local cache
 		err := r.deleteCatalogCache(ctx, catalog)
@@ -244,10 +244,10 @@ func (r *ClusterCatalogReconciler) reconcile(ctx context.Context, catalog *v1alp
 			updateStatusProgressing(&catalog.Status, catalog.GetGeneration(), storageErr)
 			return ctrl.Result{}, storageErr
 		}
-		contentURL := r.Storage.ContentURL(catalog.Name)
+		baseURL := r.Storage.BaseURL(catalog.Name)
 
 		updateStatusProgressing(&catalog.Status, catalog.GetGeneration(), nil)
-		updateStatusServing(&catalog.Status, *unpackResult, contentURL, catalog.GetGeneration())
+		updateStatusServing(&catalog.Status, *unpackResult, baseURL, catalog.GetGeneration())
 	default:
 		panic(fmt.Sprintf("unknown unpack state %q", unpackResult.State))
 	}
@@ -271,7 +271,7 @@ func (r *ClusterCatalogReconciler) getCurrentState(catalog *v1alpha1.ClusterCata
 	// Set expected status based on what we see in the stored catalog
 	clearUnknownConditions(expectedStatus)
 	if hasStoredCatalog && r.Storage.ContentExists(catalog.Name) {
-		updateStatusServing(expectedStatus, storedCatalog.unpackResult, r.Storage.ContentURL(catalog.Name), storedCatalog.observedGeneration)
+		updateStatusServing(expectedStatus, storedCatalog.unpackResult, r.Storage.BaseURL(catalog.Name), storedCatalog.observedGeneration)
 		updateStatusProgressing(expectedStatus, storedCatalog.observedGeneration, nil)
 	}
 
@@ -323,9 +323,12 @@ func updateStatusProgressing(status *v1alpha1.ClusterCatalogStatus, generation i
 	meta.SetStatusCondition(&status.Conditions, progressingCond)
 }
 
-func updateStatusServing(status *v1alpha1.ClusterCatalogStatus, result source.Result, contentURL string, generation int64) {
+func updateStatusServing(status *v1alpha1.ClusterCatalogStatus, result source.Result, baseURL string, generation int64) {
 	status.ResolvedSource = result.ResolvedSource
-	status.ContentURL = contentURL
+	if status.URLs == nil {
+		status.URLs = &v1alpha1.ClusterCatalogURLs{}
+	}
+	status.URLs.Base = baseURL
 	status.LastUnpacked = metav1.NewTime(result.UnpackTime)
 	meta.SetStatusCondition(&status.Conditions, metav1.Condition{
 		Type:               v1alpha1.TypeServing,
@@ -349,7 +352,7 @@ func updateStatusCatalogDisabled(status *v1alpha1.ClusterCatalogStatus, generati
 
 func updateStatusNotServing(status *v1alpha1.ClusterCatalogStatus, generation int64) {
 	status.ResolvedSource = nil
-	status.ContentURL = ""
+	status.URLs = nil
 	status.LastUnpacked = metav1.Time{}
 	meta.SetStatusCondition(&status.Conditions, metav1.Condition{
 		Type:               v1alpha1.TypeServing,