Added cleanup routines / Cosign update

Author: dervoeti

tools/monitor-oci-artifacts/Dockerfile

@@ -8,8 +8,8 @@ ARG RELEASE="1"
 # Update image
 RUN microdnf install -y yum \
     && yum -y update-minimal --security --sec-severity=Important --sec-severity=Critical \
-    && curl -L -O https://github.com/sigstore/cosign/releases/download/v2.1.1/cosign-2.1.1.x86_64.rpm \
-    && rpm -ivh cosign-2.1.1.x86_64.rpm \
+    && curl -L -O https://github.com/sigstore/cosign/releases/download/v2.2.3/cosign-2.2.3-1.x86_64.rpm \
+    && rpm -ivh cosign-2.2.3-1.x86_64.rpm \
     && yum clean all \
     && microdnf clean all
 

tools/monitor-oci-artifacts/src/main.rs

@@ -43,19 +43,24 @@ impl std::ops::Deref for TagList {
     }
 }
 
+#[derive(Deserialize, Debug)]
+struct ArtifactReference {
+    child_digest: String,
+}
 #[derive(Deserialize, Debug)]
 struct Artifact {
     digest: String,
     manifest_media_type: String,
     media_type: String,
     tags: Option<TagList>,
+    references: Option<Vec<ArtifactReference>>,
 }
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let registry_hostname = "oci.stackable.tech";
     let base_url = format!("https://{}/api/v2.0", registry_hostname);
-    let page_size = 10;
+    let page_size = 20;
     let mut page = 1;
     let attestation_tag_regex = Regex::new(r"^sha256-[0-9a-f]{64}.att$").unwrap();
 
@@ -73,15 +78,33 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             if project_name == "sandbox" {
                 continue;
             }
-            let artifacts: Vec<Artifact> = reqwest::get(format!(
-                "{}/projects/{}/repositories/{}/artifacts",
-                base_url,
-                encode(project_name),
-                encode(repository_name)
-            ))
-            .await?
-            .json()
-            .await?;
+
+            let mut attestations: Vec<String> = Vec::with_capacity(32);
+            let mut potentially_attested_artifacts: Vec<&str> = Vec::with_capacity(32);
+            let mut artifacts: Vec<Artifact> = Vec::with_capacity(64);
+            let mut page = 1;
+            let page_size = 20;
+            loop {
+                let artifacts_page: Vec<Artifact> = reqwest::get(format!(
+                    "{}/projects/{}/repositories/{}/artifacts?page_size={}&page={}",
+                    base_url,
+                    encode(project_name),
+                    encode(repository_name),
+                    page_size,
+                    page
+                ))
+                .await?
+                .json()
+                .await?;
+
+                let number_of_returned_artifacts = artifacts_page.len();
+                artifacts.extend(artifacts_page);
+                if number_of_returned_artifacts < page_size {
+                    break;
+                }
+                page += 1;
+            }
+
             for artifact in &artifacts {
                 if artifact
                     .tags
@@ -99,15 +122,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 {
                     // it's an attestation, attestations artifacts themselves are not signed
                     println!(
-                        "skipping attestation {}{} ({})",
+                        "skipping attestation {} {} ({})",
                         repository_name,
                         artifact.digest,
                         artifact.tags.as_ref().unwrap()
                     );
+                    attestations.push(artifact.tags.as_ref().unwrap()[0].name.clone());
                     continue;
+                } else {
+                    potentially_attested_artifacts.push(&artifact.digest[7..71]);
                 }
 
-                if project_name == "sdp" && (repository_name != "git-sync" && repository_name != "ubi8-rust-builder") {
+                if project_name == "sdp"
+                    && (repository_name != "git-sync" && repository_name != "ubi8-rust-builder")
+                {
                     if artifact.manifest_media_type
                         != "application/vnd.docker.distribution.manifest.v2+json"
                     {
@@ -159,6 +187,24 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                     exit(cmd_output.status.code().unwrap_or(1));
                 }
             }
+
+            // remove dangling attestations
+            for attestation in attestations {
+                if !potentially_attested_artifacts.contains(&&attestation[7..71]) {
+                    println!("removing dangling attestation {}", attestation);
+                    reqwest::Client::new()
+                        .delete(format!(
+                            "{}/projects/{}/repositories/{}/artifacts/{}",
+                            base_url,
+                            encode(project_name),
+                            encode(repository_name),
+                            attestation
+                        ))
+                        .basic_auth("robot$stackable-cleanup", Some("XX"))
+                        .send()
+                        .await?;
+                }
+            }
         }
 
         if repositories.len() < page_size {
@@ -168,5 +214,51 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         page += 1;
     }
 
+    let latest_rust_builder_artifact: Artifact = reqwest::Client::new()
+        .get(format!(
+            "{}/projects/sdp/repositories/ubi8-rust-builder/artifacts/latest",
+            base_url,
+        ))
+        .send()
+        .await?
+        .json()
+        .await?;
+
+    let referenced_digests = latest_rust_builder_artifact
+        .references
+        .unwrap()
+        .into_iter()
+        .map(|reference| reference.child_digest)
+        .collect::<Vec<String>>();
+
+    let rust_builder_artifacts: Vec<Artifact> = reqwest::get(format!(
+        "{}/projects/sdp/repositories/ubi8-rust-builder/artifacts?page_size=100",
+        base_url
+    ))
+    .await?
+    .json()
+    .await?;
+
+    // keep "latest" and its referenced artifacts
+    for artifact in rust_builder_artifacts {
+        if artifact.digest != latest_rust_builder_artifact.digest
+            && !referenced_digests.contains(&artifact.digest)
+        {
+            println!(
+                "removing dangling rust builder artifact {}",
+                artifact.digest
+            );
+            reqwest::Client::new()
+                .delete(format!(
+                    "{}/projects/sdp/repositories/ubi8-rust-builder/artifacts/{}",
+                    base_url, artifact.digest
+                ))
+                .basic_auth("robot$stackable-cleanup", Some("XX"))
+                .send()
+                .await?;
+        }
+    }
+
     Ok(())
 }
+// cachebust