Remove libipld and fix image builder

.github/workflows/rust.yml

@@ -31,6 +31,9 @@ env:
   #PDS_REPO_SIGNING_KEY_K256_PRIVATE_KEY_HEX: ${{ secrets.PDS_REPO_SIGNING_KEY_K256_PRIVATE_KEY_HEX }}
   PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX: fb478b39dd2ddf84bef135dd60f90381903eefadbb9df4b18a2b9b174ae72582
   PDS_REPO_SIGNING_KEY_K256_PRIVATE_KEY_HEX: 71cfcf4882a6cff494c3d0affadd3858eb3a5838e7b5e15170e696a590a4fa01
+  # Docker build configuration
+  REGISTRY: ghcr.io
+  ORGANIZATION: blacksky-algorithms
 
 jobs:
   # First determine which workspace packages need to be processed
@@ -75,8 +78,8 @@ jobs:
       - name: Run cargo check for ${{ matrix.package }}
         run: cargo check -p ${{ matrix.package }}
 
-  # Parallel build job for each package
-  build:
+  # Parallel build and test job for each package
+  build-and-test:
     needs: [determine-packages, check]
     runs-on: ubuntu-latest
     if: ${{ needs.determine-packages.outputs.packages != '[]' }}
@@ -103,25 +106,8 @@ jobs:
           shared-key: ${{ matrix.package }}
       - name: Run cargo build for ${{ matrix.package }}
         run: cargo build --release -p ${{ matrix.package }}
-
-  # Parallel test job for each package
-  test:
-    needs: [determine-packages, check, build]
-    runs-on: ubuntu-latest
-    if: ${{ needs.determine-packages.outputs.packages != '[]' }}
-    strategy:
-      fail-fast: false
-      matrix:
-        package: ${{ fromJson(needs.determine-packages.outputs.packages) }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          shared-key: ${{ matrix.package }}
       - name: Run cargo test for ${{ matrix.package }}
-        run: cargo test -p ${{ matrix.package }}
+        run: cargo test --release -p ${{ matrix.package }}
 
   # Run formatting check on the entire workspace
   formatting:
@@ -134,10 +120,114 @@ jobs:
       - name: Run cargo fmt
         run: cargo fmt -- --check
 
+  # Job to build and push Docker images for specific packages
+  docker-build-push:
+    needs: [build-and-test, determine-packages]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    strategy:
+      fail-fast: false
+      matrix:
+        service:
+          - name: rsky-pds
+            dockerfile: rsky-pds/Dockerfile
+          - name: rsky-jetstream-subscriber
+            dockerfile: rsky-jetstream-subscriber/Dockerfile
+          - name: rsky-firehose
+            dockerfile: rsky-firehose/Dockerfile
+    # We need to run job-level if to check matrix values
+    if: ${{ always() }}
+    steps:
+      - name: Check if package was in determined packages
+        id: check-package
+        run: |
+          # Convert the JSON array to a space-separated string for easier checking
+          PACKAGES="${{ needs.determine-packages.outputs.packages }}"
+          # Remove brackets and quotes, replace commas with spaces
+          PACKAGES="${PACKAGES//[\[\]\"]/}"
+          PACKAGES="${PACKAGES//,/ }"
+
+          # Check if our service name is in the list
+          if [[ $PACKAGES == *"${{ matrix.service.name }}"* ]]; then
+            echo "should_run=true" >> $GITHUB_OUTPUT
+          else
+            echo "should_run=false" >> $GITHUB_OUTPUT
+            echo "Package ${{ matrix.service.name }} not in determined packages, skipping"
+          fi
+
+      - name: Checkout code
+        if: steps.check-package.outputs.should_run == 'true'
+        uses: actions/checkout@v4
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        if: steps.check-package.outputs.should_run == 'true'
+        uses: docker/setup-buildx-action@v3
+
+      # Log in to GitHub Container Registry
+      - name: Login to GitHub Container Registry
+        if: steps.check-package.outputs.should_run == 'true'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata for Docker
+      - name: Extract Docker metadata
+        if: steps.check-package.outputs.should_run == 'true'
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/${{ matrix.service.name }}
+          tags: |
+            # Always add the git commit SHA
+            type=sha,format=long
+            # Add 'main' tag if this is on the main branch
+            type=raw,value=main,enable=${{ github.ref == 'refs/heads/main' }}
+
+      # Check if Dockerfile exists for the package
+      - name: Check for Dockerfile
+        if: steps.check-package.outputs.should_run == 'true'
+        id: dockerfile-check
+        run: |
+          if [ -f "${{ matrix.service.dockerfile }}" ]; then
+            echo "dockerfile_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "No Dockerfile found at ${{ matrix.service.dockerfile }}, skipping Docker build"
+            echo "dockerfile_exists=false" >> $GITHUB_OUTPUT
+          fi
+
+      # Build and push the Docker image
+      - name: Build and push Docker image
+        if: steps.check-package.outputs.should_run == 'true' && steps.dockerfile-check.outputs.dockerfile_exists == 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ${{ matrix.service.dockerfile }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Generate artifact attestation
+      - name: Generate artifact attestation
+        if: steps.check-package.outputs.should_run == 'true' && steps.dockerfile-check.outputs.dockerfile_exists == 'true'
+        uses: actions/attest-build-provenance@v2
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/${{ matrix.service.name }}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true
+
   # Optional: Add a final job that depends on all tests to signal success
   ci-success:
     runs-on: ubuntu-latest
-    needs: [check, build, test, formatting]
+    needs: [check, build-and-test, formatting, docker-build-push]
     if: always()
     steps:
       - name: CI Success
@@ -147,4 +237,4 @@ jobs:
         if: ${{ contains(needs.*.result, 'failure') }}
         run: |
           echo "Some CI jobs failed!"
-          exit 1
+          exit 1

Cargo.toml

@@ -8,7 +8,7 @@ serde = { version = "1.0.160", features = ["derive"] }
 serde_derive = "^1.0"
 serde_ipld_dagcbor = {  version = "0.6.1" ,features = ["codec"]}
 lexicon_cid = { package = "cid", version = "0.11.1", features = ["serde-codec"] }
-libipld = "0.16.0"
+ipld-core = "0.4.2"
 serde_cbor = "0.11.2"
 serde_bytes = "0.11.15"
 tokio = { version = "1.28.2",features = ["full"] }

rsky-common/Cargo.toml

@@ -27,7 +27,7 @@ rsky-identity = {workspace = true}
 base64ct = "1.6.0"
 urlencoding = "2.1.3"
 futures = "0.3.28"
-libipld = {workspace = true}
+ipld-core = {workspace = true}
 multihash = "0.19"
 multihash-codetable = { version = "0.1.3",features = ["sha2"]}
 indexmap = { version = "1.9.3",features = ["serde-1"] }

rsky-common/src/ipld.rs

@@ -1,13 +1,14 @@
 use anyhow::Result;
+use ipld_core::codec::Codec;
 use lexicon_cid::Cid;
-use libipld::codec::Codec;
-use libipld::raw::RawCodec;
 use multihash::Multihash;
 use serde::Serialize;
 use sha2::{Digest, Sha256};
 
 const SHA2_256: u64 = 0x12;
 const DAGCBORCODEC: u64 = 0x71;
+// https://docs.rs/libipld-core/0.16.0/src/libipld_core/raw.rs.html#19
+const RAWCODEC: u64 = 0x77;
 
 pub fn cid_for_cbor<T: Serialize>(data: &T) -> Result<Cid> {
     let bytes = crate::struct_to_cbor(data)?;
@@ -21,17 +22,10 @@ pub fn cid_for_cbor<T: Serialize>(data: &T) -> Result<Cid> {
     Ok(cid)
 }
 
-pub fn sha256_to_cid<T: Codec>(hash: Vec<u8>, codec: T) -> Cid
-where
-    u64: From<T>,
-{
+pub fn sha256_to_cid(hash: Vec<u8>) -> Cid {
     let cid = Cid::new_v1(
-        u64::from(codec),
+        RAWCODEC,
         Multihash::<64>::wrap(SHA2_256, hash.as_slice()).unwrap(),
     );
     cid
 }
-
-pub fn sha256_raw_to_cid(hash: Vec<u8>) -> Cid {
-    sha256_to_cid(hash, RawCodec)
-}

rsky-firehose/Dockerfile

@@ -1,13 +1,56 @@
 # Use the official Rust image.
 # https://hub.docker.com/_/rust
-FROM --platform=linux/amd64 rust
+# FROM rust
+
+# # Copy local code to the container image.
+# WORKDIR /usr/src/rsky
+# COPY . .
+
+# # Install production dependencies and build a release artifact.
+# RUN cargo build --release --package rsky-firehose
+
+# # Run the web service on container startup.
+# CMD["cargo", "run", "--package", "rsky-firehose"]
+
+# Use the official Rust image.
+# https://hub.docker.com/_/rust
+FROM rust AS builder
 
 # Copy local code to the container image.
 WORKDIR /usr/src/rsky
-COPY . .
 
-# Install production dependencies and build a release artifact.
-RUN cargo build --release --package rsky-firehose
+COPY Cargo.toml rust-toolchain ./
+COPY cypher cypher
+COPY rsky-common rsky-common
+COPY rsky-crypto rsky-crypto
+COPY rsky-feedgen rsky-feedgen
+COPY rsky-firehose/Cargo.toml rsky-firehose/Cargo.toml
+COPY rsky-identity rsky-identity
+COPY rsky-jetstream-subscriber rsky-jetstream-subscriber
+COPY rsky-labeler rsky-labeler
+COPY rsky-lexicon rsky-lexicon
+COPY rsky-pds rsky-pds
+COPY rsky-relay rsky-relay
+COPY rsky-repo rsky-repo
+COPY rsky-satnav rsky-satnav
+COPY rsky-syntax rsky-syntax
+
+
+# Create an empty src directory to trick Cargo into thinking it's a valid Rust project
+RUN mkdir rsky-firehose/src && echo "fn main() {}" > rsky-firehose/src/main.rs
+
+## Install production dependencies and build a release artifact.
+RUN cargo build --release --package rsky-jetstream-subscriber
+
+COPY rsky-jetstream-subscriber/src rsky-jetstream-subscriber/src
+
+RUN cargo build --release --package rsky-jetstream-subscriber
+
+
+FROM rust
+
+WORKDIR /usr/src/rsky
+
+COPY --from=builder /usr/src/rsky/target/release/rsky-firehose rsky-firehose
 
-# Run the web service on container startup.
-CMD cargo run --package rsky-firehose
+CMD ["./rsky-firehose"]