Merge pull request #836 from sigstore/store-state-in-updater

Store meta state in memory

Author: loosebazooka

sigstore-java/src/main/java/dev/sigstore/tuf/FileSystemTufStore.java

@@ -28,21 +28,21 @@
 import java.util.Optional;
 
 /** Uses a local file system directory to store the trusted TUF metadata. */
-public class FileSystemTufStore implements MutableTufStore {
+public class FileSystemTufStore implements MetaStore, TargetStore {
 
   private static final String ROOT_FILE_NAME = "root.json";
   private static final String SNAPSHOT_FILE_NAME = "snapshot.json";
   private static final String TIMESTAMP_FILE_NAME = "timestamp.json";
-  private Path repoBaseDir;
-  private Path targetsCache;
+  private final Path repoBaseDir;
+  private final Path targetsCache;
 
   @VisibleForTesting
   FileSystemTufStore(Path repoBaseDir, Path targetsCache) {
     this.repoBaseDir = repoBaseDir;
     this.targetsCache = targetsCache;
   }
 
-  public static MutableTufStore newFileSystemStore(Path repoBaseDir) throws IOException {
+  public static FileSystemTufStore newFileSystemStore(Path repoBaseDir) throws IOException {
     if (!Files.isDirectory(repoBaseDir)) {
       throw new IllegalArgumentException(repoBaseDir + " must be a file system directory.");
     }
@@ -53,7 +53,7 @@ public static MutableTufStore newFileSystemStore(Path repoBaseDir) throws IOExce
     return newFileSystemStore(repoBaseDir, defaultTargetsCache);
   }
 
-  public static MutableTufStore newFileSystemStore(Path repoBaseDir, Path targetsCache) {
+  public static FileSystemTufStore newFileSystemStore(Path repoBaseDir, Path targetsCache) {
     if (!Files.isDirectory(repoBaseDir)) {
       throw new IllegalArgumentException(repoBaseDir + " must be a file system directory.");
     }
@@ -65,50 +65,26 @@ public static MutableTufStore newFileSystemStore(Path repoBaseDir, Path targetsC
 
   @Override
   public String getIdentifier() {
-    return repoBaseDir.toAbsolutePath().toString();
+    return "Meta: " + repoBaseDir.toAbsolutePath() + ", Targets:" + targetsCache.toAbsolutePath();
   }
 
   @Override
-  public Optional<Root> loadTrustedRoot() throws IOException {
-    return loadRole(RootRole.ROOT, Root.class);
-  }
-
-  @Override
-  public Optional<Timestamp> loadTimestamp() throws IOException {
-    return loadRole(RootRole.TIMESTAMP, Timestamp.class);
-  }
-
-  @Override
-  public Optional<Snapshot> loadSnapshot() throws IOException {
-    return loadRole(RootRole.SNAPSHOT, Snapshot.class);
-  }
-
-  @Override
-  public Optional<Targets> loadTargets() throws IOException {
-    return loadRole(RootRole.TARGETS, Targets.class);
-  }
-
-  @Override
-  public Optional<Targets> loadDelegatedTargets(String roleName) throws IOException {
-    return loadRole(roleName, Targets.class);
-  }
-
-  @Override
-  public void storeTargetFile(String targetName, byte[] targetContents) throws IOException {
+  public void writeTarget(String targetName, byte[] targetContents) throws IOException {
     Files.write(targetsCache.resolve(targetName), targetContents);
   }
 
   @Override
-  public byte[] getTargetFile(String targetName) throws IOException {
+  public byte[] readTarget(String targetName) throws IOException {
     return Files.readAllBytes(targetsCache.resolve(targetName));
   }
 
   @Override
-  public void storeMeta(String roleName, SignedTufMeta<?> meta) throws IOException {
+  public void writeMeta(String roleName, SignedTufMeta<?> meta) throws IOException {
     storeRole(roleName, meta);
   }
 
-  <T extends SignedTufMeta<?>> Optional<T> loadRole(String roleName, Class<T> tClass)
+  @Override
+  public <T extends SignedTufMeta<?>> Optional<T> readMeta(String roleName, Class<T> tClass)
       throws IOException {
     Path roleFile = repoBaseDir.resolve(roleName + ".json");
     if (!roleFile.toFile().exists()) {
@@ -125,8 +101,8 @@ <T extends SignedTufMeta<?>> void storeRole(String roleName, T role) throws IOEx
   }
 
   @Override
-  public void storeTrustedRoot(Root root) throws IOException {
-    Optional<Root> trustedRoot = loadTrustedRoot();
+  public void writeRoot(Root root) throws IOException {
+    Optional<Root> trustedRoot = readMeta(RootRole.ROOT, Root.class);
     if (trustedRoot.isPresent()) {
       try {
         Files.move(

sigstore-java/src/main/java/dev/sigstore/tuf/MetaReader.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2024 The Sigstore Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package dev.sigstore.tuf;
+
+import dev.sigstore.tuf.model.SignedTufMeta;
+import dev.sigstore.tuf.model.TufMeta;
+import java.io.IOException;
+import java.util.Optional;
+
+/** Interface that defines reading meta from local storage. */
+public interface MetaReader {
+
+  /**
+   * Return a named metadata item if there is any.
+   *
+   * @param roleName the name of the role to load (root, timestamp, snapshot, targets, or a
+   *     delegated target role)
+   * @param tClass the class type
+   * @return an instance of the signed metadata for the role if it was found
+   * @throws IOException if an error occurs reading from the backing store
+   */
+  <T extends SignedTufMeta<? extends TufMeta>> Optional<T> readMeta(
+      String roleName, Class<T> tClass) throws IOException;
+}

sigstore-java/src/main/java/dev/sigstore/tuf/MutableTufStore.java renamed to sigstore-java/src/main/java/dev/sigstore/tuf/MetaStore.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 The Sigstore Authors.
+ * Copyright 2024 The Sigstore Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,28 +15,30 @@
  */
 package dev.sigstore.tuf;
 
-import dev.sigstore.tuf.model.*;
+import dev.sigstore.tuf.model.Root;
+import dev.sigstore.tuf.model.SignedTufMeta;
+import dev.sigstore.tuf.model.TufMeta;
 import java.io.IOException;
 
-/** Defines the set of actions needed to support a local repository of TUF metadata. */
-public interface MutableTufStore extends TufStore {
+/** Interface that defines a mutable meta store functionality. */
+public interface MetaStore extends MetaReader {
+
   /**
-   * Writes a TUF target to the local target store.
-   *
-   * @param targetName the name of the target file to write (e.g. ctfe.pub)
-   * @param targetContents the content of the target file as bytes
-   * @throws IOException if an error occurs
+   * A generic string for identifying the local store in debug messages. A file system based
+   * implementation might return the path being used for storage, while an in-memory store may just
+   * return something like 'in-memory'.
    */
-  void storeTargetFile(String targetName, byte[] targetContents) throws IOException;
+  String getIdentifier();
 
   /**
-   * Generic method to store one of the {@link SignedTufMeta} resources in the local tuf store.
+   * Generic method to store one of the {@link SignedTufMeta} resources in the local tuf store. Do
+   * not use for Root role, use {@link #writeRoot(Root)} instead.
    *
    * @param roleName the name of the role
    * @param meta the metadata to store
    * @throws IOException if writing the resource causes an IO error
    */
-  void storeMeta(String roleName, SignedTufMeta<?> meta) throws IOException;
+  void writeMeta(String roleName, SignedTufMeta<? extends TufMeta> meta) throws IOException;
 
   /**
    * Once you have ascertained that your root is trustworthy use this method to persist it to your
@@ -49,7 +51,7 @@ public interface MutableTufStore extends TufStore {
    * @see <a
    *     href="https://theupdateframework.github.io/specification/latest/#detailed-client-workflow">5.3.8</a>
    */
-  void storeTrustedRoot(Root root) throws IOException;
+  void writeRoot(Root root) throws IOException;
 
   /**
    * This clears out the snapshot and timestamp metadata from the store, as required when snapshot

sigstore-java/src/main/java/dev/sigstore/tuf/PassthroughCacheMetaStore.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2024 The Sigstore Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package dev.sigstore.tuf;
+
+import dev.sigstore.tuf.model.Root;
+import dev.sigstore.tuf.model.RootRole;
+import dev.sigstore.tuf.model.SignedTufMeta;
+import dev.sigstore.tuf.model.TufMeta;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+/** An in memory cache that will pass through to a provided local tuf store. */
+public class PassthroughCacheMetaStore implements MetaReader, MetaStore {
+  private final MetaStore localStore;
+  private final Map<String, SignedTufMeta<? extends TufMeta>> cache;
+
+  private PassthroughCacheMetaStore(MetaStore localStore) {
+    this.localStore = localStore;
+    this.cache = new HashMap<>();
+  }
+
+  @Override
+  public String getIdentifier() {
+    return "In memory cache backed by: " + localStore.getIdentifier();
+  }
+
+  public static PassthroughCacheMetaStore newPassthroughMetaCache(MetaStore localStore) {
+    return new PassthroughCacheMetaStore(localStore);
+  }
+
+  @Override
+  public void writeRoot(Root root) throws IOException {
+    // call writeRoot instead of generic writeMeta because it may do extra work when storing on disk
+    localStore.writeRoot(root);
+    cache.put(RootRole.ROOT, root);
+  }
+
+  @Override
+  @SuppressWarnings("unchecked")
+  public <T extends SignedTufMeta<? extends TufMeta>> Optional<T> readMeta(
+      String roleName, Class<T> tClass) throws IOException {
+    // check memory cache
+    if (cache.containsKey(roleName)) {
+      return Optional.of((T) cache.get(roleName));
+    }
+
+    // check backing storage and write to memory if found
+    var value = localStore.readMeta(roleName, tClass);
+    value.ifPresent(v -> cache.put(roleName, v));
+
+    return value;
+  }
+
+  @Override
+  public void writeMeta(String roleName, SignedTufMeta<? extends TufMeta> meta) throws IOException {
+    if (Objects.equals(roleName, RootRole.ROOT)) {
+      throw new IllegalArgumentException("Calling writeMeta on root instead of writeRoot");
+    }
+    localStore.writeMeta(roleName, meta);
+    cache.put(roleName, meta);
+  }
+
+  @Override
+  public void clearMetaDueToKeyRotation() throws IOException {
+    localStore.clearMetaDueToKeyRotation();
+    cache.remove(RootRole.TIMESTAMP);
+    cache.remove(RootRole.SNAPSHOT);
+  }
+}

sigstore-java/src/main/java/dev/sigstore/tuf/SigstoreTufClient.java

@@ -130,14 +130,18 @@ public SigstoreTufClient build() throws IOException {
           remoteMirror.toString().endsWith("/")
               ? remoteMirror
               : new URL(remoteMirror.toExternalForm() + "/");
-      var targetsLocation = new URL(normalizedRemoteMirror.toExternalForm() + "targets");
+      var remoteTargetsLocation = new URL(normalizedRemoteMirror.toExternalForm() + "targets");
+      var filesystemTufStore = FileSystemTufStore.newFileSystemStore(tufCacheLocation);
       var tufUpdater =
           Updater.builder()
               .setTrustedRootPath(trustedRoot)
-              .setLocalStore(FileSystemTufStore.newFileSystemStore(tufCacheLocation))
+              .setTrustedMetaStore(
+                  TrustedMetaStore.newTrustedMetaStore(
+                      PassthroughCacheMetaStore.newPassthroughMetaCache(filesystemTufStore)))
+              .setTargetStore(filesystemTufStore)
               .setMetaFetcher(
                   MetaFetcher.newFetcher(HttpFetcher.newFetcher(normalizedRemoteMirror)))
-              .setTargetFetcher(HttpFetcher.newFetcher(targetsLocation))
+              .setTargetFetcher(HttpFetcher.newFetcher(remoteTargetsLocation))
               .build();
       return new SigstoreTufClient(tufUpdater, cacheValidity);
     }
@@ -166,7 +170,7 @@ public void forceUpdate()
     JsonFormat.parser()
         .merge(
             new String(
-                updater.getLocalStore().getTargetFile(TRUST_ROOT_FILENAME), StandardCharsets.UTF_8),
+                updater.getTargetStore().readTarget(TRUST_ROOT_FILENAME), StandardCharsets.UTF_8),
             trustedRootBuilder);
     sigstoreTrustedRoot = SigstoreTrustedRoot.from(trustedRootBuilder.build());
   }

sigstore-java/src/main/java/dev/sigstore/tuf/TargetReader.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2024 The Sigstore Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package dev.sigstore.tuf;
+
+import java.io.IOException;
+
+/** Interface that defines reading targets from local storage. */
+public interface TargetReader {
+
+  /**
+   * Reads a TUF target file from the local TUF store
+   *
+   * @param targetName the name of the target file to read (e.g. ctfe.pub)
+   * @return the content of the file as bytes
+   * @throws IOException if an error occurs
+   */
+  byte[] readTarget(String targetName) throws IOException;
+}

sigstore-java/src/main/java/dev/sigstore/tuf/TargetStore.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2024 The Sigstore Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package dev.sigstore.tuf;
+
+import java.io.IOException;
+
+/** Interface that defines a mutable target store functionality. */
+public interface TargetStore extends TargetReader {
+
+  /**
+   * A generic string for identifying the local store in debug messages. A file system based
+   * implementation might return the path being used for storage, while an in-memory store may just
+   * return something like 'in-memory'.
+   */
+  String getIdentifier();
+
+  /**
+   * Writes a TUF target to the local target store.
+   *
+   * @param targetName the name of the target file to write (e.g. ctfe.pub)
+   * @param targetContents the content of the target file as bytes
+   * @throws IOException if an error occurs
+   */
+  void writeTarget(String targetName, byte[] targetContents) throws IOException;
+}