Skip to content

Serializing new salt fields #449

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aulme merged 5 commits into from
Apr 29, 2025
Merged

Serializing new salt fields #449

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6ee04a9
Serializing new salt fields
aulme Apr 29, 2025
b988ca1
Adapted existing tests to new fields
aulme Apr 29, 2025
08dc841
Feedback
aulme Apr 29, 2025
bf9ff09
Feedback
aulme Apr 29, 2025
7de2b77
More feedback
aulme Apr 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 22 additions & 29 deletions src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import com.uid2.shared.cloud.TaggableCloudStorage;
import com.uid2.shared.encryption.AesGcm;
import com.uid2.shared.model.CloudEncryptionKey;
import com.uid2.shared.model.SaltEntry;
import com.uid2.shared.store.CloudPath;
import com.uid2.shared.store.salt.RotatingSaltProvider;
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
Expand All @@ -15,10 +14,7 @@
import org.slf4j.LoggerFactory;
import io.vertx.core.json.JsonObject;

import java.io.BufferedWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;

public class EncryptedSaltStoreWriter extends SaltStoreWriter implements StoreWriter {
Expand All @@ -37,6 +33,19 @@ public EncryptedSaltStoreWriter(JsonObject config, RotatingSaltProvider provider
this.siteId = siteId;
}

@Override
public void upload(Object data, JsonObject extraMeta) throws Exception {
this.unEncryptedMetadataData = extraMeta;
@SuppressWarnings("unchecked")
List<RotatingSaltProvider.SaltSnapshot> snapshots = new ArrayList<>((Collection<RotatingSaltProvider.SaltSnapshot>) data);
this.buildAndUploadMetadata(snapshots);
}

@Override
public void rewriteMeta() throws Exception {

}

@Override
protected java.lang.String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) {
return scope.resolve(new CloudPath("salts.txt." + snapshot.getEffective().toEpochMilli())).toString();
Expand Down Expand Up @@ -82,13 +91,16 @@ protected boolean tryUploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snaps
return false;
}
}
StringBuilder stringBuilder = new StringBuilder();

for (SaltEntry entry: snapshot.getAllRotatingSalts()) {
stringBuilder.append(entry.id()).append(",").append(entry.lastUpdated()).append(",").append(entry.currentSalt()).append("\n");
}
var saltCsv = SaltSerializer.toCsv(snapshot.getAllRotatingSalts());
var encryptedSaltCsv = addEncryptedEnvelope(encryptionKey, saltCsv);
uploadSaltsFile(location, encryptedSaltCsv);

LOGGER.info("File encryption completed for site_id={} key_id={} store={}", siteId, encryptionKey.getId(), "salts");
return true;
}

String data = stringBuilder.toString();
private String addEncryptedEnvelope(CloudEncryptionKey encryptionKey, String data) {
JsonObject encryptedJson = new JsonObject();
if (encryptionKey != null) {
byte[] secret = Base64.getDecoder().decode(encryptionKey.getSecret());
Expand All @@ -100,13 +112,7 @@ protected boolean tryUploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snaps
throw new IllegalStateException("No Cloud Encryption keys available for encryption for site ID: " + siteId);
}

final Path newSaltsFile = Files.createTempFile("salts", ".txt");
try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) {
w.write(encryptedJson.encodePrettily());
}
this.upload(newSaltsFile.toString(), location);
LOGGER.info("File encryption completed for site_id={} key_id={} store={}", siteId, encryptionKey.getId(), "salts");
return true;
return encryptedJson.encodePrettily();
}

@Override
Expand All @@ -118,17 +124,4 @@ protected JsonObject getMetadata(){
protected Long getMetadataVersion() throws Exception {
return this.unEncryptedMetadataData.getLong("version");
}

@Override
public void upload(Object data, JsonObject extraMeta) throws Exception {
this.unEncryptedMetadataData = extraMeta;
@SuppressWarnings("unchecked")
List<RotatingSaltProvider.SaltSnapshot> snapshots = new ArrayList<>((Collection<RotatingSaltProvider.SaltSnapshot>) data);
this.buildAndUploadMetadata(snapshots);
}

@Override
public void rewriteMeta() throws Exception {

}
}
54 changes: 54 additions & 0 deletions src/main/java/com/uid2/admin/store/writer/SaltSerializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package com.uid2.admin.store.writer;

import com.uid2.shared.model.SaltEntry;

public class SaltSerializer {
public static String toCsv(SaltEntry[] entries) {
StringBuilder stringBuilder = new StringBuilder();

for (SaltEntry entry : entries) {
addLine(entry, stringBuilder);
}

return stringBuilder.toString();
}

private static void addLine(SaltEntry entry, StringBuilder stringBuilder) {
stringBuilder
.append(entry.id())
.append(",")
.append(entry.lastUpdated())
.append(",")
.append(entry.currentSalt());

stringBuilder.append(",");
stringBuilder.append(serializeNullable(entry.refreshFrom()));

stringBuilder.append(",");
stringBuilder.append(serializeNullable(entry.previousSalt()));

appendKey(stringBuilder, entry.currentKey());
appendKey(stringBuilder, entry.previousKey());

stringBuilder.append("\n");
}

private static void appendKey(StringBuilder stringBuilder, SaltEntry.KeyMaterial key) {
if (key != null) {
stringBuilder
.append(",")
.append(key.id())
.append(",")
.append(serializeNullable(key.key()))
.append(",")
.append(serializeNullable(key.salt()));
}
else {
stringBuilder.append(",,,");
}
}

public static <T> String serializeNullable(T obj) {
return obj == null ? "" : obj.toString();
}
}
66 changes: 34 additions & 32 deletions src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import com.uid2.admin.store.FileManager;
import com.uid2.admin.store.version.VersionGenerator;
import com.uid2.shared.Utils;
import com.uid2.shared.cloud.CloudStorageException;
import com.uid2.shared.cloud.TaggableCloudStorage;
import com.uid2.shared.model.SaltEntry;
Expand All @@ -14,15 +13,13 @@
import org.slf4j.LoggerFactory;

import java.io.BufferedWriter;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class SaltStoreWriter {
Expand All @@ -45,6 +42,30 @@ public SaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileMan
this.versionGenerator = versionGenerator;
}

public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
this.buildAndUploadMetadata(this.getSnapshots(data));
refreshProvider();
}

/**
* reads the metadata file, and marks each referenced file as ready for archiving
*/
public void archiveSaltLocations() throws Exception {
final JsonObject metadata = provider.getMetadata();

metadata.getJsonArray("salts").forEach(instance -> {
try {
JsonObject salt = (JsonObject) instance;
String location = salt.getString("location", "");
if (!location.isBlank()) {
this.setStatusTagToObsolete(location);
}
} catch (Exception ex) {
LOGGER.error("Error marking object as ready for archiving", ex);
}
});
}

private List<RotatingSaltProvider.SaltSnapshot> getSnapshots(RotatingSaltProvider.SaltSnapshot data){
if (provider.getSnapshots() == null) {
throw new IllegalStateException("Snapshots cannot be null");
Expand Down Expand Up @@ -126,6 +147,7 @@ protected void buildAndUploadMetadata(List<RotatingSaltProvider.SaltSnapshot> sn
protected JsonObject enrichMetadata(JsonObject metadata){
return metadata;
}

/**
* Builds snapshot metadata and uploads snapshots if they need to be updated.
* <p>
Expand All @@ -151,34 +173,10 @@ private JsonArray uploadAndGetSnapshotsMetadata(List<RotatingSaltProvider.SaltSn
return anyUploadSucceeded ? snapshotsMetadata : new JsonArray();
}

public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
this.buildAndUploadMetadata(this.getSnapshots(data));
refreshProvider();
}

private void refreshProvider() throws Exception {
provider.loadContent();
}

/**
* reads the metadata file, and marks each referenced file as ready for archiving
*/
public void archiveSaltLocations() throws Exception {
final JsonObject metadata = provider.getMetadata();

metadata.getJsonArray("salts").forEach(instance -> {
try {
JsonObject salt = (JsonObject) instance;
String location = salt.getString("location", "");
if (!location.isBlank()) {
this.setStatusTagToObsolete(location);
}
} catch (Exception ex) {
LOGGER.error("Error marking object as ready for archiving", ex);
}
});
}

protected String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) {
return saltSnapshotLocationPrefix + snapshot.getEffective().toEpochMilli();
}
Expand All @@ -200,14 +198,18 @@ protected boolean tryUploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snaps
return false;
}

final Path newSaltsFile = Files.createTempFile("operators", ".txt");
var saltCsv = SaltSerializer.toCsv(snapshot.getAllRotatingSalts());
uploadSaltsFile(location, saltCsv);

return true;
}

protected void uploadSaltsFile(String location, String data) throws Exception {
final Path newSaltsFile = Files.createTempFile("salts", ".txt");
try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) {
for (SaltEntry entry : snapshot.getAllRotatingSalts()) {
w.write(entry.id() + "," + entry.lastUpdated() + "," + entry.currentSalt() + "\n");
}
w.write(data);
}
this.upload(newSaltsFile.toString(), location);
return true;
}

protected void upload(String data, String location) throws Exception {
Expand Down
85 changes: 59 additions & 26 deletions src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.uid2.shared.cloud.TaggableCloudStorage;
import com.uid2.shared.model.CloudEncryptionKey;
import com.uid2.shared.model.SaltEntry;
import com.uid2.shared.model.SaltEntry.KeyMaterial;
import com.uid2.shared.store.CloudPath;
import com.uid2.shared.store.salt.RotatingSaltProvider;
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
Expand All @@ -29,8 +30,8 @@
import java.time.Instant;
import java.util.*;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static java.lang.Long.parseLong;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream;

Expand Down Expand Up @@ -81,31 +82,19 @@ public void setup() throws Exception {
when(rotatingCloudEncryptionKeyProvider.getEncryptionKeyForSite(SITE_ID)).thenReturn(encryptionKey);
}

private RotatingSaltProvider.SaltSnapshot makeSnapshot(Instant effective, Instant expires, int nsalts) {
SaltEntry[] entries = new SaltEntry[nsalts];
for (int i = 0; i < entries.length; ++i) {
entries[i] = new SaltEntry(i, "hashed_id", effective.toEpochMilli(), "salt", null, null, null, null);
}
return new RotatingSaltProvider.SaltSnapshot(effective, expires, entries, "test_first_level_salt");
}

private void verifyFile(String filelocation, RotatingSaltProvider.SaltSnapshot snapshot) throws IOException {
InputStream encoded = Files.newInputStream(Paths.get(filelocation));
String contents = decryptInputStream(encoded, rotatingCloudEncryptionKeyProvider, "salts");
SaltEntry[] entries = snapshot.getAllRotatingSalts();
int idx = 0;
for (String line : contents.split("\n")) {
String[] entrySplit = line.split(",");
assertEquals(entries[idx].id(), Long.parseLong(entrySplit[0]));
assertEquals(entries[idx].currentSalt(), entrySplit[2]);
idx++;
}
}

@Test
public void testUploadNew() throws Exception {
RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot(Instant.ofEpochMilli(1740607938167L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 90002), 100);
RotatingSaltProvider.SaltSnapshot snapshot2 = makeSnapshot(Instant.ofEpochMilli(1740694476392L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 130000), 10);
RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot(
Instant.ofEpochMilli(1740607938167L),
Instant.ofEpochMilli(Instant.now().toEpochMilli() + 90002),
100
); // Older snapshot
RotatingSaltProvider.SaltSnapshot snapshot2 = makeSnapshot(
Instant.ofEpochMilli(1740694476392L),
Instant.ofEpochMilli(Instant.now().toEpochMilli() + 130000),
10
); // Newer active snapshot

JsonObject metadata = new JsonObject()
.put("version", 1742770328863L)
.put("generated", 1742770328)
Expand Down Expand Up @@ -135,7 +124,7 @@ public void testUploadNew() throws Exception {
assertEquals(capturedMetadata.getString("id_prefix"), metadata.getValue("id_prefix"));
verify(taggableCloudStorage,times(2)).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any());

verifyFile(pathCaptor.getValue(), snapshot);
assertWrittenFileEquals(pathCaptor.getValue(), snapshot2);
}

@Test
Expand Down Expand Up @@ -191,4 +180,48 @@ public void testUnencryptedAndEncryptedBehavesTheSame() throws Exception {
assertEquals(10, salt.getInteger("size"), "Size should match second entries");
verify(taggableCloudStorage,atLeastOnce()).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any());
}

private RotatingSaltProvider.SaltSnapshot makeSnapshot(Instant effective, Instant expires, int nsalts) {
SaltEntry[] entries = new SaltEntry[nsalts];

for (int i = 0; i < entries.length; ++i) {
entries[i] = new SaltEntry(
i,
"hashed_id",
effective.toEpochMilli(),
"salt",
1000L,
"previous salt",
new KeyMaterial(1, "key 1", "key salt 1"),
new KeyMaterial(2, "key 2", "key salt 2")
);
}
return new RotatingSaltProvider.SaltSnapshot(effective, expires, entries, "test_first_level_salt");
}

private void assertWrittenFileEquals(String fileLocation, RotatingSaltProvider.SaltSnapshot snapshot) throws IOException {
InputStream encoded = Files.newInputStream(Paths.get(fileLocation));
String contents = decryptInputStream(encoded, rotatingCloudEncryptionKeyProvider, "salts");
SaltEntry[] entries = snapshot.getAllRotatingSalts();
var lines = contents.split("\n");
for (int i = 0; i < lines.length; i++) {
var line = lines[i];
var entry = entries[i];
String[] fields = line.split(",");

assertAll(
() -> assertEquals(entry.id(), parseLong(fields[0])),
() -> assertEquals(entry.lastUpdated(), parseLong(fields[1])),
() -> assertEquals(entry.currentSalt(), fields[2]),
() -> assertEquals(entry.refreshFrom(), parseLong(fields[3])),
() -> assertEquals(entry.previousSalt(), fields[4]),
() -> assertEquals(entry.currentKey().id(), parseLong(fields[5])),
() -> assertEquals(entry.currentKey().key(), fields[6]),
() -> assertEquals(entry.currentKey().salt(), fields[7]),
() -> assertEquals(entry.previousKey().id(), parseLong(fields[8])),
() -> assertEquals(entry.previousKey().key(), fields[9]),
() -> assertEquals(entry.previousKey().salt(), fields[10])
);
}
}
}
Loading