Skip to content

Commit 6ee04a9

Browse files
aulmeaulme
committed
Serializing new salt fields
1 parent 146249f commit 6ee04a9

File tree

4 files changed

+233
-61
lines changed

4 files changed

+233
-61
lines changed

‎src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java‎

Lines changed: 22 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import com.uid2.shared.cloud.TaggableCloudStorage;
77
import com.uid2.shared.encryption.AesGcm;
88
import com.uid2.shared.model.CloudEncryptionKey;
9-
import com.uid2.shared.model.SaltEntry;
109
import com.uid2.shared.store.CloudPath;
1110
import com.uid2.shared.store.salt.RotatingSaltProvider;
1211
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
@@ -15,10 +14,7 @@
1514
import org.slf4j.LoggerFactory;
1615
import io.vertx.core.json.JsonObject;
1716

18-
import java.io.BufferedWriter;
1917
import java.nio.charset.StandardCharsets;
20-
import java.nio.file.Files;
21-
import java.nio.file.Path;
2218
import java.util.*;
2319

2420
public class EncryptedSaltStoreWriter extends SaltStoreWriter implements StoreWriter {
@@ -37,6 +33,19 @@ public EncryptedSaltStoreWriter(JsonObject config, RotatingSaltProvider provider
3733
this.siteId = siteId;
3834
}
3935

36+
@Override
37+
public void upload(Object data, JsonObject extraMeta) throws Exception {
38+
this.unEncryptedMetadataData = extraMeta;
39+
@SuppressWarnings("unchecked")
40+
List<RotatingSaltProvider.SaltSnapshot> snapshots = new ArrayList<>((Collection<RotatingSaltProvider.SaltSnapshot>) data);
41+
this.buildAndUploadMetadata(snapshots);
42+
}
43+
44+
@Override
45+
public void rewriteMeta() throws Exception {
46+
47+
}
48+
4049
@Override
4150
protected java.lang.String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) {
4251
return scope.resolve(new CloudPath("salts.txt." + snapshot.getEffective().toEpochMilli())).toString();
@@ -82,13 +91,16 @@ protected boolean tryUploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snaps
8291
return false;
8392
}
8493
}
85-
StringBuilder stringBuilder = new StringBuilder();
8694

87-
for (SaltEntry entry: snapshot.getAllRotatingSalts()) {
88-
stringBuilder.append(entry.id()).append(",").append(entry.lastUpdated()).append(",").append(entry.currentSalt()).append("\n");
89-
}
95+
var saltCsv = SaltSerializer.toCsv(snapshot.getAllRotatingSalts());
96+
var encryptedSaltCsv = addEncryptedEnvelope(encryptionKey, saltCsv);
97+
uploadSaltsFile(location, encryptedSaltCsv);
98+
99+
LOGGER.info("File encryption completed for site_id={} key_id={} store={}", siteId, encryptionKey.getId(), "salts");
100+
return true;
101+
}
90102

91-
String data = stringBuilder.toString();
103+
private String addEncryptedEnvelope(CloudEncryptionKey encryptionKey, String data) {
92104
JsonObject encryptedJson = new JsonObject();
93105
if (encryptionKey != null) {
94106
byte[] secret = Base64.getDecoder().decode(encryptionKey.getSecret());
@@ -100,13 +112,7 @@ protected boolean tryUploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snaps
100112
throw new IllegalStateException("No Cloud Encryption keys available for encryption for site ID: " + siteId);
101113
}
102114

103-
final Path newSaltsFile = Files.createTempFile("salts", ".txt");
104-
try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) {
105-
w.write(encryptedJson.encodePrettily());
106-
}
107-
this.upload(newSaltsFile.toString(), location);
108-
LOGGER.info("File encryption completed for site_id={} key_id={} store={}", siteId, encryptionKey.getId(), "salts");
109-
return true;
115+
return encryptedJson.encodePrettily();
110116
}
111117

112118
@Override
@@ -118,17 +124,4 @@ protected JsonObject getMetadata(){
118124
protected Long getMetadataVersion() throws Exception {
119125
return this.unEncryptedMetadataData.getLong("version");
120126
}
121-
122-
@Override
123-
public void upload(Object data, JsonObject extraMeta) throws Exception {
124-
this.unEncryptedMetadataData = extraMeta;
125-
@SuppressWarnings("unchecked")
126-
List<RotatingSaltProvider.SaltSnapshot> snapshots = new ArrayList<>((Collection<RotatingSaltProvider.SaltSnapshot>) data);
127-
this.buildAndUploadMetadata(snapshots);
128-
}
129-
130-
@Override
131-
public void rewriteMeta() throws Exception {
132-
133-
}
134127
}

‎src/main/java/com/uid2/admin/store/writer/SaltSerializer.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package com.uid2.admin.store.writer;
2+
3+
import com.uid2.shared.model.SaltEntry;
4+
5+
import java.util.List;
6+
7+
public class SaltSerializer {
8+
public static String toCsv(SaltEntry[] entries) {
9+
StringBuilder stringBuilder = new StringBuilder();
10+
11+
for (SaltEntry entry : entries) {
12+
addLine(entry, stringBuilder);
13+
}
14+
15+
return stringBuilder.toString();
16+
}
17+
18+
private static void addLine(SaltEntry entry, StringBuilder stringBuilder) {
19+
stringBuilder
20+
.append(entry.id())
21+
.append(",")
22+
.append(entry.lastUpdated())
23+
.append(",")
24+
.append(entry.currentSalt());
25+
26+
stringBuilder.append(",");
27+
stringBuilder.append(serializeNullable(entry.refreshFrom()));
28+
29+
stringBuilder.append(",");
30+
stringBuilder.append(serializeNullable(entry.previousSalt()));
31+
32+
appendKey(stringBuilder, entry.currentKey());
33+
appendKey(stringBuilder, entry.previousKey());
34+
35+
stringBuilder.append("\n");
36+
}
37+
38+
private static void appendKey(StringBuilder stringBuilder, SaltEntry.KeyMaterial key) {
39+
if (key != null) {
40+
stringBuilder
41+
.append(",")
42+
.append(key.id())
43+
.append(",")
44+
.append(serializeNullable(key.key()))
45+
.append(",")
46+
.append(serializeNullable(key.salt()));
47+
}
48+
else {
49+
stringBuilder.append(",,,");
50+
}
51+
}
52+
53+
public static <T> String serializeNullable(T obj) {
54+
return obj == null ? "" : obj.toString();
55+
}
56+
}

‎src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java‎

Lines changed: 34 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import com.uid2.admin.store.FileManager;
44
import com.uid2.admin.store.version.VersionGenerator;
5-
import com.uid2.shared.Utils;
65
import com.uid2.shared.cloud.CloudStorageException;
76
import com.uid2.shared.cloud.TaggableCloudStorage;
87
import com.uid2.shared.model.SaltEntry;
@@ -14,15 +13,13 @@
1413
import org.slf4j.LoggerFactory;
1514

1615
import java.io.BufferedWriter;
17-
import java.io.InputStream;
1816
import java.nio.file.Files;
1917
import java.nio.file.Path;
2018
import java.time.Instant;
2119
import java.util.ArrayList;
2220
import java.util.Comparator;
2321
import java.util.List;
2422
import java.util.Map;
25-
import java.util.stream.Collectors;
2623
import java.util.stream.Stream;
2724

2825
public class SaltStoreWriter {
@@ -45,6 +42,30 @@ public SaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileMan
4542
this.versionGenerator = versionGenerator;
4643
}
4744

45+
public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
46+
this.buildAndUploadMetadata(this.getSnapshots(data));
47+
refreshProvider();
48+
}
49+
50+
/**
51+
* reads the metadata file, and marks each referenced file as ready for archiving
52+
*/
53+
public void archiveSaltLocations() throws Exception {
54+
final JsonObject metadata = provider.getMetadata();
55+
56+
metadata.getJsonArray("salts").forEach(instance -> {
57+
try {
58+
JsonObject salt = (JsonObject) instance;
59+
String location = salt.getString("location", "");
60+
if (!location.isBlank()) {
61+
this.setStatusTagToObsolete(location);
62+
}
63+
} catch (Exception ex) {
64+
LOGGER.error("Error marking object as ready for archiving", ex);
65+
}
66+
});
67+
}
68+
4869
private List<RotatingSaltProvider.SaltSnapshot> getSnapshots(RotatingSaltProvider.SaltSnapshot data){
4970
if (provider.getSnapshots() == null) {
5071
throw new IllegalStateException("Snapshots cannot be null");
@@ -126,6 +147,7 @@ protected void buildAndUploadMetadata(List<RotatingSaltProvider.SaltSnapshot> sn
126147
protected JsonObject enrichMetadata(JsonObject metadata){
127148
return metadata;
128149
}
150+
129151
/**
130152
* Builds snapshot metadata and uploads snapshots if they need to be updated.
131153
* <p>
@@ -151,34 +173,10 @@ private JsonArray uploadAndGetSnapshotsMetadata(List<RotatingSaltProvider.SaltSn
151173
return anyUploadSucceeded ? snapshotsMetadata : new JsonArray();
152174
}
153175

154-
public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
155-
this.buildAndUploadMetadata(this.getSnapshots(data));
156-
refreshProvider();
157-
}
158-
159176
private void refreshProvider() throws Exception {
160177
provider.loadContent();
161178
}
162179

163-
/**
164-
* reads the metadata file, and marks each referenced file as ready for archiving
165-
*/
166-
public void archiveSaltLocations() throws Exception {
167-
final JsonObject metadata = provider.getMetadata();
168-
169-
metadata.getJsonArray("salts").forEach(instance -> {
170-
try {
171-
JsonObject salt = (JsonObject) instance;
172-
String location = salt.getString("location", "");
173-
if (!location.isBlank()) {
174-
this.setStatusTagToObsolete(location);
175-
}
176-
} catch (Exception ex) {
177-
LOGGER.error("Error marking object as ready for archiving", ex);
178-
}
179-
});
180-
}
181-
182180
protected String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) {
183181
return saltSnapshotLocationPrefix + snapshot.getEffective().toEpochMilli();
184182
}
@@ -200,14 +198,18 @@ protected boolean tryUploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snaps
200198
return false;
201199
}
202200

203-
final Path newSaltsFile = Files.createTempFile("operators", ".txt");
201+
var saltCsv = SaltSerializer.toCsv(snapshot.getAllRotatingSalts());
202+
uploadSaltsFile(location, saltCsv);
203+
204+
return true;
205+
}
206+
207+
protected void uploadSaltsFile(String location, String data) throws Exception {
208+
final Path newSaltsFile = Files.createTempFile("salts", ".txt");
204209
try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) {
205-
for (SaltEntry entry : snapshot.getAllRotatingSalts()) {
206-
w.write(entry.id() + "," + entry.lastUpdated() + "," + entry.currentSalt() + "\n");
207-
}
210+
w.write(data);
208211
}
209212
this.upload(newSaltsFile.toString(), location);
210-
return true;
211213
}
212214

213215
protected void upload(String data, String location) throws Exception {

‎src/test/java/com/uid2/admin/store/writer/SaltSerializerTest.java‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
package com.uid2.admin.store.writer;
2+
3+
import com.uid2.shared.model.SaltEntry;
4+
import com.uid2.shared.model.SaltEntry.KeyMaterial;
5+
import org.junit.jupiter.api.Test;
6+
7+
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
8+
9+
class SaltSerializerTest {
10+
@Test
11+
void toCsv_serializesNoSalts() {
12+
var expected = "";
13+
14+
var salts = new SaltEntry[]{};
15+
var actual = SaltSerializer.toCsv(salts);
16+
17+
assertThat(actual).isEqualTo(expected);
18+
}
19+
20+
@Test
21+
void toCsv_serializesSaltsWithNoOptionalFields() {
22+
var expected = """
23+
1,100,salt1,,,,,,,,
24+
2,200,salt2,,,,,,,,
25+
""";
26+
27+
var salts = new SaltEntry[]{
28+
new SaltEntry(1, "hashedId1", 100, "salt1", null, null, null, null),
29+
new SaltEntry(2, "hashedId2", 200, "salt2", null, null, null, null),
30+
};
31+
var actual = SaltSerializer.toCsv(salts);
32+
33+
assertThat(actual).isEqualTo(expected);
34+
}
35+
36+
@Test
37+
void toCsv_serializesSaltsWithAllOptionalFields() {
38+
var expected = """
39+
1,100,salt1,1000,previousSalt1,11,key1,keySalt1,111,key11,keySalt11
40+
2,200,salt2,2000,previousSalt2,22,key2,keySalt2,222,key22,keySalt22
41+
""";
42+
43+
var salts = new SaltEntry[]{
44+
new SaltEntry(1,
45+
"hashedId1",
46+
100,
47+
"salt1",
48+
1000L,
49+
"previousSalt1",
50+
new KeyMaterial(11, "key1", "keySalt1"),
51+
new KeyMaterial(111, "key11", "keySalt11")
52+
),
53+
new SaltEntry(2,
54+
"hashedId2",
55+
200,
56+
"salt2",
57+
2000L,
58+
"previousSalt2",
59+
new KeyMaterial(22, "key2", "keySalt2"),
60+
new KeyMaterial(222, "key22", "keySalt22")
61+
),
62+
};
63+
var actual = SaltSerializer.toCsv(salts);
64+
65+
assertThat(actual).isEqualTo(expected);
66+
}
67+
68+
@Test
69+
void toCsv_serializesSaltsWithV3IdentityMapFields() {
70+
var expected = """
71+
1,100,salt1,1000,previousSalt1,,,,,,
72+
2,200,salt2,2000,previousSalt2,,,,,,
73+
""";
74+
75+
var salts = new SaltEntry[]{
76+
new SaltEntry(1,
77+
"hashedId1",
78+
100,
79+
"salt1",
80+
1000L,
81+
"previousSalt1",
82+
null,
83+
null
84+
),
85+
new SaltEntry(2,
86+
"hashedId2",
87+
200,
88+
"salt2",
89+
2000L,
90+
"previousSalt2",
91+
null,
92+
null
93+
),
94+
};
95+
var actual = SaltSerializer.toCsv(salts);
96+
97+
assertThat(actual).isEqualTo(expected);
98+
}
99+
100+
@Test
101+
void toCsv_toleratesNullsInKeys() {
102+
var expected = """
103+
1,100,salt1,1000,previousSalt1,11,,,111,,
104+
""";
105+
106+
var salts = new SaltEntry[]{
107+
new SaltEntry(1,
108+
"hashedId1",
109+
100,
110+
"salt1",
111+
1000L,
112+
"previousSalt1",
113+
new KeyMaterial(11, null, null),
114+
new KeyMaterial(111, null, null)
115+
),
116+
};
117+
var actual = SaltSerializer.toCsv(salts);
118+
119+
assertThat(actual).isEqualTo(expected);
120+
}
121+
}

0 commit comments

Comments
 (0)