Skip to content

Commit db3e5c3

Browse files
aulmeaulme
authored
Merge pull request #449 from IABTechLab/aul-UID2-5338-serialize-new-salt-fields
Serializing new salt fields
2 parents 146249f + 7de2b77 commit db3e5c3

File tree

5 files changed

+326
-96
lines changed

5 files changed

+326
-96
lines changed

‎src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java‎

Lines changed: 22 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import com.uid2.shared.cloud.TaggableCloudStorage;
77
import com.uid2.shared.encryption.AesGcm;
88
import com.uid2.shared.model.CloudEncryptionKey;
9-
import com.uid2.shared.model.SaltEntry;
109
import com.uid2.shared.store.CloudPath;
1110
import com.uid2.shared.store.salt.RotatingSaltProvider;
1211
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
@@ -15,10 +14,7 @@
1514
import org.slf4j.LoggerFactory;
1615
import io.vertx.core.json.JsonObject;
1716

18-
import java.io.BufferedWriter;
1917
import java.nio.charset.StandardCharsets;
20-
import java.nio.file.Files;
21-
import java.nio.file.Path;
2218
import java.util.*;
2319

2420
public class EncryptedSaltStoreWriter extends SaltStoreWriter implements StoreWriter {
@@ -37,6 +33,19 @@ public EncryptedSaltStoreWriter(JsonObject config, RotatingSaltProvider provider
3733
this.siteId = siteId;
3834
}
3935

36+
@Override
37+
public void upload(Object data, JsonObject extraMeta) throws Exception {
38+
this.unEncryptedMetadataData = extraMeta;
39+
@SuppressWarnings("unchecked")
40+
List<RotatingSaltProvider.SaltSnapshot> snapshots = new ArrayList<>((Collection<RotatingSaltProvider.SaltSnapshot>) data);
41+
this.buildAndUploadMetadata(snapshots);
42+
}
43+
44+
@Override
45+
public void rewriteMeta() throws Exception {
46+
// No rewrite_metadata endpoint for salts
47+
}
48+
4049
@Override
4150
protected java.lang.String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) {
4251
return scope.resolve(new CloudPath("salts.txt." + snapshot.getEffective().toEpochMilli())).toString();
@@ -82,13 +91,16 @@ protected boolean tryUploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snaps
8291
return false;
8392
}
8493
}
85-
StringBuilder stringBuilder = new StringBuilder();
8694

87-
for (SaltEntry entry: snapshot.getAllRotatingSalts()) {
88-
stringBuilder.append(entry.id()).append(",").append(entry.lastUpdated()).append(",").append(entry.currentSalt()).append("\n");
89-
}
95+
var saltCsv = SaltSerializer.toCsv(snapshot.getAllRotatingSalts());
96+
var encryptedSaltCsv = addEncryptedEnvelope(encryptionKey, saltCsv);
97+
uploadSaltsFile(location, encryptedSaltCsv);
98+
99+
LOGGER.info("File encryption completed for site_id={} key_id={} store={}", siteId, encryptionKey.getId(), "salts");
100+
return true;
101+
}
90102

91-
String data = stringBuilder.toString();
103+
private String addEncryptedEnvelope(CloudEncryptionKey encryptionKey, String data) {
92104
JsonObject encryptedJson = new JsonObject();
93105
if (encryptionKey != null) {
94106
byte[] secret = Base64.getDecoder().decode(encryptionKey.getSecret());
@@ -100,13 +112,7 @@ protected boolean tryUploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snaps
100112
throw new IllegalStateException("No Cloud Encryption keys available for encryption for site ID: " + siteId);
101113
}
102114

103-
final Path newSaltsFile = Files.createTempFile("salts", ".txt");
104-
try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) {
105-
w.write(encryptedJson.encodePrettily());
106-
}
107-
this.upload(newSaltsFile.toString(), location);
108-
LOGGER.info("File encryption completed for site_id={} key_id={} store={}", siteId, encryptionKey.getId(), "salts");
109-
return true;
115+
return encryptedJson.encodePrettily();
110116
}
111117

112118
@Override
@@ -118,17 +124,4 @@ protected JsonObject getMetadata(){
118124
protected Long getMetadataVersion() throws Exception {
119125
return this.unEncryptedMetadataData.getLong("version");
120126
}
121-
122-
@Override
123-
public void upload(Object data, JsonObject extraMeta) throws Exception {
124-
this.unEncryptedMetadataData = extraMeta;
125-
@SuppressWarnings("unchecked")
126-
List<RotatingSaltProvider.SaltSnapshot> snapshots = new ArrayList<>((Collection<RotatingSaltProvider.SaltSnapshot>) data);
127-
this.buildAndUploadMetadata(snapshots);
128-
}
129-
130-
@Override
131-
public void rewriteMeta() throws Exception {
132-
133-
}
134127
}

‎src/main/java/com/uid2/admin/store/writer/SaltSerializer.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package com.uid2.admin.store.writer;
2+
3+
import com.uid2.shared.model.SaltEntry;
4+
5+
public final class SaltSerializer {
6+
private SaltSerializer() {}
7+
8+
public static String toCsv(SaltEntry[] entries) {
9+
StringBuilder stringBuilder = new StringBuilder();
10+
11+
for (SaltEntry entry : entries) {
12+
addLine(entry, stringBuilder);
13+
}
14+
15+
return stringBuilder.toString();
16+
}
17+
18+
private static void addLine(SaltEntry entry, StringBuilder stringBuilder) {
19+
stringBuilder
20+
.append(entry.id())
21+
.append(",")
22+
.append(entry.lastUpdated())
23+
.append(",")
24+
.append(entry.currentSalt());
25+
26+
stringBuilder.append(",");
27+
stringBuilder.append(serializeNullable(entry.refreshFrom()));
28+
29+
stringBuilder.append(",");
30+
stringBuilder.append(serializeNullable(entry.previousSalt()));
31+
32+
appendKey(stringBuilder, entry.currentKey());
33+
appendKey(stringBuilder, entry.previousKey());
34+
35+
stringBuilder.append("\n");
36+
}
37+
38+
private static void appendKey(StringBuilder stringBuilder, SaltEntry.KeyMaterial key) {
39+
if (key != null) {
40+
stringBuilder
41+
.append(",")
42+
.append(key.id())
43+
.append(",")
44+
.append(serializeNullable(key.key()))
45+
.append(",")
46+
.append(serializeNullable(key.salt()));
47+
}
48+
else {
49+
stringBuilder.append(",,,");
50+
}
51+
}
52+
53+
public static <T> String serializeNullable(T obj) {
54+
return obj == null ? "" : obj.toString();
55+
}
56+
}

‎src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java‎

Lines changed: 34 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import com.uid2.admin.store.FileManager;
44
import com.uid2.admin.store.version.VersionGenerator;
5-
import com.uid2.shared.Utils;
65
import com.uid2.shared.cloud.CloudStorageException;
76
import com.uid2.shared.cloud.TaggableCloudStorage;
87
import com.uid2.shared.model.SaltEntry;
@@ -14,15 +13,13 @@
1413
import org.slf4j.LoggerFactory;
1514

1615
import java.io.BufferedWriter;
17-
import java.io.InputStream;
1816
import java.nio.file.Files;
1917
import java.nio.file.Path;
2018
import java.time.Instant;
2119
import java.util.ArrayList;
2220
import java.util.Comparator;
2321
import java.util.List;
2422
import java.util.Map;
25-
import java.util.stream.Collectors;
2623
import java.util.stream.Stream;
2724

2825
public class SaltStoreWriter {
@@ -45,6 +42,30 @@ public SaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileMan
4542
this.versionGenerator = versionGenerator;
4643
}
4744

45+
public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
46+
this.buildAndUploadMetadata(this.getSnapshots(data));
47+
refreshProvider();
48+
}
49+
50+
/**
51+
* reads the metadata file, and marks each referenced file as ready for archiving
52+
*/
53+
public void archiveSaltLocations() throws Exception {
54+
final JsonObject metadata = provider.getMetadata();
55+
56+
metadata.getJsonArray("salts").forEach(instance -> {
57+
try {
58+
JsonObject salt = (JsonObject) instance;
59+
String location = salt.getString("location", "");
60+
if (!location.isBlank()) {
61+
this.setStatusTagToObsolete(location);
62+
}
63+
} catch (Exception ex) {
64+
LOGGER.error("Error marking object as ready for archiving", ex);
65+
}
66+
});
67+
}
68+
4869
private List<RotatingSaltProvider.SaltSnapshot> getSnapshots(RotatingSaltProvider.SaltSnapshot data){
4970
if (provider.getSnapshots() == null) {
5071
throw new IllegalStateException("Snapshots cannot be null");
@@ -126,6 +147,7 @@ protected void buildAndUploadMetadata(List<RotatingSaltProvider.SaltSnapshot> sn
126147
protected JsonObject enrichMetadata(JsonObject metadata){
127148
return metadata;
128149
}
150+
129151
/**
130152
* Builds snapshot metadata and uploads snapshots if they need to be updated.
131153
* <p>
@@ -151,34 +173,10 @@ private JsonArray uploadAndGetSnapshotsMetadata(List<RotatingSaltProvider.SaltSn
151173
return anyUploadSucceeded ? snapshotsMetadata : new JsonArray();
152174
}
153175

154-
public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
155-
this.buildAndUploadMetadata(this.getSnapshots(data));
156-
refreshProvider();
157-
}
158-
159176
private void refreshProvider() throws Exception {
160177
provider.loadContent();
161178
}
162179

163-
/**
164-
* reads the metadata file, and marks each referenced file as ready for archiving
165-
*/
166-
public void archiveSaltLocations() throws Exception {
167-
final JsonObject metadata = provider.getMetadata();
168-
169-
metadata.getJsonArray("salts").forEach(instance -> {
170-
try {
171-
JsonObject salt = (JsonObject) instance;
172-
String location = salt.getString("location", "");
173-
if (!location.isBlank()) {
174-
this.setStatusTagToObsolete(location);
175-
}
176-
} catch (Exception ex) {
177-
LOGGER.error("Error marking object as ready for archiving", ex);
178-
}
179-
});
180-
}
181-
182180
protected String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) {
183181
return saltSnapshotLocationPrefix + snapshot.getEffective().toEpochMilli();
184182
}
@@ -200,14 +198,18 @@ protected boolean tryUploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snaps
200198
return false;
201199
}
202200

203-
final Path newSaltsFile = Files.createTempFile("operators", ".txt");
201+
var saltCsv = SaltSerializer.toCsv(snapshot.getAllRotatingSalts());
202+
uploadSaltsFile(location, saltCsv);
203+
204+
return true;
205+
}
206+
207+
protected void uploadSaltsFile(String location, String data) throws Exception {
208+
final Path newSaltsFile = Files.createTempFile("salts", ".txt");
204209
try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) {
205-
for (SaltEntry entry : snapshot.getAllRotatingSalts()) {
206-
w.write(entry.id() + "," + entry.lastUpdated() + "," + entry.currentSalt() + "\n");
207-
}
210+
w.write(data);
208211
}
209212
this.upload(newSaltsFile.toString(), location);
210-
return true;
211213
}
212214

213215
protected void upload(String data, String location) throws Exception {

0 commit comments

Comments
 (0)