Skip to content

Commit 7ae43b9

Browse files
cody-constine-ttdcody-constine-ttd
authored
Merge pull request #355 from IABTechLab/cbc-UID2-4571-cloud-encryption-cleanup
Cbc UI d2 4571 cloud encryption cleanup
2 parents 415cf4e + 73cf984 commit 7ae43b9

11 files changed

+575
-77
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
<groupId>com.uid2</groupId>
77
<artifactId>uid2-shared</artifactId>
8-
<version>8.0.9</version>
8+
<version>8.0.15-alpha-177-SNAPSHOT</version>
99
<name>${project.groupId}:${project.artifactId}</name>
1010
<description>Library for all the shared uid2 operations</description>
1111
<url>https://github.com/IABTechLab/uid2docs</url>

‎src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package com.uid2.shared.store;
2+
3+
import com.uid2.shared.Const;
4+
import com.uid2.shared.cloud.DownloadCloudStorage;
5+
import com.uid2.shared.model.SaltEntry;
6+
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
7+
import com.uid2.shared.store.scope.StoreScope;
8+
9+
import java.io.IOException;
10+
import java.io.InputStream;
11+
import java.util.Collection;
12+
13+
import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream;
14+
15+
public class EncryptedRotatingSaltProvider extends RotatingSaltProvider {
16+
private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider;
17+
18+
public EncryptedRotatingSaltProvider(DownloadCloudStorage fileStreamProvider, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider, StoreScope scope) {
19+
super(fileStreamProvider, scope.getMetadataPath().toString());
20+
this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider;
21+
}
22+
23+
@Override
24+
protected SaltEntry[] readInputStream(InputStream inputStream, SaltEntryBuilder entryBuilder, Integer size) throws IOException {
25+
String decrypted = decryptInputStream(inputStream, cloudEncryptionKeyProvider);
26+
SaltEntry[] entries = new SaltEntry[size];
27+
int idx = 0;
28+
for (String line : decrypted.split("\n")) {
29+
final SaltEntry entry = entryBuilder.toEntry(line);
30+
entries[idx] = entry;
31+
idx++;
32+
}
33+
return entries;
34+
}
35+
}

‎src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java‎

Lines changed: 3 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,18 @@
11
package com.uid2.shared.store;
22

33
import com.uid2.shared.cloud.DownloadCloudStorage;
4-
import com.uid2.shared.model.CloudEncryptionKey;
54
import com.uid2.shared.store.parser.Parser;
65
import com.uid2.shared.store.parser.ParsingResult;
76
import com.uid2.shared.store.scope.StoreScope;
87
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
9-
import io.vertx.core.json.JsonObject;
108
import org.slf4j.Logger;
119
import org.slf4j.LoggerFactory;
1210

1311
import java.io.*;
1412

15-
import com.uid2.shared.encryption.AesGcm;
16-
1713
import java.nio.charset.StandardCharsets;
18-
import java.util.Base64;
19-
import java.util.Map;
14+
15+
import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream;
2016

2117
public class EncryptedScopedStoreReader<T> extends ScopedStoreReader<T> {
2218
private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedScopedStoreReader.class);
@@ -31,8 +27,7 @@ public EncryptedScopedStoreReader(DownloadCloudStorage fileStreamProvider, Store
3127
@Override
3228
protected long loadContent(String path) throws Exception {
3329
try (InputStream inputStream = this.contentStreamProvider.download(path)) {
34-
String encryptedContent = inputStreamToString(inputStream);
35-
String decryptedContent = getDecryptedContent(encryptedContent);
30+
String decryptedContent = decryptInputStream(inputStream, cloudEncryptionKeyProvider);
3631
ParsingResult<T> parsed = this.parser.deserialize(new ByteArrayInputStream(decryptedContent.getBytes(StandardCharsets.UTF_8)));
3732
latestSnapshot.set(parsed.getData());
3833

@@ -45,39 +40,4 @@ protected long loadContent(String path) throws Exception {
4540
throw e;
4641
}
4742
}
48-
49-
protected String getDecryptedContent(String encryptedContent) throws Exception {
50-
JsonObject json = new JsonObject(encryptedContent);
51-
int keyId = json.getInteger("key_id");
52-
String encryptedPayload = json.getString("encrypted_payload");
53-
Map<Integer, CloudEncryptionKey> cloudEncryptionKeys = cloudEncryptionKeyProvider.getAll();
54-
CloudEncryptionKey decryptionKey = null;
55-
for (CloudEncryptionKey key : cloudEncryptionKeys.values()) {
56-
if (key.getId() == keyId) {
57-
decryptionKey = key;
58-
break;
59-
}
60-
}
61-
62-
if (decryptionKey == null) {
63-
throw new IllegalStateException("No matching S3 key found for decryption for key ID: " + keyId);
64-
}
65-
66-
byte[] secret = Base64.getDecoder().decode(decryptionKey.getSecret());
67-
byte[] encryptedBytes = Base64.getDecoder().decode(encryptedPayload);
68-
byte[] decryptedBytes = AesGcm.decrypt(encryptedBytes, 0, secret);
69-
70-
return new String(decryptedBytes, StandardCharsets.UTF_8);
71-
}
72-
73-
public static String inputStreamToString(InputStream inputStream) throws IOException {
74-
try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
75-
StringBuilder stringBuilder = new StringBuilder();
76-
String line;
77-
while ((line = reader.readLine()) != null) {
78-
stringBuilder.append(line);
79-
}
80-
return stringBuilder.toString();
81-
}
82-
}
8343
}

‎src/main/java/com/uid2/shared/store/RotatingSaltProvider.java‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.hashids.Hashids;
1414

1515
import java.io.BufferedReader;
16+
import java.io.IOException;
1617
import java.io.InputStream;
1718
import java.io.InputStreamReader;
1819
import java.nio.charset.StandardCharsets;
@@ -21,6 +22,7 @@
2122
import java.util.*;
2223
import java.util.concurrent.atomic.AtomicReference;
2324
import java.util.stream.Collectors;
25+
import java.util.stream.Stream;
2426

2527
/*
2628
1. metadata.json format
@@ -130,20 +132,25 @@ private SaltSnapshot loadSnapshot(JsonObject spec, String firstLevelSalt, SaltEn
130132
final Instant expires = Instant.ofEpochMilli(spec.getLong("expires", defaultExpires.toEpochMilli()));
131133

132134
final String path = spec.getString("location");
133-
int idx = 0;
134-
final SaltEntry[] entries = new SaltEntry[spec.getInteger("size")];
135-
136-
try (InputStream inputStream = this.contentStreamProvider.download(path);
137-
InputStreamReader inputStreamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
138-
BufferedReader reader = new BufferedReader(inputStreamReader)) {
139-
for (String l; (l = reader.readLine()) != null; ++idx) {
140-
final SaltEntry entry = entryBuilder.toEntry(l);
135+
Integer size = spec.getInteger("size");
136+
SaltEntry[] entries = readInputStream(this.contentStreamProvider.download(path), entryBuilder, size);
137+
138+
LOGGER.info("Loaded " + size + " salts");
139+
return new SaltSnapshot(effective, expires, entries, firstLevelSalt);
140+
}
141+
142+
protected SaltEntry[] readInputStream(InputStream inputStream, SaltEntryBuilder entryBuilder, Integer size) throws IOException {
143+
try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
144+
String line;
145+
SaltEntry[] entries = new SaltEntry[size];
146+
int idx = 0;
147+
while ((line = reader.readLine()) != null) {
148+
final SaltEntry entry = entryBuilder.toEntry(line);
141149
entries[idx] = entry;
150+
idx++;
142151
}
152+
return entries;
143153
}
144-
145-
LOGGER.info("Loaded " + idx + " salts");
146-
return new SaltSnapshot(effective, expires, entries, firstLevelSalt);
147154
}
148155

149156
public static class SaltSnapshot implements ISaltSnapshot {
@@ -214,7 +221,7 @@ public String encode(long id) {
214221
}
215222
}
216223

217-
static final class SaltEntryBuilder {
224+
protected static final class SaltEntryBuilder {
218225
private final IdHashingScheme idHashingScheme;
219226

220227
public SaltEntryBuilder(IdHashingScheme idHashingScheme) {

‎src/main/java/com/uid2/shared/store/reader/RotatingClientSideKeypairStore.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.uid2.shared.cloud.DownloadCloudStorage;
44
import com.uid2.shared.model.ClientSideKeypair;
55
import com.uid2.shared.store.CloudPath;
6+
import com.uid2.shared.store.EncryptedScopedStoreReader;
67
import com.uid2.shared.store.IClientSideKeypairStore;
78
import com.uid2.shared.store.ScopedStoreReader;
89
import com.uid2.shared.store.parser.ClientSideKeypairParser;
@@ -19,6 +20,10 @@ public RotatingClientSideKeypairStore(DownloadCloudStorage fileStreamProvider, S
1920
this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new ClientSideKeypairParser(), "client_side_keypairs");
2021
}
2122

23+
public RotatingClientSideKeypairStore(DownloadCloudStorage fileStreamProvider, StoreScope scope, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
24+
this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new ClientSideKeypairParser(), "client_side_keypairs", cloudEncryptionKeyProvider);
25+
}
26+
2227
@Override
2328
public long getVersion(JsonObject metadata) {
2429
return metadata.getLong("version");

‎src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import java.time.Instant;
2525

2626
public class RotatingCloudEncryptionKeyProvider implements StoreReader<Map<Integer, CloudEncryptionKey>> {
27-
ScopedStoreReader<Map<Integer, CloudEncryptionKey>> reader;
27+
protected ScopedStoreReader<Map<Integer, CloudEncryptionKey>> reader;
2828

2929
private static final Logger LOGGER = LoggerFactory.getLogger(RotatingCloudEncryptionKeyProvider.class);
3030
public Map<Integer, List<CloudEncryptionKey>> siteToKeysMap = new HashMap<>();
@@ -33,6 +33,11 @@ public RotatingCloudEncryptionKeyProvider(DownloadCloudStorage fileStreamProvide
3333
this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys");
3434
}
3535

36+
37+
public RotatingCloudEncryptionKeyProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope, ScopedStoreReader<Map<Integer, CloudEncryptionKey>> reader) {
38+
this.reader = reader;
39+
}
40+
3641
@Override
3742
public JsonObject getMetadata() throws Exception {
3843
return reader.getMetadata();
@@ -61,6 +66,15 @@ public Map<Integer, CloudEncryptionKey> getAll() {
6166
return keys != null ? keys : new HashMap<>();
6267
}
6368

69+
public CloudEncryptionKey getKey(int id) {
70+
Map<Integer, CloudEncryptionKey> snapshot = reader.getSnapshot();
71+
if(snapshot == null) {
72+
return null;
73+
}
74+
75+
return snapshot.get(id);
76+
}
77+
6478
public void updateSiteToKeysMapping() {
6579
Map<Integer, CloudEncryptionKey> allKeys = getAll();
6680
siteToKeysMap.clear();

‎src/main/java/com/uid2/shared/util/CloudEncryptionHelpers.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package com.uid2.shared.util;
2+
3+
import java.io.InputStream;
4+
5+
import com.fasterxml.jackson.core.JsonFactory;
6+
import com.fasterxml.jackson.core.JsonParser;
7+
import com.fasterxml.jackson.core.JsonToken;
8+
import com.uid2.shared.encryption.AesGcm;
9+
import com.uid2.shared.model.CloudEncryptionKey;
10+
11+
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
12+
import io.vertx.core.json.JsonObject;
13+
import java.nio.charset.StandardCharsets;
14+
import java.util.Base64;
15+
16+
import java.io.*;
17+
18+
public class CloudEncryptionHelpers {
19+
public static String decryptInputStream(InputStream inputStream, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) throws IOException {
20+
JsonFactory factory = new JsonFactory();
21+
JsonParser parser = factory.createParser(inputStream);
22+
int keyId = -1;
23+
byte[] encryptedPayload = null;
24+
parser.nextToken();
25+
while (parser.nextToken() != JsonToken.END_OBJECT) {
26+
String fieldName = parser.getCurrentName();
27+
if(fieldName.equals("key_id")) {
28+
parser.nextToken();
29+
keyId = parser.getIntValue();
30+
}
31+
if(fieldName.equals("encrypted_payload")) {
32+
parser.nextToken();
33+
encryptedPayload = parser.getBinaryValue();
34+
}
35+
}
36+
37+
if(keyId == -1 || encryptedPayload == null) {
38+
throw new IllegalStateException("failed to parse json");
39+
}
40+
41+
CloudEncryptionKey decryptionKey = cloudEncryptionKeyProvider.getKey(keyId);
42+
43+
if (decryptionKey == null) {
44+
throw new IllegalStateException("No matching S3 key found for decryption for key ID: " + keyId);
45+
}
46+
47+
byte[] secret = Base64.getDecoder().decode(decryptionKey.getSecret());
48+
byte[] encryptedBytes = encryptedPayload;
49+
byte[] decryptedBytes = AesGcm.decrypt(encryptedBytes, 0, secret);
50+
51+
return new String(decryptedBytes, StandardCharsets.UTF_8);
52+
}
53+
}

0 commit comments

Comments
 (0)