Skip to content

Commit 38edb8d

Browse files
MMcMMMcM
committed
Pull out transformer state and prefix bit manipulation.
Use varint encoding for prefix.
1 parent 80d5e7c commit 38edb8d

File tree

5 files changed

+309
-135
lines changed

5 files changed

+309
-135
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/common/TransformedRecordSerializer.java

Lines changed: 35 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,13 @@
2727
import com.apple.foundationdb.record.logging.LogMessageKeys;
2828
import com.apple.foundationdb.record.metadata.RecordType;
2929
import com.apple.foundationdb.tuple.Tuple;
30-
import com.google.common.annotations.VisibleForTesting;
3130
import com.google.protobuf.Message;
32-
import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings;
3331

3432
import javax.annotation.Nonnull;
3533
import javax.annotation.Nullable;
3634
import java.nio.ByteBuffer;
3735
import java.nio.ByteOrder;
3836
import java.security.GeneralSecurityException;
39-
import java.util.Arrays;
4037
import java.util.concurrent.ThreadLocalRandom;
4138
import java.util.zip.DataFormatException;
4239
import java.util.zip.Deflater;
@@ -50,7 +47,8 @@
5047
* added in the future.
5148
*
5249
* <p>
53-
* This serializer will begin each serialized string with a one-byte prefix
50+
* This serializer will begin each serialized string with a prefix
51+
* (see {@link TransformedRecordSerializerPrefix} for details)
5452
* containing information about which transformations were performed. This
5553
* way, when deserializing, it can detect which transformations were applied
5654
* so it knows which ones it needs to use to restore the original record.
@@ -78,15 +76,6 @@
7876
*/
7977
@API(API.Status.UNSTABLE)
8078
public class TransformedRecordSerializer<M extends Message> implements RecordSerializer<M> {
81-
@VisibleForTesting
82-
protected static final int ENCODING_ENCRYPTED = 1;
83-
@VisibleForTesting
84-
protected static final int ENCODING_CLEAR = 2;
85-
@VisibleForTesting
86-
protected static final int ENCODING_COMPRESSED = 4;
87-
// TODO: Can remove this after transition to write everything with _CLEAR.
88-
protected static final int ENCODING_PROTO_MESSAGE_FIELD = 0x02;
89-
protected static final int ENCODING_PROTO_TYPE_MASK = 0x07;
9079
protected static final int DEFAULT_COMPRESSION_LEVEL = Deflater.BEST_COMPRESSION;
9180
protected static final int MIN_COMPRESSION_VERSION = 1;
9281
protected static final int MAX_COMPRESSION_VERSION = 1;
@@ -110,53 +99,7 @@ protected TransformedRecordSerializer(@Nonnull RecordSerializer<M> inner,
11099
this.writeValidationRatio = writeValidationRatio;
111100
}
112101

113-
@SpotBugsSuppressWarnings("EI_EXPOSE_REP")
114-
protected static class TransformState {
115-
public boolean compressed;
116-
public boolean encrypted;
117-
118-
@Nonnull public byte[] data;
119-
public int offset;
120-
public int length;
121-
122-
public TransformState(@Nonnull byte[] data) {
123-
this(data, 0, data.length);
124-
}
125-
126-
public TransformState(@Nonnull byte[] data, int offset, int length) {
127-
this.compressed = false;
128-
this.encrypted = false;
129-
this.data = data;
130-
this.offset = offset;
131-
this.length = length;
132-
}
133-
134-
@Nonnull
135-
public byte[] getDataArray() {
136-
if (offset == 0 && length == data.length) {
137-
return data;
138-
} else {
139-
byte[] newData = Arrays.copyOfRange(data, offset, offset + length);
140-
offset = 0;
141-
length = newData.length;
142-
data = newData;
143-
return newData;
144-
}
145-
}
146-
147-
148-
public void setDataArray(@Nonnull byte[] data) {
149-
setDataArray(data, 0, data.length);
150-
}
151-
152-
public void setDataArray(@Nonnull byte[] data, int offset, int length) {
153-
this.data = data;
154-
this.offset = offset;
155-
this.length = length;
156-
}
157-
}
158-
159-
protected void compress(@Nonnull TransformState state, @Nullable StoreTimer timer) {
102+
protected void compress(@Nonnull TransformedRecordSerializerState state, @Nullable StoreTimer timer) {
160103
long startTime = System.nanoTime();
161104

162105
increment(timer, Counts.RECORD_BYTES_BEFORE_COMPRESSION, state.length);
@@ -209,7 +152,7 @@ private void increment(@Nullable StoreTimer timer, StoreTimer.Count counter, int
209152
}
210153
}
211154

212-
protected void encrypt(@Nonnull TransformState state, @Nullable StoreTimer timer) throws GeneralSecurityException {
155+
protected void encrypt(@Nonnull TransformedRecordSerializerState state, @Nullable StoreTimer timer) throws GeneralSecurityException {
213156
throw new RecordSerializationException("this serializer cannot encrypt");
214157
}
215158

@@ -225,7 +168,7 @@ public byte[] serialize(@Nonnull RecordMetaData metaData,
225168
@Nullable StoreTimer timer) {
226169
byte[] innerSerialized = inner.serialize(metaData, recordType, rec, timer);
227170

228-
TransformState state = new TransformState(innerSerialized);
171+
TransformedRecordSerializerState state = new TransformedRecordSerializerState(innerSerialized);
229172

230173
if (compressWhenSerializing) {
231174
compress(state, timer);
@@ -241,32 +184,16 @@ public byte[] serialize(@Nonnull RecordMetaData metaData,
241184
}
242185
}
243186

244-
int code;
245-
if (state.compressed || state.encrypted) {
246-
code = 0;
247-
if (state.compressed) {
248-
code = code | ENCODING_COMPRESSED;
249-
}
250-
if (state.encrypted) {
251-
code = code | ENCODING_ENCRYPTED;
252-
}
253-
} else {
254-
code = ENCODING_CLEAR;
255-
}
256-
257-
int size = state.length + 1;
258-
byte[] serialized = new byte[size];
259-
serialized[0] = (byte) code;
260-
System.arraycopy(state.data, state.offset, serialized, 1, state.length);
187+
TransformedRecordSerializerPrefix.encodePrefix(state);
261188

262189
if (shouldValidateSerialization()) {
263-
validateSerialization(metaData, recordType, rec, serialized, timer);
190+
validateSerialization(metaData, recordType, rec, state.getDataArray(), timer);
264191
}
265192

266-
return serialized;
193+
return state.getDataArray();
267194
}
268195

269-
protected void decompress(@Nonnull TransformState state, @Nullable StoreTimer timer) throws DataFormatException {
196+
protected void decompress(@Nonnull TransformedRecordSerializerState state, @Nullable StoreTimer timer) throws DataFormatException {
270197
final long startTime = System.nanoTime();
271198

272199
// At the moment, there is only one compression version, so
@@ -305,7 +232,7 @@ protected void decompress(@Nonnull TransformState state, @Nullable StoreTimer ti
305232
}
306233
}
307234

308-
protected void decrypt(@Nonnull TransformState state, @Nullable StoreTimer timer) throws GeneralSecurityException {
235+
protected void decrypt(@Nonnull TransformedRecordSerializerState state, @Nullable StoreTimer timer) throws GeneralSecurityException {
309236
throw new RecordSerializationException("this serializer cannot decrypt");
310237
}
311238

@@ -316,52 +243,35 @@ public M deserialize(@Nonnull RecordMetaData metaData,
316243
@Nonnull Tuple primaryKey,
317244
@Nonnull byte[] serialized,
318245
@Nullable StoreTimer timer) {
319-
int encoding = serialized[0];
320-
if (encoding != ENCODING_CLEAR && (encoding & ENCODING_PROTO_TYPE_MASK) == ENCODING_PROTO_MESSAGE_FIELD) {
321-
// TODO: Can remove this after transition to write everything with _CLEAR.
246+
TransformedRecordSerializerState state = new TransformedRecordSerializerState(serialized);
247+
if (!TransformedRecordSerializerPrefix.decodePrefix(state, primaryKey)) {
322248
return inner.deserialize(metaData, primaryKey, serialized, timer);
323-
} else {
324-
TransformState state = new TransformState(serialized, 1, serialized.length - 1);
325-
if (encoding != ENCODING_CLEAR) {
326-
if ((encoding & ENCODING_COMPRESSED) == ENCODING_COMPRESSED) {
327-
state.compressed = true;
328-
}
329-
if ((encoding & ENCODING_ENCRYPTED) == ENCODING_ENCRYPTED) {
330-
state.encrypted = true;
331-
}
332-
if ((encoding & ~(ENCODING_COMPRESSED | ENCODING_ENCRYPTED)) != 0) {
333-
throw new RecordSerializationException("unrecognized transformation encoding")
334-
.addLogInfo(LogMessageKeys.META_DATA_VERSION, metaData.getVersion())
335-
.addLogInfo(LogMessageKeys.PRIMARY_KEY, primaryKey)
336-
.addLogInfo("encoding", encoding);
337-
}
338-
}
339-
if (state.encrypted) {
340-
try {
341-
decrypt(state, timer);
342-
} catch (RecordCoreException ex) {
343-
throw ex.addLogInfo(LogMessageKeys.META_DATA_VERSION, metaData.getVersion())
344-
.addLogInfo(LogMessageKeys.PRIMARY_KEY, primaryKey);
345-
} catch (GeneralSecurityException ex) {
346-
throw new RecordSerializationException("decryption error", ex)
347-
.addLogInfo(LogMessageKeys.META_DATA_VERSION, metaData.getVersion())
348-
.addLogInfo(LogMessageKeys.PRIMARY_KEY, primaryKey);
349-
}
249+
}
250+
if (state.encrypted) {
251+
try {
252+
decrypt(state, timer);
253+
} catch (RecordCoreException ex) {
254+
throw ex.addLogInfo(LogMessageKeys.META_DATA_VERSION, metaData.getVersion())
255+
.addLogInfo(LogMessageKeys.PRIMARY_KEY, primaryKey);
256+
} catch (GeneralSecurityException ex) {
257+
throw new RecordSerializationException("decryption error", ex)
258+
.addLogInfo(LogMessageKeys.META_DATA_VERSION, metaData.getVersion())
259+
.addLogInfo(LogMessageKeys.PRIMARY_KEY, primaryKey);
350260
}
351-
if (state.compressed) {
352-
try {
353-
decompress(state, timer);
354-
} catch (RecordCoreException ex) {
355-
throw ex.addLogInfo(LogMessageKeys.META_DATA_VERSION, metaData.getVersion())
356-
.addLogInfo(LogMessageKeys.PRIMARY_KEY, primaryKey);
357-
} catch (DataFormatException ex) {
358-
throw new RecordSerializationException("decompression error", ex)
359-
.addLogInfo(LogMessageKeys.META_DATA_VERSION, metaData.getVersion())
360-
.addLogInfo(LogMessageKeys.PRIMARY_KEY, primaryKey);
361-
}
261+
}
262+
if (state.compressed) {
263+
try {
264+
decompress(state, timer);
265+
} catch (RecordCoreException ex) {
266+
throw ex.addLogInfo(LogMessageKeys.META_DATA_VERSION, metaData.getVersion())
267+
.addLogInfo(LogMessageKeys.PRIMARY_KEY, primaryKey);
268+
} catch (DataFormatException ex) {
269+
throw new RecordSerializationException("decompression error", ex)
270+
.addLogInfo(LogMessageKeys.META_DATA_VERSION, metaData.getVersion())
271+
.addLogInfo(LogMessageKeys.PRIMARY_KEY, primaryKey);
362272
}
363-
return inner.deserialize(metaData, primaryKey, state.getDataArray(), timer);
364273
}
274+
return inner.deserialize(metaData, primaryKey, state.getDataArray(), timer);
365275
}
366276

367277
@Nonnull

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/common/TransformedRecordSerializerJCE.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ protected TransformedRecordSerializerJCE(@Nonnull RecordSerializer<M> inner,
6161
}
6262

6363
@Override
64-
protected void encrypt(@Nonnull TransformState state, @Nullable StoreTimer timer) throws GeneralSecurityException {
64+
protected void encrypt(@Nonnull TransformedRecordSerializerState state, @Nullable StoreTimer timer) throws GeneralSecurityException {
6565
if (cipherName == null || encryptionKey == null || secureRandom == null) {
6666
throw new RecordSerializationException("attempted to encrypt without setting cipher name and key");
6767
}
@@ -92,7 +92,7 @@ protected void encrypt(@Nonnull TransformState state, @Nullable StoreTimer timer
9292
}
9393

9494
@Override
95-
protected void decrypt(@Nonnull TransformState state, @Nullable StoreTimer timer) throws GeneralSecurityException {
95+
protected void decrypt(@Nonnull TransformedRecordSerializerState state, @Nullable StoreTimer timer) throws GeneralSecurityException {
9696
if (cipherName == null || encryptionKey == null || secureRandom == null) {
9797
throw new RecordSerializationException("missing encryption key or provider during decryption");
9898
}

0 commit comments

Comments
 (0)