diff --git a/protobuf-api/src/main/java/com/google/protobuf/ByteString.java b/protobuf-api/src/main/java/com/google/protobuf/ByteString.java index 8ba729c..99823f5 100644 --- a/protobuf-api/src/main/java/com/google/protobuf/ByteString.java +++ b/protobuf-api/src/main/java/com/google/protobuf/ByteString.java @@ -11,22 +11,21 @@ import static java.lang.Integer.toHexString; import static java.lang.System.identityHashCode; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.InvalidObjectException; -import java.io.ObjectInputStream; import java.io.OutputStream; import java.io.Serializable; import java.io.UnsupportedEncodingException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.nio.charset.UnsupportedCharsetException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -69,8 +68,22 @@ public abstract class ByteString implements Iterable, Serializable { static final int MAX_READ_FROM_CHUNK_SIZE = 0x2000; // 8k + private static final Class LITERAL_BYTE_STRING = getClassForName("com.google.protobuf.LiteralByteString"); + private static final Class LEAF_BYTE_STRING = getClassForName("com.google.protobuf.LeafByteString"); + private static final Class NIO_BYTE_STRING = getClassForName("com.google.protobuf.NioByteString"); + private static final Class ROPE_BYTE_STRING = getClassForName("com.google.protobuf.RopeByteString"); + private static final Class BOUNDED_BYTE_STRING = getClassForName("com.google.protobuf.BoundedByteString"); + + private static Class getClassForName(String name) { + try { + return (Class) Class.forName(name); + } catch (Throwable e) { + return null; + } + } + /** Empty {@code ByteString}. */ - public static final ByteString EMPTY = new LiteralByteString(Internal.EMPTY_BYTE_ARRAY); + public static final ByteString EMPTY = newLiteralByteString(Internal.EMPTY_BYTE_ARRAY); /** * An interface to efficiently copy {@code byte[]}. @@ -379,7 +392,21 @@ public static ByteString fromHex(@CompileTimeConstant String hexString) { int d2 = extractHexDigit(hexString, 2 * i + 1); bytes[i] = (byte) (d1 << 4 | d2); } - return new LiteralByteString(bytes); + + return newLiteralByteString(bytes); + } + + private static ByteString newLiteralByteString(byte[] bytes) { + try { + Constructor constructor = LITERAL_BYTE_STRING.getDeclaredConstructors()[0]; + return (ByteString) constructor.newInstance(bytes); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } } // ================================================================= @@ -396,7 +423,7 @@ public static ByteString fromHex(@CompileTimeConstant String hexString) { */ public static ByteString copyFrom(byte[] bytes, int offset, int size) { checkRange(offset, offset + size, bytes.length); - return new LiteralByteString(byteArrayCopier.copyFrom(bytes, offset, size)); + return newLiteralByteString(byteArrayCopier.copyFrom(bytes, offset, size)); } /** @@ -418,7 +445,16 @@ static ByteString wrap(ByteBuffer buffer) { final int offset = buffer.arrayOffset(); return ByteString.wrap(buffer.array(), offset + buffer.position(), buffer.remaining()); } else { - return new NioByteString(buffer); + try { + Constructor constructor = NIO_BYTE_STRING.getDeclaredConstructors()[0]; + return (ByteString) constructor.newInstance(buffer); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } } } @@ -428,7 +464,7 @@ static ByteString wrap(ByteBuffer buffer) { */ static ByteString wrap(byte[] bytes) { // TODO: Return EMPTY when bytes are empty to reduce allocations? - return new LiteralByteString(bytes); + return newLiteralByteString(bytes); } /** @@ -436,7 +472,16 @@ static ByteString wrap(byte[] bytes) { * to force a classload of ByteString before BoundedByteString and LiteralByteString. */ static ByteString wrap(byte[] bytes, int offset, int length) { - return new BoundedByteString(bytes, offset, length); + try { + Constructor constructor = BOUNDED_BYTE_STRING.getDeclaredConstructors()[0]; + return (ByteString) constructor.newInstance(bytes, offset, length); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } } /** @@ -452,7 +497,8 @@ public static ByteString copyFrom(ByteBuffer bytes, int size) { checkRange(0, size, bytes.remaining()); byte[] copy = new byte[size]; bytes.get(copy); - return new LiteralByteString(copy); + return newLiteralByteString(copy); + } /** @@ -476,7 +522,7 @@ public static ByteString copyFrom(ByteBuffer bytes) { */ public static ByteString copyFrom(String text, String charsetName) throws UnsupportedEncodingException { - return new LiteralByteString(text.getBytes(charsetName)); + return newLiteralByteString(text.getBytes(charsetName)); } /** @@ -488,7 +534,7 @@ public static ByteString copyFrom(String text, String charsetName) * @return new {@code ByteString} */ public static ByteString copyFrom(String text, Charset charset) { - return new LiteralByteString(text.getBytes(charset)); + return newLiteralByteString(text.getBytes(charset)); } /** @@ -499,7 +545,7 @@ public static ByteString copyFrom(String text, Charset charset) { * @return new {@code ByteString} */ public static ByteString copyFromUtf8(String text) { - return new LiteralByteString(text.getBytes(Internal.UTF_8)); + return newLiteralByteString(text.getBytes(Internal.UTF_8)); } // ================================================================= @@ -623,8 +669,16 @@ public final ByteString concat(ByteString other) { throw new IllegalArgumentException( "ByteString would be too long: " + size() + "+" + other.size()); } - - return RopeByteString.concatenate(this, other); + try { + Method concatenateMethod = ROPE_BYTE_STRING.getMethod("concatenate", ByteString.class, ByteString.class); + return (ByteString) concatenateMethod.invoke(null, this, other); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } } /** @@ -924,37 +978,6 @@ public final String toStringUtf8() { @Override public abstract boolean equals(Object o); - /** Base class for leaf {@link ByteString}s (i.e. non-ropes). */ - abstract static class LeafByteString extends ByteString { - private static final long serialVersionUID = 1L; - - @Override - protected final int getTreeDepth() { - return 0; - } - - @Override - protected final boolean isBalanced() { - return true; - } - - @Override - void writeToReverse(ByteOutput byteOutput) throws IOException { - writeTo(byteOutput); - } - - /** - * Check equality of the substring of given length of this object starting at zero with another - * {@code ByteString} substring starting at offset. - * - * @param other what to compare a substring in - * @param offset offset into other - * @param length number of bytes to compare - * @return true for equality of substrings, else false. - */ - abstract boolean equalsRange(ByteString other, int offset, int length); - } - /** * Compute the hashCode using the traditional algorithm from {@link ByteString}. * @@ -1168,7 +1191,7 @@ public String toString() { * buffer whose size is at least the specified minimum size. */ private void flushFullBuffer(int minSize) { - flushedBuffers.add(new LiteralByteString(buffer)); + flushedBuffers.add(newLiteralByteString(buffer)); flushedBuffersTotalBytes += buffer.length; // We want to increase our total capacity by 50%, but as a minimum, // the new buffer should also at least be >= minSize and @@ -1186,12 +1209,12 @@ private void flushLastBuffer() { if (bufferPos < buffer.length) { if (bufferPos > 0) { byte[] bufferCopy = Arrays.copyOf(buffer, bufferPos); - flushedBuffers.add(new LiteralByteString(bufferCopy)); + flushedBuffers.add(newLiteralByteString(bufferCopy)); } // We reuse this buffer for further writes. } else { // Buffer is completely full. Huzzah. - flushedBuffers.add(new LiteralByteString(buffer)); + flushedBuffers.add(newLiteralByteString(buffer)); // 99% of the time, we're not going to use this OutputStream again. // We set buffer to an empty byte stream so that we're handling this // case without wasting space. In the rare case that more writes @@ -1236,7 +1259,7 @@ public ByteString build() { // We can be confident that the CodedOutputStream will not modify the // underlying bytes anymore because it already wrote all of them. So, // no need to make a copy. - return new LiteralByteString(buffer); + return newLiteralByteString(buffer); } public CodedOutputStream getCodedOutput() { @@ -1343,323 +1366,7 @@ private String truncateAndEscapeForDisplay() { return size() <= limit ? escapeBytes(this) : escapeBytes(substring(0, limit - 3)) + "..."; } - /** - * This class implements a {@link com.google.protobuf.ByteString} backed by a single array of - * bytes, contiguous in memory. It supports substring by pointing to only a sub-range of the - * underlying byte array, meaning that a substring will reference the full byte-array of the - * string it's made from, exactly as with {@link String}. - * - * @author carlanton@google.com (Carl Haverl) - */ - // Keep this class private to avoid deadlocks in classloading across threads as ByteString's - // static initializer loads LiteralByteString and another thread loads LiteralByteString. - private static class LiteralByteString extends ByteString.LeafByteString { - private static final long serialVersionUID = 1L; - - protected final byte[] bytes; - - /** - * Creates a {@code LiteralByteString} backed by the given array, without copying. - * - * @param bytes array to wrap - */ - LiteralByteString(byte[] bytes) { - if (bytes == null) { - throw new NullPointerException(); - } - this.bytes = bytes; - } - - @Override - public byte byteAt(int index) { - // Unlike most methods in this class, this one is a direct implementation - // ignoring the potential offset because we need to do range-checking in the - // substring case anyway. - return bytes[index]; - } - - @Override - byte internalByteAt(int index) { - return bytes[index]; - } - - @Override - public int size() { - return bytes.length; - } - - // ================================================================= - // ByteString -> substring - - @Override - public final ByteString substring(int beginIndex, int endIndex) { - final int length = checkRange(beginIndex, endIndex, size()); - - if (length == 0) { - return ByteString.EMPTY; - } - - return new BoundedByteString(bytes, getOffsetIntoBytes() + beginIndex, length); - } - - // ================================================================= - // ByteString -> byte[] - - @Override - protected void copyToInternal( - byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { - // Optimized form, not for subclasses, since we don't call - // getOffsetIntoBytes() or check the 'numberToCopy' parameter. - // TODO: Is not calling getOffsetIntoBytes really saving that much? - System.arraycopy(bytes, sourceOffset, target, targetOffset, numberToCopy); - } - - @Override - public final void copyTo(ByteBuffer target) { - target.put(bytes, getOffsetIntoBytes(), size()); // Copies bytes - } - - @Override - public final ByteBuffer asReadOnlyByteBuffer() { - return ByteBuffer.wrap(bytes, getOffsetIntoBytes(), size()).asReadOnlyBuffer(); - } - - @Override - public final List asReadOnlyByteBufferList() { - return Collections.singletonList(asReadOnlyByteBuffer()); - } - - @Override - public final void writeTo(OutputStream outputStream) throws IOException { - outputStream.write(toByteArray()); - } - - @Override - final void writeToInternal(OutputStream outputStream, int sourceOffset, int numberToWrite) - throws IOException { - outputStream.write(bytes, getOffsetIntoBytes() + sourceOffset, numberToWrite); - } - - @Override - final void writeTo(ByteOutput output) throws IOException { - output.writeLazy(bytes, getOffsetIntoBytes(), size()); - } - - @Override - protected final String toStringInternal(Charset charset) { - return new String(bytes, getOffsetIntoBytes(), size(), charset); - } - - // ================================================================= - // UTF-8 decoding - - @Override - public final boolean isValidUtf8() { - int offset = getOffsetIntoBytes(); - return Utf8.isValidUtf8(bytes, offset, offset + size()); - } - - @Override - protected final int partialIsValidUtf8(int state, int offset, int length) { - int index = getOffsetIntoBytes() + offset; - return Utf8.partialIsValidUtf8(state, bytes, index, index + length); - } - - // ================================================================= - // equals() and hashCode() - - @Override - public final boolean equals(Object other) { - if (other == this) { - return true; - } - if (!(other instanceof ByteString)) { - return false; - } - - if (size() != ((ByteString) other).size()) { - return false; - } - if (size() == 0) { - return true; - } - - if (other instanceof LiteralByteString) { - LiteralByteString otherAsLiteral = (LiteralByteString) other; - // If we know the hash codes and they are not equal, we know the byte - // strings are not equal. - int thisHash = peekCachedHashCode(); - int thatHash = otherAsLiteral.peekCachedHashCode(); - if (thisHash != 0 && thatHash != 0 && thisHash != thatHash) { - return false; - } - - return equalsRange((LiteralByteString) other, 0, size()); - } else { - // RopeByteString and NioByteString. - return other.equals(this); - } - } - - /** - * Check equality of the substring of given length of this object starting at zero with another - * {@code LiteralByteString} substring starting at offset. - * - * @param other what to compare a substring in - * @param offset offset into other - * @param length number of bytes to compare - * @return true for equality of substrings, else false. - */ - @Override - final boolean equalsRange(ByteString other, int offset, int length) { - if (length > other.size()) { - throw new IllegalArgumentException("Length too large: " + length + size()); - } - if (offset + length > other.size()) { - throw new IllegalArgumentException( - "Ran off end of other: " + offset + ", " + length + ", " + other.size()); - } - - if (other instanceof LiteralByteString) { - LiteralByteString lbsOther = (LiteralByteString) other; - byte[] thisBytes = bytes; - byte[] otherBytes = lbsOther.bytes; - int thisLimit = getOffsetIntoBytes() + length; - for (int thisIndex = getOffsetIntoBytes(), - otherIndex = lbsOther.getOffsetIntoBytes() + offset; - (thisIndex < thisLimit); - ++thisIndex, ++otherIndex) { - if (thisBytes[thisIndex] != otherBytes[otherIndex]) { - return false; - } - } - return true; - } - - return other.substring(offset, offset + length).equals(substring(0, length)); - } - - @Override - protected final int partialHash(int h, int offset, int length) { - return Internal.partialHash(h, bytes, getOffsetIntoBytes() + offset, length); - } - - // ================================================================= - // Input stream - - @Override - public final InputStream newInput() { - return new ByteArrayInputStream(bytes, getOffsetIntoBytes(), size()); // No copy - } - - @Override - public final CodedInputStream newCodedInput() { - // We trust CodedInputStream not to modify the bytes, or to give anyone - // else access to them. - return CodedInputStream.newInstance( - bytes, getOffsetIntoBytes(), size(), /* bufferIsImmutable= */ true); - } - - // ================================================================= - // Internal methods - - /** - * Offset into {@code bytes[]} to use, non-zero for substrings. - * - * @return always 0 for this class - */ - protected int getOffsetIntoBytes() { - return 0; - } - } - - /** - * This class is used to represent the substring of a {@link ByteString} over a single byte array. - * In terms of the public API of {@link ByteString}, you end up here by calling {@link - * ByteString#copyFrom(byte[])} followed by {@link ByteString#substring(int, int)}. - * - *

This class contains most of the overhead involved in creating a substring from a {@link - * LiteralByteString}. The overhead involves some range-checking and two extra fields. - * - * @author carlanton@google.com (Carl Haverl) - */ - // Keep this class private to avoid deadlocks in classloading across threads as ByteString's - // static initializer loads LiteralByteString and another thread loads BoundedByteString. - private static final class BoundedByteString extends LiteralByteString { - private final int bytesOffset; - private final int bytesLength; - - /** - * Creates a {@code BoundedByteString} backed by the sub-range of given array, without copying. - * - * @param bytes array to wrap - * @param offset index to first byte to use in bytes - * @param length number of bytes to use from bytes - * @throws IllegalArgumentException if {@code offset < 0}, {@code length < 0}, or if {@code - * offset + length > bytes.length}. - */ - BoundedByteString(byte[] bytes, int offset, int length) { - super(bytes); - checkRange(offset, offset + length, bytes.length); - - this.bytesOffset = offset; - this.bytesLength = length; - } - - /** - * Gets the byte at the given index. Throws {@link ArrayIndexOutOfBoundsException} for - * backwards-compatibility reasons although it would more properly be {@link - * IndexOutOfBoundsException}. - * - * @param index index of byte - * @return the value - * @throws ArrayIndexOutOfBoundsException {@code index} is < 0 or >= size - */ - @Override - public byte byteAt(int index) { - // We must check the index ourselves as we cannot rely on Java array index - // checking for substrings. - checkIndex(index, size()); - return bytes[bytesOffset + index]; - } - - @Override - byte internalByteAt(int index) { - return bytes[bytesOffset + index]; - } - @Override - public int size() { - return bytesLength; - } - @Override - protected int getOffsetIntoBytes() { - return bytesOffset; - } - // ================================================================= - // ByteString -> byte[] - - @Override - protected void copyToInternal( - byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { - System.arraycopy( - bytes, getOffsetIntoBytes() + sourceOffset, target, targetOffset, numberToCopy); - } - - // ================================================================= - // Serializable - - private static final long serialVersionUID = 1L; - - Object writeReplace() { - return ByteString.wrap(toByteArray()); - } - - private void readObject(@SuppressWarnings("unused") ObjectInputStream in) throws IOException { - throw new InvalidObjectException( - "BoundedByteStream instances are not to be serialized directly"); - } - } } diff --git a/protobuf-api/src/main/java/com/google/protobuf/CodedInputStream.java b/protobuf-api/src/main/java/com/google/protobuf/CodedInputStream.java index fd29913..ae4ad1c 100644 --- a/protobuf-api/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/protobuf-api/src/main/java/com/google/protobuf/CodedInputStream.java @@ -8,22 +8,13 @@ package com.google.protobuf; import static com.google.protobuf.Internal.EMPTY_BYTE_ARRAY; -import static com.google.protobuf.Internal.EMPTY_BYTE_BUFFER; -import static com.google.protobuf.Internal.UTF_8; import static com.google.protobuf.Internal.checkNotNull; -import static com.google.protobuf.WireFormat.FIXED32_SIZE; -import static com.google.protobuf.WireFormat.FIXED64_SIZE; -import static com.google.protobuf.WireFormat.MAX_VARINT_SIZE; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.Buffer; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; /** * Reads and decodes protocol message fields. @@ -37,11 +28,24 @@ * @author kenton@google.com Kenton Varda */ public abstract class CodedInputStream { - private static final int DEFAULT_BUFFER_SIZE = 4096; + static final int DEFAULT_BUFFER_SIZE = 4096; // Integer.MAX_VALUE == 0x7FFFFFF == INT_MAX from limits.h private static final int DEFAULT_SIZE_LIMIT = Integer.MAX_VALUE; private static volatile int defaultRecursionLimit = 100; + private static final Class ARRAY_DECODER = getClassForName("com.google.protobuf.ArrayDecoder"); + private static final Class ITERABLE_DIRECT_BYTE_BUFFER_DECODER = getClassForName("com.google.protobuf.IterableDirectByteBufferDecoder"); + private static final Class STREAM_DECODER = getClassForName("com.google.protobuf.StreamDecoder"); + private static final Class UNSAFE_DIRECT_NIO_DECODER = getClassForName("com.google.protobuf.UnsafeDirectNioDecoder"); + + private static Class getClassForName(String name) { + try { + return (Class) Class.forName(name); + } catch (Throwable e) { + return null; + } + } + /** Visible for subclasses. See setRecursionLimit() */ int recursionDepth; @@ -67,12 +71,21 @@ public static CodedInputStream newInstance(final InputStream input, int bufferSi // Ideally we would throw here. This is done for backward compatibility. return newInstance(EMPTY_BYTE_ARRAY); } - return new StreamDecoder(input, bufferSize); + try { + Constructor constructor = STREAM_DECODER.getDeclaredConstructors()[0]; + return (CodedInputStream) constructor.newInstance(input, bufferSize); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } } /** Create a new CodedInputStream wrapping the given {@code Iterable }. */ public static CodedInputStream newInstance(final Iterable input) { - if (!UnsafeDirectNioDecoder.isSupported()) { + if (!UnsafeUtil.hasUnsafeByteBufferOperations()) { return newInstance(new IterableByteBufferInputStream(input)); } return newInstance(input, false); @@ -100,7 +113,16 @@ static CodedInputStream newInstance( } } if (flag == 2) { - return new IterableDirectByteBufferDecoder(bufs, totalSize, bufferIsImmutable); + try { + Constructor constructor = ITERABLE_DIRECT_BYTE_BUFFER_DECODER.getDeclaredConstructors()[0]; + return (CodedInputStream) constructor.newInstance(bufs, totalSize, bufferIsImmutable); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } } else { // TODO: add another decoders to deal case 1 and 3. return newInstance(new IterableByteBufferInputStream(bufs)); @@ -120,14 +142,24 @@ public static CodedInputStream newInstance(final byte[] buf, final int off, fina /** Create a new CodedInputStream wrapping the given byte array slice. */ static CodedInputStream newInstance( final byte[] buf, final int off, final int len, final boolean bufferIsImmutable) { - ArrayDecoder result = new ArrayDecoder(buf, off, len, bufferIsImmutable); + CodedInputStream arrayDecoder; + try { + Constructor constructor = ARRAY_DECODER.getDeclaredConstructors()[0]; + arrayDecoder = (CodedInputStream) constructor.newInstance(buf, off, len, bufferIsImmutable); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } try { // Some uses of CodedInputStream can be more efficient if they know // exactly how many bytes are available. By pushing the end point of the // buffer as a limit, we allow them to get this information via // getBytesUntilLimit(). Pushing a limit that we know is at the end of // the stream can never hurt, since we can never past that point anyway. - result.pushLimit(len); + arrayDecoder.pushLimit(len); } catch (InvalidProtocolBufferException ex) { // The only reason pushLimit() might throw an exception here is if len // is negative. Normally pushLimit()'s parameter comes directly off the @@ -138,7 +170,7 @@ static CodedInputStream newInstance( // appropriate. throw new IllegalArgumentException(ex); } - return result; + return arrayDecoder; } /** @@ -160,8 +192,17 @@ static CodedInputStream newInstance(ByteBuffer buf, boolean bufferIsImmutable) { buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(), bufferIsImmutable); } - if (buf.isDirect() && UnsafeDirectNioDecoder.isSupported()) { - return new UnsafeDirectNioDecoder(buf, bufferIsImmutable); + if (buf.isDirect() && UnsafeUtil.hasUnsafeByteBufferOperations()) { + try { + Constructor constructor = UNSAFE_DIRECT_NIO_DECODER.getDeclaredConstructors()[0]; + return (CodedInputStream) constructor.newInstance(buf, bufferIsImmutable); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } } // The buffer is non-direct and does not expose the underlying array. Using the ByteBuffer API @@ -178,7 +219,7 @@ public void checkRecursionLimit() throws InvalidProtocolBufferException { } } /** Disable construction/inheritance outside of this class. */ - private CodedInputStream() {} + CodedInputStream() {} // ----------------------------------------------------------------- @@ -598,3309 +639,4 @@ static int readRawVarint32(final InputStream input) throws IOException { } return readRawVarint32(firstByte, input); } - - /** A {@link CodedInputStream} implementation that uses a backing array as the input. */ - private static final class ArrayDecoder extends CodedInputStream { - private final byte[] buffer; - private final boolean immutable; - private int limit; - private int bufferSizeAfterLimit; - private int pos; - private int startPos; - private int lastTag; - private boolean enableAliasing; - - /** The absolute position of the end of the current message. */ - private int currentLimit = Integer.MAX_VALUE; - - private ArrayDecoder(final byte[] buffer, final int offset, final int len, boolean immutable) { - this.buffer = buffer; - limit = offset + len; - pos = offset; - startPos = pos; - this.immutable = immutable; - } - - @Override - public int readTag() throws IOException { - if (isAtEnd()) { - lastTag = 0; - return 0; - } - - lastTag = readRawVarint32(); - if (WireFormat.getTagFieldNumber(lastTag) == 0) { - // If we actually read zero (or any tag number corresponding to field - // number zero), that's not a valid tag. - throw InvalidProtocolBufferException.invalidTag(); - } - return lastTag; - } - - @Override - public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { - if (lastTag != value) { - throw InvalidProtocolBufferException.invalidEndTag(); - } - } - - @Override - public int getLastTag() { - return lastTag; - } - - @Override - public boolean skipField(final int tag) throws IOException { - switch (WireFormat.getTagWireType(tag)) { - case WireFormat.WIRETYPE_VARINT: - skipRawVarint(); - return true; - case WireFormat.WIRETYPE_FIXED64: - skipRawBytes(FIXED64_SIZE); - return true; - case WireFormat.WIRETYPE_LENGTH_DELIMITED: - skipRawBytes(readRawVarint32()); - return true; - case WireFormat.WIRETYPE_START_GROUP: - skipMessage(); - checkLastTagWas( - WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); - return true; - case WireFormat.WIRETYPE_END_GROUP: - return false; - case WireFormat.WIRETYPE_FIXED32: - skipRawBytes(FIXED32_SIZE); - return true; - default: - throw InvalidProtocolBufferException.invalidWireType(); - } - } - - @Override - public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { - switch (WireFormat.getTagWireType(tag)) { - case WireFormat.WIRETYPE_VARINT: - { - long value = readInt64(); - output.writeUInt32NoTag(tag); - output.writeUInt64NoTag(value); - return true; - } - case WireFormat.WIRETYPE_FIXED64: - { - long value = readRawLittleEndian64(); - output.writeUInt32NoTag(tag); - output.writeFixed64NoTag(value); - return true; - } - case WireFormat.WIRETYPE_LENGTH_DELIMITED: - { - ByteString value = readBytes(); - output.writeUInt32NoTag(tag); - output.writeBytesNoTag(value); - return true; - } - case WireFormat.WIRETYPE_START_GROUP: - { - output.writeUInt32NoTag(tag); - skipMessage(output); - int endtag = - WireFormat.makeTag( - WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); - checkLastTagWas(endtag); - output.writeUInt32NoTag(endtag); - return true; - } - case WireFormat.WIRETYPE_END_GROUP: - { - return false; - } - case WireFormat.WIRETYPE_FIXED32: - { - int value = readRawLittleEndian32(); - output.writeUInt32NoTag(tag); - output.writeFixed32NoTag(value); - return true; - } - default: - throw InvalidProtocolBufferException.invalidWireType(); - } - } - - // ----------------------------------------------------------------- - - @Override - public double readDouble() throws IOException { - return Double.longBitsToDouble(readRawLittleEndian64()); - } - - @Override - public float readFloat() throws IOException { - return Float.intBitsToFloat(readRawLittleEndian32()); - } - - @Override - public long readUInt64() throws IOException { - return readRawVarint64(); - } - - @Override - public long readInt64() throws IOException { - return readRawVarint64(); - } - - @Override - public int readInt32() throws IOException { - return readRawVarint32(); - } - - @Override - public long readFixed64() throws IOException { - return readRawLittleEndian64(); - } - - @Override - public int readFixed32() throws IOException { - return readRawLittleEndian32(); - } - - @Override - public boolean readBool() throws IOException { - return readRawVarint64() != 0; - } - - @Override - public String readString() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= (limit - pos)) { - // Fast path: We already have the bytes in a contiguous buffer, so - // just copy directly from it. - final String result = new String(buffer, pos, size, UTF_8); - pos += size; - return result; - } - - if (size == 0) { - return ""; - } - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public String readStringRequireUtf8() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= (limit - pos)) { - String result = Utf8.decodeUtf8(buffer, pos, size); - pos += size; - return result; - } - - if (size == 0) { - return ""; - } - if (size <= 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public void readGroup( - final int fieldNumber, - final MessageLite.Builder builder, - final ExtensionRegistryLite extensionRegistry) - throws IOException { - checkRecursionLimit(); - ++recursionDepth; - builder.mergeFrom(this, extensionRegistry); - checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); - --recursionDepth; - } - - @Override - public T readGroup( - final int fieldNumber, - final Parser parser, - final ExtensionRegistryLite extensionRegistry) - throws IOException { - checkRecursionLimit(); - ++recursionDepth; - T result = parser.parsePartialFrom(this, extensionRegistry); - checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); - --recursionDepth; - return result; - } - - @Deprecated - @Override - public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) - throws IOException { - readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry()); - } - - @Override - public void readMessage( - final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) - throws IOException { - final int length = readRawVarint32(); - checkRecursionLimit(); - final int oldLimit = pushLimit(length); - ++recursionDepth; - builder.mergeFrom(this, extensionRegistry); - checkLastTagWas(0); - --recursionDepth; - if (getBytesUntilLimit() != 0) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - popLimit(oldLimit); - } - - @Override - public T readMessage( - final Parser parser, final ExtensionRegistryLite extensionRegistry) throws IOException { - int length = readRawVarint32(); - checkRecursionLimit(); - final int oldLimit = pushLimit(length); - ++recursionDepth; - T result = parser.parsePartialFrom(this, extensionRegistry); - checkLastTagWas(0); - --recursionDepth; - if (getBytesUntilLimit() != 0) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - popLimit(oldLimit); - return result; - } - - @Override - public ByteString readBytes() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= (limit - pos)) { - // Fast path: We already have the bytes in a contiguous buffer, so - // just copy directly from it. - final ByteString result = - immutable && enableAliasing - ? ByteString.wrap(buffer, pos, size) - : ByteString.copyFrom(buffer, pos, size); - pos += size; - return result; - } - if (size == 0) { - return ByteString.EMPTY; - } - // Slow path: Build a byte array first then copy it. - return ByteString.wrap(readRawBytes(size)); - } - - @Override - public byte[] readByteArray() throws IOException { - final int size = readRawVarint32(); - return readRawBytes(size); - } - - @Override - public ByteBuffer readByteBuffer() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= (limit - pos)) { - // Fast path: We already have the bytes in a contiguous buffer. - // When aliasing is enabled, we can return a ByteBuffer pointing directly - // into the underlying byte array without copy if the CodedInputStream is - // constructed from a byte array. If aliasing is disabled or the input is - // from an InputStream or ByteString, we have to make a copy of the bytes. - ByteBuffer result = - !immutable && enableAliasing - ? ByteBuffer.wrap(buffer, pos, size).slice() - : ByteBuffer.wrap(Arrays.copyOfRange(buffer, pos, pos + size)); - pos += size; - // TODO: Investigate making the ByteBuffer be made read-only - return result; - } - - if (size == 0) { - return EMPTY_BYTE_BUFFER; - } - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public int readUInt32() throws IOException { - return readRawVarint32(); - } - - @Override - public int readEnum() throws IOException { - return readRawVarint32(); - } - - @Override - public int readSFixed32() throws IOException { - return readRawLittleEndian32(); - } - - @Override - public long readSFixed64() throws IOException { - return readRawLittleEndian64(); - } - - @Override - public int readSInt32() throws IOException { - return decodeZigZag32(readRawVarint32()); - } - - @Override - public long readSInt64() throws IOException { - return decodeZigZag64(readRawVarint64()); - } - - // ================================================================= - - @Override - public int readRawVarint32() throws IOException { - // See implementation notes for readRawVarint64 - fastpath: - { - int tempPos = pos; - - if (limit == tempPos) { - break fastpath; - } - - final byte[] buffer = this.buffer; - int x; - if ((x = buffer[tempPos++]) >= 0) { - pos = tempPos; - return x; - } else if (limit - tempPos < 9) { - break fastpath; - } else if ((x ^= (buffer[tempPos++] << 7)) < 0) { - x ^= (~0 << 7); - } else if ((x ^= (buffer[tempPos++] << 14)) >= 0) { - x ^= (~0 << 7) ^ (~0 << 14); - } else if ((x ^= (buffer[tempPos++] << 21)) < 0) { - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); - } else { - int y = buffer[tempPos++]; - x ^= y << 28; - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); - if (y < 0 - && buffer[tempPos++] < 0 - && buffer[tempPos++] < 0 - && buffer[tempPos++] < 0 - && buffer[tempPos++] < 0 - && buffer[tempPos++] < 0) { - break fastpath; // Will throw malformedVarint() - } - } - pos = tempPos; - return x; - } - return (int) readRawVarint64SlowPath(); - } - - private void skipRawVarint() throws IOException { - if (limit - pos >= MAX_VARINT_SIZE) { - skipRawVarintFastPath(); - } else { - skipRawVarintSlowPath(); - } - } - - private void skipRawVarintFastPath() throws IOException { - for (int i = 0; i < MAX_VARINT_SIZE; i++) { - if (buffer[pos++] >= 0) { - return; - } - } - throw InvalidProtocolBufferException.malformedVarint(); - } - - private void skipRawVarintSlowPath() throws IOException { - for (int i = 0; i < MAX_VARINT_SIZE; i++) { - if (readRawByte() >= 0) { - return; - } - } - throw InvalidProtocolBufferException.malformedVarint(); - } - - @Override - public long readRawVarint64() throws IOException { - // Implementation notes: - // - // Optimized for one-byte values, expected to be common. - // The particular code below was selected from various candidates - // empirically, by winning VarintBenchmark. - // - // Sign extension of (signed) Java bytes is usually a nuisance, but - // we exploit it here to more easily obtain the sign of bytes read. - // Instead of cleaning up the sign extension bits by masking eagerly, - // we delay until we find the final (positive) byte, when we clear all - // accumulated bits with one xor. We depend on javac to constant fold. - fastpath: - { - int tempPos = pos; - - if (limit == tempPos) { - break fastpath; - } - - final byte[] buffer = this.buffer; - long x; - int y; - if ((y = buffer[tempPos++]) >= 0) { - pos = tempPos; - return y; - } else if (limit - tempPos < 9) { - break fastpath; - } else if ((y ^= (buffer[tempPos++] << 7)) < 0) { - x = y ^ (~0 << 7); - } else if ((y ^= (buffer[tempPos++] << 14)) >= 0) { - x = y ^ ((~0 << 7) ^ (~0 << 14)); - } else if ((y ^= (buffer[tempPos++] << 21)) < 0) { - x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); - } else if ((x = y ^ ((long) buffer[tempPos++] << 28)) >= 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); - } else if ((x ^= ((long) buffer[tempPos++] << 35)) < 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); - } else if ((x ^= ((long) buffer[tempPos++] << 42)) >= 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); - } else if ((x ^= ((long) buffer[tempPos++] << 49)) < 0L) { - x ^= - (~0L << 7) - ^ (~0L << 14) - ^ (~0L << 21) - ^ (~0L << 28) - ^ (~0L << 35) - ^ (~0L << 42) - ^ (~0L << 49); - } else { - x ^= ((long) buffer[tempPos++] << 56); - x ^= - (~0L << 7) - ^ (~0L << 14) - ^ (~0L << 21) - ^ (~0L << 28) - ^ (~0L << 35) - ^ (~0L << 42) - ^ (~0L << 49) - ^ (~0L << 56); - if (x < 0L) { - if (buffer[tempPos++] < 0L) { - break fastpath; // Will throw malformedVarint() - } - } - } - pos = tempPos; - return x; - } - return readRawVarint64SlowPath(); - } - - @Override - long readRawVarint64SlowPath() throws IOException { - long result = 0; - for (int shift = 0; shift < 64; shift += 7) { - final byte b = readRawByte(); - result |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - return result; - } - } - throw InvalidProtocolBufferException.malformedVarint(); - } - - @Override - public int readRawLittleEndian32() throws IOException { - int tempPos = pos; - - if (limit - tempPos < FIXED32_SIZE) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - - final byte[] buffer = this.buffer; - pos = tempPos + FIXED32_SIZE; - return ((buffer[tempPos] & 0xff) - | ((buffer[tempPos + 1] & 0xff) << 8) - | ((buffer[tempPos + 2] & 0xff) << 16) - | ((buffer[tempPos + 3] & 0xff) << 24)); - } - - @Override - public long readRawLittleEndian64() throws IOException { - int tempPos = pos; - - if (limit - tempPos < FIXED64_SIZE) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - - final byte[] buffer = this.buffer; - pos = tempPos + FIXED64_SIZE; - return ((buffer[tempPos] & 0xffL) - | ((buffer[tempPos + 1] & 0xffL) << 8) - | ((buffer[tempPos + 2] & 0xffL) << 16) - | ((buffer[tempPos + 3] & 0xffL) << 24) - | ((buffer[tempPos + 4] & 0xffL) << 32) - | ((buffer[tempPos + 5] & 0xffL) << 40) - | ((buffer[tempPos + 6] & 0xffL) << 48) - | ((buffer[tempPos + 7] & 0xffL) << 56)); - } - - @Override - public void enableAliasing(boolean enabled) { - this.enableAliasing = enabled; - } - - @Override - public void resetSizeCounter() { - startPos = pos; - } - - @Override - public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { - if (byteLimit < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - byteLimit += getTotalBytesRead(); - if (byteLimit < 0) { - throw InvalidProtocolBufferException.parseFailure(); - } - final int oldLimit = currentLimit; - if (byteLimit > oldLimit) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - currentLimit = byteLimit; - - recomputeBufferSizeAfterLimit(); - - return oldLimit; - } - - private void recomputeBufferSizeAfterLimit() { - limit += bufferSizeAfterLimit; - final int bufferEnd = limit - startPos; - if (bufferEnd > currentLimit) { - // Limit is in current buffer. - bufferSizeAfterLimit = bufferEnd - currentLimit; - limit -= bufferSizeAfterLimit; - } else { - bufferSizeAfterLimit = 0; - } - } - - @Override - public void popLimit(final int oldLimit) { - currentLimit = oldLimit; - recomputeBufferSizeAfterLimit(); - } - - @Override - public int getBytesUntilLimit() { - if (currentLimit == Integer.MAX_VALUE) { - return -1; - } - - return currentLimit - getTotalBytesRead(); - } - - @Override - public boolean isAtEnd() throws IOException { - return pos == limit; - } - - @Override - public int getTotalBytesRead() { - return pos - startPos; - } - - @Override - public byte readRawByte() throws IOException { - if (pos == limit) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - return buffer[pos++]; - } - - @Override - public byte[] readRawBytes(final int length) throws IOException { - if (length > 0 && length <= (limit - pos)) { - final int tempPos = pos; - pos += length; - return Arrays.copyOfRange(buffer, tempPos, pos); - } - - if (length <= 0) { - if (length == 0) { - return Internal.EMPTY_BYTE_ARRAY; - } else { - throw InvalidProtocolBufferException.negativeSize(); - } - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public void skipRawBytes(final int length) throws IOException { - if (length >= 0 && length <= (limit - pos)) { - // We have all the bytes we need already. - pos += length; - return; - } - - if (length < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - } - - /** - * A {@link CodedInputStream} implementation that uses a backing direct ByteBuffer as the input. - * Requires the use of {@code sun.misc.Unsafe} to perform fast reads on the buffer. - */ - private static final class UnsafeDirectNioDecoder extends CodedInputStream { - /** The direct buffer that is backing this stream. */ - private final ByteBuffer buffer; - - /** - * If {@code true}, indicates that the buffer is backing a {@link ByteString} and is therefore - * considered to be an immutable input source. - */ - private final boolean immutable; - - /** The unsafe address of the content of {@link #buffer}. */ - private final long address; - - /** The unsafe address of the current read limit of the buffer. */ - private long limit; - - /** The unsafe address of the current read position of the buffer. */ - private long pos; - - /** The unsafe address of the starting read position. */ - private long startPos; - - /** The amount of available data in the buffer beyond {@link #limit}. */ - private int bufferSizeAfterLimit; - - /** The last tag that was read from this stream. */ - private int lastTag; - - /** - * If {@code true}, indicates that calls to read {@link ByteString} or {@code byte[]} - * may return slices of the underlying buffer, rather than copies. - */ - private boolean enableAliasing; - - /** The absolute position of the end of the current message. */ - private int currentLimit = Integer.MAX_VALUE; - - static boolean isSupported() { - return UnsafeUtil.hasUnsafeByteBufferOperations(); - } - - private UnsafeDirectNioDecoder(ByteBuffer buffer, boolean immutable) { - this.buffer = buffer; - address = UnsafeUtil.addressOffset(buffer); - limit = address + buffer.limit(); - pos = address + buffer.position(); - startPos = pos; - this.immutable = immutable; - } - - @Override - public int readTag() throws IOException { - if (isAtEnd()) { - lastTag = 0; - return 0; - } - - lastTag = readRawVarint32(); - if (WireFormat.getTagFieldNumber(lastTag) == 0) { - // If we actually read zero (or any tag number corresponding to field - // number zero), that's not a valid tag. - throw InvalidProtocolBufferException.invalidTag(); - } - return lastTag; - } - - @Override - public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { - if (lastTag != value) { - throw InvalidProtocolBufferException.invalidEndTag(); - } - } - - @Override - public int getLastTag() { - return lastTag; - } - - @Override - public boolean skipField(final int tag) throws IOException { - switch (WireFormat.getTagWireType(tag)) { - case WireFormat.WIRETYPE_VARINT: - skipRawVarint(); - return true; - case WireFormat.WIRETYPE_FIXED64: - skipRawBytes(FIXED64_SIZE); - return true; - case WireFormat.WIRETYPE_LENGTH_DELIMITED: - skipRawBytes(readRawVarint32()); - return true; - case WireFormat.WIRETYPE_START_GROUP: - skipMessage(); - checkLastTagWas( - WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); - return true; - case WireFormat.WIRETYPE_END_GROUP: - return false; - case WireFormat.WIRETYPE_FIXED32: - skipRawBytes(FIXED32_SIZE); - return true; - default: - throw InvalidProtocolBufferException.invalidWireType(); - } - } - - @Override - public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { - switch (WireFormat.getTagWireType(tag)) { - case WireFormat.WIRETYPE_VARINT: - { - long value = readInt64(); - output.writeUInt32NoTag(tag); - output.writeUInt64NoTag(value); - return true; - } - case WireFormat.WIRETYPE_FIXED64: - { - long value = readRawLittleEndian64(); - output.writeUInt32NoTag(tag); - output.writeFixed64NoTag(value); - return true; - } - case WireFormat.WIRETYPE_LENGTH_DELIMITED: - { - ByteString value = readBytes(); - output.writeUInt32NoTag(tag); - output.writeBytesNoTag(value); - return true; - } - case WireFormat.WIRETYPE_START_GROUP: - { - output.writeUInt32NoTag(tag); - skipMessage(output); - int endtag = - WireFormat.makeTag( - WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); - checkLastTagWas(endtag); - output.writeUInt32NoTag(endtag); - return true; - } - case WireFormat.WIRETYPE_END_GROUP: - { - return false; - } - case WireFormat.WIRETYPE_FIXED32: - { - int value = readRawLittleEndian32(); - output.writeUInt32NoTag(tag); - output.writeFixed32NoTag(value); - return true; - } - default: - throw InvalidProtocolBufferException.invalidWireType(); - } - } - - // ----------------------------------------------------------------- - - @Override - public double readDouble() throws IOException { - return Double.longBitsToDouble(readRawLittleEndian64()); - } - - @Override - public float readFloat() throws IOException { - return Float.intBitsToFloat(readRawLittleEndian32()); - } - - @Override - public long readUInt64() throws IOException { - return readRawVarint64(); - } - - @Override - public long readInt64() throws IOException { - return readRawVarint64(); - } - - @Override - public int readInt32() throws IOException { - return readRawVarint32(); - } - - @Override - public long readFixed64() throws IOException { - return readRawLittleEndian64(); - } - - @Override - public int readFixed32() throws IOException { - return readRawLittleEndian32(); - } - - @Override - public boolean readBool() throws IOException { - return readRawVarint64() != 0; - } - - @Override - public String readString() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= remaining()) { - // TODO: Is there a way to avoid this copy? - // TODO: It might be possible to share the optimized loop with - // readStringRequireUtf8 by implementing Java replacement logic there. - // The same as readBytes' logic - byte[] bytes = new byte[size]; - UnsafeUtil.copyMemory(pos, bytes, 0, size); - String result = new String(bytes, UTF_8); - pos += size; - return result; - } - - if (size == 0) { - return ""; - } - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public String readStringRequireUtf8() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= remaining()) { - final int bufferPos = bufferPos(pos); - String result = Utf8.decodeUtf8(buffer, bufferPos, size); - pos += size; - return result; - } - - if (size == 0) { - return ""; - } - if (size <= 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public void readGroup( - final int fieldNumber, - final MessageLite.Builder builder, - final ExtensionRegistryLite extensionRegistry) - throws IOException { - checkRecursionLimit(); - ++recursionDepth; - builder.mergeFrom(this, extensionRegistry); - checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); - --recursionDepth; - } - - @Override - public T readGroup( - final int fieldNumber, - final Parser parser, - final ExtensionRegistryLite extensionRegistry) - throws IOException { - checkRecursionLimit(); - ++recursionDepth; - T result = parser.parsePartialFrom(this, extensionRegistry); - checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); - --recursionDepth; - return result; - } - - @Deprecated - @Override - public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) - throws IOException { - readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry()); - } - - @Override - public void readMessage( - final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) - throws IOException { - final int length = readRawVarint32(); - checkRecursionLimit(); - final int oldLimit = pushLimit(length); - ++recursionDepth; - builder.mergeFrom(this, extensionRegistry); - checkLastTagWas(0); - --recursionDepth; - if (getBytesUntilLimit() != 0) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - popLimit(oldLimit); - } - - @Override - public T readMessage( - final Parser parser, final ExtensionRegistryLite extensionRegistry) throws IOException { - int length = readRawVarint32(); - checkRecursionLimit(); - final int oldLimit = pushLimit(length); - ++recursionDepth; - T result = parser.parsePartialFrom(this, extensionRegistry); - checkLastTagWas(0); - --recursionDepth; - if (getBytesUntilLimit() != 0) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - popLimit(oldLimit); - return result; - } - - @Override - public ByteString readBytes() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= remaining()) { - if (immutable && enableAliasing) { - final ByteBuffer result = slice(pos, pos + size); - pos += size; - return ByteString.wrap(result); - } else { - // Use UnsafeUtil to copy the memory to bytes instead of using ByteBuffer ways. - byte[] bytes = new byte[size]; - UnsafeUtil.copyMemory(pos, bytes, 0, size); - pos += size; - return ByteString.wrap(bytes); - } - } - - if (size == 0) { - return ByteString.EMPTY; - } - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public byte[] readByteArray() throws IOException { - return readRawBytes(readRawVarint32()); - } - - @Override - public ByteBuffer readByteBuffer() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= remaining()) { - // "Immutable" implies that buffer is backing a ByteString. - // Disallow slicing in this case to prevent the caller from modifying the contents - // of the ByteString. - if (!immutable && enableAliasing) { - final ByteBuffer result = slice(pos, pos + size); - pos += size; - return result; - } else { - // The same as readBytes' logic - byte[] bytes = new byte[size]; - UnsafeUtil.copyMemory(pos, bytes, 0, size); - pos += size; - return ByteBuffer.wrap(bytes); - } - // TODO: Investigate making the ByteBuffer be made read-only - } - - if (size == 0) { - return EMPTY_BYTE_BUFFER; - } - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public int readUInt32() throws IOException { - return readRawVarint32(); - } - - @Override - public int readEnum() throws IOException { - return readRawVarint32(); - } - - @Override - public int readSFixed32() throws IOException { - return readRawLittleEndian32(); - } - - @Override - public long readSFixed64() throws IOException { - return readRawLittleEndian64(); - } - - @Override - public int readSInt32() throws IOException { - return decodeZigZag32(readRawVarint32()); - } - - @Override - public long readSInt64() throws IOException { - return decodeZigZag64(readRawVarint64()); - } - - // ================================================================= - - @Override - public int readRawVarint32() throws IOException { - // See implementation notes for readRawVarint64 - fastpath: - { - long tempPos = pos; - - if (limit == tempPos) { - break fastpath; - } - - int x; - if ((x = UnsafeUtil.getByte(tempPos++)) >= 0) { - pos = tempPos; - return x; - } else if (limit - tempPos < 9) { - break fastpath; - } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) { - x ^= (~0 << 7); - } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) { - x ^= (~0 << 7) ^ (~0 << 14); - } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) { - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); - } else { - int y = UnsafeUtil.getByte(tempPos++); - x ^= y << 28; - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); - if (y < 0 - && UnsafeUtil.getByte(tempPos++) < 0 - && UnsafeUtil.getByte(tempPos++) < 0 - && UnsafeUtil.getByte(tempPos++) < 0 - && UnsafeUtil.getByte(tempPos++) < 0 - && UnsafeUtil.getByte(tempPos++) < 0) { - break fastpath; // Will throw malformedVarint() - } - } - pos = tempPos; - return x; - } - return (int) readRawVarint64SlowPath(); - } - - private void skipRawVarint() throws IOException { - if (remaining() >= MAX_VARINT_SIZE) { - skipRawVarintFastPath(); - } else { - skipRawVarintSlowPath(); - } - } - - private void skipRawVarintFastPath() throws IOException { - for (int i = 0; i < MAX_VARINT_SIZE; i++) { - if (UnsafeUtil.getByte(pos++) >= 0) { - return; - } - } - throw InvalidProtocolBufferException.malformedVarint(); - } - - private void skipRawVarintSlowPath() throws IOException { - for (int i = 0; i < MAX_VARINT_SIZE; i++) { - if (readRawByte() >= 0) { - return; - } - } - throw InvalidProtocolBufferException.malformedVarint(); - } - - @Override - public long readRawVarint64() throws IOException { - // Implementation notes: - // - // Optimized for one-byte values, expected to be common. - // The particular code below was selected from various candidates - // empirically, by winning VarintBenchmark. - // - // Sign extension of (signed) Java bytes is usually a nuisance, but - // we exploit it here to more easily obtain the sign of bytes read. - // Instead of cleaning up the sign extension bits by masking eagerly, - // we delay until we find the final (positive) byte, when we clear all - // accumulated bits with one xor. We depend on javac to constant fold. - fastpath: - { - long tempPos = pos; - - if (limit == tempPos) { - break fastpath; - } - - long x; - int y; - if ((y = UnsafeUtil.getByte(tempPos++)) >= 0) { - pos = tempPos; - return y; - } else if (limit - tempPos < 9) { - break fastpath; - } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) { - x = y ^ (~0 << 7); - } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) { - x = y ^ ((~0 << 7) ^ (~0 << 14)); - } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) { - x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); - } else if ((x = y ^ ((long) UnsafeUtil.getByte(tempPos++) << 28)) >= 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); - } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 35)) < 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); - } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 42)) >= 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); - } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 49)) < 0L) { - x ^= - (~0L << 7) - ^ (~0L << 14) - ^ (~0L << 21) - ^ (~0L << 28) - ^ (~0L << 35) - ^ (~0L << 42) - ^ (~0L << 49); - } else { - x ^= ((long) UnsafeUtil.getByte(tempPos++) << 56); - x ^= - (~0L << 7) - ^ (~0L << 14) - ^ (~0L << 21) - ^ (~0L << 28) - ^ (~0L << 35) - ^ (~0L << 42) - ^ (~0L << 49) - ^ (~0L << 56); - if (x < 0L) { - if (UnsafeUtil.getByte(tempPos++) < 0L) { - break fastpath; // Will throw malformedVarint() - } - } - } - pos = tempPos; - return x; - } - return readRawVarint64SlowPath(); - } - - @Override - long readRawVarint64SlowPath() throws IOException { - long result = 0; - for (int shift = 0; shift < 64; shift += 7) { - final byte b = readRawByte(); - result |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - return result; - } - } - throw InvalidProtocolBufferException.malformedVarint(); - } - - @Override - public int readRawLittleEndian32() throws IOException { - long tempPos = pos; - - if (limit - tempPos < FIXED32_SIZE) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - - pos = tempPos + FIXED32_SIZE; - return ((UnsafeUtil.getByte(tempPos) & 0xff) - | ((UnsafeUtil.getByte(tempPos + 1) & 0xff) << 8) - | ((UnsafeUtil.getByte(tempPos + 2) & 0xff) << 16) - | ((UnsafeUtil.getByte(tempPos + 3) & 0xff) << 24)); - } - - @Override - public long readRawLittleEndian64() throws IOException { - long tempPos = pos; - - if (limit - tempPos < FIXED64_SIZE) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - - pos = tempPos + FIXED64_SIZE; - return ((UnsafeUtil.getByte(tempPos) & 0xffL) - | ((UnsafeUtil.getByte(tempPos + 1) & 0xffL) << 8) - | ((UnsafeUtil.getByte(tempPos + 2) & 0xffL) << 16) - | ((UnsafeUtil.getByte(tempPos + 3) & 0xffL) << 24) - | ((UnsafeUtil.getByte(tempPos + 4) & 0xffL) << 32) - | ((UnsafeUtil.getByte(tempPos + 5) & 0xffL) << 40) - | ((UnsafeUtil.getByte(tempPos + 6) & 0xffL) << 48) - | ((UnsafeUtil.getByte(tempPos + 7) & 0xffL) << 56)); - } - - @Override - public void enableAliasing(boolean enabled) { - this.enableAliasing = enabled; - } - - @Override - public void resetSizeCounter() { - startPos = pos; - } - - @Override - public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { - if (byteLimit < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - byteLimit += getTotalBytesRead(); - final int oldLimit = currentLimit; - if (byteLimit > oldLimit) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - currentLimit = byteLimit; - - recomputeBufferSizeAfterLimit(); - - return oldLimit; - } - - @Override - public void popLimit(final int oldLimit) { - currentLimit = oldLimit; - recomputeBufferSizeAfterLimit(); - } - - @Override - public int getBytesUntilLimit() { - if (currentLimit == Integer.MAX_VALUE) { - return -1; - } - - return currentLimit - getTotalBytesRead(); - } - - @Override - public boolean isAtEnd() throws IOException { - return pos == limit; - } - - @Override - public int getTotalBytesRead() { - return (int) (pos - startPos); - } - - @Override - public byte readRawByte() throws IOException { - if (pos == limit) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - return UnsafeUtil.getByte(pos++); - } - - @Override - public byte[] readRawBytes(final int length) throws IOException { - if (length >= 0 && length <= remaining()) { - byte[] bytes = new byte[length]; - slice(pos, pos + length).get(bytes); - pos += length; - return bytes; - } - - if (length <= 0) { - if (length == 0) { - return EMPTY_BYTE_ARRAY; - } else { - throw InvalidProtocolBufferException.negativeSize(); - } - } - - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public void skipRawBytes(final int length) throws IOException { - if (length >= 0 && length <= remaining()) { - // We have all the bytes we need already. - pos += length; - return; - } - - if (length < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - private void recomputeBufferSizeAfterLimit() { - limit += bufferSizeAfterLimit; - final int bufferEnd = (int) (limit - startPos); - if (bufferEnd > currentLimit) { - // Limit is in current buffer. - bufferSizeAfterLimit = bufferEnd - currentLimit; - limit -= bufferSizeAfterLimit; - } else { - bufferSizeAfterLimit = 0; - } - } - - private int remaining() { - return (int) (limit - pos); - } - - private int bufferPos(long pos) { - return (int) (pos - address); - } - - private ByteBuffer slice(long begin, long end) throws IOException { - int prevPos = buffer.position(); - int prevLimit = buffer.limit(); - // View ByteBuffer as Buffer to avoid cross-Java version issues. - // See https://issues.apache.org/jira/browse/MRESOLVER-85 - Buffer asBuffer = buffer; - try { - asBuffer.position(bufferPos(begin)); - asBuffer.limit(bufferPos(end)); - return buffer.slice(); - } catch (IllegalArgumentException e) { - InvalidProtocolBufferException ex = InvalidProtocolBufferException.truncatedMessage(); - ex.initCause(e); - throw ex; - } finally { - asBuffer.position(prevPos); - asBuffer.limit(prevLimit); - } - } - } - - /** - * Implementation of {@link CodedInputStream} that uses an {@link InputStream} as the data source. - */ - private static final class StreamDecoder extends CodedInputStream { - private final InputStream input; - private final byte[] buffer; - /** bufferSize represents how many bytes are currently filled in the buffer */ - private int bufferSize; - - private int bufferSizeAfterLimit; - private int pos; - private int lastTag; - - /** - * The total number of bytes read before the current buffer. The total bytes read up to the - * current position can be computed as {@code totalBytesRetired + pos}. This value may be - * negative if reading started in the middle of the current buffer (e.g. if the constructor that - * takes a byte array and an offset was used). - */ - private int totalBytesRetired; - - /** The absolute position of the end of the current message. */ - private int currentLimit = Integer.MAX_VALUE; - - private StreamDecoder(final InputStream input, int bufferSize) { - checkNotNull(input, "input"); - this.input = input; - this.buffer = new byte[bufferSize]; - this.bufferSize = 0; - pos = 0; - totalBytesRetired = 0; - } - - /* - * The following wrapper methods exist so that InvalidProtocolBufferExceptions thrown by the - * InputStream can be differentiated from ones thrown by CodedInputStream itself. Each call to - * an InputStream method that can throw IOException must be wrapped like this. We do this - * because we sometimes need to modify IPBE instances after they are thrown far away from where - * they are thrown (ex. to add unfinished messages) and we use this signal elsewhere in the - * exception catch chain to know when to perform these operations directly or to wrap the - * exception in their own IPBE so the extra information can be communicated without trampling - * downstream information. - */ - private static int read(InputStream input, byte[] data, int offset, int length) - throws IOException { - try { - return input.read(data, offset, length); - } catch (InvalidProtocolBufferException e) { - e.setThrownFromInputStream(); - throw e; - } - } - - private static long skip(InputStream input, long length) throws IOException { - try { - return input.skip(length); - } catch (InvalidProtocolBufferException e) { - e.setThrownFromInputStream(); - throw e; - } - } - - private static int available(InputStream input) throws IOException { - try { - return input.available(); - } catch (InvalidProtocolBufferException e) { - e.setThrownFromInputStream(); - throw e; - } - } - - @Override - public int readTag() throws IOException { - if (isAtEnd()) { - lastTag = 0; - return 0; - } - - lastTag = readRawVarint32(); - if (WireFormat.getTagFieldNumber(lastTag) == 0) { - // If we actually read zero (or any tag number corresponding to field - // number zero), that's not a valid tag. - throw InvalidProtocolBufferException.invalidTag(); - } - return lastTag; - } - - @Override - public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { - if (lastTag != value) { - throw InvalidProtocolBufferException.invalidEndTag(); - } - } - - @Override - public int getLastTag() { - return lastTag; - } - - @Override - public boolean skipField(final int tag) throws IOException { - switch (WireFormat.getTagWireType(tag)) { - case WireFormat.WIRETYPE_VARINT: - skipRawVarint(); - return true; - case WireFormat.WIRETYPE_FIXED64: - skipRawBytes(FIXED64_SIZE); - return true; - case WireFormat.WIRETYPE_LENGTH_DELIMITED: - skipRawBytes(readRawVarint32()); - return true; - case WireFormat.WIRETYPE_START_GROUP: - skipMessage(); - checkLastTagWas( - WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); - return true; - case WireFormat.WIRETYPE_END_GROUP: - return false; - case WireFormat.WIRETYPE_FIXED32: - skipRawBytes(FIXED32_SIZE); - return true; - default: - throw InvalidProtocolBufferException.invalidWireType(); - } - } - - @Override - public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { - switch (WireFormat.getTagWireType(tag)) { - case WireFormat.WIRETYPE_VARINT: - { - long value = readInt64(); - output.writeUInt32NoTag(tag); - output.writeUInt64NoTag(value); - return true; - } - case WireFormat.WIRETYPE_FIXED64: - { - long value = readRawLittleEndian64(); - output.writeUInt32NoTag(tag); - output.writeFixed64NoTag(value); - return true; - } - case WireFormat.WIRETYPE_LENGTH_DELIMITED: - { - ByteString value = readBytes(); - output.writeUInt32NoTag(tag); - output.writeBytesNoTag(value); - return true; - } - case WireFormat.WIRETYPE_START_GROUP: - { - output.writeUInt32NoTag(tag); - skipMessage(output); - int endtag = - WireFormat.makeTag( - WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); - checkLastTagWas(endtag); - output.writeUInt32NoTag(endtag); - return true; - } - case WireFormat.WIRETYPE_END_GROUP: - { - return false; - } - case WireFormat.WIRETYPE_FIXED32: - { - int value = readRawLittleEndian32(); - output.writeUInt32NoTag(tag); - output.writeFixed32NoTag(value); - return true; - } - default: - throw InvalidProtocolBufferException.invalidWireType(); - } - } - - /** Collects the bytes skipped and returns the data in a ByteBuffer. */ - private class SkippedDataSink implements RefillCallback { - private int lastPos = pos; - private ByteArrayOutputStream byteArrayStream; - - @Override - public void onRefill() { - if (byteArrayStream == null) { - byteArrayStream = new ByteArrayOutputStream(); - } - byteArrayStream.write(buffer, lastPos, pos - lastPos); - lastPos = 0; - } - - /** Gets skipped data in a ByteBuffer. This method should only be called once. */ - ByteBuffer getSkippedData() { - if (byteArrayStream == null) { - return ByteBuffer.wrap(buffer, lastPos, pos - lastPos); - } else { - byteArrayStream.write(buffer, lastPos, pos); - return ByteBuffer.wrap(byteArrayStream.toByteArray()); - } - } - } - - // ----------------------------------------------------------------- - - @Override - public double readDouble() throws IOException { - return Double.longBitsToDouble(readRawLittleEndian64()); - } - - @Override - public float readFloat() throws IOException { - return Float.intBitsToFloat(readRawLittleEndian32()); - } - - @Override - public long readUInt64() throws IOException { - return readRawVarint64(); - } - - @Override - public long readInt64() throws IOException { - return readRawVarint64(); - } - - @Override - public int readInt32() throws IOException { - return readRawVarint32(); - } - - @Override - public long readFixed64() throws IOException { - return readRawLittleEndian64(); - } - - @Override - public int readFixed32() throws IOException { - return readRawLittleEndian32(); - } - - @Override - public boolean readBool() throws IOException { - return readRawVarint64() != 0; - } - - @Override - public String readString() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= (bufferSize - pos)) { - // Fast path: We already have the bytes in a contiguous buffer, so - // just copy directly from it. - final String result = new String(buffer, pos, size, UTF_8); - pos += size; - return result; - } - if (size == 0) { - return ""; - } - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - if (size <= bufferSize) { - refillBuffer(size); - String result = new String(buffer, pos, size, UTF_8); - pos += size; - return result; - } - // Slow path: Build a byte array first then copy it. - return new String(readRawBytesSlowPath(size, /* ensureNoLeakedReferences= */ false), UTF_8); - } - - @Override - public String readStringRequireUtf8() throws IOException { - final int size = readRawVarint32(); - final byte[] bytes; - final int oldPos = pos; - final int tempPos; - if (size <= (bufferSize - oldPos) && size > 0) { - // Fast path: We already have the bytes in a contiguous buffer, so - // just copy directly from it. - bytes = buffer; - pos = oldPos + size; - tempPos = oldPos; - } else if (size == 0) { - return ""; - } else if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } else if (size <= bufferSize) { - refillBuffer(size); - bytes = buffer; - tempPos = 0; - pos = tempPos + size; - } else { - // Slow path: Build a byte array first then copy it. - bytes = readRawBytesSlowPath(size, /* ensureNoLeakedReferences= */ false); - tempPos = 0; - } - return Utf8.decodeUtf8(bytes, tempPos, size); - } - - @Override - public void readGroup( - final int fieldNumber, - final MessageLite.Builder builder, - final ExtensionRegistryLite extensionRegistry) - throws IOException { - checkRecursionLimit(); - ++recursionDepth; - builder.mergeFrom(this, extensionRegistry); - checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); - --recursionDepth; - } - - @Override - public T readGroup( - final int fieldNumber, - final Parser parser, - final ExtensionRegistryLite extensionRegistry) - throws IOException { - checkRecursionLimit(); - ++recursionDepth; - T result = parser.parsePartialFrom(this, extensionRegistry); - checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); - --recursionDepth; - return result; - } - - @Deprecated - @Override - public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) - throws IOException { - readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry()); - } - - @Override - public void readMessage( - final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) - throws IOException { - final int length = readRawVarint32(); - checkRecursionLimit(); - final int oldLimit = pushLimit(length); - ++recursionDepth; - builder.mergeFrom(this, extensionRegistry); - checkLastTagWas(0); - --recursionDepth; - if (getBytesUntilLimit() != 0) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - popLimit(oldLimit); - } - - @Override - public T readMessage( - final Parser parser, final ExtensionRegistryLite extensionRegistry) throws IOException { - int length = readRawVarint32(); - checkRecursionLimit(); - final int oldLimit = pushLimit(length); - ++recursionDepth; - T result = parser.parsePartialFrom(this, extensionRegistry); - checkLastTagWas(0); - --recursionDepth; - if (getBytesUntilLimit() != 0) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - popLimit(oldLimit); - return result; - } - - @Override - public ByteString readBytes() throws IOException { - final int size = readRawVarint32(); - if (size <= (bufferSize - pos) && size > 0) { - // Fast path: We already have the bytes in a contiguous buffer, so - // just copy directly from it. - final ByteString result = ByteString.copyFrom(buffer, pos, size); - pos += size; - return result; - } - if (size == 0) { - return ByteString.EMPTY; - } - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - return readBytesSlowPath(size); - } - - @Override - public byte[] readByteArray() throws IOException { - final int size = readRawVarint32(); - if (size <= (bufferSize - pos) && size > 0) { - // Fast path: We already have the bytes in a contiguous buffer, so - // just copy directly from it. - final byte[] result = Arrays.copyOfRange(buffer, pos, pos + size); - pos += size; - return result; - } else if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } else { - // Slow path: Build a byte array first then copy it. - // TODO: Do we want to protect from malicious input streams here? - return readRawBytesSlowPath(size, /* ensureNoLeakedReferences= */ false); - } - } - - @Override - public ByteBuffer readByteBuffer() throws IOException { - final int size = readRawVarint32(); - if (size <= (bufferSize - pos) && size > 0) { - // Fast path: We already have the bytes in a contiguous buffer. - ByteBuffer result = ByteBuffer.wrap(Arrays.copyOfRange(buffer, pos, pos + size)); - pos += size; - return result; - } - if (size == 0) { - return Internal.EMPTY_BYTE_BUFFER; - } - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - // Slow path: Build a byte array first then copy it. - - // We must copy as the byte array was handed off to the InputStream and a malicious - // implementation could retain a reference. - return ByteBuffer.wrap(readRawBytesSlowPath(size, /* ensureNoLeakedReferences= */ true)); - } - - @Override - public int readUInt32() throws IOException { - return readRawVarint32(); - } - - @Override - public int readEnum() throws IOException { - return readRawVarint32(); - } - - @Override - public int readSFixed32() throws IOException { - return readRawLittleEndian32(); - } - - @Override - public long readSFixed64() throws IOException { - return readRawLittleEndian64(); - } - - @Override - public int readSInt32() throws IOException { - return decodeZigZag32(readRawVarint32()); - } - - @Override - public long readSInt64() throws IOException { - return decodeZigZag64(readRawVarint64()); - } - - // ================================================================= - - @Override - public int readRawVarint32() throws IOException { - // See implementation notes for readRawVarint64 - fastpath: - { - int tempPos = pos; - - if (bufferSize == tempPos) { - break fastpath; - } - - final byte[] buffer = this.buffer; - int x; - if ((x = buffer[tempPos++]) >= 0) { - pos = tempPos; - return x; - } else if (bufferSize - tempPos < 9) { - break fastpath; - } else if ((x ^= (buffer[tempPos++] << 7)) < 0) { - x ^= (~0 << 7); - } else if ((x ^= (buffer[tempPos++] << 14)) >= 0) { - x ^= (~0 << 7) ^ (~0 << 14); - } else if ((x ^= (buffer[tempPos++] << 21)) < 0) { - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); - } else { - int y = buffer[tempPos++]; - x ^= y << 28; - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); - if (y < 0 - && buffer[tempPos++] < 0 - && buffer[tempPos++] < 0 - && buffer[tempPos++] < 0 - && buffer[tempPos++] < 0 - && buffer[tempPos++] < 0) { - break fastpath; // Will throw malformedVarint() - } - } - pos = tempPos; - return x; - } - return (int) readRawVarint64SlowPath(); - } - - private void skipRawVarint() throws IOException { - if (bufferSize - pos >= MAX_VARINT_SIZE) { - skipRawVarintFastPath(); - } else { - skipRawVarintSlowPath(); - } - } - - private void skipRawVarintFastPath() throws IOException { - for (int i = 0; i < MAX_VARINT_SIZE; i++) { - if (buffer[pos++] >= 0) { - return; - } - } - throw InvalidProtocolBufferException.malformedVarint(); - } - - private void skipRawVarintSlowPath() throws IOException { - for (int i = 0; i < MAX_VARINT_SIZE; i++) { - if (readRawByte() >= 0) { - return; - } - } - throw InvalidProtocolBufferException.malformedVarint(); - } - - @Override - public long readRawVarint64() throws IOException { - // Implementation notes: - // - // Optimized for one-byte values, expected to be common. - // The particular code below was selected from various candidates - // empirically, by winning VarintBenchmark. - // - // Sign extension of (signed) Java bytes is usually a nuisance, but - // we exploit it here to more easily obtain the sign of bytes read. - // Instead of cleaning up the sign extension bits by masking eagerly, - // we delay until we find the final (positive) byte, when we clear all - // accumulated bits with one xor. We depend on javac to constant fold. - fastpath: - { - int tempPos = pos; - - if (bufferSize == tempPos) { - break fastpath; - } - - final byte[] buffer = this.buffer; - long x; - int y; - if ((y = buffer[tempPos++]) >= 0) { - pos = tempPos; - return y; - } else if (bufferSize - tempPos < 9) { - break fastpath; - } else if ((y ^= (buffer[tempPos++] << 7)) < 0) { - x = y ^ (~0 << 7); - } else if ((y ^= (buffer[tempPos++] << 14)) >= 0) { - x = y ^ ((~0 << 7) ^ (~0 << 14)); - } else if ((y ^= (buffer[tempPos++] << 21)) < 0) { - x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); - } else if ((x = y ^ ((long) buffer[tempPos++] << 28)) >= 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); - } else if ((x ^= ((long) buffer[tempPos++] << 35)) < 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); - } else if ((x ^= ((long) buffer[tempPos++] << 42)) >= 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); - } else if ((x ^= ((long) buffer[tempPos++] << 49)) < 0L) { - x ^= - (~0L << 7) - ^ (~0L << 14) - ^ (~0L << 21) - ^ (~0L << 28) - ^ (~0L << 35) - ^ (~0L << 42) - ^ (~0L << 49); - } else { - x ^= ((long) buffer[tempPos++] << 56); - x ^= - (~0L << 7) - ^ (~0L << 14) - ^ (~0L << 21) - ^ (~0L << 28) - ^ (~0L << 35) - ^ (~0L << 42) - ^ (~0L << 49) - ^ (~0L << 56); - if (x < 0L) { - if (buffer[tempPos++] < 0L) { - break fastpath; // Will throw malformedVarint() - } - } - } - pos = tempPos; - return x; - } - return readRawVarint64SlowPath(); - } - - @Override - long readRawVarint64SlowPath() throws IOException { - long result = 0; - for (int shift = 0; shift < 64; shift += 7) { - final byte b = readRawByte(); - result |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - return result; - } - } - throw InvalidProtocolBufferException.malformedVarint(); - } - - @Override - public int readRawLittleEndian32() throws IOException { - int tempPos = pos; - - if (bufferSize - tempPos < FIXED32_SIZE) { - refillBuffer(FIXED32_SIZE); - tempPos = pos; - } - - final byte[] buffer = this.buffer; - pos = tempPos + FIXED32_SIZE; - return ((buffer[tempPos] & 0xff) - | ((buffer[tempPos + 1] & 0xff) << 8) - | ((buffer[tempPos + 2] & 0xff) << 16) - | ((buffer[tempPos + 3] & 0xff) << 24)); - } - - @Override - public long readRawLittleEndian64() throws IOException { - int tempPos = pos; - - if (bufferSize - tempPos < FIXED64_SIZE) { - refillBuffer(FIXED64_SIZE); - tempPos = pos; - } - - final byte[] buffer = this.buffer; - pos = tempPos + FIXED64_SIZE; - return (((buffer[tempPos] & 0xffL)) - | ((buffer[tempPos + 1] & 0xffL) << 8) - | ((buffer[tempPos + 2] & 0xffL) << 16) - | ((buffer[tempPos + 3] & 0xffL) << 24) - | ((buffer[tempPos + 4] & 0xffL) << 32) - | ((buffer[tempPos + 5] & 0xffL) << 40) - | ((buffer[tempPos + 6] & 0xffL) << 48) - | ((buffer[tempPos + 7] & 0xffL) << 56)); - } - - // ----------------------------------------------------------------- - - @Override - public void enableAliasing(boolean enabled) { - // TODO: Ideally we should throw here. Do nothing for backward compatibility. - } - - @Override - public void resetSizeCounter() { - totalBytesRetired = -pos; - } - - @Override - public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { - if (byteLimit < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - byteLimit += totalBytesRetired + pos; - final int oldLimit = currentLimit; - if (byteLimit > oldLimit) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - currentLimit = byteLimit; - - recomputeBufferSizeAfterLimit(); - - return oldLimit; - } - - private void recomputeBufferSizeAfterLimit() { - bufferSize += bufferSizeAfterLimit; - final int bufferEnd = totalBytesRetired + bufferSize; - if (bufferEnd > currentLimit) { - // Limit is in current buffer. - bufferSizeAfterLimit = bufferEnd - currentLimit; - bufferSize -= bufferSizeAfterLimit; - } else { - bufferSizeAfterLimit = 0; - } - } - - @Override - public void popLimit(final int oldLimit) { - currentLimit = oldLimit; - recomputeBufferSizeAfterLimit(); - } - - @Override - public int getBytesUntilLimit() { - if (currentLimit == Integer.MAX_VALUE) { - return -1; - } - - final int currentAbsolutePosition = totalBytesRetired + pos; - return currentLimit - currentAbsolutePosition; - } - - @Override - public boolean isAtEnd() throws IOException { - return pos == bufferSize && !tryRefillBuffer(1); - } - - @Override - public int getTotalBytesRead() { - return totalBytesRetired + pos; - } - - private interface RefillCallback { - void onRefill(); - } - - private RefillCallback refillCallback = null; - - /** - * Reads more bytes from the input, making at least {@code n} bytes available in the buffer. - * Caller must ensure that the requested space is not yet available, and that the requested - * space is less than BUFFER_SIZE. - * - * @throws InvalidProtocolBufferException The end of the stream or the current limit was - * reached. - */ - private void refillBuffer(int n) throws IOException { - if (!tryRefillBuffer(n)) { - // We have to distinguish the exception between sizeLimitExceeded and truncatedMessage. So - // we just throw an sizeLimitExceeded exception here if it exceeds the sizeLimit - if (n > sizeLimit - totalBytesRetired - pos) { - throw InvalidProtocolBufferException.sizeLimitExceeded(); - } else { - throw InvalidProtocolBufferException.truncatedMessage(); - } - } - } - - /** - * Tries to read more bytes from the input, making at least {@code n} bytes available in the - * buffer. Caller must ensure that the requested space is not yet available, and that the - * requested space is less than BUFFER_SIZE. - * - * @return {@code true} If the bytes could be made available; {@code false} 1. Current at the - * end of the stream 2. The current limit was reached 3. The total size limit was reached - */ - private boolean tryRefillBuffer(int n) throws IOException { - if (pos + n <= bufferSize) { - throw new IllegalStateException( - "refillBuffer() called when " + n + " bytes were already available in buffer"); - } - - // Check whether the size of total message needs to read is bigger than the size limit. - // We shouldn't throw an exception here as isAtEnd() function needs to get this function's - // return as the result. - if (n > sizeLimit - totalBytesRetired - pos) { - return false; - } - - // Shouldn't throw the exception here either. - if (totalBytesRetired + pos + n > currentLimit) { - // Oops, we hit a limit. - return false; - } - - if (refillCallback != null) { - refillCallback.onRefill(); - } - - int tempPos = pos; - if (tempPos > 0) { - if (bufferSize > tempPos) { - System.arraycopy(buffer, tempPos, buffer, 0, bufferSize - tempPos); - } - totalBytesRetired += tempPos; - bufferSize -= tempPos; - pos = 0; - } - - // Here we should refill the buffer as many bytes as possible. - int bytesRead = - read( - input, - buffer, - bufferSize, - Math.min( - // the size of allocated but unused bytes in the buffer - buffer.length - bufferSize, - // do not exceed the total bytes limit - sizeLimit - totalBytesRetired - bufferSize)); - if (bytesRead == 0 || bytesRead < -1 || bytesRead > buffer.length) { - throw new IllegalStateException( - input.getClass() - + "#read(byte[]) returned invalid result: " - + bytesRead - + "\nThe InputStream implementation is buggy."); - } - if (bytesRead > 0) { - bufferSize += bytesRead; - recomputeBufferSizeAfterLimit(); - return (bufferSize >= n) ? true : tryRefillBuffer(n); - } - - return false; - } - - @Override - public byte readRawByte() throws IOException { - if (pos == bufferSize) { - refillBuffer(1); - } - return buffer[pos++]; - } - - @Override - public byte[] readRawBytes(final int size) throws IOException { - final int tempPos = pos; - if (size <= (bufferSize - tempPos) && size > 0) { - pos = tempPos + size; - return Arrays.copyOfRange(buffer, tempPos, tempPos + size); - } else { - // TODO: Do we want to protect from malicious input streams here? - return readRawBytesSlowPath(size, /* ensureNoLeakedReferences= */ false); - } - } - - /** - * Exactly like readRawBytes, but caller must have already checked the fast path: (size <= - * (bufferSize - pos) && size > 0) - * - * If ensureNoLeakedReferences is true, the value is guaranteed to have not escaped to - * untrusted code. - */ - private byte[] readRawBytesSlowPath( - final int size, boolean ensureNoLeakedReferences) throws IOException { - // Attempt to read the data in one byte array when it's safe to do. - byte[] result = readRawBytesSlowPathOneChunk(size); - if (result != null) { - return ensureNoLeakedReferences ? result.clone() : result; - } - - final int originalBufferPos = pos; - final int bufferedBytes = bufferSize - pos; - - // Mark the current buffer consumed. - totalBytesRetired += bufferSize; - pos = 0; - bufferSize = 0; - - // Determine the number of bytes we need to read from the input stream. - int sizeLeft = size - bufferedBytes; - - // The size is very large. For security reasons we read them in small - // chunks. - List chunks = readRawBytesSlowPathRemainingChunks(sizeLeft); - - // OK, got everything. Now concatenate it all into one buffer. - final byte[] bytes = new byte[size]; - - // Start by copying the leftover bytes from this.buffer. - System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); - - // And now all the chunks. - int tempPos = bufferedBytes; - for (final byte[] chunk : chunks) { - System.arraycopy(chunk, 0, bytes, tempPos, chunk.length); - tempPos += chunk.length; - } - - // Done. - return bytes; - } - - /** - * Attempts to read the data in one byte array when it's safe to do. Returns null if the size to - * read is too large and needs to be allocated in smaller chunks for security reasons. - * - *

Returns a byte[] that may have escaped to user code via InputStream APIs. - */ - private byte[] readRawBytesSlowPathOneChunk(final int size) throws IOException { - if (size == 0) { - return Internal.EMPTY_BYTE_ARRAY; - } - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - - // Integer-overflow-conscious check that the message size so far has not exceeded sizeLimit. - int currentMessageSize = totalBytesRetired + pos + size; - if (currentMessageSize - sizeLimit > 0) { - throw InvalidProtocolBufferException.sizeLimitExceeded(); - } - - // Verify that the message size so far has not exceeded currentLimit. - if (currentMessageSize > currentLimit) { - // Read to the end of the stream anyway. - skipRawBytes(currentLimit - totalBytesRetired - pos); - throw InvalidProtocolBufferException.truncatedMessage(); - } - - final int bufferedBytes = bufferSize - pos; - // Determine the number of bytes we need to read from the input stream. - int sizeLeft = size - bufferedBytes; - // TODO: Consider using a value larger than DEFAULT_BUFFER_SIZE. - if (sizeLeft < DEFAULT_BUFFER_SIZE || sizeLeft <= available(input)) { - // Either the bytes we need are known to be available, or the required buffer is - // within an allowed threshold - go ahead and allocate the buffer now. - final byte[] bytes = new byte[size]; - - // Copy all of the buffered bytes to the result buffer. - System.arraycopy(buffer, pos, bytes, 0, bufferedBytes); - totalBytesRetired += bufferSize; - pos = 0; - bufferSize = 0; - - // Fill the remaining bytes from the input stream. - int tempPos = bufferedBytes; - while (tempPos < bytes.length) { - int n = read(input, bytes, tempPos, size - tempPos); - if (n == -1) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - totalBytesRetired += n; - tempPos += n; - } - - return bytes; - } - - return null; - } - - /** - * Reads the remaining data in small chunks from the input stream. - * - * Returns a byte[] that may have escaped to user code via InputStream APIs. - */ - private List readRawBytesSlowPathRemainingChunks(int sizeLeft) throws IOException { - // The size is very large. For security reasons, we can't allocate the - // entire byte array yet. The size comes directly from the input, so a - // maliciously-crafted message could provide a bogus very large size in - // order to trick the app into allocating a lot of memory. We avoid this - // by allocating and reading only a small chunk at a time, so that the - // malicious message must actually *be* extremely large to cause - // problems. Meanwhile, we limit the allowed size of a message elsewhere. - final List chunks = new ArrayList<>(); - - while (sizeLeft > 0) { - // TODO: Consider using a value larger than DEFAULT_BUFFER_SIZE. - final byte[] chunk = new byte[Math.min(sizeLeft, DEFAULT_BUFFER_SIZE)]; - int tempPos = 0; - while (tempPos < chunk.length) { - final int n = input.read(chunk, tempPos, chunk.length - tempPos); - if (n == -1) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - totalBytesRetired += n; - tempPos += n; - } - sizeLeft -= chunk.length; - chunks.add(chunk); - } - - return chunks; - } - - /** - * Like readBytes, but caller must have already checked the fast path: (size <= (bufferSize - - * pos) && size > 0 || size == 0) - */ - private ByteString readBytesSlowPath(final int size) throws IOException { - final byte[] result = readRawBytesSlowPathOneChunk(size); - if (result != null) { - // We must copy as the byte array was handed off to the InputStream and a malicious - // implementation could retain a reference. - return ByteString.copyFrom(result); - } - - final int originalBufferPos = pos; - final int bufferedBytes = bufferSize - pos; - - // Mark the current buffer consumed. - totalBytesRetired += bufferSize; - pos = 0; - bufferSize = 0; - - // Determine the number of bytes we need to read from the input stream. - int sizeLeft = size - bufferedBytes; - - // The size is very large. For security reasons we read them in small - // chunks. - List chunks = readRawBytesSlowPathRemainingChunks(sizeLeft); - - // OK, got everything. Now concatenate it all into one buffer. - final byte[] bytes = new byte[size]; - - // Start by copying the leftover bytes from this.buffer. - System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); - - // And now all the chunks. - int tempPos = bufferedBytes; - for (final byte[] chunk : chunks) { - System.arraycopy(chunk, 0, bytes, tempPos, chunk.length); - tempPos += chunk.length; - } - - return ByteString.wrap(bytes); - } - - @Override - public void skipRawBytes(final int size) throws IOException { - if (size <= (bufferSize - pos) && size >= 0) { - // We have all the bytes we need already. - pos += size; - } else { - skipRawBytesSlowPath(size); - } - } - - /** - * Exactly like skipRawBytes, but caller must have already checked the fast path: (size <= - * (bufferSize - pos) && size >= 0) - */ - private void skipRawBytesSlowPath(final int size) throws IOException { - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - - if (totalBytesRetired + pos + size > currentLimit) { - // Read to the end of the stream anyway. - skipRawBytes(currentLimit - totalBytesRetired - pos); - // Then fail. - throw InvalidProtocolBufferException.truncatedMessage(); - } - - int totalSkipped = 0; - if (refillCallback == null) { - // Skipping more bytes than are in the buffer. First skip what we have. - totalBytesRetired += pos; - totalSkipped = bufferSize - pos; - bufferSize = 0; - pos = 0; - - try { - while (totalSkipped < size) { - int toSkip = size - totalSkipped; - long skipped = skip(input, toSkip); - if (skipped < 0 || skipped > toSkip) { - throw new IllegalStateException( - input.getClass() - + "#skip returned invalid result: " - + skipped - + "\nThe InputStream implementation is buggy."); - } else if (skipped == 0) { - // The API contract of skip() permits an inputstream to skip zero bytes for any reason - // it wants. In particular, ByteArrayInputStream will just return zero over and over - // when it's at the end of its input. In order to actually confirm that we've hit the - // end of input, we need to issue a read call via the other path. - break; - } - totalSkipped += (int) skipped; - } - } finally { - totalBytesRetired += totalSkipped; - recomputeBufferSizeAfterLimit(); - } - } - if (totalSkipped < size) { - // Skipping more bytes than are in the buffer. First skip what we have. - int tempPos = bufferSize - pos; - pos = bufferSize; - - // Keep refilling the buffer until we get to the point we wanted to skip to. - // This has the side effect of ensuring the limits are updated correctly. - refillBuffer(1); - while (size - tempPos > bufferSize) { - tempPos += bufferSize; - pos = bufferSize; - refillBuffer(1); - } - - pos = size - tempPos; - } - } - } - - /** - * Implementation of {@link CodedInputStream} that uses an {@link Iterable } as the - * data source. Requires the use of {@code sun.misc.Unsafe} to perform fast reads on the buffer. - */ - private static final class IterableDirectByteBufferDecoder extends CodedInputStream { - /** The object that need to decode. */ - private final Iterable input; - /** The {@link Iterator} with type {@link ByteBuffer} of {@code input} */ - private final Iterator iterator; - /** The current ByteBuffer; */ - private ByteBuffer currentByteBuffer; - /** - * If {@code true}, indicates that all the buffers are backing a {@link ByteString} and are - * therefore considered to be an immutable input source. - */ - private final boolean immutable; - /** - * If {@code true}, indicates that calls to read {@link ByteString} or {@code byte[]} - * may return slices of the underlying buffer, rather than copies. - */ - private boolean enableAliasing; - /** The global total message length limit */ - private int totalBufferSize; - /** The amount of available data in the input beyond {@link #currentLimit}. */ - private int bufferSizeAfterCurrentLimit; - /** The absolute position of the end of the current message. */ - private int currentLimit = Integer.MAX_VALUE; - /** The last tag that was read from this stream. */ - private int lastTag; - /** Total Bytes have been Read from the {@link Iterable} {@link ByteBuffer} */ - private int totalBytesRead; - /** The start position offset of the whole message, used as to reset the totalBytesRead */ - private int startOffset; - /** The current position for current ByteBuffer */ - private long currentByteBufferPos; - - private long currentByteBufferStartPos; - /** - * If the current ByteBuffer is unsafe-direct based, currentAddress is the start address of this - * ByteBuffer; otherwise should be zero. - */ - private long currentAddress; - /** The limit position for current ByteBuffer */ - private long currentByteBufferLimit; - - /** - * The constructor of {@code Iterable} decoder. - * - * @param inputBufs The input data. - * @param size The total size of the input data. - * @param immutableFlag whether the input data is immutable. - */ - private IterableDirectByteBufferDecoder( - Iterable inputBufs, int size, boolean immutableFlag) { - totalBufferSize = size; - input = inputBufs; - iterator = input.iterator(); - immutable = immutableFlag; - startOffset = totalBytesRead = 0; - if (size == 0) { - currentByteBuffer = EMPTY_BYTE_BUFFER; - currentByteBufferPos = 0; - currentByteBufferStartPos = 0; - currentByteBufferLimit = 0; - currentAddress = 0; - } else { - tryGetNextByteBuffer(); - } - } - - /** To get the next ByteBuffer from {@code input}, and then update the parameters */ - private void getNextByteBuffer() throws InvalidProtocolBufferException { - if (!iterator.hasNext()) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - tryGetNextByteBuffer(); - } - - private void tryGetNextByteBuffer() { - currentByteBuffer = iterator.next(); - totalBytesRead += (int) (currentByteBufferPos - currentByteBufferStartPos); - currentByteBufferPos = currentByteBuffer.position(); - currentByteBufferStartPos = currentByteBufferPos; - currentByteBufferLimit = currentByteBuffer.limit(); - currentAddress = UnsafeUtil.addressOffset(currentByteBuffer); - currentByteBufferPos += currentAddress; - currentByteBufferStartPos += currentAddress; - currentByteBufferLimit += currentAddress; - } - - @Override - public int readTag() throws IOException { - if (isAtEnd()) { - lastTag = 0; - return 0; - } - - lastTag = readRawVarint32(); - if (WireFormat.getTagFieldNumber(lastTag) == 0) { - // If we actually read zero (or any tag number corresponding to field - // number zero), that's not a valid tag. - throw InvalidProtocolBufferException.invalidTag(); - } - return lastTag; - } - - @Override - public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { - if (lastTag != value) { - throw InvalidProtocolBufferException.invalidEndTag(); - } - } - - @Override - public int getLastTag() { - return lastTag; - } - - @Override - public boolean skipField(final int tag) throws IOException { - switch (WireFormat.getTagWireType(tag)) { - case WireFormat.WIRETYPE_VARINT: - skipRawVarint(); - return true; - case WireFormat.WIRETYPE_FIXED64: - skipRawBytes(FIXED64_SIZE); - return true; - case WireFormat.WIRETYPE_LENGTH_DELIMITED: - skipRawBytes(readRawVarint32()); - return true; - case WireFormat.WIRETYPE_START_GROUP: - skipMessage(); - checkLastTagWas( - WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); - return true; - case WireFormat.WIRETYPE_END_GROUP: - return false; - case WireFormat.WIRETYPE_FIXED32: - skipRawBytes(FIXED32_SIZE); - return true; - default: - throw InvalidProtocolBufferException.invalidWireType(); - } - } - - @Override - public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { - switch (WireFormat.getTagWireType(tag)) { - case WireFormat.WIRETYPE_VARINT: - { - long value = readInt64(); - output.writeUInt32NoTag(tag); - output.writeUInt64NoTag(value); - return true; - } - case WireFormat.WIRETYPE_FIXED64: - { - long value = readRawLittleEndian64(); - output.writeUInt32NoTag(tag); - output.writeFixed64NoTag(value); - return true; - } - case WireFormat.WIRETYPE_LENGTH_DELIMITED: - { - ByteString value = readBytes(); - output.writeUInt32NoTag(tag); - output.writeBytesNoTag(value); - return true; - } - case WireFormat.WIRETYPE_START_GROUP: - { - output.writeUInt32NoTag(tag); - skipMessage(output); - int endtag = - WireFormat.makeTag( - WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); - checkLastTagWas(endtag); - output.writeUInt32NoTag(endtag); - return true; - } - case WireFormat.WIRETYPE_END_GROUP: - { - return false; - } - case WireFormat.WIRETYPE_FIXED32: - { - int value = readRawLittleEndian32(); - output.writeUInt32NoTag(tag); - output.writeFixed32NoTag(value); - return true; - } - default: - throw InvalidProtocolBufferException.invalidWireType(); - } - } - - // ----------------------------------------------------------------- - - @Override - public double readDouble() throws IOException { - return Double.longBitsToDouble(readRawLittleEndian64()); - } - - @Override - public float readFloat() throws IOException { - return Float.intBitsToFloat(readRawLittleEndian32()); - } - - @Override - public long readUInt64() throws IOException { - return readRawVarint64(); - } - - @Override - public long readInt64() throws IOException { - return readRawVarint64(); - } - - @Override - public int readInt32() throws IOException { - return readRawVarint32(); - } - - @Override - public long readFixed64() throws IOException { - return readRawLittleEndian64(); - } - - @Override - public int readFixed32() throws IOException { - return readRawLittleEndian32(); - } - - @Override - public boolean readBool() throws IOException { - return readRawVarint64() != 0; - } - - @Override - public String readString() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) { - byte[] bytes = new byte[size]; - UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size); - String result = new String(bytes, UTF_8); - currentByteBufferPos += size; - return result; - } else if (size > 0 && size <= remaining()) { - // TODO: To use an underlying bytes[] instead of allocating a new bytes[] - byte[] bytes = new byte[size]; - readRawBytesTo(bytes, 0, size); - String result = new String(bytes, UTF_8); - return result; - } - - if (size == 0) { - return ""; - } - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public String readStringRequireUtf8() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) { - final int bufferPos = (int) (currentByteBufferPos - currentByteBufferStartPos); - String result = Utf8.decodeUtf8(currentByteBuffer, bufferPos, size); - currentByteBufferPos += size; - return result; - } - if (size >= 0 && size <= remaining()) { - byte[] bytes = new byte[size]; - readRawBytesTo(bytes, 0, size); - return Utf8.decodeUtf8(bytes, 0, size); - } - - if (size == 0) { - return ""; - } - if (size <= 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public void readGroup( - final int fieldNumber, - final MessageLite.Builder builder, - final ExtensionRegistryLite extensionRegistry) - throws IOException { - checkRecursionLimit(); - ++recursionDepth; - builder.mergeFrom(this, extensionRegistry); - checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); - --recursionDepth; - } - - @Override - public T readGroup( - final int fieldNumber, - final Parser parser, - final ExtensionRegistryLite extensionRegistry) - throws IOException { - checkRecursionLimit(); - ++recursionDepth; - T result = parser.parsePartialFrom(this, extensionRegistry); - checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); - --recursionDepth; - return result; - } - - @Deprecated - @Override - public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) - throws IOException { - readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry()); - } - - @Override - public void readMessage( - final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) - throws IOException { - final int length = readRawVarint32(); - checkRecursionLimit(); - final int oldLimit = pushLimit(length); - ++recursionDepth; - builder.mergeFrom(this, extensionRegistry); - checkLastTagWas(0); - --recursionDepth; - if (getBytesUntilLimit() != 0) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - popLimit(oldLimit); - } - - @Override - public T readMessage( - final Parser parser, final ExtensionRegistryLite extensionRegistry) throws IOException { - int length = readRawVarint32(); - checkRecursionLimit(); - final int oldLimit = pushLimit(length); - ++recursionDepth; - T result = parser.parsePartialFrom(this, extensionRegistry); - checkLastTagWas(0); - --recursionDepth; - if (getBytesUntilLimit() != 0) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - popLimit(oldLimit); - return result; - } - - @Override - public ByteString readBytes() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) { - if (immutable && enableAliasing) { - final int idx = (int) (currentByteBufferPos - currentAddress); - final ByteString result = ByteString.wrap(slice(idx, idx + size)); - currentByteBufferPos += size; - return result; - } else { - byte[] bytes = new byte[size]; - UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size); - currentByteBufferPos += size; - return ByteString.wrap(bytes); - } - } else if (size > 0 && size <= remaining()) { - if (immutable && enableAliasing) { - ArrayList byteStrings = new ArrayList<>(); - int l = size; - while (l > 0) { - if (currentRemaining() == 0) { - getNextByteBuffer(); - } - int bytesToCopy = Math.min(l, (int) currentRemaining()); - int idx = (int) (currentByteBufferPos - currentAddress); - byteStrings.add(ByteString.wrap(slice(idx, idx + bytesToCopy))); - l -= bytesToCopy; - currentByteBufferPos += bytesToCopy; - } - return ByteString.copyFrom(byteStrings); - } else { - byte[] temp = new byte[size]; - readRawBytesTo(temp, 0, size); - return ByteString.wrap(temp); - } - } - - if (size == 0) { - return ByteString.EMPTY; - } - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public byte[] readByteArray() throws IOException { - return readRawBytes(readRawVarint32()); - } - - @Override - public ByteBuffer readByteBuffer() throws IOException { - final int size = readRawVarint32(); - if (size > 0 && size <= currentRemaining()) { - if (!immutable && enableAliasing) { - currentByteBufferPos += size; - return slice( - (int) (currentByteBufferPos - currentAddress - size), - (int) (currentByteBufferPos - currentAddress)); - } else { - byte[] bytes = new byte[size]; - UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size); - currentByteBufferPos += size; - return ByteBuffer.wrap(bytes); - } - } else if (size > 0 && size <= remaining()) { - byte[] temp = new byte[size]; - readRawBytesTo(temp, 0, size); - return ByteBuffer.wrap(temp); - } - - if (size == 0) { - return EMPTY_BYTE_BUFFER; - } - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public int readUInt32() throws IOException { - return readRawVarint32(); - } - - @Override - public int readEnum() throws IOException { - return readRawVarint32(); - } - - @Override - public int readSFixed32() throws IOException { - return readRawLittleEndian32(); - } - - @Override - public long readSFixed64() throws IOException { - return readRawLittleEndian64(); - } - - @Override - public int readSInt32() throws IOException { - return decodeZigZag32(readRawVarint32()); - } - - @Override - public long readSInt64() throws IOException { - return decodeZigZag64(readRawVarint64()); - } - - @Override - public int readRawVarint32() throws IOException { - fastpath: - { - long tempPos = currentByteBufferPos; - - if (currentByteBufferLimit == currentByteBufferPos) { - break fastpath; - } - - int x; - if ((x = UnsafeUtil.getByte(tempPos++)) >= 0) { - currentByteBufferPos++; - return x; - } else if (currentByteBufferLimit - currentByteBufferPos < 10) { - break fastpath; - } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) { - x ^= (~0 << 7); - } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) { - x ^= (~0 << 7) ^ (~0 << 14); - } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) { - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); - } else { - int y = UnsafeUtil.getByte(tempPos++); - x ^= y << 28; - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); - if (y < 0 - && UnsafeUtil.getByte(tempPos++) < 0 - && UnsafeUtil.getByte(tempPos++) < 0 - && UnsafeUtil.getByte(tempPos++) < 0 - && UnsafeUtil.getByte(tempPos++) < 0 - && UnsafeUtil.getByte(tempPos++) < 0) { - break fastpath; // Will throw malformedVarint() - } - } - currentByteBufferPos = tempPos; - return x; - } - return (int) readRawVarint64SlowPath(); - } - - @Override - public long readRawVarint64() throws IOException { - fastpath: - { - long tempPos = currentByteBufferPos; - - if (currentByteBufferLimit == currentByteBufferPos) { - break fastpath; - } - - long x; - int y; - if ((y = UnsafeUtil.getByte(tempPos++)) >= 0) { - currentByteBufferPos++; - return y; - } else if (currentByteBufferLimit - currentByteBufferPos < 10) { - break fastpath; - } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) { - x = y ^ (~0 << 7); - } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) { - x = y ^ ((~0 << 7) ^ (~0 << 14)); - } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) { - x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); - } else if ((x = y ^ ((long) UnsafeUtil.getByte(tempPos++) << 28)) >= 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); - } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 35)) < 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); - } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 42)) >= 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); - } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 49)) < 0L) { - x ^= - (~0L << 7) - ^ (~0L << 14) - ^ (~0L << 21) - ^ (~0L << 28) - ^ (~0L << 35) - ^ (~0L << 42) - ^ (~0L << 49); - } else { - x ^= ((long) UnsafeUtil.getByte(tempPos++) << 56); - x ^= - (~0L << 7) - ^ (~0L << 14) - ^ (~0L << 21) - ^ (~0L << 28) - ^ (~0L << 35) - ^ (~0L << 42) - ^ (~0L << 49) - ^ (~0L << 56); - if (x < 0L) { - if (UnsafeUtil.getByte(tempPos++) < 0L) { - break fastpath; // Will throw malformedVarint() - } - } - } - currentByteBufferPos = tempPos; - return x; - } - return readRawVarint64SlowPath(); - } - - @Override - long readRawVarint64SlowPath() throws IOException { - long result = 0; - for (int shift = 0; shift < 64; shift += 7) { - final byte b = readRawByte(); - result |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - return result; - } - } - throw InvalidProtocolBufferException.malformedVarint(); - } - - @Override - public int readRawLittleEndian32() throws IOException { - if (currentRemaining() >= FIXED32_SIZE) { - long tempPos = currentByteBufferPos; - currentByteBufferPos += FIXED32_SIZE; - return ((UnsafeUtil.getByte(tempPos) & 0xff) - | ((UnsafeUtil.getByte(tempPos + 1) & 0xff) << 8) - | ((UnsafeUtil.getByte(tempPos + 2) & 0xff) << 16) - | ((UnsafeUtil.getByte(tempPos + 3) & 0xff) << 24)); - } - return ((readRawByte() & 0xff) - | ((readRawByte() & 0xff) << 8) - | ((readRawByte() & 0xff) << 16) - | ((readRawByte() & 0xff) << 24)); - } - - @Override - public long readRawLittleEndian64() throws IOException { - if (currentRemaining() >= FIXED64_SIZE) { - long tempPos = currentByteBufferPos; - currentByteBufferPos += FIXED64_SIZE; - return ((UnsafeUtil.getByte(tempPos) & 0xffL) - | ((UnsafeUtil.getByte(tempPos + 1) & 0xffL) << 8) - | ((UnsafeUtil.getByte(tempPos + 2) & 0xffL) << 16) - | ((UnsafeUtil.getByte(tempPos + 3) & 0xffL) << 24) - | ((UnsafeUtil.getByte(tempPos + 4) & 0xffL) << 32) - | ((UnsafeUtil.getByte(tempPos + 5) & 0xffL) << 40) - | ((UnsafeUtil.getByte(tempPos + 6) & 0xffL) << 48) - | ((UnsafeUtil.getByte(tempPos + 7) & 0xffL) << 56)); - } - return ((readRawByte() & 0xffL) - | ((readRawByte() & 0xffL) << 8) - | ((readRawByte() & 0xffL) << 16) - | ((readRawByte() & 0xffL) << 24) - | ((readRawByte() & 0xffL) << 32) - | ((readRawByte() & 0xffL) << 40) - | ((readRawByte() & 0xffL) << 48) - | ((readRawByte() & 0xffL) << 56)); - } - - @Override - public void enableAliasing(boolean enabled) { - this.enableAliasing = enabled; - } - - @Override - public void resetSizeCounter() { - startOffset = (int) (totalBytesRead + currentByteBufferPos - currentByteBufferStartPos); - } - - @Override - public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { - if (byteLimit < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - byteLimit += getTotalBytesRead(); - final int oldLimit = currentLimit; - if (byteLimit > oldLimit) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - currentLimit = byteLimit; - - recomputeBufferSizeAfterLimit(); - - return oldLimit; - } - - private void recomputeBufferSizeAfterLimit() { - totalBufferSize += bufferSizeAfterCurrentLimit; - final int bufferEnd = totalBufferSize - startOffset; - if (bufferEnd > currentLimit) { - // Limit is in current buffer. - bufferSizeAfterCurrentLimit = bufferEnd - currentLimit; - totalBufferSize -= bufferSizeAfterCurrentLimit; - } else { - bufferSizeAfterCurrentLimit = 0; - } - } - - @Override - public void popLimit(final int oldLimit) { - currentLimit = oldLimit; - recomputeBufferSizeAfterLimit(); - } - - @Override - public int getBytesUntilLimit() { - if (currentLimit == Integer.MAX_VALUE) { - return -1; - } - - return currentLimit - getTotalBytesRead(); - } - - @Override - public boolean isAtEnd() throws IOException { - return totalBytesRead + currentByteBufferPos - currentByteBufferStartPos == totalBufferSize; - } - - @Override - public int getTotalBytesRead() { - return (int) - (totalBytesRead - startOffset + currentByteBufferPos - currentByteBufferStartPos); - } - - @Override - public byte readRawByte() throws IOException { - if (currentRemaining() == 0) { - getNextByteBuffer(); - } - return UnsafeUtil.getByte(currentByteBufferPos++); - } - - @Override - public byte[] readRawBytes(final int length) throws IOException { - if (length >= 0 && length <= currentRemaining()) { - byte[] bytes = new byte[length]; - UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, length); - currentByteBufferPos += length; - return bytes; - } - if (length >= 0 && length <= remaining()) { - byte[] bytes = new byte[length]; - readRawBytesTo(bytes, 0, length); - return bytes; - } - - if (length <= 0) { - if (length == 0) { - return EMPTY_BYTE_ARRAY; - } else { - throw InvalidProtocolBufferException.negativeSize(); - } - } - - throw InvalidProtocolBufferException.truncatedMessage(); - } - - /** - * Try to get raw bytes from {@code input} with the size of {@code length} and copy to {@code - * bytes} array. If the size is bigger than the number of remaining bytes in the input, then - * throw {@code truncatedMessage} exception. - */ - private void readRawBytesTo(byte[] bytes, int offset, final int length) throws IOException { - if (length >= 0 && length <= remaining()) { - int l = length; - while (l > 0) { - if (currentRemaining() == 0) { - getNextByteBuffer(); - } - int bytesToCopy = Math.min(l, (int) currentRemaining()); - UnsafeUtil.copyMemory(currentByteBufferPos, bytes, length - l + offset, bytesToCopy); - l -= bytesToCopy; - currentByteBufferPos += bytesToCopy; - } - return; - } - - if (length <= 0) { - if (length == 0) { - return; - } else { - throw InvalidProtocolBufferException.negativeSize(); - } - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - @Override - public void skipRawBytes(final int length) throws IOException { - if (length >= 0 - && length - <= (totalBufferSize - - totalBytesRead - - currentByteBufferPos - + currentByteBufferStartPos)) { - // We have all the bytes we need already. - int l = length; - while (l > 0) { - if (currentRemaining() == 0) { - getNextByteBuffer(); - } - int rl = Math.min(l, (int) currentRemaining()); - l -= rl; - currentByteBufferPos += rl; - } - return; - } - - if (length < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - throw InvalidProtocolBufferException.truncatedMessage(); - } - - // TODO: optimize to fastpath - private void skipRawVarint() throws IOException { - for (int i = 0; i < MAX_VARINT_SIZE; i++) { - if (readRawByte() >= 0) { - return; - } - } - throw InvalidProtocolBufferException.malformedVarint(); - } - - /** - * Try to get the number of remaining bytes in {@code input}. - * - * @return the number of remaining bytes in {@code input}. - */ - private int remaining() { - return (int) - (totalBufferSize - totalBytesRead - currentByteBufferPos + currentByteBufferStartPos); - } - - /** - * Try to get the number of remaining bytes in {@code currentByteBuffer}. - * - * @return the number of remaining bytes in {@code currentByteBuffer} - */ - private long currentRemaining() { - return (currentByteBufferLimit - currentByteBufferPos); - } - - private ByteBuffer slice(int begin, int end) throws IOException { - int prevPos = currentByteBuffer.position(); - int prevLimit = currentByteBuffer.limit(); - // View ByteBuffer as Buffer to avoid cross-Java version issues. - // See https://issues.apache.org/jira/browse/MRESOLVER-85 - Buffer asBuffer = currentByteBuffer; - try { - asBuffer.position(begin); - asBuffer.limit(end); - return currentByteBuffer.slice(); - } catch (IllegalArgumentException e) { - throw InvalidProtocolBufferException.truncatedMessage(); - } finally { - asBuffer.position(prevPos); - asBuffer.limit(prevLimit); - } - } - } } diff --git a/protobuf-api/src/main/java/com/google/protobuf/CodedOutputStream.java b/protobuf-api/src/main/java/com/google/protobuf/CodedOutputStream.java index 37bb44a..66d928e 100644 --- a/protobuf-api/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/protobuf-api/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -7,18 +7,12 @@ package com.google.protobuf; -import static com.google.protobuf.WireFormat.FIXED32_SIZE; -import static com.google.protobuf.WireFormat.FIXED64_SIZE; -import static com.google.protobuf.WireFormat.MAX_VARINT32_SIZE; -import static com.google.protobuf.WireFormat.MAX_VARINT_SIZE; -import static java.lang.Math.max; - import com.google.protobuf.Utf8.UnpairedSurrogateException; import java.io.IOException; import java.io.OutputStream; -import java.nio.BufferOverflowException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.logging.Level; import java.util.logging.Logger; @@ -35,17 +29,44 @@ */ public abstract class CodedOutputStream extends ByteOutput { private static final Logger logger = Logger.getLogger(CodedOutputStream.class.getName()); - private static final boolean HAS_UNSAFE_ARRAY_OPERATIONS = UnsafeUtil.hasUnsafeArrayOperations(); - + static final boolean HAS_UNSAFE_ARRAY_OPERATIONS = UnsafeUtil.hasUnsafeArrayOperations(); + private static final Class ARRAY_ENCODER = getClassForName("com.google.protobuf.ArrayEncoder"); + private static final Class BYTE_OUTPUT_ENCODER = getClassForName("com.google.protobuf.ByteOutputEncoder"); + private static final Class HEAP_NIO_ENCODER = getClassForName("com.google.protobuf.HeapNioEncoder"); + private static final Class OUTPUT_STREAM_ENCODER = getClassForName("com.google.protobuf.OutputStreamEncoder"); + private static final Class SAFE_DIRECT_NIO_ENCODER = getClassForName("com.google.protobuf.SafeDirectNioEncoder"); + private static final Class UNSAFE_DIRECT_NIO_ENCODER = getClassForName("com.google.protobuf.UnsafeDirectNioEncoder"); /** Used to adapt to the experimental {@link Writer} interface. */ CodedOutputStreamWriter wrapper; + // Field numbers for fields in MessageSet wire format. + static final int MESSAGE_SET_ITEM = 1; + static final int MESSAGE_SET_TYPE_ID = 2; + static final int MESSAGE_SET_MESSAGE = 3; + + static final int TAG_TYPE_BITS = 3; + + static final int FIXED32_SIZE = 4; + static final int FIXED64_SIZE = 8; + static final int MAX_VARINT_SIZE = 10; + + public static final int WIRETYPE_START_GROUP = 3; + public static final int WIRETYPE_END_GROUP = 4; + + /** @deprecated Use {@link #computeFixed32SizeNoTag(int)} instead. */ @Deprecated public static final int LITTLE_ENDIAN_32_SIZE = FIXED32_SIZE; /** The buffer size used in {@link #newInstance(OutputStream)}. */ public static final int DEFAULT_BUFFER_SIZE = 4096; + private static Class getClassForName(String name) { + try { + return (Class) Class.forName(name); + } catch (Throwable e) { + return null; + } + } /** * Returns the buffer size to efficiently write dataLength bytes to this CodedOutputStream. Used * by AbstractMessageLite. @@ -79,7 +100,16 @@ public static CodedOutputStream newInstance(final OutputStream output) { * debug. */ public static CodedOutputStream newInstance(final OutputStream output, final int bufferSize) { - return new OutputStreamEncoder(output, bufferSize); + try { + Constructor constructor = OUTPUT_STREAM_ENCODER.getDeclaredConstructors()[0]; + return (CodedOutputStream)constructor.newInstance(output, bufferSize); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } } /** @@ -100,16 +130,34 @@ public static CodedOutputStream newInstance(final byte[] flatArray) { */ public static CodedOutputStream newInstance( final byte[] flatArray, final int offset, final int length) { - return new ArrayEncoder(flatArray, offset, length); + try { + Constructor constructor = ARRAY_ENCODER.getDeclaredConstructors()[0]; + return (CodedOutputStream)constructor.newInstance(flatArray, offset, length); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } } /** Create a new {@code CodedOutputStream} that writes to the given {@link ByteBuffer}. */ public static CodedOutputStream newInstance(ByteBuffer buffer) { if (buffer.hasArray()) { - return new HeapNioEncoder(buffer); + try { + Constructor constructor = HEAP_NIO_ENCODER.getDeclaredConstructors()[0]; + return (CodedOutputStream)constructor.newInstance(buffer); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } } if (buffer.isDirect() && !buffer.isReadOnly()) { - return UnsafeDirectNioEncoder.isSupported() + return UnsafeUtil.hasUnsafeByteBufferOperations() ? newUnsafeInstance(buffer) : newSafeInstance(buffer); } @@ -118,12 +166,30 @@ public static CodedOutputStream newInstance(ByteBuffer buffer) { /** For testing purposes only. */ static CodedOutputStream newUnsafeInstance(ByteBuffer buffer) { - return new UnsafeDirectNioEncoder(buffer); + try { + Constructor constructor = UNSAFE_DIRECT_NIO_ENCODER.getDeclaredConstructors()[0]; + return (CodedOutputStream)constructor.newInstance(buffer); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } } /** For testing purposes only. */ static CodedOutputStream newSafeInstance(ByteBuffer buffer) { - return new SafeDirectNioEncoder(buffer); + try { + Constructor constructor = SAFE_DIRECT_NIO_ENCODER.getDeclaredConstructors()[0]; + return (CodedOutputStream)constructor.newInstance(buffer); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } } /** @@ -193,12 +259,20 @@ static CodedOutputStream newInstance(ByteOutput byteOutput, int bufferSize) { if (bufferSize < 0) { throw new IllegalArgumentException("bufferSize must be positive"); } - - return new ByteOutputEncoder(byteOutput, bufferSize); + try { + Constructor constructor = BYTE_OUTPUT_ENCODER.getDeclaredConstructors()[0]; + return (CodedOutputStream)constructor.newInstance(byteOutput, bufferSize); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } } // Disallow construction outside of this class. - private CodedOutputStream() {} + CodedOutputStream() {} // ----------------------------------------------------------------- @@ -652,9 +726,9 @@ static int computeMessageSize( * stream. For historical reasons, the wire format differs from normal fields. */ public static int computeMessageSetExtensionSize(final int fieldNumber, final MessageLite value) { - return computeTagSize(WireFormat.MESSAGE_SET_ITEM) * 2 - + computeUInt32Size(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber) - + computeMessageSize(WireFormat.MESSAGE_SET_MESSAGE, value); + return computeTagSize(MESSAGE_SET_ITEM) * 2 + + computeUInt32Size(MESSAGE_SET_TYPE_ID, fieldNumber) + + computeMessageSize(MESSAGE_SET_MESSAGE, value); } /** @@ -663,9 +737,9 @@ public static int computeMessageSetExtensionSize(final int fieldNumber, final Me */ public static int computeRawMessageSetExtensionSize( final int fieldNumber, final ByteString value) { - return computeTagSize(WireFormat.MESSAGE_SET_ITEM) * 2 - + computeUInt32Size(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber) - + computeBytesSize(WireFormat.MESSAGE_SET_MESSAGE, value); + return computeTagSize(MESSAGE_SET_ITEM) * 2 + + computeUInt32Size(MESSAGE_SET_TYPE_ID, fieldNumber) + + computeBytesSize(MESSAGE_SET_MESSAGE, value); } /** @@ -675,16 +749,20 @@ public static int computeRawMessageSetExtensionSize( */ public static int computeLazyFieldMessageSetExtensionSize( final int fieldNumber, final LazyFieldLite value) { - return computeTagSize(WireFormat.MESSAGE_SET_ITEM) * 2 - + computeUInt32Size(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber) - + computeLazyFieldSize(WireFormat.MESSAGE_SET_MESSAGE, value); + return computeTagSize(MESSAGE_SET_ITEM) * 2 + + computeUInt32Size(MESSAGE_SET_TYPE_ID, fieldNumber) + + computeLazyFieldSize(MESSAGE_SET_MESSAGE, value); } // ----------------------------------------------------------------- /** Compute the number of bytes that would be needed to encode a tag. */ public static int computeTagSize(final int fieldNumber) { - return computeUInt32SizeNoTag(WireFormat.makeTag(fieldNumber, 0)); + return computeUInt32SizeNoTag(makeTag(fieldNumber, 0)); + } + + static int makeTag(final int fieldNumber, final int wireType) { + return (fieldNumber << TAG_TYPE_BITS) | wireType; } /** @@ -856,7 +934,7 @@ public static int computeMessageSizeNoTag(final MessageLite value) { /** Compute the number of bytes that would be needed to encode an embedded message field. */ static int computeMessageSizeNoTag(final MessageLite value, final Schema schema) { - return computeLengthDelimitedFieldSize(((AbstractMessageLite) value).getSerializedSize(schema)); + return computeLengthDelimitedFieldSize(getSerializedSize(value, schema)); } static int computeLengthDelimitedFieldSize(int fieldLength) { @@ -985,9 +1063,9 @@ final void inefficientWriteStringNoTag(String value, UnpairedSurrogateException */ @Deprecated public final void writeGroup(final int fieldNumber, final MessageLite value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_START_GROUP); + writeTag(fieldNumber, WIRETYPE_START_GROUP); writeGroupNoTag(value); - writeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP); + writeTag(fieldNumber, WIRETYPE_END_GROUP); } /** @@ -998,9 +1076,9 @@ public final void writeGroup(final int fieldNumber, final MessageLite value) thr @Deprecated final void writeGroup(final int fieldNumber, final MessageLite value, Schema schema) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_START_GROUP); + writeTag(fieldNumber, WIRETYPE_START_GROUP); writeGroupNoTag(value, schema); - writeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP); + writeTag(fieldNumber, WIRETYPE_END_GROUP); } /** @@ -1052,10 +1130,24 @@ public static int computeGroupSizeNoTag(final MessageLite value) { return value.getSerializedSize(); } + private static final Class ABSTRACT_MESSAGE_LITE = getClassForName("com.google.protobuf.AbstractMessageLite"); + /** Compute the number of bytes that would be needed to encode a {@code group} field. */ @Deprecated static int computeGroupSizeNoTag(final MessageLite value, Schema schema) { - return ((AbstractMessageLite) value).getSerializedSize(schema); + return getSerializedSize(value, schema); + } + + private static int getSerializedSize(MessageLite value, Schema schema) { + try { + return (int) ABSTRACT_MESSAGE_LITE.getMethod("getSerializedSize", Schema.class).invoke(value, schema); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } } /** @@ -1130,1893 +1222,4 @@ public final void writeRawLittleEndian64(final long value) throws IOException { writeFixed64NoTag(value); } - // ================================================================= - - /** A {@link CodedOutputStream} that writes directly to a byte array. */ - private static class ArrayEncoder extends CodedOutputStream { - private final byte[] buffer; - private final int offset; - private final int limit; - private int position; - - ArrayEncoder(byte[] buffer, int offset, int length) { - if (buffer == null) { - throw new NullPointerException("buffer"); - } - if ((offset | length | (buffer.length - (offset + length))) < 0) { - throw new IllegalArgumentException( - String.format( - "Array range is invalid. Buffer.length=%d, offset=%d, length=%d", - buffer.length, offset, length)); - } - this.buffer = buffer; - this.offset = offset; - position = offset; - limit = offset + length; - } - - @Override - public final void writeTag(final int fieldNumber, final int wireType) throws IOException { - writeUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); - } - - @Override - public final void writeInt32(final int fieldNumber, final int value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeInt32NoTag(value); - } - - @Override - public final void writeUInt32(final int fieldNumber, final int value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeUInt32NoTag(value); - } - - @Override - public final void writeFixed32(final int fieldNumber, final int value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); - writeFixed32NoTag(value); - } - - @Override - public final void writeUInt64(final int fieldNumber, final long value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeUInt64NoTag(value); - } - - @Override - public final void writeFixed64(final int fieldNumber, final long value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); - writeFixed64NoTag(value); - } - - @Override - public final void writeBool(final int fieldNumber, final boolean value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - write((byte) (value ? 1 : 0)); - } - - @Override - public final void writeString(final int fieldNumber, final String value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeStringNoTag(value); - } - - @Override - public final void writeBytes(final int fieldNumber, final ByteString value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeBytesNoTag(value); - } - - @Override - public final void writeByteArray(final int fieldNumber, final byte[] value) throws IOException { - writeByteArray(fieldNumber, value, 0, value.length); - } - - @Override - public final void writeByteArray( - final int fieldNumber, final byte[] value, final int offset, final int length) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeByteArrayNoTag(value, offset, length); - } - - @Override - public final void writeByteBuffer(final int fieldNumber, final ByteBuffer value) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeUInt32NoTag(value.capacity()); - writeRawBytes(value); - } - - @Override - public final void writeBytesNoTag(final ByteString value) throws IOException { - writeUInt32NoTag(value.size()); - value.writeTo(this); - } - - @Override - public final void writeByteArrayNoTag(final byte[] value, int offset, int length) - throws IOException { - writeUInt32NoTag(length); - write(value, offset, length); - } - - @Override - public final void writeRawBytes(final ByteBuffer value) throws IOException { - if (value.hasArray()) { - write(value.array(), value.arrayOffset(), value.capacity()); - } else { - ByteBuffer duplicated = value.duplicate(); - Java8Compatibility.clear(duplicated); - write(duplicated); - } - } - - @Override - public final void writeMessage(final int fieldNumber, final MessageLite value) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeMessageNoTag(value); - } - - @Override - final void writeMessage(final int fieldNumber, final MessageLite value, Schema schema) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeUInt32NoTag(((AbstractMessageLite) value).getSerializedSize(schema)); - schema.writeTo(value, wrapper); - } - - @Override - public final void writeMessageSetExtension(final int fieldNumber, final MessageLite value) - throws IOException { - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); - writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); - writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value); - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); - } - - @Override - public final void writeRawMessageSetExtension(final int fieldNumber, final ByteString value) - throws IOException { - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); - writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); - writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value); - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); - } - - @Override - public final void writeMessageNoTag(final MessageLite value) throws IOException { - writeUInt32NoTag(value.getSerializedSize()); - value.writeTo(this); - } - - @Override - final void writeMessageNoTag(final MessageLite value, Schema schema) throws IOException { - writeUInt32NoTag(((AbstractMessageLite) value).getSerializedSize(schema)); - schema.writeTo(value, wrapper); - } - - @Override - public final void write(byte value) throws IOException { - try { - buffer[position++] = value; - } catch (IndexOutOfBoundsException e) { - throw new OutOfSpaceException( - String.format("Pos: %d, limit: %d, len: %d", position, limit, 1), e); - } - } - - @Override - public final void writeInt32NoTag(int value) throws IOException { - if (value >= 0) { - writeUInt32NoTag(value); - } else { - // Must sign-extend. - writeUInt64NoTag(value); - } - } - - @Override - public final void writeUInt32NoTag(int value) throws IOException { - try { - while (true) { - if ((value & ~0x7F) == 0) { - buffer[position++] = (byte) value; - return; - } else { - buffer[position++] = (byte) ((value & 0x7F) | 0x80); - value >>>= 7; - } - } - } catch (IndexOutOfBoundsException e) { - throw new OutOfSpaceException( - String.format("Pos: %d, limit: %d, len: %d", position, limit, 1), e); - } - } - - @Override - public final void writeFixed32NoTag(int value) throws IOException { - try { - buffer[position++] = (byte) (value & 0xFF); - buffer[position++] = (byte) ((value >> 8) & 0xFF); - buffer[position++] = (byte) ((value >> 16) & 0xFF); - buffer[position++] = (byte) ((value >> 24) & 0xFF); - } catch (IndexOutOfBoundsException e) { - throw new OutOfSpaceException( - String.format("Pos: %d, limit: %d, len: %d", position, limit, 1), e); - } - } - - @Override - public final void writeUInt64NoTag(long value) throws IOException { - if (HAS_UNSAFE_ARRAY_OPERATIONS && spaceLeft() >= MAX_VARINT_SIZE) { - while (true) { - if ((value & ~0x7FL) == 0) { - UnsafeUtil.putByte(buffer, position++, (byte) value); - return; - } else { - UnsafeUtil.putByte(buffer, position++, (byte) (((int) value & 0x7F) | 0x80)); - value >>>= 7; - } - } - } else { - try { - while (true) { - if ((value & ~0x7FL) == 0) { - buffer[position++] = (byte) value; - return; - } else { - buffer[position++] = (byte) (((int) value & 0x7F) | 0x80); - value >>>= 7; - } - } - } catch (IndexOutOfBoundsException e) { - throw new OutOfSpaceException( - String.format("Pos: %d, limit: %d, len: %d", position, limit, 1), e); - } - } - } - - @Override - public final void writeFixed64NoTag(long value) throws IOException { - try { - buffer[position++] = (byte) ((int) (value) & 0xFF); - buffer[position++] = (byte) ((int) (value >> 8) & 0xFF); - buffer[position++] = (byte) ((int) (value >> 16) & 0xFF); - buffer[position++] = (byte) ((int) (value >> 24) & 0xFF); - buffer[position++] = (byte) ((int) (value >> 32) & 0xFF); - buffer[position++] = (byte) ((int) (value >> 40) & 0xFF); - buffer[position++] = (byte) ((int) (value >> 48) & 0xFF); - buffer[position++] = (byte) ((int) (value >> 56) & 0xFF); - } catch (IndexOutOfBoundsException e) { - throw new OutOfSpaceException( - String.format("Pos: %d, limit: %d, len: %d", position, limit, 1), e); - } - } - - @Override - public final void write(byte[] value, int offset, int length) throws IOException { - try { - System.arraycopy(value, offset, buffer, position, length); - position += length; - } catch (IndexOutOfBoundsException e) { - throw new OutOfSpaceException( - String.format("Pos: %d, limit: %d, len: %d", position, limit, length), e); - } - } - - @Override - public final void writeLazy(byte[] value, int offset, int length) throws IOException { - write(value, offset, length); - } - - @Override - public final void write(ByteBuffer value) throws IOException { - final int length = value.remaining(); - try { - value.get(buffer, position, length); - position += length; - } catch (IndexOutOfBoundsException e) { - throw new OutOfSpaceException( - String.format("Pos: %d, limit: %d, len: %d", position, limit, length), e); - } - } - - @Override - public final void writeLazy(ByteBuffer value) throws IOException { - write(value); - } - - @Override - public final void writeStringNoTag(String value) throws IOException { - final int oldPosition = position; - try { - // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), - // and at most 3 times of it. We take advantage of this in both branches below. - final int maxLength = value.length() * Utf8.MAX_BYTES_PER_CHAR; - final int maxLengthVarIntSize = computeUInt32SizeNoTag(maxLength); - final int minLengthVarIntSize = computeUInt32SizeNoTag(value.length()); - if (minLengthVarIntSize == maxLengthVarIntSize) { - position = oldPosition + minLengthVarIntSize; - int newPosition = Utf8.encode(value, buffer, position, spaceLeft()); - // Since this class is stateful and tracks the position, we rewind and store the state, - // prepend the length, then reset it back to the end of the string. - position = oldPosition; - int length = newPosition - oldPosition - minLengthVarIntSize; - writeUInt32NoTag(length); - position = newPosition; - } else { - int length = Utf8.encodedLength(value); - writeUInt32NoTag(length); - position = Utf8.encode(value, buffer, position, spaceLeft()); - } - } catch (UnpairedSurrogateException e) { - // Roll back the change - we fall back to inefficient path. - position = oldPosition; - - // TODO: We should throw an IOException here instead. - inefficientWriteStringNoTag(value, e); - } catch (IndexOutOfBoundsException e) { - throw new OutOfSpaceException(e); - } - } - - @Override - public void flush() { - // Do nothing. - } - - @Override - public final int spaceLeft() { - return limit - position; - } - - @Override - public final int getTotalBytesWritten() { - return position - offset; - } - } - - /** - * A {@link CodedOutputStream} that writes directly to a heap {@link ByteBuffer}. Writes are done - * directly to the underlying array. The buffer position is only updated after a flush. - */ - private static final class HeapNioEncoder extends ArrayEncoder { - private final ByteBuffer byteBuffer; - private int initialPosition; - - HeapNioEncoder(ByteBuffer byteBuffer) { - super( - byteBuffer.array(), - byteBuffer.arrayOffset() + byteBuffer.position(), - byteBuffer.remaining()); - this.byteBuffer = byteBuffer; - this.initialPosition = byteBuffer.position(); - } - - @Override - public void flush() { - // Update the position on the buffer. - Java8Compatibility.position(byteBuffer, initialPosition + getTotalBytesWritten()); - } - } - - /** - * A {@link CodedOutputStream} that writes directly to a direct {@link ByteBuffer}, using only - * safe operations.. - */ - private static final class SafeDirectNioEncoder extends CodedOutputStream { - private final ByteBuffer originalBuffer; - private final ByteBuffer buffer; - private final int initialPosition; - - SafeDirectNioEncoder(ByteBuffer buffer) { - this.originalBuffer = buffer; - this.buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); - initialPosition = buffer.position(); - } - - @Override - public void writeTag(final int fieldNumber, final int wireType) throws IOException { - writeUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); - } - - @Override - public void writeInt32(final int fieldNumber, final int value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeInt32NoTag(value); - } - - @Override - public void writeUInt32(final int fieldNumber, final int value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeUInt32NoTag(value); - } - - @Override - public void writeFixed32(final int fieldNumber, final int value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); - writeFixed32NoTag(value); - } - - @Override - public void writeUInt64(final int fieldNumber, final long value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeUInt64NoTag(value); - } - - @Override - public void writeFixed64(final int fieldNumber, final long value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); - writeFixed64NoTag(value); - } - - @Override - public void writeBool(final int fieldNumber, final boolean value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - write((byte) (value ? 1 : 0)); - } - - @Override - public void writeString(final int fieldNumber, final String value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeStringNoTag(value); - } - - @Override - public void writeBytes(final int fieldNumber, final ByteString value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeBytesNoTag(value); - } - - @Override - public void writeByteArray(final int fieldNumber, final byte[] value) throws IOException { - writeByteArray(fieldNumber, value, 0, value.length); - } - - @Override - public void writeByteArray( - final int fieldNumber, final byte[] value, final int offset, final int length) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeByteArrayNoTag(value, offset, length); - } - - @Override - public void writeByteBuffer(final int fieldNumber, final ByteBuffer value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeUInt32NoTag(value.capacity()); - writeRawBytes(value); - } - - @Override - public void writeMessage(final int fieldNumber, final MessageLite value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeMessageNoTag(value); - } - - @Override - void writeMessage(final int fieldNumber, final MessageLite value, Schema schema) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeMessageNoTag(value, schema); - } - - @Override - public void writeMessageSetExtension(final int fieldNumber, final MessageLite value) - throws IOException { - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); - writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); - writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value); - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); - } - - @Override - public void writeRawMessageSetExtension(final int fieldNumber, final ByteString value) - throws IOException { - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); - writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); - writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value); - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); - } - - @Override - public void writeMessageNoTag(final MessageLite value) throws IOException { - writeUInt32NoTag(value.getSerializedSize()); - value.writeTo(this); - } - - @Override - void writeMessageNoTag(final MessageLite value, Schema schema) throws IOException { - writeUInt32NoTag(((AbstractMessageLite) value).getSerializedSize(schema)); - schema.writeTo(value, wrapper); - } - - @Override - public void write(byte value) throws IOException { - try { - buffer.put(value); - } catch (BufferOverflowException e) { - throw new OutOfSpaceException(e); - } - } - - @Override - public void writeBytesNoTag(final ByteString value) throws IOException { - writeUInt32NoTag(value.size()); - value.writeTo(this); - } - - @Override - public void writeByteArrayNoTag(final byte[] value, int offset, int length) throws IOException { - writeUInt32NoTag(length); - write(value, offset, length); - } - - @Override - public void writeRawBytes(final ByteBuffer value) throws IOException { - if (value.hasArray()) { - write(value.array(), value.arrayOffset(), value.capacity()); - } else { - ByteBuffer duplicated = value.duplicate(); - Java8Compatibility.clear(duplicated); - write(duplicated); - } - } - - @Override - public void writeInt32NoTag(int value) throws IOException { - if (value >= 0) { - writeUInt32NoTag(value); - } else { - // Must sign-extend. - writeUInt64NoTag(value); - } - } - - @Override - public void writeUInt32NoTag(int value) throws IOException { - try { - while (true) { - if ((value & ~0x7F) == 0) { - buffer.put((byte) value); - return; - } else { - buffer.put((byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - } - } catch (BufferOverflowException e) { - throw new OutOfSpaceException(e); - } - } - - @Override - public void writeFixed32NoTag(int value) throws IOException { - try { - buffer.putInt(value); - } catch (BufferOverflowException e) { - throw new OutOfSpaceException(e); - } - } - - @Override - public void writeUInt64NoTag(long value) throws IOException { - try { - while (true) { - if ((value & ~0x7FL) == 0) { - buffer.put((byte) value); - return; - } else { - buffer.put((byte) (((int) value & 0x7F) | 0x80)); - value >>>= 7; - } - } - } catch (BufferOverflowException e) { - throw new OutOfSpaceException(e); - } - } - - @Override - public void writeFixed64NoTag(long value) throws IOException { - try { - buffer.putLong(value); - } catch (BufferOverflowException e) { - throw new OutOfSpaceException(e); - } - } - - @Override - public void write(byte[] value, int offset, int length) throws IOException { - try { - buffer.put(value, offset, length); - } catch (IndexOutOfBoundsException e) { - throw new OutOfSpaceException(e); - } catch (BufferOverflowException e) { - throw new OutOfSpaceException(e); - } - } - - @Override - public void writeLazy(byte[] value, int offset, int length) throws IOException { - write(value, offset, length); - } - - @Override - public void write(ByteBuffer value) throws IOException { - try { - buffer.put(value); - } catch (BufferOverflowException e) { - throw new OutOfSpaceException(e); - } - } - - @Override - public void writeLazy(ByteBuffer value) throws IOException { - write(value); - } - - @Override - public void writeStringNoTag(String value) throws IOException { - final int startPos = buffer.position(); - try { - // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), - // and at most 3 times of it. We take advantage of this in both branches below. - final int maxEncodedSize = value.length() * Utf8.MAX_BYTES_PER_CHAR; - final int maxLengthVarIntSize = computeUInt32SizeNoTag(maxEncodedSize); - final int minLengthVarIntSize = computeUInt32SizeNoTag(value.length()); - if (minLengthVarIntSize == maxLengthVarIntSize) { - // Save the current position and increment past the length field. We'll come back - // and write the length field after the encoding is complete. - final int startOfBytes = buffer.position() + minLengthVarIntSize; - Java8Compatibility.position(buffer, startOfBytes); - - // Encode the string. - encode(value); - - // Now go back to the beginning and write the length. - int endOfBytes = buffer.position(); - Java8Compatibility.position(buffer, startPos); - writeUInt32NoTag(endOfBytes - startOfBytes); - - // Reposition the buffer past the written data. - Java8Compatibility.position(buffer, endOfBytes); - } else { - final int length = Utf8.encodedLength(value); - writeUInt32NoTag(length); - encode(value); - } - } catch (UnpairedSurrogateException e) { - // Roll back the change and convert to an IOException. - Java8Compatibility.position(buffer, startPos); - - // TODO: We should throw an IOException here instead. - inefficientWriteStringNoTag(value, e); - } catch (IllegalArgumentException e) { - // Thrown by buffer.position() if out of range. - throw new OutOfSpaceException(e); - } - } - - @Override - public void flush() { - // Update the position of the original buffer. - Java8Compatibility.position(originalBuffer, buffer.position()); - } - - @Override - public int spaceLeft() { - return buffer.remaining(); - } - - @Override - public int getTotalBytesWritten() { - return buffer.position() - initialPosition; - } - - private void encode(String value) throws IOException { - try { - Utf8.encodeUtf8(value, buffer); - } catch (IndexOutOfBoundsException e) { - throw new OutOfSpaceException(e); - } - } - } - - /** - * A {@link CodedOutputStream} that writes directly to a direct {@link ByteBuffer} using {@code - * sun.misc.Unsafe}. - */ - private static final class UnsafeDirectNioEncoder extends CodedOutputStream { - private final ByteBuffer originalBuffer; - private final ByteBuffer buffer; - private final long address; - private final long initialPosition; - private final long limit; - private final long oneVarintLimit; - private long position; - - UnsafeDirectNioEncoder(ByteBuffer buffer) { - this.originalBuffer = buffer; - this.buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); - address = UnsafeUtil.addressOffset(buffer); - initialPosition = address + buffer.position(); - limit = address + buffer.limit(); - oneVarintLimit = limit - MAX_VARINT_SIZE; - position = initialPosition; - } - - static boolean isSupported() { - return UnsafeUtil.hasUnsafeByteBufferOperations(); - } - - @Override - public void writeTag(int fieldNumber, int wireType) throws IOException { - writeUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); - } - - @Override - public void writeInt32(int fieldNumber, int value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeInt32NoTag(value); - } - - @Override - public void writeUInt32(int fieldNumber, int value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeUInt32NoTag(value); - } - - @Override - public void writeFixed32(int fieldNumber, int value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); - writeFixed32NoTag(value); - } - - @Override - public void writeUInt64(int fieldNumber, long value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeUInt64NoTag(value); - } - - @Override - public void writeFixed64(int fieldNumber, long value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); - writeFixed64NoTag(value); - } - - @Override - public void writeBool(int fieldNumber, boolean value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - write((byte) (value ? 1 : 0)); - } - - @Override - public void writeString(int fieldNumber, String value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeStringNoTag(value); - } - - @Override - public void writeBytes(int fieldNumber, ByteString value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeBytesNoTag(value); - } - - @Override - public void writeByteArray(int fieldNumber, byte[] value) throws IOException { - writeByteArray(fieldNumber, value, 0, value.length); - } - - @Override - public void writeByteArray(int fieldNumber, byte[] value, int offset, int length) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeByteArrayNoTag(value, offset, length); - } - - @Override - public void writeByteBuffer(int fieldNumber, ByteBuffer value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeUInt32NoTag(value.capacity()); - writeRawBytes(value); - } - - @Override - public void writeMessage(int fieldNumber, MessageLite value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeMessageNoTag(value); - } - - @Override - void writeMessage(int fieldNumber, MessageLite value, Schema schema) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeMessageNoTag(value, schema); - } - - @Override - public void writeMessageSetExtension(int fieldNumber, MessageLite value) throws IOException { - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); - writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); - writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value); - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); - } - - @Override - public void writeRawMessageSetExtension(int fieldNumber, ByteString value) throws IOException { - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); - writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); - writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value); - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); - } - - @Override - public void writeMessageNoTag(MessageLite value) throws IOException { - writeUInt32NoTag(value.getSerializedSize()); - value.writeTo(this); - } - - @Override - void writeMessageNoTag(MessageLite value, Schema schema) throws IOException { - writeUInt32NoTag(((AbstractMessageLite) value).getSerializedSize(schema)); - schema.writeTo(value, wrapper); - } - - @Override - public void write(byte value) throws IOException { - if (position >= limit) { - throw new OutOfSpaceException( - String.format("Pos: %d, limit: %d, len: %d", position, limit, 1)); - } - UnsafeUtil.putByte(position++, value); - } - - @Override - public void writeBytesNoTag(ByteString value) throws IOException { - writeUInt32NoTag(value.size()); - value.writeTo(this); - } - - @Override - public void writeByteArrayNoTag(byte[] value, int offset, int length) throws IOException { - writeUInt32NoTag(length); - write(value, offset, length); - } - - @Override - public void writeRawBytes(ByteBuffer value) throws IOException { - if (value.hasArray()) { - write(value.array(), value.arrayOffset(), value.capacity()); - } else { - ByteBuffer duplicated = value.duplicate(); - Java8Compatibility.clear(duplicated); - write(duplicated); - } - } - - @Override - public void writeInt32NoTag(int value) throws IOException { - if (value >= 0) { - writeUInt32NoTag(value); - } else { - // Must sign-extend. - writeUInt64NoTag(value); - } - } - - @Override - public void writeUInt32NoTag(int value) throws IOException { - if (position <= oneVarintLimit) { - // Optimization to avoid bounds checks on each iteration. - while (true) { - if ((value & ~0x7F) == 0) { - UnsafeUtil.putByte(position++, (byte) value); - return; - } else { - UnsafeUtil.putByte(position++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - } - } else { - while (position < limit) { - if ((value & ~0x7F) == 0) { - UnsafeUtil.putByte(position++, (byte) value); - return; - } else { - UnsafeUtil.putByte(position++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - } - throw new OutOfSpaceException( - String.format("Pos: %d, limit: %d, len: %d", position, limit, 1)); - } - } - - @Override - public void writeFixed32NoTag(int value) throws IOException { - buffer.putInt(bufferPos(position), value); - position += FIXED32_SIZE; - } - - @Override - public void writeUInt64NoTag(long value) throws IOException { - if (position <= oneVarintLimit) { - // Optimization to avoid bounds checks on each iteration. - while (true) { - if ((value & ~0x7FL) == 0) { - UnsafeUtil.putByte(position++, (byte) value); - return; - } else { - UnsafeUtil.putByte(position++, (byte) (((int) value & 0x7F) | 0x80)); - value >>>= 7; - } - } - } else { - while (position < limit) { - if ((value & ~0x7FL) == 0) { - UnsafeUtil.putByte(position++, (byte) value); - return; - } else { - UnsafeUtil.putByte(position++, (byte) (((int) value & 0x7F) | 0x80)); - value >>>= 7; - } - } - throw new OutOfSpaceException( - String.format("Pos: %d, limit: %d, len: %d", position, limit, 1)); - } - } - - @Override - public void writeFixed64NoTag(long value) throws IOException { - buffer.putLong(bufferPos(position), value); - position += FIXED64_SIZE; - } - - @Override - public void write(byte[] value, int offset, int length) throws IOException { - if (value == null - || offset < 0 - || length < 0 - || (value.length - length) < offset - || (limit - length) < position) { - if (value == null) { - throw new NullPointerException("value"); - } - throw new OutOfSpaceException( - String.format("Pos: %d, limit: %d, len: %d", position, limit, length)); - } - - UnsafeUtil.copyMemory(value, offset, position, length); - position += length; - } - - @Override - public void writeLazy(byte[] value, int offset, int length) throws IOException { - write(value, offset, length); - } - - @Override - public void write(ByteBuffer value) throws IOException { - try { - int length = value.remaining(); - repositionBuffer(position); - buffer.put(value); - position += length; - } catch (BufferOverflowException e) { - throw new OutOfSpaceException(e); - } - } - - @Override - public void writeLazy(ByteBuffer value) throws IOException { - write(value); - } - - @Override - public void writeStringNoTag(String value) throws IOException { - long prevPos = position; - try { - // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), - // and at most 3 times of it. We take advantage of this in both branches below. - int maxEncodedSize = value.length() * Utf8.MAX_BYTES_PER_CHAR; - int maxLengthVarIntSize = computeUInt32SizeNoTag(maxEncodedSize); - int minLengthVarIntSize = computeUInt32SizeNoTag(value.length()); - if (minLengthVarIntSize == maxLengthVarIntSize) { - // Save the current position and increment past the length field. We'll come back - // and write the length field after the encoding is complete. - int stringStart = bufferPos(position) + minLengthVarIntSize; - Java8Compatibility.position(buffer, stringStart); - - // Encode the string. - Utf8.encodeUtf8(value, buffer); - - // Write the length and advance the position. - int length = buffer.position() - stringStart; - writeUInt32NoTag(length); - position += length; - } else { - // Calculate and write the encoded length. - int length = Utf8.encodedLength(value); - writeUInt32NoTag(length); - - // Write the string and advance the position. - repositionBuffer(position); - Utf8.encodeUtf8(value, buffer); - position += length; - } - } catch (UnpairedSurrogateException e) { - // Roll back the change and convert to an IOException. - position = prevPos; - repositionBuffer(position); - - // TODO: We should throw an IOException here instead. - inefficientWriteStringNoTag(value, e); - } catch (IllegalArgumentException e) { - // Thrown by buffer.position() if out of range. - throw new OutOfSpaceException(e); - } catch (IndexOutOfBoundsException e) { - throw new OutOfSpaceException(e); - } - } - - @Override - public void flush() { - // Update the position of the original buffer. - Java8Compatibility.position(originalBuffer, bufferPos(position)); - } - - @Override - public int spaceLeft() { - return (int) (limit - position); - } - - @Override - public int getTotalBytesWritten() { - return (int) (position - initialPosition); - } - - private void repositionBuffer(long pos) { - Java8Compatibility.position(buffer, bufferPos(pos)); - } - - private int bufferPos(long pos) { - return (int) (pos - address); - } - } - - /** Abstract base class for buffered encoders. */ - private abstract static class AbstractBufferedEncoder extends CodedOutputStream { - final byte[] buffer; - final int limit; - int position; - int totalBytesWritten; - - AbstractBufferedEncoder(int bufferSize) { - if (bufferSize < 0) { - throw new IllegalArgumentException("bufferSize must be >= 0"); - } - // As an optimization, we require that the buffer be able to store at least 2 - // varints so that we can buffer any integer write (tag + value). This reduces the - // number of range checks for a single write to 1 (i.e. if there is not enough space - // to buffer the tag+value, flush and then buffer it). - this.buffer = new byte[max(bufferSize, MAX_VARINT_SIZE * 2)]; - this.limit = buffer.length; - } - - @Override - public final int spaceLeft() { - throw new UnsupportedOperationException( - "spaceLeft() can only be called on CodedOutputStreams that are " - + "writing to a flat array or ByteBuffer."); - } - - @Override - public final int getTotalBytesWritten() { - return totalBytesWritten; - } - - /** - * This method does not perform bounds checking on the array. Checking array bounds is the - * responsibility of the caller. - */ - final void buffer(byte value) { - buffer[position++] = value; - totalBytesWritten++; - } - - /** - * This method does not perform bounds checking on the array. Checking array bounds is the - * responsibility of the caller. - */ - final void bufferTag(final int fieldNumber, final int wireType) { - bufferUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); - } - - /** - * This method does not perform bounds checking on the array. Checking array bounds is the - * responsibility of the caller. - */ - final void bufferInt32NoTag(final int value) { - if (value >= 0) { - bufferUInt32NoTag(value); - } else { - // Must sign-extend. - bufferUInt64NoTag(value); - } - } - - /** - * This method does not perform bounds checking on the array. Checking array bounds is the - * responsibility of the caller. - */ - final void bufferUInt32NoTag(int value) { - if (HAS_UNSAFE_ARRAY_OPERATIONS) { - final long originalPos = position; - while (true) { - if ((value & ~0x7F) == 0) { - UnsafeUtil.putByte(buffer, position++, (byte) value); - break; - } else { - UnsafeUtil.putByte(buffer, position++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - } - int delta = (int) (position - originalPos); - totalBytesWritten += delta; - } else { - while (true) { - if ((value & ~0x7F) == 0) { - buffer[position++] = (byte) value; - totalBytesWritten++; - return; - } else { - buffer[position++] = (byte) ((value & 0x7F) | 0x80); - totalBytesWritten++; - value >>>= 7; - } - } - } - } - - /** - * This method does not perform bounds checking on the array. Checking array bounds is the - * responsibility of the caller. - */ - final void bufferUInt64NoTag(long value) { - if (HAS_UNSAFE_ARRAY_OPERATIONS) { - final long originalPos = position; - while (true) { - if ((value & ~0x7FL) == 0) { - UnsafeUtil.putByte(buffer, position++, (byte) value); - break; - } else { - UnsafeUtil.putByte(buffer, position++, (byte) (((int) value & 0x7F) | 0x80)); - value >>>= 7; - } - } - int delta = (int) (position - originalPos); - totalBytesWritten += delta; - } else { - while (true) { - if ((value & ~0x7FL) == 0) { - buffer[position++] = (byte) value; - totalBytesWritten++; - return; - } else { - buffer[position++] = (byte) (((int) value & 0x7F) | 0x80); - totalBytesWritten++; - value >>>= 7; - } - } - } - } - - /** - * This method does not perform bounds checking on the array. Checking array bounds is the - * responsibility of the caller. - */ - final void bufferFixed32NoTag(int value) { - buffer[position++] = (byte) (value & 0xFF); - buffer[position++] = (byte) ((value >> 8) & 0xFF); - buffer[position++] = (byte) ((value >> 16) & 0xFF); - buffer[position++] = (byte) ((value >> 24) & 0xFF); - totalBytesWritten += FIXED32_SIZE; - } - - /** - * This method does not perform bounds checking on the array. Checking array bounds is the - * responsibility of the caller. - */ - final void bufferFixed64NoTag(long value) { - buffer[position++] = (byte) (value & 0xFF); - buffer[position++] = (byte) ((value >> 8) & 0xFF); - buffer[position++] = (byte) ((value >> 16) & 0xFF); - buffer[position++] = (byte) ((value >> 24) & 0xFF); - buffer[position++] = (byte) ((int) (value >> 32) & 0xFF); - buffer[position++] = (byte) ((int) (value >> 40) & 0xFF); - buffer[position++] = (byte) ((int) (value >> 48) & 0xFF); - buffer[position++] = (byte) ((int) (value >> 56) & 0xFF); - totalBytesWritten += FIXED64_SIZE; - } - } - - /** - * A {@link CodedOutputStream} that decorates a {@link ByteOutput}. It internal buffer only to - * support string encoding operations. All other writes are just passed through to the {@link - * ByteOutput}. - */ - private static final class ByteOutputEncoder extends AbstractBufferedEncoder { - private final ByteOutput out; - - ByteOutputEncoder(ByteOutput out, int bufferSize) { - super(bufferSize); - if (out == null) { - throw new NullPointerException("out"); - } - this.out = out; - } - - @Override - public void writeTag(final int fieldNumber, final int wireType) throws IOException { - writeUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); - } - - @Override - public void writeInt32(final int fieldNumber, final int value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE * 2); - bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - bufferInt32NoTag(value); - } - - @Override - public void writeUInt32(final int fieldNumber, final int value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE * 2); - bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - bufferUInt32NoTag(value); - } - - @Override - public void writeFixed32(final int fieldNumber, final int value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE + FIXED32_SIZE); - bufferTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); - bufferFixed32NoTag(value); - } - - @Override - public void writeUInt64(final int fieldNumber, final long value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE * 2); - bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - bufferUInt64NoTag(value); - } - - @Override - public void writeFixed64(final int fieldNumber, final long value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE + FIXED64_SIZE); - bufferTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); - bufferFixed64NoTag(value); - } - - @Override - public void writeBool(final int fieldNumber, final boolean value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE + 1); - bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - buffer((byte) (value ? 1 : 0)); - } - - @Override - public void writeString(final int fieldNumber, final String value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeStringNoTag(value); - } - - @Override - public void writeBytes(final int fieldNumber, final ByteString value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeBytesNoTag(value); - } - - @Override - public void writeByteArray(final int fieldNumber, final byte[] value) throws IOException { - writeByteArray(fieldNumber, value, 0, value.length); - } - - @Override - public void writeByteArray( - final int fieldNumber, final byte[] value, final int offset, final int length) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeByteArrayNoTag(value, offset, length); - } - - @Override - public void writeByteBuffer(final int fieldNumber, final ByteBuffer value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeUInt32NoTag(value.capacity()); - writeRawBytes(value); - } - - @Override - public void writeBytesNoTag(final ByteString value) throws IOException { - writeUInt32NoTag(value.size()); - value.writeTo(this); - } - - @Override - public void writeByteArrayNoTag(final byte[] value, int offset, int length) throws IOException { - writeUInt32NoTag(length); - write(value, offset, length); - } - - @Override - public void writeRawBytes(final ByteBuffer value) throws IOException { - if (value.hasArray()) { - write(value.array(), value.arrayOffset(), value.capacity()); - } else { - ByteBuffer duplicated = value.duplicate(); - Java8Compatibility.clear(duplicated); - write(duplicated); - } - } - - @Override - public void writeMessage(final int fieldNumber, final MessageLite value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeMessageNoTag(value); - } - - @Override - void writeMessage(final int fieldNumber, final MessageLite value, Schema schema) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeMessageNoTag(value, schema); - } - - @Override - public void writeMessageSetExtension(final int fieldNumber, final MessageLite value) - throws IOException { - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); - writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); - writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value); - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); - } - - @Override - public void writeRawMessageSetExtension(final int fieldNumber, final ByteString value) - throws IOException { - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); - writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); - writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value); - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); - } - - @Override - public void writeMessageNoTag(final MessageLite value) throws IOException { - writeUInt32NoTag(value.getSerializedSize()); - value.writeTo(this); - } - - @Override - void writeMessageNoTag(final MessageLite value, Schema schema) throws IOException { - writeUInt32NoTag(((AbstractMessageLite) value).getSerializedSize(schema)); - schema.writeTo(value, wrapper); - } - - @Override - public void write(byte value) throws IOException { - if (position == limit) { - doFlush(); - } - - buffer(value); - } - - @Override - public void writeInt32NoTag(int value) throws IOException { - if (value >= 0) { - writeUInt32NoTag(value); - } else { - // Must sign-extend. - writeUInt64NoTag(value); - } - } - - @Override - public void writeUInt32NoTag(int value) throws IOException { - flushIfNotAvailable(MAX_VARINT32_SIZE); - bufferUInt32NoTag(value); - } - - @Override - public void writeFixed32NoTag(final int value) throws IOException { - flushIfNotAvailable(FIXED32_SIZE); - bufferFixed32NoTag(value); - } - - @Override - public void writeUInt64NoTag(long value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE); - bufferUInt64NoTag(value); - } - - @Override - public void writeFixed64NoTag(final long value) throws IOException { - flushIfNotAvailable(FIXED64_SIZE); - bufferFixed64NoTag(value); - } - - @Override - public void writeStringNoTag(String value) throws IOException { - // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), - // and at most 3 times of it. We take advantage of this in both branches below. - final int maxLength = value.length() * Utf8.MAX_BYTES_PER_CHAR; - final int maxLengthVarIntSize = computeUInt32SizeNoTag(maxLength); - - // If we are streaming and the potential length is too big to fit in our buffer, we take the - // slower path. - if (maxLengthVarIntSize + maxLength > limit) { - // Allocate a byte[] that we know can fit the string and encode into it. String.getBytes() - // does the same internally and then does *another copy* to return a byte[] of exactly the - // right size. We can skip that copy and just writeRawBytes up to the actualLength of the - // UTF-8 encoded bytes. - final byte[] encodedBytes = new byte[maxLength]; - int actualLength = Utf8.encode(value, encodedBytes, 0, maxLength); - writeUInt32NoTag(actualLength); - writeLazy(encodedBytes, 0, actualLength); - return; - } - - // Fast path: we have enough space available in our buffer for the string... - if (maxLengthVarIntSize + maxLength > limit - position) { - // Flush to free up space. - doFlush(); - } - - final int oldPosition = position; - try { - // Optimize for the case where we know this length results in a constant varint length as - // this saves a pass for measuring the length of the string. - final int minLengthVarIntSize = computeUInt32SizeNoTag(value.length()); - - if (minLengthVarIntSize == maxLengthVarIntSize) { - position = oldPosition + minLengthVarIntSize; - int newPosition = Utf8.encode(value, buffer, position, limit - position); - // Since this class is stateful and tracks the position, we rewind and store the state, - // prepend the length, then reset it back to the end of the string. - position = oldPosition; - int length = newPosition - oldPosition - minLengthVarIntSize; - bufferUInt32NoTag(length); - position = newPosition; - totalBytesWritten += length; - } else { - int length = Utf8.encodedLength(value); - bufferUInt32NoTag(length); - position = Utf8.encode(value, buffer, position, length); - totalBytesWritten += length; - } - } catch (UnpairedSurrogateException e) { - // Roll back the change and convert to an IOException. - totalBytesWritten -= position - oldPosition; - position = oldPosition; - - // TODO: We should throw an IOException here instead. - inefficientWriteStringNoTag(value, e); - } catch (IndexOutOfBoundsException e) { - throw new OutOfSpaceException(e); - } - } - - @Override - public void flush() throws IOException { - if (position > 0) { - // Flush the buffer. - doFlush(); - } - } - - @Override - public void write(byte[] value, int offset, int length) throws IOException { - flush(); - out.write(value, offset, length); - totalBytesWritten += length; - } - - @Override - public void writeLazy(byte[] value, int offset, int length) throws IOException { - flush(); - out.writeLazy(value, offset, length); - totalBytesWritten += length; - } - - @Override - public void write(ByteBuffer value) throws IOException { - flush(); - int length = value.remaining(); - out.write(value); - totalBytesWritten += length; - } - - @Override - public void writeLazy(ByteBuffer value) throws IOException { - flush(); - int length = value.remaining(); - out.writeLazy(value); - totalBytesWritten += length; - } - - private void flushIfNotAvailable(int requiredSize) throws IOException { - if (limit - position < requiredSize) { - doFlush(); - } - } - - private void doFlush() throws IOException { - out.write(buffer, 0, position); - position = 0; - } - } - - /** - * An {@link CodedOutputStream} that decorates an {@link OutputStream}. It performs internal - * buffering to optimize writes to the {@link OutputStream}. - */ - private static final class OutputStreamEncoder extends AbstractBufferedEncoder { - private final OutputStream out; - - OutputStreamEncoder(OutputStream out, int bufferSize) { - super(bufferSize); - if (out == null) { - throw new NullPointerException("out"); - } - this.out = out; - } - - @Override - public void writeTag(final int fieldNumber, final int wireType) throws IOException { - writeUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); - } - - @Override - public void writeInt32(final int fieldNumber, final int value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE * 2); - bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - bufferInt32NoTag(value); - } - - @Override - public void writeUInt32(final int fieldNumber, final int value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE * 2); - bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - bufferUInt32NoTag(value); - } - - @Override - public void writeFixed32(final int fieldNumber, final int value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE + FIXED32_SIZE); - bufferTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); - bufferFixed32NoTag(value); - } - - @Override - public void writeUInt64(final int fieldNumber, final long value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE * 2); - bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - bufferUInt64NoTag(value); - } - - @Override - public void writeFixed64(final int fieldNumber, final long value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE + FIXED64_SIZE); - bufferTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); - bufferFixed64NoTag(value); - } - - @Override - public void writeBool(final int fieldNumber, final boolean value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE + 1); - bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - buffer((byte) (value ? 1 : 0)); - } - - @Override - public void writeString(final int fieldNumber, final String value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeStringNoTag(value); - } - - @Override - public void writeBytes(final int fieldNumber, final ByteString value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeBytesNoTag(value); - } - - @Override - public void writeByteArray(final int fieldNumber, final byte[] value) throws IOException { - writeByteArray(fieldNumber, value, 0, value.length); - } - - @Override - public void writeByteArray( - final int fieldNumber, final byte[] value, final int offset, final int length) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeByteArrayNoTag(value, offset, length); - } - - @Override - public void writeByteBuffer(final int fieldNumber, final ByteBuffer value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeUInt32NoTag(value.capacity()); - writeRawBytes(value); - } - - @Override - public void writeBytesNoTag(final ByteString value) throws IOException { - writeUInt32NoTag(value.size()); - value.writeTo(this); - } - - @Override - public void writeByteArrayNoTag(final byte[] value, int offset, int length) throws IOException { - writeUInt32NoTag(length); - write(value, offset, length); - } - - @Override - public void writeRawBytes(final ByteBuffer value) throws IOException { - if (value.hasArray()) { - write(value.array(), value.arrayOffset(), value.capacity()); - } else { - ByteBuffer duplicated = value.duplicate(); - Java8Compatibility.clear(duplicated); - write(duplicated); - } - } - - @Override - public void writeMessage(final int fieldNumber, final MessageLite value) throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeMessageNoTag(value); - } - - @Override - void writeMessage(final int fieldNumber, final MessageLite value, Schema schema) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeMessageNoTag(value, schema); - } - - @Override - public void writeMessageSetExtension(final int fieldNumber, final MessageLite value) - throws IOException { - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); - writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); - writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value); - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); - } - - @Override - public void writeRawMessageSetExtension(final int fieldNumber, final ByteString value) - throws IOException { - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); - writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); - writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value); - writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); - } - - @Override - public void writeMessageNoTag(final MessageLite value) throws IOException { - writeUInt32NoTag(value.getSerializedSize()); - value.writeTo(this); - } - - @Override - void writeMessageNoTag(final MessageLite value, Schema schema) throws IOException { - writeUInt32NoTag(((AbstractMessageLite) value).getSerializedSize(schema)); - schema.writeTo(value, wrapper); - } - - @Override - public void write(byte value) throws IOException { - if (position == limit) { - doFlush(); - } - - buffer(value); - } - - @Override - public void writeInt32NoTag(int value) throws IOException { - if (value >= 0) { - writeUInt32NoTag(value); - } else { - // Must sign-extend. - writeUInt64NoTag(value); - } - } - - @Override - public void writeUInt32NoTag(int value) throws IOException { - flushIfNotAvailable(MAX_VARINT32_SIZE); - bufferUInt32NoTag(value); - } - - @Override - public void writeFixed32NoTag(final int value) throws IOException { - flushIfNotAvailable(FIXED32_SIZE); - bufferFixed32NoTag(value); - } - - @Override - public void writeUInt64NoTag(long value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE); - bufferUInt64NoTag(value); - } - - @Override - public void writeFixed64NoTag(final long value) throws IOException { - flushIfNotAvailable(FIXED64_SIZE); - bufferFixed64NoTag(value); - } - - @Override - public void writeStringNoTag(String value) throws IOException { - try { - // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), - // and at most 3 times of it. We take advantage of this in both branches below. - final int maxLength = value.length() * Utf8.MAX_BYTES_PER_CHAR; - final int maxLengthVarIntSize = computeUInt32SizeNoTag(maxLength); - - // If we are streaming and the potential length is too big to fit in our buffer, we take the - // slower path. - if (maxLengthVarIntSize + maxLength > limit) { - // Allocate a byte[] that we know can fit the string and encode into it. String.getBytes() - // does the same internally and then does *another copy* to return a byte[] of exactly the - // right size. We can skip that copy and just writeRawBytes up to the actualLength of the - // UTF-8 encoded bytes. - final byte[] encodedBytes = new byte[maxLength]; - int actualLength = Utf8.encode(value, encodedBytes, 0, maxLength); - writeUInt32NoTag(actualLength); - writeLazy(encodedBytes, 0, actualLength); - return; - } - - // Fast path: we have enough space available in our buffer for the string... - if (maxLengthVarIntSize + maxLength > limit - position) { - // Flush to free up space. - doFlush(); - } - - // Optimize for the case where we know this length results in a constant varint length as - // this saves a pass for measuring the length of the string. - final int minLengthVarIntSize = computeUInt32SizeNoTag(value.length()); - int oldPosition = position; - final int length; - try { - if (minLengthVarIntSize == maxLengthVarIntSize) { - position = oldPosition + minLengthVarIntSize; - int newPosition = Utf8.encode(value, buffer, position, limit - position); - // Since this class is stateful and tracks the position, we rewind and store the - // state, prepend the length, then reset it back to the end of the string. - position = oldPosition; - length = newPosition - oldPosition - minLengthVarIntSize; - bufferUInt32NoTag(length); - position = newPosition; - } else { - length = Utf8.encodedLength(value); - bufferUInt32NoTag(length); - position = Utf8.encode(value, buffer, position, length); - } - totalBytesWritten += length; - } catch (UnpairedSurrogateException e) { - // Be extra careful and restore the original position for retrying the write with the - // less efficient path. - totalBytesWritten -= position - oldPosition; - position = oldPosition; - throw e; - } catch (ArrayIndexOutOfBoundsException e) { - throw new OutOfSpaceException(e); - } - } catch (UnpairedSurrogateException e) { - inefficientWriteStringNoTag(value, e); - } - } - - @Override - public void flush() throws IOException { - if (position > 0) { - // Flush the buffer. - doFlush(); - } - } - - @Override - public void write(byte[] value, int offset, int length) throws IOException { - if (limit - position >= length) { - // We have room in the current buffer. - System.arraycopy(value, offset, buffer, position, length); - position += length; - totalBytesWritten += length; - } else { - // Write extends past current buffer. Fill the rest of this buffer and - // flush. - final int bytesWritten = limit - position; - System.arraycopy(value, offset, buffer, position, bytesWritten); - offset += bytesWritten; - length -= bytesWritten; - position = limit; - totalBytesWritten += bytesWritten; - doFlush(); - - // Now deal with the rest. - // Since we have an output stream, this is our buffer - // and buffer offset == 0 - if (length <= limit) { - // Fits in new buffer. - System.arraycopy(value, offset, buffer, 0, length); - position = length; - } else { - // Write is very big. Let's do it all at once. - out.write(value, offset, length); - } - totalBytesWritten += length; - } - } - - @Override - public void writeLazy(byte[] value, int offset, int length) throws IOException { - write(value, offset, length); - } - - @Override - public void write(ByteBuffer value) throws IOException { - int length = value.remaining(); - if (limit - position >= length) { - // We have room in the current buffer. - value.get(buffer, position, length); - position += length; - totalBytesWritten += length; - } else { - // Write extends past current buffer. Fill the rest of this buffer and - // flush. - final int bytesWritten = limit - position; - value.get(buffer, position, bytesWritten); - length -= bytesWritten; - position = limit; - totalBytesWritten += bytesWritten; - doFlush(); - - // Now deal with the rest. - // Since we have an output stream, this is our buffer - // and buffer offset == 0 - while (length > limit) { - // Copy data into the buffer before writing it to OutputStream. - value.get(buffer, 0, limit); - out.write(buffer, 0, limit); - length -= limit; - totalBytesWritten += limit; - } - value.get(buffer, 0, length); - position = length; - totalBytesWritten += length; - } - } - - @Override - public void writeLazy(ByteBuffer value) throws IOException { - write(value); - } - - private void flushIfNotAvailable(int requiredSize) throws IOException { - if (limit - position < requiredSize) { - doFlush(); - } - } - - private void doFlush() throws IOException { - out.write(buffer, 0, position); - position = 0; - } - } } diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/AbstractBufferedEncoder.java b/protobuf-sdk/src/main/java/com/google/protobuf/AbstractBufferedEncoder.java new file mode 100644 index 0000000..cafbf01 --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/AbstractBufferedEncoder.java @@ -0,0 +1,160 @@ +package com.google.protobuf; + +import static com.google.protobuf.WireFormat.*; +import static java.lang.Math.max; + +/** Abstract base class for buffered encoders. */ +abstract class AbstractBufferedEncoder extends CodedOutputStream { + final byte[] buffer; + final int limit; + int position; + int totalBytesWritten; + + AbstractBufferedEncoder(int bufferSize) { + if (bufferSize < 0) { + throw new IllegalArgumentException("bufferSize must be >= 0"); + } + // As an optimization, we require that the buffer be able to store at least 2 + // varints so that we can buffer any integer write (tag + value). This reduces the + // number of range checks for a single write to 1 (i.e. if there is not enough space + // to buffer the tag+value, flush and then buffer it). + this.buffer = new byte[max(bufferSize, MAX_VARINT_SIZE * 2)]; + this.limit = buffer.length; + } + + @Override + public final int spaceLeft() { + throw new UnsupportedOperationException( + "spaceLeft() can only be called on CodedOutputStreams that are " + + "writing to a flat array or ByteBuffer."); + } + + @Override + public final int getTotalBytesWritten() { + return totalBytesWritten; + } + + /** + * This method does not perform bounds checking on the array. Checking array bounds is the + * responsibility of the caller. + */ + final void buffer(byte value) { + buffer[position++] = value; + totalBytesWritten++; + } + + /** + * This method does not perform bounds checking on the array. Checking array bounds is the + * responsibility of the caller. + */ + final void bufferTag(final int fieldNumber, final int wireType) { + bufferUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); + } + + /** + * This method does not perform bounds checking on the array. Checking array bounds is the + * responsibility of the caller. + */ + final void bufferInt32NoTag(final int value) { + if (value >= 0) { + bufferUInt32NoTag(value); + } else { + // Must sign-extend. + bufferUInt64NoTag(value); + } + } + + /** + * This method does not perform bounds checking on the array. Checking array bounds is the + * responsibility of the caller. + */ + final void bufferUInt32NoTag(int value) { + if (HAS_UNSAFE_ARRAY_OPERATIONS) { + final long originalPos = position; + while (true) { + if ((value & ~0x7F) == 0) { + UnsafeUtil.putByte(buffer, position++, (byte) value); + break; + } else { + UnsafeUtil.putByte(buffer, position++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + } + int delta = (int) (position - originalPos); + totalBytesWritten += delta; + } else { + while (true) { + if ((value & ~0x7F) == 0) { + buffer[position++] = (byte) value; + totalBytesWritten++; + return; + } else { + buffer[position++] = (byte) ((value & 0x7F) | 0x80); + totalBytesWritten++; + value >>>= 7; + } + } + } + } + + /** + * This method does not perform bounds checking on the array. Checking array bounds is the + * responsibility of the caller. + */ + final void bufferUInt64NoTag(long value) { + if (HAS_UNSAFE_ARRAY_OPERATIONS) { + final long originalPos = position; + while (true) { + if ((value & ~0x7FL) == 0) { + UnsafeUtil.putByte(buffer, position++, (byte) value); + break; + } else { + UnsafeUtil.putByte(buffer, position++, (byte) (((int) value & 0x7F) | 0x80)); + value >>>= 7; + } + } + int delta = (int) (position - originalPos); + totalBytesWritten += delta; + } else { + while (true) { + if ((value & ~0x7FL) == 0) { + buffer[position++] = (byte) value; + totalBytesWritten++; + return; + } else { + buffer[position++] = (byte) (((int) value & 0x7F) | 0x80); + totalBytesWritten++; + value >>>= 7; + } + } + } + } + + /** + * This method does not perform bounds checking on the array. Checking array bounds is the + * responsibility of the caller. + */ + final void bufferFixed32NoTag(int value) { + buffer[position++] = (byte) (value & 0xFF); + buffer[position++] = (byte) ((value >> 8) & 0xFF); + buffer[position++] = (byte) ((value >> 16) & 0xFF); + buffer[position++] = (byte) ((value >> 24) & 0xFF); + totalBytesWritten += FIXED32_SIZE; + } + + /** + * This method does not perform bounds checking on the array. Checking array bounds is the + * responsibility of the caller. + */ + final void bufferFixed64NoTag(long value) { + buffer[position++] = (byte) (value & 0xFF); + buffer[position++] = (byte) ((value >> 8) & 0xFF); + buffer[position++] = (byte) ((value >> 16) & 0xFF); + buffer[position++] = (byte) ((value >> 24) & 0xFF); + buffer[position++] = (byte) ((int) (value >> 32) & 0xFF); + buffer[position++] = (byte) ((int) (value >> 40) & 0xFF); + buffer[position++] = (byte) ((int) (value >> 48) & 0xFF); + buffer[position++] = (byte) ((int) (value >> 56) & 0xFF); + totalBytesWritten += FIXED64_SIZE; + } +} \ No newline at end of file diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/ArrayDecoder.java b/protobuf-sdk/src/main/java/com/google/protobuf/ArrayDecoder.java new file mode 100644 index 0000000..f61f939 --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/ArrayDecoder.java @@ -0,0 +1,672 @@ +package com.google.protobuf; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; + +import static com.google.protobuf.Internal.EMPTY_BYTE_BUFFER; +import static com.google.protobuf.Internal.UTF_8; +import static com.google.protobuf.WireFormat.*; +import static com.google.protobuf.WireFormat.FIXED64_SIZE; + + +/** A {@link CodedInputStream} implementation that uses a backing array as the input. */ +final class ArrayDecoder extends CodedInputStream { + private final byte[] buffer; + private final boolean immutable; + private int limit; + private int bufferSizeAfterLimit; + private int pos; + private int startPos; + private int lastTag; + private boolean enableAliasing; + + /** The absolute position of the end of the current message. */ + private int currentLimit = Integer.MAX_VALUE; + + ArrayDecoder(final byte[] buffer, final int offset, final int len, boolean immutable) { + super(); + this.buffer = buffer; + limit = offset + len; + pos = offset; + startPos = pos; + this.immutable = immutable; + } + + @Override + public int readTag() throws IOException { + if (isAtEnd()) { + lastTag = 0; + return 0; + } + + lastTag = readRawVarint32(); + if (WireFormat.getTagFieldNumber(lastTag) == 0) { + // If we actually read zero (or any tag number corresponding to field + // number zero), that's not a valid tag. + throw InvalidProtocolBufferException.invalidTag(); + } + return lastTag; + } + + @Override + public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { + if (lastTag != value) { + throw InvalidProtocolBufferException.invalidEndTag(); + } + } + + @Override + public int getLastTag() { + return lastTag; + } + + @Override + public boolean skipField(final int tag) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + skipRawVarint(); + return true; + case WireFormat.WIRETYPE_FIXED64: + skipRawBytes(FIXED64_SIZE); + return true; + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + skipRawBytes(readRawVarint32()); + return true; + case WireFormat.WIRETYPE_START_GROUP: + skipMessage(); + checkLastTagWas( + WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); + return true; + case WireFormat.WIRETYPE_END_GROUP: + return false; + case WireFormat.WIRETYPE_FIXED32: + skipRawBytes(FIXED32_SIZE); + return true; + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + @Override + public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + { + long value = readInt64(); + output.writeUInt32NoTag(tag); + output.writeUInt64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_FIXED64: + { + long value = readRawLittleEndian64(); + output.writeUInt32NoTag(tag); + output.writeFixed64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + { + ByteString value = readBytes(); + output.writeUInt32NoTag(tag); + output.writeBytesNoTag(value); + return true; + } + case WireFormat.WIRETYPE_START_GROUP: + { + output.writeUInt32NoTag(tag); + skipMessage(output); + int endtag = + WireFormat.makeTag( + WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); + checkLastTagWas(endtag); + output.writeUInt32NoTag(endtag); + return true; + } + case WireFormat.WIRETYPE_END_GROUP: + { + return false; + } + case WireFormat.WIRETYPE_FIXED32: + { + int value = readRawLittleEndian32(); + output.writeUInt32NoTag(tag); + output.writeFixed32NoTag(value); + return true; + } + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + // ----------------------------------------------------------------- + + @Override + public double readDouble() throws IOException { + return Double.longBitsToDouble(readRawLittleEndian64()); + } + + @Override + public float readFloat() throws IOException { + return Float.intBitsToFloat(readRawLittleEndian32()); + } + + @Override + public long readUInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public long readInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public int readInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public long readFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readFixed32() throws IOException { + return readRawLittleEndian32(); + } + + @Override + public boolean readBool() throws IOException { + return readRawVarint64() != 0; + } + + @Override + public String readString() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= (limit - pos)) { + // Fast path: We already have the bytes in a contiguous buffer, so + // just copy directly from it. + final String result = new String(buffer, pos, size, UTF_8); + pos += size; + return result; + } + + if (size == 0) { + return ""; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public String readStringRequireUtf8() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= (limit - pos)) { + String result = Utf8.decodeUtf8(buffer, pos, size); + pos += size; + return result; + } + + if (size == 0) { + return ""; + } + if (size <= 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public void readGroup( + final int fieldNumber, + final MessageLite.Builder builder, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + checkRecursionLimit(); + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + } + + @Override + public T readGroup( + final int fieldNumber, + final Parser parser, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + checkRecursionLimit(); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + return result; + } + + @Deprecated + @Override + public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) + throws IOException { + readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry()); + } + + @Override + public void readMessage( + final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) + throws IOException { + final int length = readRawVarint32(); + checkRecursionLimit(); + final int oldLimit = pushLimit(length); + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + if (getBytesUntilLimit() != 0) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + popLimit(oldLimit); + } + + @Override + public T readMessage( + final Parser parser, final ExtensionRegistryLite extensionRegistry) throws IOException { + int length = readRawVarint32(); + checkRecursionLimit(); + final int oldLimit = pushLimit(length); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + if (getBytesUntilLimit() != 0) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + popLimit(oldLimit); + return result; + } + + @Override + public ByteString readBytes() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= (limit - pos)) { + // Fast path: We already have the bytes in a contiguous buffer, so + // just copy directly from it. + final ByteString result = + immutable && enableAliasing + ? ByteString.wrap(buffer, pos, size) + : ByteString.copyFrom(buffer, pos, size); + pos += size; + return result; + } + if (size == 0) { + return ByteString.EMPTY; + } + // Slow path: Build a byte array first then copy it. + return ByteString.wrap(readRawBytes(size)); + } + + @Override + public byte[] readByteArray() throws IOException { + final int size = readRawVarint32(); + return readRawBytes(size); + } + + @Override + public ByteBuffer readByteBuffer() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= (limit - pos)) { + // Fast path: We already have the bytes in a contiguous buffer. + // When aliasing is enabled, we can return a ByteBuffer pointing directly + // into the underlying byte array without copy if the CodedInputStream is + // constructed from a byte array. If aliasing is disabled or the input is + // from an InputStream or ByteString, we have to make a copy of the bytes. + ByteBuffer result = + !immutable && enableAliasing + ? ByteBuffer.wrap(buffer, pos, size).slice() + : ByteBuffer.wrap(Arrays.copyOfRange(buffer, pos, pos + size)); + pos += size; + // TODO: Investigate making the ByteBuffer be made read-only + return result; + } + + if (size == 0) { + return EMPTY_BYTE_BUFFER; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public int readUInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public int readEnum() throws IOException { + return readRawVarint32(); + } + + @Override + public int readSFixed32() throws IOException { + return readRawLittleEndian32(); + } + + @Override + public long readSFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readSInt32() throws IOException { + return decodeZigZag32(readRawVarint32()); + } + + @Override + public long readSInt64() throws IOException { + return decodeZigZag64(readRawVarint64()); + } + + // ================================================================= + + @Override + public int readRawVarint32() throws IOException { + // See implementation notes for readRawVarint64 + fastpath: + { + int tempPos = pos; + + if (limit == tempPos) { + break fastpath; + } + + final byte[] buffer = this.buffer; + int x; + if ((x = buffer[tempPos++]) >= 0) { + pos = tempPos; + return x; + } else if (limit - tempPos < 9) { + break fastpath; + } else if ((x ^= (buffer[tempPos++] << 7)) < 0) { + x ^= (~0 << 7); + } else if ((x ^= (buffer[tempPos++] << 14)) >= 0) { + x ^= (~0 << 7) ^ (~0 << 14); + } else if ((x ^= (buffer[tempPos++] << 21)) < 0) { + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); + } else { + int y = buffer[tempPos++]; + x ^= y << 28; + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); + if (y < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0) { + break fastpath; // Will throw malformedVarint() + } + } + pos = tempPos; + return x; + } + return (int) readRawVarint64SlowPath(); + } + + private void skipRawVarint() throws IOException { + if (limit - pos >= MAX_VARINT_SIZE) { + skipRawVarintFastPath(); + } else { + skipRawVarintSlowPath(); + } + } + + private void skipRawVarintFastPath() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (buffer[pos++] >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + private void skipRawVarintSlowPath() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (readRawByte() >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + @Override + public long readRawVarint64() throws IOException { + // Implementation notes: + // + // Optimized for one-byte values, expected to be common. + // The particular code below was selected from various candidates + // empirically, by winning VarintBenchmark. + // + // Sign extension of (signed) Java bytes is usually a nuisance, but + // we exploit it here to more easily obtain the sign of bytes read. + // Instead of cleaning up the sign extension bits by masking eagerly, + // we delay until we find the final (positive) byte, when we clear all + // accumulated bits with one xor. We depend on javac to constant fold. + fastpath: + { + int tempPos = pos; + + if (limit == tempPos) { + break fastpath; + } + + final byte[] buffer = this.buffer; + long x; + int y; + if ((y = buffer[tempPos++]) >= 0) { + pos = tempPos; + return y; + } else if (limit - tempPos < 9) { + break fastpath; + } else if ((y ^= (buffer[tempPos++] << 7)) < 0) { + x = y ^ (~0 << 7); + } else if ((y ^= (buffer[tempPos++] << 14)) >= 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14)); + } else if ((y ^= (buffer[tempPos++] << 21)) < 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); + } else if ((x = y ^ ((long) buffer[tempPos++] << 28)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); + } else if ((x ^= ((long) buffer[tempPos++] << 35)) < 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); + } else if ((x ^= ((long) buffer[tempPos++] << 42)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); + } else if ((x ^= ((long) buffer[tempPos++] << 49)) < 0L) { + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49); + } else { + x ^= ((long) buffer[tempPos++] << 56); + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49) + ^ (~0L << 56); + if (x < 0L) { + if (buffer[tempPos++] < 0L) { + break fastpath; // Will throw malformedVarint() + } + } + } + pos = tempPos; + return x; + } + return readRawVarint64SlowPath(); + } + + @Override + long readRawVarint64SlowPath() throws IOException { + long result = 0; + for (int shift = 0; shift < 64; shift += 7) { + final byte b = readRawByte(); + result |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + @Override + public int readRawLittleEndian32() throws IOException { + int tempPos = pos; + + if (limit - tempPos < FIXED32_SIZE) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + + final byte[] buffer = this.buffer; + pos = tempPos + FIXED32_SIZE; + return ((buffer[tempPos] & 0xff) + | ((buffer[tempPos + 1] & 0xff) << 8) + | ((buffer[tempPos + 2] & 0xff) << 16) + | ((buffer[tempPos + 3] & 0xff) << 24)); + } + + @Override + public long readRawLittleEndian64() throws IOException { + int tempPos = pos; + + if (limit - tempPos < FIXED64_SIZE) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + + final byte[] buffer = this.buffer; + pos = tempPos + FIXED64_SIZE; + return ((buffer[tempPos] & 0xffL) + | ((buffer[tempPos + 1] & 0xffL) << 8) + | ((buffer[tempPos + 2] & 0xffL) << 16) + | ((buffer[tempPos + 3] & 0xffL) << 24) + | ((buffer[tempPos + 4] & 0xffL) << 32) + | ((buffer[tempPos + 5] & 0xffL) << 40) + | ((buffer[tempPos + 6] & 0xffL) << 48) + | ((buffer[tempPos + 7] & 0xffL) << 56)); + } + + @Override + public void enableAliasing(boolean enabled) { + this.enableAliasing = enabled; + } + + @Override + public void resetSizeCounter() { + startPos = pos; + } + + @Override + public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { + if (byteLimit < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + byteLimit += getTotalBytesRead(); + if (byteLimit < 0) { + throw InvalidProtocolBufferException.parseFailure(); + } + final int oldLimit = currentLimit; + if (byteLimit > oldLimit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + currentLimit = byteLimit; + + recomputeBufferSizeAfterLimit(); + + return oldLimit; + } + + private void recomputeBufferSizeAfterLimit() { + limit += bufferSizeAfterLimit; + final int bufferEnd = limit - startPos; + if (bufferEnd > currentLimit) { + // Limit is in current buffer. + bufferSizeAfterLimit = bufferEnd - currentLimit; + limit -= bufferSizeAfterLimit; + } else { + bufferSizeAfterLimit = 0; + } + } + + @Override + public void popLimit(final int oldLimit) { + currentLimit = oldLimit; + recomputeBufferSizeAfterLimit(); + } + + @Override + public int getBytesUntilLimit() { + if (currentLimit == Integer.MAX_VALUE) { + return -1; + } + + return currentLimit - getTotalBytesRead(); + } + + @Override + public boolean isAtEnd() throws IOException { + return pos == limit; + } + + @Override + public int getTotalBytesRead() { + return pos - startPos; + } + + @Override + public byte readRawByte() throws IOException { + if (pos == limit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + return buffer[pos++]; + } + + @Override + public byte[] readRawBytes(final int length) throws IOException { + if (length > 0 && length <= (limit - pos)) { + final int tempPos = pos; + pos += length; + return Arrays.copyOfRange(buffer, tempPos, pos); + } + + if (length <= 0) { + if (length == 0) { + return Internal.EMPTY_BYTE_ARRAY; + } else { + throw InvalidProtocolBufferException.negativeSize(); + } + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public void skipRawBytes(final int length) throws IOException { + if (length >= 0 && length <= (limit - pos)) { + // We have all the bytes we need already. + pos += length; + return; + } + + if (length < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } +} \ No newline at end of file diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/ArrayEncoder.java b/protobuf-sdk/src/main/java/com/google/protobuf/ArrayEncoder.java new file mode 100644 index 0000000..efd2860 --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/ArrayEncoder.java @@ -0,0 +1,354 @@ +package com.google.protobuf; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static com.google.protobuf.WireFormat.MAX_VARINT_SIZE; + +/** A {@link CodedOutputStream} that writes directly to a byte array. */ +class ArrayEncoder extends CodedOutputStream { + private final byte[] buffer; + private final int offset; + private final int limit; + private int position; + + ArrayEncoder(byte[] buffer, int offset, int length) { + super(); + if (buffer == null) { + throw new NullPointerException("buffer"); + } + if ((offset | length | (buffer.length - (offset + length))) < 0) { + throw new IllegalArgumentException( + String.format( + "Array range is invalid. Buffer.length=%d, offset=%d, length=%d", + buffer.length, offset, length)); + } + this.buffer = buffer; + this.offset = offset; + position = offset; + limit = offset + length; + } + + @Override + public final void writeTag(final int fieldNumber, final int wireType) throws IOException { + writeUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); + } + + @Override + public final void writeInt32(final int fieldNumber, final int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeInt32NoTag(value); + } + + @Override + public final void writeUInt32(final int fieldNumber, final int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeUInt32NoTag(value); + } + + @Override + public final void writeFixed32(final int fieldNumber, final int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); + writeFixed32NoTag(value); + } + + @Override + public final void writeUInt64(final int fieldNumber, final long value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeUInt64NoTag(value); + } + + @Override + public final void writeFixed64(final int fieldNumber, final long value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); + writeFixed64NoTag(value); + } + + @Override + public final void writeBool(final int fieldNumber, final boolean value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + write((byte) (value ? 1 : 0)); + } + + @Override + public final void writeString(final int fieldNumber, final String value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeStringNoTag(value); + } + + @Override + public final void writeBytes(final int fieldNumber, final ByteString value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeBytesNoTag(value); + } + + @Override + public final void writeByteArray(final int fieldNumber, final byte[] value) throws IOException { + writeByteArray(fieldNumber, value, 0, value.length); + } + + @Override + public final void writeByteArray( + final int fieldNumber, final byte[] value, final int offset, final int length) + throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeByteArrayNoTag(value, offset, length); + } + + @Override + public final void writeByteBuffer(final int fieldNumber, final ByteBuffer value) + throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeUInt32NoTag(value.capacity()); + writeRawBytes(value); + } + + @Override + public final void writeBytesNoTag(final ByteString value) throws IOException { + writeUInt32NoTag(value.size()); + value.writeTo(this); + } + + @Override + public final void writeByteArrayNoTag(final byte[] value, int offset, int length) + throws IOException { + writeUInt32NoTag(length); + write(value, offset, length); + } + + @Override + public final void writeRawBytes(final ByteBuffer value) throws IOException { + if (value.hasArray()) { + write(value.array(), value.arrayOffset(), value.capacity()); + } else { + ByteBuffer duplicated = value.duplicate(); + Java8Compatibility.clear(duplicated); + write(duplicated); + } + } + + @Override + public final void writeMessage(final int fieldNumber, final MessageLite value) + throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeMessageNoTag(value); + } + + @Override + final void writeMessage(final int fieldNumber, final MessageLite value, Schema schema) + throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeUInt32NoTag(((AbstractMessageLite) value).getSerializedSize(schema)); + schema.writeTo(value, wrapper); + } + + @Override + public final void writeMessageSetExtension(final int fieldNumber, final MessageLite value) + throws IOException { + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); + writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); + writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value); + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); + } + + @Override + public final void writeRawMessageSetExtension(final int fieldNumber, final ByteString value) + throws IOException { + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); + writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); + writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value); + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); + } + + @Override + public final void writeMessageNoTag(final MessageLite value) throws IOException { + writeUInt32NoTag(value.getSerializedSize()); + ((AbstractMessageLite) value).writeTo(this); + } + + @Override + final void writeMessageNoTag(final MessageLite value, Schema schema) throws IOException { + writeUInt32NoTag(((AbstractMessageLite) value).getSerializedSize(schema)); + schema.writeTo(value, wrapper); + } + + @Override + public final void write(byte value) throws IOException { + try { + buffer[position++] = value; + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, 1), e); + } + } + + @Override + public final void writeInt32NoTag(int value) throws IOException { + if (value >= 0) { + writeUInt32NoTag(value); + } else { + // Must sign-extend. + writeUInt64NoTag(value); + } + } + + @Override + public final void writeUInt32NoTag(int value) throws IOException { + try { + while (true) { + if ((value & ~0x7F) == 0) { + buffer[position++] = (byte) value; + return; + } else { + buffer[position++] = (byte) ((value & 0x7F) | 0x80); + value >>>= 7; + } + } + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, 1), e); + } + } + + @Override + public final void writeFixed32NoTag(int value) throws IOException { + try { + buffer[position++] = (byte) (value & 0xFF); + buffer[position++] = (byte) ((value >> 8) & 0xFF); + buffer[position++] = (byte) ((value >> 16) & 0xFF); + buffer[position++] = (byte) ((value >> 24) & 0xFF); + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, 1), e); + } + } + + @Override + public final void writeUInt64NoTag(long value) throws IOException { + if (HAS_UNSAFE_ARRAY_OPERATIONS && spaceLeft() >= MAX_VARINT_SIZE) { + while (true) { + if ((value & ~0x7FL) == 0) { + UnsafeUtil.putByte(buffer, position++, (byte) value); + return; + } else { + UnsafeUtil.putByte(buffer, position++, (byte) (((int) value & 0x7F) | 0x80)); + value >>>= 7; + } + } + } else { + try { + while (true) { + if ((value & ~0x7FL) == 0) { + buffer[position++] = (byte) value; + return; + } else { + buffer[position++] = (byte) (((int) value & 0x7F) | 0x80); + value >>>= 7; + } + } + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, 1), e); + } + } + } + + @Override + public final void writeFixed64NoTag(long value) throws IOException { + try { + buffer[position++] = (byte) ((int) (value) & 0xFF); + buffer[position++] = (byte) ((int) (value >> 8) & 0xFF); + buffer[position++] = (byte) ((int) (value >> 16) & 0xFF); + buffer[position++] = (byte) ((int) (value >> 24) & 0xFF); + buffer[position++] = (byte) ((int) (value >> 32) & 0xFF); + buffer[position++] = (byte) ((int) (value >> 40) & 0xFF); + buffer[position++] = (byte) ((int) (value >> 48) & 0xFF); + buffer[position++] = (byte) ((int) (value >> 56) & 0xFF); + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, 1), e); + } + } + + @Override + public final void write(byte[] value, int offset, int length) throws IOException { + try { + System.arraycopy(value, offset, buffer, position, length); + position += length; + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, length), e); + } + } + + @Override + public final void writeLazy(byte[] value, int offset, int length) throws IOException { + write(value, offset, length); + } + + @Override + public final void write(ByteBuffer value) throws IOException { + final int length = value.remaining(); + try { + value.get(buffer, position, length); + position += length; + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, length), e); + } + } + + @Override + public final void writeLazy(ByteBuffer value) throws IOException { + write(value); + } + + @Override + public final void writeStringNoTag(String value) throws IOException { + final int oldPosition = position; + try { + // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), + // and at most 3 times of it. We take advantage of this in both branches below. + final int maxLength = value.length() * Utf8.MAX_BYTES_PER_CHAR; + final int maxLengthVarIntSize = computeUInt32SizeNoTag(maxLength); + final int minLengthVarIntSize = computeUInt32SizeNoTag(value.length()); + if (minLengthVarIntSize == maxLengthVarIntSize) { + position = oldPosition + minLengthVarIntSize; + int newPosition = Utf8.encode(value, buffer, position, spaceLeft()); + // Since this class is stateful and tracks the position, we rewind and store the state, + // prepend the length, then reset it back to the end of the string. + position = oldPosition; + int length = newPosition - oldPosition - minLengthVarIntSize; + writeUInt32NoTag(length); + position = newPosition; + } else { + int length = Utf8.encodedLength(value); + writeUInt32NoTag(length); + position = Utf8.encode(value, buffer, position, spaceLeft()); + } + } catch (Utf8.UnpairedSurrogateException e) { + // Roll back the change - we fall back to inefficient path. + position = oldPosition; + + // TODO: We should throw an IOException here instead. + inefficientWriteStringNoTag(value, e); + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void flush() { + // Do nothing. + } + + @Override + public final int spaceLeft() { + return limit - position; + } + + @Override + public final int getTotalBytesWritten() { + return position - offset; + } +} \ No newline at end of file diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/BoundedByteString.java b/protobuf-sdk/src/main/java/com/google/protobuf/BoundedByteString.java new file mode 100644 index 0000000..9da632d --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/BoundedByteString.java @@ -0,0 +1,98 @@ +package com.google.protobuf; + +import java.io.IOException; +import java.io.InvalidObjectException; +import java.io.ObjectInputStream; + +import static com.google.protobuf.ByteString.checkIndex; +import static com.google.protobuf.ByteString.checkRange; + +/** + * This class is used to represent the substring of a {@link ByteString} over a single byte array. + * In terms of the public API of {@link ByteString}, you end up here by calling {@link + * ByteString#copyFrom(byte[])} followed by {@link ByteString#substring(int, int)}. + * + *

This class contains most of the overhead involved in creating a substring from a {@link + * LiteralByteString}. The overhead involves some range-checking and two extra fields. + * + * @author carlanton@google.com (Carl Haverl) + */ +// Keep this class private to avoid deadlocks in classloading across threads as ByteString's +// static initializer loads LiteralByteString and another thread loads BoundedByteString. +final class BoundedByteString extends LiteralByteString { + private final int bytesOffset; + private final int bytesLength; + + /** + * Creates a {@code BoundedByteString} backed by the sub-range of given array, without copying. + * + * @param bytes array to wrap + * @param offset index to first byte to use in bytes + * @param length number of bytes to use from bytes + * @throws IllegalArgumentException if {@code offset < 0}, {@code length < 0}, or if {@code + * offset + length > bytes.length}. + */ + BoundedByteString(byte[] bytes, int offset, int length) { + super(bytes); + checkRange(offset, offset + length, bytes.length); + + this.bytesOffset = offset; + this.bytesLength = length; + } + + /** + * Gets the byte at the given index. Throws {@link ArrayIndexOutOfBoundsException} for + * backwards-compatibility reasons although it would more properly be {@link + * IndexOutOfBoundsException}. + * + * @param index index of byte + * @return the value + * @throws ArrayIndexOutOfBoundsException {@code index} is < 0 or >= size + */ + @Override + public byte byteAt(int index) { + // We must check the index ourselves as we cannot rely on Java array index + // checking for substrings. + checkIndex(index, size()); + return bytes[bytesOffset + index]; + } + + @Override + byte internalByteAt(int index) { + return bytes[bytesOffset + index]; + } + + @Override + public int size() { + return bytesLength; + } + + @Override + protected int getOffsetIntoBytes() { + return bytesOffset; + } + + // ================================================================= + // ByteString -> byte[] + + @Override + protected void copyToInternal( + byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { + System.arraycopy( + bytes, getOffsetIntoBytes() + sourceOffset, target, targetOffset, numberToCopy); + } + + // ================================================================= + // Serializable + + private static final long serialVersionUID = 1L; + + Object writeReplace() { + return ByteString.wrap(toByteArray()); + } + + private void readObject(@SuppressWarnings("unused") ObjectInputStream in) throws IOException { + throw new InvalidObjectException( + "BoundedByteStream instances are not to be serialized directly"); + } +} \ No newline at end of file diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/ByteOutputEncoder.java b/protobuf-sdk/src/main/java/com/google/protobuf/ByteOutputEncoder.java new file mode 100644 index 0000000..060d095 --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/ByteOutputEncoder.java @@ -0,0 +1,322 @@ +package com.google.protobuf; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static com.google.protobuf.WireFormat.*; +import static com.google.protobuf.WireFormat.FIXED64_SIZE; + +/** + * A {@link CodedOutputStream} that decorates a {@link ByteOutput}. It internal buffer only to + * support string encoding operations. All other writes are just passed through to the {@link + * ByteOutput}. + */ +final class ByteOutputEncoder extends AbstractBufferedEncoder { + private final ByteOutput out; + + ByteOutputEncoder(ByteOutput out, int bufferSize) { + super(bufferSize); + if (out == null) { + throw new NullPointerException("out"); + } + this.out = out; + } + + @Override + public void writeTag(final int fieldNumber, final int wireType) throws IOException { + writeUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); + } + + @Override + public void writeInt32(final int fieldNumber, final int value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE * 2); + bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + bufferInt32NoTag(value); + } + + @Override + public void writeUInt32(final int fieldNumber, final int value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE * 2); + bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + bufferUInt32NoTag(value); + } + + @Override + public void writeFixed32(final int fieldNumber, final int value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE + FIXED32_SIZE); + bufferTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); + bufferFixed32NoTag(value); + } + + @Override + public void writeUInt64(final int fieldNumber, final long value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE * 2); + bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + bufferUInt64NoTag(value); + } + + @Override + public void writeFixed64(final int fieldNumber, final long value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE + FIXED64_SIZE); + bufferTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); + bufferFixed64NoTag(value); + } + + @Override + public void writeBool(final int fieldNumber, final boolean value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE + 1); + bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + buffer((byte) (value ? 1 : 0)); + } + + @Override + public void writeString(final int fieldNumber, final String value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeStringNoTag(value); + } + + @Override + public void writeBytes(final int fieldNumber, final ByteString value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeBytesNoTag(value); + } + + @Override + public void writeByteArray(final int fieldNumber, final byte[] value) throws IOException { + writeByteArray(fieldNumber, value, 0, value.length); + } + + @Override + public void writeByteArray( + final int fieldNumber, final byte[] value, final int offset, final int length) + throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeByteArrayNoTag(value, offset, length); + } + + @Override + public void writeByteBuffer(final int fieldNumber, final ByteBuffer value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeUInt32NoTag(value.capacity()); + writeRawBytes(value); + } + + @Override + public void writeBytesNoTag(final ByteString value) throws IOException { + writeUInt32NoTag(value.size()); + value.writeTo(this); + } + + @Override + public void writeByteArrayNoTag(final byte[] value, int offset, int length) throws IOException { + writeUInt32NoTag(length); + write(value, offset, length); + } + + @Override + public void writeRawBytes(final ByteBuffer value) throws IOException { + if (value.hasArray()) { + write(value.array(), value.arrayOffset(), value.capacity()); + } else { + ByteBuffer duplicated = value.duplicate(); + Java8Compatibility.clear(duplicated); + write(duplicated); + } + } + + @Override + public void writeMessage(final int fieldNumber, final MessageLite value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeMessageNoTag(value); + } + + @Override + void writeMessage(final int fieldNumber, final MessageLite value, Schema schema) + throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeMessageNoTag(value, schema); + } + + @Override + public void writeMessageSetExtension(final int fieldNumber, final MessageLite value) + throws IOException { + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); + writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); + writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value); + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); + } + + @Override + public void writeRawMessageSetExtension(final int fieldNumber, final ByteString value) + throws IOException { + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); + writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); + writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value); + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); + } + + @Override + public void writeMessageNoTag(final MessageLite value) throws IOException { + writeUInt32NoTag(value.getSerializedSize()); + value.writeTo(this); + } + + @Override + void writeMessageNoTag(final MessageLite value, Schema schema) throws IOException { + writeUInt32NoTag(((AbstractMessageLite) value).getSerializedSize(schema)); + schema.writeTo(value, wrapper); + } + + @Override + public void write(byte value) throws IOException { + if (position == limit) { + doFlush(); + } + + buffer(value); + } + + @Override + public void writeInt32NoTag(int value) throws IOException { + if (value >= 0) { + writeUInt32NoTag(value); + } else { + // Must sign-extend. + writeUInt64NoTag(value); + } + } + + @Override + public void writeUInt32NoTag(int value) throws IOException { + flushIfNotAvailable(MAX_VARINT32_SIZE); + bufferUInt32NoTag(value); + } + + @Override + public void writeFixed32NoTag(final int value) throws IOException { + flushIfNotAvailable(FIXED32_SIZE); + bufferFixed32NoTag(value); + } + + @Override + public void writeUInt64NoTag(long value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE); + bufferUInt64NoTag(value); + } + + @Override + public void writeFixed64NoTag(final long value) throws IOException { + flushIfNotAvailable(FIXED64_SIZE); + bufferFixed64NoTag(value); + } + + @Override + public void writeStringNoTag(String value) throws IOException { + // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), + // and at most 3 times of it. We take advantage of this in both branches below. + final int maxLength = value.length() * Utf8.MAX_BYTES_PER_CHAR; + final int maxLengthVarIntSize = computeUInt32SizeNoTag(maxLength); + + // If we are streaming and the potential length is too big to fit in our buffer, we take the + // slower path. + if (maxLengthVarIntSize + maxLength > limit) { + // Allocate a byte[] that we know can fit the string and encode into it. String.getBytes() + // does the same internally and then does *another copy* to return a byte[] of exactly the + // right size. We can skip that copy and just writeRawBytes up to the actualLength of the + // UTF-8 encoded bytes. + final byte[] encodedBytes = new byte[maxLength]; + int actualLength = Utf8.encode(value, encodedBytes, 0, maxLength); + writeUInt32NoTag(actualLength); + writeLazy(encodedBytes, 0, actualLength); + return; + } + + // Fast path: we have enough space available in our buffer for the string... + if (maxLengthVarIntSize + maxLength > limit - position) { + // Flush to free up space. + doFlush(); + } + + final int oldPosition = position; + try { + // Optimize for the case where we know this length results in a constant varint length as + // this saves a pass for measuring the length of the string. + final int minLengthVarIntSize = computeUInt32SizeNoTag(value.length()); + + if (minLengthVarIntSize == maxLengthVarIntSize) { + position = oldPosition + minLengthVarIntSize; + int newPosition = Utf8.encode(value, buffer, position, limit - position); + // Since this class is stateful and tracks the position, we rewind and store the state, + // prepend the length, then reset it back to the end of the string. + position = oldPosition; + int length = newPosition - oldPosition - minLengthVarIntSize; + bufferUInt32NoTag(length); + position = newPosition; + totalBytesWritten += length; + } else { + int length = Utf8.encodedLength(value); + bufferUInt32NoTag(length); + position = Utf8.encode(value, buffer, position, length); + totalBytesWritten += length; + } + } catch (Utf8.UnpairedSurrogateException e) { + // Roll back the change and convert to an IOException. + totalBytesWritten -= position - oldPosition; + position = oldPosition; + + // TODO: We should throw an IOException here instead. + inefficientWriteStringNoTag(value, e); + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void flush() throws IOException { + if (position > 0) { + // Flush the buffer. + doFlush(); + } + } + + @Override + public void write(byte[] value, int offset, int length) throws IOException { + flush(); + out.write(value, offset, length); + totalBytesWritten += length; + } + + @Override + public void writeLazy(byte[] value, int offset, int length) throws IOException { + flush(); + out.writeLazy(value, offset, length); + totalBytesWritten += length; + } + + @Override + public void write(ByteBuffer value) throws IOException { + flush(); + int length = value.remaining(); + out.write(value); + totalBytesWritten += length; + } + + @Override + public void writeLazy(ByteBuffer value) throws IOException { + flush(); + int length = value.remaining(); + out.writeLazy(value); + totalBytesWritten += length; + } + + private void flushIfNotAvailable(int requiredSize) throws IOException { + if (limit - position < requiredSize) { + doFlush(); + } + } + + private void doFlush() throws IOException { + out.write(buffer, 0, position); + position = 0; + } +} \ No newline at end of file diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/HeapNioEncoder.java b/protobuf-sdk/src/main/java/com/google/protobuf/HeapNioEncoder.java new file mode 100644 index 0000000..0c0bdd0 --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/HeapNioEncoder.java @@ -0,0 +1,27 @@ +package com.google.protobuf; + +import java.nio.ByteBuffer; + +/** + * A {@link CodedOutputStream} that writes directly to a heap {@link ByteBuffer}. Writes are done + * directly to the underlying array. The buffer position is only updated after a flush. + */ +final class HeapNioEncoder extends ArrayEncoder { + private final ByteBuffer byteBuffer; + private int initialPosition; + + HeapNioEncoder(ByteBuffer byteBuffer) { + super( + byteBuffer.array(), + byteBuffer.arrayOffset() + byteBuffer.position(), + byteBuffer.remaining()); + this.byteBuffer = byteBuffer; + this.initialPosition = byteBuffer.position(); + } + + @Override + public void flush() { + // Update the position on the buffer. + Java8Compatibility.position(byteBuffer, initialPosition + getTotalBytesWritten()); + } +} \ No newline at end of file diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/IterableDirectByteBufferDecoder.java b/protobuf-sdk/src/main/java/com/google/protobuf/IterableDirectByteBufferDecoder.java new file mode 100644 index 0000000..af8e6d9 --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/IterableDirectByteBufferDecoder.java @@ -0,0 +1,835 @@ +package com.google.protobuf; + +import java.io.IOException; +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Iterator; + +import static com.google.protobuf.Internal.*; +import static com.google.protobuf.WireFormat.*; + +/** + * Implementation of {@link CodedInputStream} that uses an {@link Iterable } as the + * data source. Requires the use of {@code sun.misc.Unsafe} to perform fast reads on the buffer. + */ +final class IterableDirectByteBufferDecoder extends CodedInputStream { + /** The object that need to decode. */ + private final Iterable input; + /** The {@link Iterator} with type {@link ByteBuffer} of {@code input} */ + private final Iterator iterator; + /** The current ByteBuffer; */ + private ByteBuffer currentByteBuffer; + /** + * If {@code true}, indicates that all the buffers are backing a {@link ByteString} and are + * therefore considered to be an immutable input source. + */ + private final boolean immutable; + /** + * If {@code true}, indicates that calls to read {@link ByteString} or {@code byte[]} + * may return slices of the underlying buffer, rather than copies. + */ + private boolean enableAliasing; + /** The global total message length limit */ + private int totalBufferSize; + /** The amount of available data in the input beyond {@link #currentLimit}. */ + private int bufferSizeAfterCurrentLimit; + /** The absolute position of the end of the current message. */ + private int currentLimit = Integer.MAX_VALUE; + /** The last tag that was read from this stream. */ + private int lastTag; + /** Total Bytes have been Read from the {@link Iterable} {@link ByteBuffer} */ + private int totalBytesRead; + /** The start position offset of the whole message, used as to reset the totalBytesRead */ + private int startOffset; + /** The current position for current ByteBuffer */ + private long currentByteBufferPos; + + private long currentByteBufferStartPos; + /** + * If the current ByteBuffer is unsafe-direct based, currentAddress is the start address of this + * ByteBuffer; otherwise should be zero. + */ + private long currentAddress; + /** The limit position for current ByteBuffer */ + private long currentByteBufferLimit; + + /** + * The constructor of {@code Iterable} decoder. + * + * @param inputBufs The input data. + * @param size The total size of the input data. + * @param immutableFlag whether the input data is immutable. + */ + IterableDirectByteBufferDecoder( + Iterable inputBufs, int size, boolean immutableFlag) { + totalBufferSize = size; + input = inputBufs; + iterator = input.iterator(); + immutable = immutableFlag; + startOffset = totalBytesRead = 0; + if (size == 0) { + currentByteBuffer = EMPTY_BYTE_BUFFER; + currentByteBufferPos = 0; + currentByteBufferStartPos = 0; + currentByteBufferLimit = 0; + currentAddress = 0; + } else { + tryGetNextByteBuffer(); + } + } + + /** To get the next ByteBuffer from {@code input}, and then update the parameters */ + private void getNextByteBuffer() throws InvalidProtocolBufferException { + if (!iterator.hasNext()) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + tryGetNextByteBuffer(); + } + + private void tryGetNextByteBuffer() { + currentByteBuffer = iterator.next(); + totalBytesRead += (int) (currentByteBufferPos - currentByteBufferStartPos); + currentByteBufferPos = currentByteBuffer.position(); + currentByteBufferStartPos = currentByteBufferPos; + currentByteBufferLimit = currentByteBuffer.limit(); + currentAddress = UnsafeUtil.addressOffset(currentByteBuffer); + currentByteBufferPos += currentAddress; + currentByteBufferStartPos += currentAddress; + currentByteBufferLimit += currentAddress; + } + + @Override + public int readTag() throws IOException { + if (isAtEnd()) { + lastTag = 0; + return 0; + } + + lastTag = readRawVarint32(); + if (WireFormat.getTagFieldNumber(lastTag) == 0) { + // If we actually read zero (or any tag number corresponding to field + // number zero), that's not a valid tag. + throw InvalidProtocolBufferException.invalidTag(); + } + return lastTag; + } + + @Override + public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { + if (lastTag != value) { + throw InvalidProtocolBufferException.invalidEndTag(); + } + } + + @Override + public int getLastTag() { + return lastTag; + } + + @Override + public boolean skipField(final int tag) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + skipRawVarint(); + return true; + case WireFormat.WIRETYPE_FIXED64: + skipRawBytes(FIXED64_SIZE); + return true; + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + skipRawBytes(readRawVarint32()); + return true; + case WireFormat.WIRETYPE_START_GROUP: + skipMessage(); + checkLastTagWas( + WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); + return true; + case WireFormat.WIRETYPE_END_GROUP: + return false; + case WireFormat.WIRETYPE_FIXED32: + skipRawBytes(FIXED32_SIZE); + return true; + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + @Override + public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + { + long value = readInt64(); + output.writeUInt32NoTag(tag); + output.writeUInt64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_FIXED64: + { + long value = readRawLittleEndian64(); + output.writeUInt32NoTag(tag); + output.writeFixed64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + { + ByteString value = readBytes(); + output.writeUInt32NoTag(tag); + output.writeBytesNoTag(value); + return true; + } + case WireFormat.WIRETYPE_START_GROUP: + { + output.writeUInt32NoTag(tag); + skipMessage(output); + int endtag = + WireFormat.makeTag( + WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); + checkLastTagWas(endtag); + output.writeUInt32NoTag(endtag); + return true; + } + case WireFormat.WIRETYPE_END_GROUP: + { + return false; + } + case WireFormat.WIRETYPE_FIXED32: + { + int value = readRawLittleEndian32(); + output.writeUInt32NoTag(tag); + output.writeFixed32NoTag(value); + return true; + } + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + // ----------------------------------------------------------------- + + @Override + public double readDouble() throws IOException { + return Double.longBitsToDouble(readRawLittleEndian64()); + } + + @Override + public float readFloat() throws IOException { + return Float.intBitsToFloat(readRawLittleEndian32()); + } + + @Override + public long readUInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public long readInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public int readInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public long readFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readFixed32() throws IOException { + return readRawLittleEndian32(); + } + + @Override + public boolean readBool() throws IOException { + return readRawVarint64() != 0; + } + + @Override + public String readString() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) { + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size); + String result = new String(bytes, UTF_8); + currentByteBufferPos += size; + return result; + } else if (size > 0 && size <= remaining()) { + // TODO: To use an underlying bytes[] instead of allocating a new bytes[] + byte[] bytes = new byte[size]; + readRawBytesTo(bytes, 0, size); + String result = new String(bytes, UTF_8); + return result; + } + + if (size == 0) { + return ""; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public String readStringRequireUtf8() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) { + final int bufferPos = (int) (currentByteBufferPos - currentByteBufferStartPos); + String result = Utf8.decodeUtf8(currentByteBuffer, bufferPos, size); + currentByteBufferPos += size; + return result; + } + if (size >= 0 && size <= remaining()) { + byte[] bytes = new byte[size]; + readRawBytesTo(bytes, 0, size); + return Utf8.decodeUtf8(bytes, 0, size); + } + + if (size == 0) { + return ""; + } + if (size <= 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public void readGroup( + final int fieldNumber, + final MessageLite.Builder builder, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + checkRecursionLimit(); + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + } + + @Override + public T readGroup( + final int fieldNumber, + final Parser parser, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + checkRecursionLimit(); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + return result; + } + + @Deprecated + @Override + public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) + throws IOException { + readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry()); + } + + @Override + public void readMessage( + final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) + throws IOException { + final int length = readRawVarint32(); + checkRecursionLimit(); + final int oldLimit = pushLimit(length); + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + if (getBytesUntilLimit() != 0) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + popLimit(oldLimit); + } + + @Override + public T readMessage( + final Parser parser, final ExtensionRegistryLite extensionRegistry) throws IOException { + int length = readRawVarint32(); + checkRecursionLimit(); + final int oldLimit = pushLimit(length); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + if (getBytesUntilLimit() != 0) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + popLimit(oldLimit); + return result; + } + + @Override + public ByteString readBytes() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) { + if (immutable && enableAliasing) { + final int idx = (int) (currentByteBufferPos - currentAddress); + final ByteString result = ByteString.wrap(slice(idx, idx + size)); + currentByteBufferPos += size; + return result; + } else { + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size); + currentByteBufferPos += size; + return ByteString.wrap(bytes); + } + } else if (size > 0 && size <= remaining()) { + if (immutable && enableAliasing) { + ArrayList byteStrings = new ArrayList<>(); + int l = size; + while (l > 0) { + if (currentRemaining() == 0) { + getNextByteBuffer(); + } + int bytesToCopy = Math.min(l, (int) currentRemaining()); + int idx = (int) (currentByteBufferPos - currentAddress); + byteStrings.add(ByteString.wrap(slice(idx, idx + bytesToCopy))); + l -= bytesToCopy; + currentByteBufferPos += bytesToCopy; + } + return ByteString.copyFrom(byteStrings); + } else { + byte[] temp = new byte[size]; + readRawBytesTo(temp, 0, size); + return ByteString.wrap(temp); + } + } + + if (size == 0) { + return ByteString.EMPTY; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public byte[] readByteArray() throws IOException { + return readRawBytes(readRawVarint32()); + } + + @Override + public ByteBuffer readByteBuffer() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= currentRemaining()) { + if (!immutable && enableAliasing) { + currentByteBufferPos += size; + return slice( + (int) (currentByteBufferPos - currentAddress - size), + (int) (currentByteBufferPos - currentAddress)); + } else { + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size); + currentByteBufferPos += size; + return ByteBuffer.wrap(bytes); + } + } else if (size > 0 && size <= remaining()) { + byte[] temp = new byte[size]; + readRawBytesTo(temp, 0, size); + return ByteBuffer.wrap(temp); + } + + if (size == 0) { + return EMPTY_BYTE_BUFFER; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public int readUInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public int readEnum() throws IOException { + return readRawVarint32(); + } + + @Override + public int readSFixed32() throws IOException { + return readRawLittleEndian32(); + } + + @Override + public long readSFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readSInt32() throws IOException { + return decodeZigZag32(readRawVarint32()); + } + + @Override + public long readSInt64() throws IOException { + return decodeZigZag64(readRawVarint64()); + } + + @Override + public int readRawVarint32() throws IOException { + fastpath: + { + long tempPos = currentByteBufferPos; + + if (currentByteBufferLimit == currentByteBufferPos) { + break fastpath; + } + + int x; + if ((x = UnsafeUtil.getByte(tempPos++)) >= 0) { + currentByteBufferPos++; + return x; + } else if (currentByteBufferLimit - currentByteBufferPos < 10) { + break fastpath; + } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) { + x ^= (~0 << 7); + } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) { + x ^= (~0 << 7) ^ (~0 << 14); + } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) { + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); + } else { + int y = UnsafeUtil.getByte(tempPos++); + x ^= y << 28; + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); + if (y < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0) { + break fastpath; // Will throw malformedVarint() + } + } + currentByteBufferPos = tempPos; + return x; + } + return (int) readRawVarint64SlowPath(); + } + + @Override + public long readRawVarint64() throws IOException { + fastpath: + { + long tempPos = currentByteBufferPos; + + if (currentByteBufferLimit == currentByteBufferPos) { + break fastpath; + } + + long x; + int y; + if ((y = UnsafeUtil.getByte(tempPos++)) >= 0) { + currentByteBufferPos++; + return y; + } else if (currentByteBufferLimit - currentByteBufferPos < 10) { + break fastpath; + } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) { + x = y ^ (~0 << 7); + } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14)); + } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); + } else if ((x = y ^ ((long) UnsafeUtil.getByte(tempPos++) << 28)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); + } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 35)) < 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); + } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 42)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); + } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 49)) < 0L) { + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49); + } else { + x ^= ((long) UnsafeUtil.getByte(tempPos++) << 56); + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49) + ^ (~0L << 56); + if (x < 0L) { + if (UnsafeUtil.getByte(tempPos++) < 0L) { + break fastpath; // Will throw malformedVarint() + } + } + } + currentByteBufferPos = tempPos; + return x; + } + return readRawVarint64SlowPath(); + } + + @Override + long readRawVarint64SlowPath() throws IOException { + long result = 0; + for (int shift = 0; shift < 64; shift += 7) { + final byte b = readRawByte(); + result |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + @Override + public int readRawLittleEndian32() throws IOException { + if (currentRemaining() >= FIXED32_SIZE) { + long tempPos = currentByteBufferPos; + currentByteBufferPos += FIXED32_SIZE; + return ((UnsafeUtil.getByte(tempPos) & 0xff) + | ((UnsafeUtil.getByte(tempPos + 1) & 0xff) << 8) + | ((UnsafeUtil.getByte(tempPos + 2) & 0xff) << 16) + | ((UnsafeUtil.getByte(tempPos + 3) & 0xff) << 24)); + } + return ((readRawByte() & 0xff) + | ((readRawByte() & 0xff) << 8) + | ((readRawByte() & 0xff) << 16) + | ((readRawByte() & 0xff) << 24)); + } + + @Override + public long readRawLittleEndian64() throws IOException { + if (currentRemaining() >= FIXED64_SIZE) { + long tempPos = currentByteBufferPos; + currentByteBufferPos += FIXED64_SIZE; + return ((UnsafeUtil.getByte(tempPos) & 0xffL) + | ((UnsafeUtil.getByte(tempPos + 1) & 0xffL) << 8) + | ((UnsafeUtil.getByte(tempPos + 2) & 0xffL) << 16) + | ((UnsafeUtil.getByte(tempPos + 3) & 0xffL) << 24) + | ((UnsafeUtil.getByte(tempPos + 4) & 0xffL) << 32) + | ((UnsafeUtil.getByte(tempPos + 5) & 0xffL) << 40) + | ((UnsafeUtil.getByte(tempPos + 6) & 0xffL) << 48) + | ((UnsafeUtil.getByte(tempPos + 7) & 0xffL) << 56)); + } + return ((readRawByte() & 0xffL) + | ((readRawByte() & 0xffL) << 8) + | ((readRawByte() & 0xffL) << 16) + | ((readRawByte() & 0xffL) << 24) + | ((readRawByte() & 0xffL) << 32) + | ((readRawByte() & 0xffL) << 40) + | ((readRawByte() & 0xffL) << 48) + | ((readRawByte() & 0xffL) << 56)); + } + + @Override + public void enableAliasing(boolean enabled) { + this.enableAliasing = enabled; + } + + @Override + public void resetSizeCounter() { + startOffset = (int) (totalBytesRead + currentByteBufferPos - currentByteBufferStartPos); + } + + @Override + public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { + if (byteLimit < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + byteLimit += getTotalBytesRead(); + final int oldLimit = currentLimit; + if (byteLimit > oldLimit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + currentLimit = byteLimit; + + recomputeBufferSizeAfterLimit(); + + return oldLimit; + } + + private void recomputeBufferSizeAfterLimit() { + totalBufferSize += bufferSizeAfterCurrentLimit; + final int bufferEnd = totalBufferSize - startOffset; + if (bufferEnd > currentLimit) { + // Limit is in current buffer. + bufferSizeAfterCurrentLimit = bufferEnd - currentLimit; + totalBufferSize -= bufferSizeAfterCurrentLimit; + } else { + bufferSizeAfterCurrentLimit = 0; + } + } + + @Override + public void popLimit(final int oldLimit) { + currentLimit = oldLimit; + recomputeBufferSizeAfterLimit(); + } + + @Override + public int getBytesUntilLimit() { + if (currentLimit == Integer.MAX_VALUE) { + return -1; + } + + return currentLimit - getTotalBytesRead(); + } + + @Override + public boolean isAtEnd() throws IOException { + return totalBytesRead + currentByteBufferPos - currentByteBufferStartPos == totalBufferSize; + } + + @Override + public int getTotalBytesRead() { + return (int) + (totalBytesRead - startOffset + currentByteBufferPos - currentByteBufferStartPos); + } + + @Override + public byte readRawByte() throws IOException { + if (currentRemaining() == 0) { + getNextByteBuffer(); + } + return UnsafeUtil.getByte(currentByteBufferPos++); + } + + @Override + public byte[] readRawBytes(final int length) throws IOException { + if (length >= 0 && length <= currentRemaining()) { + byte[] bytes = new byte[length]; + UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, length); + currentByteBufferPos += length; + return bytes; + } + if (length >= 0 && length <= remaining()) { + byte[] bytes = new byte[length]; + readRawBytesTo(bytes, 0, length); + return bytes; + } + + if (length <= 0) { + if (length == 0) { + return EMPTY_BYTE_ARRAY; + } else { + throw InvalidProtocolBufferException.negativeSize(); + } + } + + throw InvalidProtocolBufferException.truncatedMessage(); + } + + /** + * Try to get raw bytes from {@code input} with the size of {@code length} and copy to {@code + * bytes} array. If the size is bigger than the number of remaining bytes in the input, then + * throw {@code truncatedMessage} exception. + */ + private void readRawBytesTo(byte[] bytes, int offset, final int length) throws IOException { + if (length >= 0 && length <= remaining()) { + int l = length; + while (l > 0) { + if (currentRemaining() == 0) { + getNextByteBuffer(); + } + int bytesToCopy = Math.min(l, (int) currentRemaining()); + UnsafeUtil.copyMemory(currentByteBufferPos, bytes, length - l + offset, bytesToCopy); + l -= bytesToCopy; + currentByteBufferPos += bytesToCopy; + } + return; + } + + if (length <= 0) { + if (length == 0) { + return; + } else { + throw InvalidProtocolBufferException.negativeSize(); + } + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public void skipRawBytes(final int length) throws IOException { + if (length >= 0 + && length + <= (totalBufferSize + - totalBytesRead + - currentByteBufferPos + + currentByteBufferStartPos)) { + // We have all the bytes we need already. + int l = length; + while (l > 0) { + if (currentRemaining() == 0) { + getNextByteBuffer(); + } + int rl = Math.min(l, (int) currentRemaining()); + l -= rl; + currentByteBufferPos += rl; + } + return; + } + + if (length < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + // TODO: optimize to fastpath + private void skipRawVarint() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (readRawByte() >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + /** + * Try to get the number of remaining bytes in {@code input}. + * + * @return the number of remaining bytes in {@code input}. + */ + private int remaining() { + return (int) + (totalBufferSize - totalBytesRead - currentByteBufferPos + currentByteBufferStartPos); + } + + /** + * Try to get the number of remaining bytes in {@code currentByteBuffer}. + * + * @return the number of remaining bytes in {@code currentByteBuffer} + */ + private long currentRemaining() { + return (currentByteBufferLimit - currentByteBufferPos); + } + + private ByteBuffer slice(int begin, int end) throws IOException { + int prevPos = currentByteBuffer.position(); + int prevLimit = currentByteBuffer.limit(); + // View ByteBuffer as Buffer to avoid cross-Java version issues. + // See https://issues.apache.org/jira/browse/MRESOLVER-85 + Buffer asBuffer = currentByteBuffer; + try { + asBuffer.position(begin); + asBuffer.limit(end); + return currentByteBuffer.slice(); + } catch (IllegalArgumentException e) { + throw InvalidProtocolBufferException.truncatedMessage(); + } finally { + asBuffer.position(prevPos); + asBuffer.limit(prevLimit); + } + } +} \ No newline at end of file diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/LeafByteString.java b/protobuf-sdk/src/main/java/com/google/protobuf/LeafByteString.java new file mode 100644 index 0000000..2f70f6e --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/LeafByteString.java @@ -0,0 +1,34 @@ +package com.google.protobuf; + +import java.io.IOException; + +/** Base class for leaf {@link ByteString}s (i.e. non-ropes). */ +abstract class LeafByteString extends ByteString { + private static final long serialVersionUID = 1L; + + @Override + protected final int getTreeDepth() { + return 0; + } + + @Override + protected final boolean isBalanced() { + return true; + } + + @Override + void writeToReverse(ByteOutput byteOutput) throws IOException { + writeTo(byteOutput); + } + + /** + * Check equality of the substring of given length of this object starting at zero with another + * {@code ByteString} substring starting at offset. + * + * @param other what to compare a substring in + * @param offset offset into other + * @param length number of bytes to compare + * @return true for equality of substrings, else false. + */ + abstract boolean equalsRange(ByteString other, int offset, int length); +} \ No newline at end of file diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/LiteralByteString.java b/protobuf-sdk/src/main/java/com/google/protobuf/LiteralByteString.java new file mode 100644 index 0000000..5872e29 --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/LiteralByteString.java @@ -0,0 +1,240 @@ +package com.google.protobuf; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.Collections; +import java.util.List; + +/** + * This class implements a {@link com.google.protobuf.ByteString} backed by a single array of + * bytes, contiguous in memory. It supports substring by pointing to only a sub-range of the + * underlying byte array, meaning that a substring will reference the full byte-array of the + * string it's made from, exactly as with {@link String}. + * + * @author carlanton@google.com (Carl Haverl) + */ +// Keep this class private to avoid deadlocks in classloading across threads as ByteString's +// static initializer loads LiteralByteString and another thread loads LiteralByteString. +class LiteralByteString extends LeafByteString { + private static final long serialVersionUID = 1L; + + protected final byte[] bytes; + + /** + * Creates a {@code LiteralByteString} backed by the given array, without copying. + * + * @param bytes array to wrap + */ + LiteralByteString(byte[] bytes) { + if (bytes == null) { + throw new NullPointerException(); + } + this.bytes = bytes; + } + + @Override + public byte byteAt(int index) { + // Unlike most methods in this class, this one is a direct implementation + // ignoring the potential offset because we need to do range-checking in the + // substring case anyway. + return bytes[index]; + } + + @Override + byte internalByteAt(int index) { + return bytes[index]; + } + + @Override + public int size() { + return bytes.length; + } + + // ================================================================= + // ByteString -> substring + + @Override + public final ByteString substring(int beginIndex, int endIndex) { + final int length = checkRange(beginIndex, endIndex, size()); + + if (length == 0) { + return ByteString.EMPTY; + } + + return new BoundedByteString(bytes, getOffsetIntoBytes() + beginIndex, length); + } + + // ================================================================= + // ByteString -> byte[] + + @Override + protected void copyToInternal( + byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { + // Optimized form, not for subclasses, since we don't call + // getOffsetIntoBytes() or check the 'numberToCopy' parameter. + // TODO: Is not calling getOffsetIntoBytes really saving that much? + System.arraycopy(bytes, sourceOffset, target, targetOffset, numberToCopy); + } + + @Override + public final void copyTo(ByteBuffer target) { + target.put(bytes, getOffsetIntoBytes(), size()); // Copies bytes + } + + @Override + public final ByteBuffer asReadOnlyByteBuffer() { + return ByteBuffer.wrap(bytes, getOffsetIntoBytes(), size()).asReadOnlyBuffer(); + } + + @Override + public final List asReadOnlyByteBufferList() { + return Collections.singletonList(asReadOnlyByteBuffer()); + } + + @Override + public final void writeTo(OutputStream outputStream) throws IOException { + outputStream.write(toByteArray()); + } + + @Override + final void writeToInternal(OutputStream outputStream, int sourceOffset, int numberToWrite) + throws IOException { + outputStream.write(bytes, getOffsetIntoBytes() + sourceOffset, numberToWrite); + } + + @Override + final void writeTo(ByteOutput output) throws IOException { + output.writeLazy(bytes, getOffsetIntoBytes(), size()); + } + + @Override + protected final String toStringInternal(Charset charset) { + return new String(bytes, getOffsetIntoBytes(), size(), charset); + } + + // ================================================================= + // UTF-8 decoding + + @Override + public final boolean isValidUtf8() { + int offset = getOffsetIntoBytes(); + return Utf8.isValidUtf8(bytes, offset, offset + size()); + } + + @Override + protected final int partialIsValidUtf8(int state, int offset, int length) { + int index = getOffsetIntoBytes() + offset; + return Utf8.partialIsValidUtf8(state, bytes, index, index + length); + } + + // ================================================================= + // equals() and hashCode() + + @Override + public final boolean equals(Object other) { + if (other == this) { + return true; + } + if (!(other instanceof ByteString)) { + return false; + } + + if (size() != ((ByteString) other).size()) { + return false; + } + if (size() == 0) { + return true; + } + + if (other instanceof LiteralByteString) { + LiteralByteString otherAsLiteral = (LiteralByteString) other; + // If we know the hash codes and they are not equal, we know the byte + // strings are not equal. + int thisHash = peekCachedHashCode(); + int thatHash = otherAsLiteral.peekCachedHashCode(); + if (thisHash != 0 && thatHash != 0 && thisHash != thatHash) { + return false; + } + + return equalsRange((LiteralByteString) other, 0, size()); + } else { + // RopeByteString and NioByteString. + return other.equals(this); + } + } + + /** + * Check equality of the substring of given length of this object starting at zero with another + * {@code LiteralByteString} substring starting at offset. + * + * @param other what to compare a substring in + * @param offset offset into other + * @param length number of bytes to compare + * @return true for equality of substrings, else false. + */ + @Override + final boolean equalsRange(ByteString other, int offset, int length) { + if (length > other.size()) { + throw new IllegalArgumentException("Length too large: " + length + size()); + } + if (offset + length > other.size()) { + throw new IllegalArgumentException( + "Ran off end of other: " + offset + ", " + length + ", " + other.size()); + } + + if (other instanceof LiteralByteString) { + LiteralByteString lbsOther = (LiteralByteString) other; + byte[] thisBytes = bytes; + byte[] otherBytes = lbsOther.bytes; + int thisLimit = getOffsetIntoBytes() + length; + for (int thisIndex = getOffsetIntoBytes(), + otherIndex = lbsOther.getOffsetIntoBytes() + offset; + (thisIndex < thisLimit); + ++thisIndex, ++otherIndex) { + if (thisBytes[thisIndex] != otherBytes[otherIndex]) { + return false; + } + } + return true; + } + + return other.substring(offset, offset + length).equals(substring(0, length)); + } + + @Override + protected final int partialHash(int h, int offset, int length) { + return Internal.partialHash(h, bytes, getOffsetIntoBytes() + offset, length); + } + + // ================================================================= + // Input stream + + @Override + public final InputStream newInput() { + return new ByteArrayInputStream(bytes, getOffsetIntoBytes(), size()); // No copy + } + + @Override + public final CodedInputStream newCodedInput() { + // We trust CodedInputStream not to modify the bytes, or to give anyone + // else access to them. + return CodedInputStream.newInstance( + bytes, getOffsetIntoBytes(), size(), /* bufferIsImmutable= */ true); + } + + // ================================================================= + // Internal methods + + /** + * Offset into {@code bytes[]} to use, non-zero for substrings. + * + * @return always 0 for this class + */ + protected int getOffsetIntoBytes() { + return 0; + } +} diff --git a/protobuf-api/src/main/java/com/google/protobuf/NioByteString.java b/protobuf-sdk/src/main/java/com/google/protobuf/NioByteString.java similarity index 99% rename from protobuf-api/src/main/java/com/google/protobuf/NioByteString.java rename to protobuf-sdk/src/main/java/com/google/protobuf/NioByteString.java index 56c0de9..110ed5f 100644 --- a/protobuf-api/src/main/java/com/google/protobuf/NioByteString.java +++ b/protobuf-sdk/src/main/java/com/google/protobuf/NioByteString.java @@ -22,7 +22,7 @@ import java.util.List; /** A {@link ByteString} that wraps around a {@link ByteBuffer}. */ -final class NioByteString extends ByteString.LeafByteString { +final class NioByteString extends LeafByteString { private final ByteBuffer buffer; NioByteString(ByteBuffer buffer) { diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/OutputStreamEncoder.java b/protobuf-sdk/src/main/java/com/google/protobuf/OutputStreamEncoder.java new file mode 100644 index 0000000..04e6c04 --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/OutputStreamEncoder.java @@ -0,0 +1,371 @@ +package com.google.protobuf; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +import static com.google.protobuf.WireFormat.*; +import static com.google.protobuf.WireFormat.FIXED64_SIZE; + +/** + * An {@link CodedOutputStream} that decorates an {@link OutputStream}. It performs internal + * buffering to optimize writes to the {@link OutputStream}. + */ +final class OutputStreamEncoder extends AbstractBufferedEncoder { + private final OutputStream out; + + OutputStreamEncoder(OutputStream out, int bufferSize) { + super(bufferSize); + if (out == null) { + throw new NullPointerException("out"); + } + this.out = out; + } + + @Override + public void writeTag(final int fieldNumber, final int wireType) throws IOException { + writeUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); + } + + @Override + public void writeInt32(final int fieldNumber, final int value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE * 2); + bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + bufferInt32NoTag(value); + } + + @Override + public void writeUInt32(final int fieldNumber, final int value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE * 2); + bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + bufferUInt32NoTag(value); + } + + @Override + public void writeFixed32(final int fieldNumber, final int value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE + FIXED32_SIZE); + bufferTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); + bufferFixed32NoTag(value); + } + + @Override + public void writeUInt64(final int fieldNumber, final long value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE * 2); + bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + bufferUInt64NoTag(value); + } + + @Override + public void writeFixed64(final int fieldNumber, final long value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE + FIXED64_SIZE); + bufferTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); + bufferFixed64NoTag(value); + } + + @Override + public void writeBool(final int fieldNumber, final boolean value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE + 1); + bufferTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + buffer((byte) (value ? 1 : 0)); + } + + @Override + public void writeString(final int fieldNumber, final String value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeStringNoTag(value); + } + + @Override + public void writeBytes(final int fieldNumber, final ByteString value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeBytesNoTag(value); + } + + @Override + public void writeByteArray(final int fieldNumber, final byte[] value) throws IOException { + writeByteArray(fieldNumber, value, 0, value.length); + } + + @Override + public void writeByteArray( + final int fieldNumber, final byte[] value, final int offset, final int length) + throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeByteArrayNoTag(value, offset, length); + } + + @Override + public void writeByteBuffer(final int fieldNumber, final ByteBuffer value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeUInt32NoTag(value.capacity()); + writeRawBytes(value); + } + + @Override + public void writeBytesNoTag(final ByteString value) throws IOException { + writeUInt32NoTag(value.size()); + value.writeTo(this); + } + + @Override + public void writeByteArrayNoTag(final byte[] value, int offset, int length) throws IOException { + writeUInt32NoTag(length); + write(value, offset, length); + } + + @Override + public void writeRawBytes(final ByteBuffer value) throws IOException { + if (value.hasArray()) { + write(value.array(), value.arrayOffset(), value.capacity()); + } else { + ByteBuffer duplicated = value.duplicate(); + Java8Compatibility.clear(duplicated); + write(duplicated); + } + } + + @Override + public void writeMessage(final int fieldNumber, final MessageLite value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeMessageNoTag(value); + } + + @Override + void writeMessage(final int fieldNumber, final MessageLite value, Schema schema) + throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeMessageNoTag(value, schema); + } + + @Override + public void writeMessageSetExtension(final int fieldNumber, final MessageLite value) + throws IOException { + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); + writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); + writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value); + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); + } + + @Override + public void writeRawMessageSetExtension(final int fieldNumber, final ByteString value) + throws IOException { + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); + writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); + writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value); + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); + } + + @Override + public void writeMessageNoTag(final MessageLite value) throws IOException { + writeUInt32NoTag(value.getSerializedSize()); + value.writeTo(this); + } + + @Override + void writeMessageNoTag(final MessageLite value, Schema schema) throws IOException { + writeUInt32NoTag(((AbstractMessageLite) value).getSerializedSize(schema)); + schema.writeTo(value, wrapper); + } + + @Override + public void write(byte value) throws IOException { + if (position == limit) { + doFlush(); + } + + buffer(value); + } + + @Override + public void writeInt32NoTag(int value) throws IOException { + if (value >= 0) { + writeUInt32NoTag(value); + } else { + // Must sign-extend. + writeUInt64NoTag(value); + } + } + + @Override + public void writeUInt32NoTag(int value) throws IOException { + flushIfNotAvailable(MAX_VARINT32_SIZE); + bufferUInt32NoTag(value); + } + + @Override + public void writeFixed32NoTag(final int value) throws IOException { + flushIfNotAvailable(FIXED32_SIZE); + bufferFixed32NoTag(value); + } + + @Override + public void writeUInt64NoTag(long value) throws IOException { + flushIfNotAvailable(MAX_VARINT_SIZE); + bufferUInt64NoTag(value); + } + + @Override + public void writeFixed64NoTag(final long value) throws IOException { + flushIfNotAvailable(FIXED64_SIZE); + bufferFixed64NoTag(value); + } + + @Override + public void writeStringNoTag(String value) throws IOException { + try { + // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), + // and at most 3 times of it. We take advantage of this in both branches below. + final int maxLength = value.length() * Utf8.MAX_BYTES_PER_CHAR; + final int maxLengthVarIntSize = computeUInt32SizeNoTag(maxLength); + + // If we are streaming and the potential length is too big to fit in our buffer, we take the + // slower path. + if (maxLengthVarIntSize + maxLength > limit) { + // Allocate a byte[] that we know can fit the string and encode into it. String.getBytes() + // does the same internally and then does *another copy* to return a byte[] of exactly the + // right size. We can skip that copy and just writeRawBytes up to the actualLength of the + // UTF-8 encoded bytes. + final byte[] encodedBytes = new byte[maxLength]; + int actualLength = Utf8.encode(value, encodedBytes, 0, maxLength); + writeUInt32NoTag(actualLength); + writeLazy(encodedBytes, 0, actualLength); + return; + } + + // Fast path: we have enough space available in our buffer for the string... + if (maxLengthVarIntSize + maxLength > limit - position) { + // Flush to free up space. + doFlush(); + } + + // Optimize for the case where we know this length results in a constant varint length as + // this saves a pass for measuring the length of the string. + final int minLengthVarIntSize = computeUInt32SizeNoTag(value.length()); + int oldPosition = position; + final int length; + try { + if (minLengthVarIntSize == maxLengthVarIntSize) { + position = oldPosition + minLengthVarIntSize; + int newPosition = Utf8.encode(value, buffer, position, limit - position); + // Since this class is stateful and tracks the position, we rewind and store the + // state, prepend the length, then reset it back to the end of the string. + position = oldPosition; + length = newPosition - oldPosition - minLengthVarIntSize; + bufferUInt32NoTag(length); + position = newPosition; + } else { + length = Utf8.encodedLength(value); + bufferUInt32NoTag(length); + position = Utf8.encode(value, buffer, position, length); + } + totalBytesWritten += length; + } catch (Utf8.UnpairedSurrogateException e) { + // Be extra careful and restore the original position for retrying the write with the + // less efficient path. + totalBytesWritten -= position - oldPosition; + position = oldPosition; + throw e; + } catch (ArrayIndexOutOfBoundsException e) { + throw new OutOfSpaceException(e); + } + } catch (Utf8.UnpairedSurrogateException e) { + inefficientWriteStringNoTag(value, e); + } + } + + @Override + public void flush() throws IOException { + if (position > 0) { + // Flush the buffer. + doFlush(); + } + } + + @Override + public void write(byte[] value, int offset, int length) throws IOException { + if (limit - position >= length) { + // We have room in the current buffer. + System.arraycopy(value, offset, buffer, position, length); + position += length; + totalBytesWritten += length; + } else { + // Write extends past current buffer. Fill the rest of this buffer and + // flush. + final int bytesWritten = limit - position; + System.arraycopy(value, offset, buffer, position, bytesWritten); + offset += bytesWritten; + length -= bytesWritten; + position = limit; + totalBytesWritten += bytesWritten; + doFlush(); + + // Now deal with the rest. + // Since we have an output stream, this is our buffer + // and buffer offset == 0 + if (length <= limit) { + // Fits in new buffer. + System.arraycopy(value, offset, buffer, 0, length); + position = length; + } else { + // Write is very big. Let's do it all at once. + out.write(value, offset, length); + } + totalBytesWritten += length; + } + } + + @Override + public void writeLazy(byte[] value, int offset, int length) throws IOException { + write(value, offset, length); + } + + @Override + public void write(ByteBuffer value) throws IOException { + int length = value.remaining(); + if (limit - position >= length) { + // We have room in the current buffer. + value.get(buffer, position, length); + position += length; + totalBytesWritten += length; + } else { + // Write extends past current buffer. Fill the rest of this buffer and + // flush. + final int bytesWritten = limit - position; + value.get(buffer, position, bytesWritten); + length -= bytesWritten; + position = limit; + totalBytesWritten += bytesWritten; + doFlush(); + + // Now deal with the rest. + // Since we have an output stream, this is our buffer + // and buffer offset == 0 + while (length > limit) { + // Copy data into the buffer before writing it to OutputStream. + value.get(buffer, 0, limit); + out.write(buffer, 0, limit); + length -= limit; + totalBytesWritten += limit; + } + value.get(buffer, 0, length); + position = length; + totalBytesWritten += length; + } + } + + @Override + public void writeLazy(ByteBuffer value) throws IOException { + write(value); + } + + private void flushIfNotAvailable(int requiredSize) throws IOException { + if (limit - position < requiredSize) { + doFlush(); + } + } + + private void doFlush() throws IOException { + out.write(buffer, 0, position); + position = 0; + } +} \ No newline at end of file diff --git a/protobuf-api/src/main/java/com/google/protobuf/RopeByteString.java b/protobuf-sdk/src/main/java/com/google/protobuf/RopeByteString.java similarity index 100% rename from protobuf-api/src/main/java/com/google/protobuf/RopeByteString.java rename to protobuf-sdk/src/main/java/com/google/protobuf/RopeByteString.java diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/SafeDirectNioEncoder.java b/protobuf-sdk/src/main/java/com/google/protobuf/SafeDirectNioEncoder.java new file mode 100644 index 0000000..3b879d0 --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/SafeDirectNioEncoder.java @@ -0,0 +1,328 @@ +package com.google.protobuf; + +import java.io.IOException; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * A {@link CodedOutputStream} that writes directly to a direct {@link ByteBuffer}, using only + * safe operations.. + */ +final class SafeDirectNioEncoder extends CodedOutputStream { + private final ByteBuffer originalBuffer; + private final ByteBuffer buffer; + private final int initialPosition; + + SafeDirectNioEncoder(ByteBuffer buffer) { + this.originalBuffer = buffer; + this.buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + initialPosition = buffer.position(); + } + + @Override + public void writeTag(final int fieldNumber, final int wireType) throws IOException { + writeUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); + } + + @Override + public void writeInt32(final int fieldNumber, final int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeInt32NoTag(value); + } + + @Override + public void writeUInt32(final int fieldNumber, final int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeUInt32NoTag(value); + } + + @Override + public void writeFixed32(final int fieldNumber, final int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); + writeFixed32NoTag(value); + } + + @Override + public void writeUInt64(final int fieldNumber, final long value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeUInt64NoTag(value); + } + + @Override + public void writeFixed64(final int fieldNumber, final long value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); + writeFixed64NoTag(value); + } + + @Override + public void writeBool(final int fieldNumber, final boolean value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + write((byte) (value ? 1 : 0)); + } + + @Override + public void writeString(final int fieldNumber, final String value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeStringNoTag(value); + } + + @Override + public void writeBytes(final int fieldNumber, final ByteString value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeBytesNoTag(value); + } + + @Override + public void writeByteArray(final int fieldNumber, final byte[] value) throws IOException { + writeByteArray(fieldNumber, value, 0, value.length); + } + + @Override + public void writeByteArray( + final int fieldNumber, final byte[] value, final int offset, final int length) + throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeByteArrayNoTag(value, offset, length); + } + + @Override + public void writeByteBuffer(final int fieldNumber, final ByteBuffer value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeUInt32NoTag(value.capacity()); + writeRawBytes(value); + } + + @Override + public void writeMessage(final int fieldNumber, final MessageLite value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeMessageNoTag(value); + } + + @Override + void writeMessage(final int fieldNumber, final MessageLite value, Schema schema) + throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeMessageNoTag(value, schema); + } + + @Override + public void writeMessageSetExtension(final int fieldNumber, final MessageLite value) + throws IOException { + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); + writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); + writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value); + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); + } + + @Override + public void writeRawMessageSetExtension(final int fieldNumber, final ByteString value) + throws IOException { + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); + writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); + writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value); + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); + } + + @Override + public void writeMessageNoTag(final MessageLite value) throws IOException { + writeUInt32NoTag(value.getSerializedSize()); + value.writeTo(this); + } + + @Override + void writeMessageNoTag(final MessageLite value, Schema schema) throws IOException { + writeUInt32NoTag(((AbstractMessageLite) value).getSerializedSize(schema)); + schema.writeTo(value, wrapper); + } + + @Override + public void write(byte value) throws IOException { + try { + buffer.put(value); + } catch (BufferOverflowException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void writeBytesNoTag(final ByteString value) throws IOException { + writeUInt32NoTag(value.size()); + value.writeTo(this); + } + + @Override + public void writeByteArrayNoTag(final byte[] value, int offset, int length) throws IOException { + writeUInt32NoTag(length); + write(value, offset, length); + } + + @Override + public void writeRawBytes(final ByteBuffer value) throws IOException { + if (value.hasArray()) { + write(value.array(), value.arrayOffset(), value.capacity()); + } else { + ByteBuffer duplicated = value.duplicate(); + Java8Compatibility.clear(duplicated); + write(duplicated); + } + } + + @Override + public void writeInt32NoTag(int value) throws IOException { + if (value >= 0) { + writeUInt32NoTag(value); + } else { + // Must sign-extend. + writeUInt64NoTag(value); + } + } + + @Override + public void writeUInt32NoTag(int value) throws IOException { + try { + while (true) { + if ((value & ~0x7F) == 0) { + buffer.put((byte) value); + return; + } else { + buffer.put((byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + } + } catch (BufferOverflowException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void writeFixed32NoTag(int value) throws IOException { + try { + buffer.putInt(value); + } catch (BufferOverflowException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void writeUInt64NoTag(long value) throws IOException { + try { + while (true) { + if ((value & ~0x7FL) == 0) { + buffer.put((byte) value); + return; + } else { + buffer.put((byte) (((int) value & 0x7F) | 0x80)); + value >>>= 7; + } + } + } catch (BufferOverflowException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void writeFixed64NoTag(long value) throws IOException { + try { + buffer.putLong(value); + } catch (BufferOverflowException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void write(byte[] value, int offset, int length) throws IOException { + try { + buffer.put(value, offset, length); + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException(e); + } catch (BufferOverflowException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void writeLazy(byte[] value, int offset, int length) throws IOException { + write(value, offset, length); + } + + @Override + public void write(ByteBuffer value) throws IOException { + try { + buffer.put(value); + } catch (BufferOverflowException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void writeLazy(ByteBuffer value) throws IOException { + write(value); + } + + @Override + public void writeStringNoTag(String value) throws IOException { + final int startPos = buffer.position(); + try { + // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), + // and at most 3 times of it. We take advantage of this in both branches below. + final int maxEncodedSize = value.length() * Utf8.MAX_BYTES_PER_CHAR; + final int maxLengthVarIntSize = computeUInt32SizeNoTag(maxEncodedSize); + final int minLengthVarIntSize = computeUInt32SizeNoTag(value.length()); + if (minLengthVarIntSize == maxLengthVarIntSize) { + // Save the current position and increment past the length field. We'll come back + // and write the length field after the encoding is complete. + final int startOfBytes = buffer.position() + minLengthVarIntSize; + Java8Compatibility.position(buffer, startOfBytes); + + // Encode the string. + encode(value); + + // Now go back to the beginning and write the length. + int endOfBytes = buffer.position(); + Java8Compatibility.position(buffer, startPos); + writeUInt32NoTag(endOfBytes - startOfBytes); + + // Reposition the buffer past the written data. + Java8Compatibility.position(buffer, endOfBytes); + } else { + final int length = Utf8.encodedLength(value); + writeUInt32NoTag(length); + encode(value); + } + } catch (Utf8.UnpairedSurrogateException e) { + // Roll back the change and convert to an IOException. + Java8Compatibility.position(buffer, startPos); + + // TODO: We should throw an IOException here instead. + inefficientWriteStringNoTag(value, e); + } catch (IllegalArgumentException e) { + // Thrown by buffer.position() if out of range. + throw new OutOfSpaceException(e); + } + } + + @Override + public void flush() { + // Update the position of the original buffer. + Java8Compatibility.position(originalBuffer, buffer.position()); + } + + @Override + public int spaceLeft() { + return buffer.remaining(); + } + + @Override + public int getTotalBytesWritten() { + return buffer.position() - initialPosition; + } + + private void encode(String value) throws IOException { + try { + Utf8.encodeUtf8(value, buffer); + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException(e); + } + } +} \ No newline at end of file diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/StreamDecoder.java b/protobuf-sdk/src/main/java/com/google/protobuf/StreamDecoder.java new file mode 100644 index 0000000..9ce5786 --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/StreamDecoder.java @@ -0,0 +1,1106 @@ +package com.google.protobuf; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static com.google.protobuf.Internal.UTF_8; +import static com.google.protobuf.Internal.checkNotNull; +import static com.google.protobuf.WireFormat.*; +import static com.google.protobuf.WireFormat.FIXED64_SIZE; + +/** + * Implementation of {@link CodedInputStream} that uses an {@link InputStream} as the data source. + */ +final class StreamDecoder extends CodedInputStream { + private final InputStream input; + private final byte[] buffer; + /** bufferSize represents how many bytes are currently filled in the buffer */ + private int bufferSize; + + private int bufferSizeAfterLimit; + private int pos; + private int lastTag; + + /** + * The total number of bytes read before the current buffer. The total bytes read up to the + * current position can be computed as {@code totalBytesRetired + pos}. This value may be + * negative if reading started in the middle of the current buffer (e.g. if the constructor that + * takes a byte array and an offset was used). + */ + private int totalBytesRetired; + + /** The absolute position of the end of the current message. */ + private int currentLimit = Integer.MAX_VALUE; + + StreamDecoder(final InputStream input, int bufferSize) { + checkNotNull(input, "input"); + this.input = input; + this.buffer = new byte[bufferSize]; + this.bufferSize = 0; + pos = 0; + totalBytesRetired = 0; + } + + /* + * The following wrapper methods exist so that InvalidProtocolBufferExceptions thrown by the + * InputStream can be differentiated from ones thrown by CodedInputStream itself. Each call to + * an InputStream method that can throw IOException must be wrapped like this. We do this + * because we sometimes need to modify IPBE instances after they are thrown far away from where + * they are thrown (ex. to add unfinished messages) and we use this signal elsewhere in the + * exception catch chain to know when to perform these operations directly or to wrap the + * exception in their own IPBE so the extra information can be communicated without trampling + * downstream information. + */ + private static int read(InputStream input, byte[] data, int offset, int length) + throws IOException { + try { + return input.read(data, offset, length); + } catch (InvalidProtocolBufferException e) { + e.setThrownFromInputStream(); + throw e; + } + } + + private static long skip(InputStream input, long length) throws IOException { + try { + return input.skip(length); + } catch (InvalidProtocolBufferException e) { + e.setThrownFromInputStream(); + throw e; + } + } + + private static int available(InputStream input) throws IOException { + try { + return input.available(); + } catch (InvalidProtocolBufferException e) { + e.setThrownFromInputStream(); + throw e; + } + } + + @Override + public int readTag() throws IOException { + if (isAtEnd()) { + lastTag = 0; + return 0; + } + + lastTag = readRawVarint32(); + if (WireFormat.getTagFieldNumber(lastTag) == 0) { + // If we actually read zero (or any tag number corresponding to field + // number zero), that's not a valid tag. + throw InvalidProtocolBufferException.invalidTag(); + } + return lastTag; + } + + @Override + public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { + if (lastTag != value) { + throw InvalidProtocolBufferException.invalidEndTag(); + } + } + + @Override + public int getLastTag() { + return lastTag; + } + + @Override + public boolean skipField(final int tag) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + skipRawVarint(); + return true; + case WireFormat.WIRETYPE_FIXED64: + skipRawBytes(FIXED64_SIZE); + return true; + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + skipRawBytes(readRawVarint32()); + return true; + case WireFormat.WIRETYPE_START_GROUP: + skipMessage(); + checkLastTagWas( + WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); + return true; + case WireFormat.WIRETYPE_END_GROUP: + return false; + case WireFormat.WIRETYPE_FIXED32: + skipRawBytes(FIXED32_SIZE); + return true; + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + @Override + public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + { + long value = readInt64(); + output.writeUInt32NoTag(tag); + output.writeUInt64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_FIXED64: + { + long value = readRawLittleEndian64(); + output.writeUInt32NoTag(tag); + output.writeFixed64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + { + ByteString value = readBytes(); + output.writeUInt32NoTag(tag); + output.writeBytesNoTag(value); + return true; + } + case WireFormat.WIRETYPE_START_GROUP: + { + output.writeUInt32NoTag(tag); + skipMessage(output); + int endtag = + WireFormat.makeTag( + WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); + checkLastTagWas(endtag); + output.writeUInt32NoTag(endtag); + return true; + } + case WireFormat.WIRETYPE_END_GROUP: + { + return false; + } + case WireFormat.WIRETYPE_FIXED32: + { + int value = readRawLittleEndian32(); + output.writeUInt32NoTag(tag); + output.writeFixed32NoTag(value); + return true; + } + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + /** Collects the bytes skipped and returns the data in a ByteBuffer. */ + private class SkippedDataSink implements RefillCallback { + private int lastPos = pos; + private ByteArrayOutputStream byteArrayStream; + + @Override + public void onRefill() { + if (byteArrayStream == null) { + byteArrayStream = new ByteArrayOutputStream(); + } + byteArrayStream.write(buffer, lastPos, pos - lastPos); + lastPos = 0; + } + + /** Gets skipped data in a ByteBuffer. This method should only be called once. */ + ByteBuffer getSkippedData() { + if (byteArrayStream == null) { + return ByteBuffer.wrap(buffer, lastPos, pos - lastPos); + } else { + byteArrayStream.write(buffer, lastPos, pos); + return ByteBuffer.wrap(byteArrayStream.toByteArray()); + } + } + } + + // ----------------------------------------------------------------- + + @Override + public double readDouble() throws IOException { + return Double.longBitsToDouble(readRawLittleEndian64()); + } + + @Override + public float readFloat() throws IOException { + return Float.intBitsToFloat(readRawLittleEndian32()); + } + + @Override + public long readUInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public long readInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public int readInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public long readFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readFixed32() throws IOException { + return readRawLittleEndian32(); + } + + @Override + public boolean readBool() throws IOException { + return readRawVarint64() != 0; + } + + @Override + public String readString() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= (bufferSize - pos)) { + // Fast path: We already have the bytes in a contiguous buffer, so + // just copy directly from it. + final String result = new String(buffer, pos, size, UTF_8); + pos += size; + return result; + } + if (size == 0) { + return ""; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + if (size <= bufferSize) { + refillBuffer(size); + String result = new String(buffer, pos, size, UTF_8); + pos += size; + return result; + } + // Slow path: Build a byte array first then copy it. + return new String(readRawBytesSlowPath(size, /* ensureNoLeakedReferences= */ false), UTF_8); + } + + @Override + public String readStringRequireUtf8() throws IOException { + final int size = readRawVarint32(); + final byte[] bytes; + final int oldPos = pos; + final int tempPos; + if (size <= (bufferSize - oldPos) && size > 0) { + // Fast path: We already have the bytes in a contiguous buffer, so + // just copy directly from it. + bytes = buffer; + pos = oldPos + size; + tempPos = oldPos; + } else if (size == 0) { + return ""; + } else if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } else if (size <= bufferSize) { + refillBuffer(size); + bytes = buffer; + tempPos = 0; + pos = tempPos + size; + } else { + // Slow path: Build a byte array first then copy it. + bytes = readRawBytesSlowPath(size, /* ensureNoLeakedReferences= */ false); + tempPos = 0; + } + return Utf8.decodeUtf8(bytes, tempPos, size); + } + + @Override + public void readGroup( + final int fieldNumber, + final MessageLite.Builder builder, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + checkRecursionLimit(); + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + } + + @Override + public T readGroup( + final int fieldNumber, + final Parser parser, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + checkRecursionLimit(); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + return result; + } + + @Deprecated + @Override + public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) + throws IOException { + readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry()); + } + + @Override + public void readMessage( + final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) + throws IOException { + final int length = readRawVarint32(); + checkRecursionLimit(); + final int oldLimit = pushLimit(length); + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + if (getBytesUntilLimit() != 0) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + popLimit(oldLimit); + } + + @Override + public T readMessage( + final Parser parser, final ExtensionRegistryLite extensionRegistry) throws IOException { + int length = readRawVarint32(); + checkRecursionLimit(); + final int oldLimit = pushLimit(length); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + if (getBytesUntilLimit() != 0) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + popLimit(oldLimit); + return result; + } + + @Override + public ByteString readBytes() throws IOException { + final int size = readRawVarint32(); + if (size <= (bufferSize - pos) && size > 0) { + // Fast path: We already have the bytes in a contiguous buffer, so + // just copy directly from it. + final ByteString result = ByteString.copyFrom(buffer, pos, size); + pos += size; + return result; + } + if (size == 0) { + return ByteString.EMPTY; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + return readBytesSlowPath(size); + } + + @Override + public byte[] readByteArray() throws IOException { + final int size = readRawVarint32(); + if (size <= (bufferSize - pos) && size > 0) { + // Fast path: We already have the bytes in a contiguous buffer, so + // just copy directly from it. + final byte[] result = Arrays.copyOfRange(buffer, pos, pos + size); + pos += size; + return result; + } else if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } else { + // Slow path: Build a byte array first then copy it. + // TODO: Do we want to protect from malicious input streams here? + return readRawBytesSlowPath(size, /* ensureNoLeakedReferences= */ false); + } + } + + @Override + public ByteBuffer readByteBuffer() throws IOException { + final int size = readRawVarint32(); + if (size <= (bufferSize - pos) && size > 0) { + // Fast path: We already have the bytes in a contiguous buffer. + ByteBuffer result = ByteBuffer.wrap(Arrays.copyOfRange(buffer, pos, pos + size)); + pos += size; + return result; + } + if (size == 0) { + return Internal.EMPTY_BYTE_BUFFER; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + // Slow path: Build a byte array first then copy it. + + // We must copy as the byte array was handed off to the InputStream and a malicious + // implementation could retain a reference. + return ByteBuffer.wrap(readRawBytesSlowPath(size, /* ensureNoLeakedReferences= */ true)); + } + + @Override + public int readUInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public int readEnum() throws IOException { + return readRawVarint32(); + } + + @Override + public int readSFixed32() throws IOException { + return readRawLittleEndian32(); + } + + @Override + public long readSFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readSInt32() throws IOException { + return decodeZigZag32(readRawVarint32()); + } + + @Override + public long readSInt64() throws IOException { + return decodeZigZag64(readRawVarint64()); + } + + // ================================================================= + + @Override + public int readRawVarint32() throws IOException { + // See implementation notes for readRawVarint64 + fastpath: + { + int tempPos = pos; + + if (bufferSize == tempPos) { + break fastpath; + } + + final byte[] buffer = this.buffer; + int x; + if ((x = buffer[tempPos++]) >= 0) { + pos = tempPos; + return x; + } else if (bufferSize - tempPos < 9) { + break fastpath; + } else if ((x ^= (buffer[tempPos++] << 7)) < 0) { + x ^= (~0 << 7); + } else if ((x ^= (buffer[tempPos++] << 14)) >= 0) { + x ^= (~0 << 7) ^ (~0 << 14); + } else if ((x ^= (buffer[tempPos++] << 21)) < 0) { + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); + } else { + int y = buffer[tempPos++]; + x ^= y << 28; + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); + if (y < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0) { + break fastpath; // Will throw malformedVarint() + } + } + pos = tempPos; + return x; + } + return (int) readRawVarint64SlowPath(); + } + + private void skipRawVarint() throws IOException { + if (bufferSize - pos >= MAX_VARINT_SIZE) { + skipRawVarintFastPath(); + } else { + skipRawVarintSlowPath(); + } + } + + private void skipRawVarintFastPath() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (buffer[pos++] >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + private void skipRawVarintSlowPath() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (readRawByte() >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + @Override + public long readRawVarint64() throws IOException { + // Implementation notes: + // + // Optimized for one-byte values, expected to be common. + // The particular code below was selected from various candidates + // empirically, by winning VarintBenchmark. + // + // Sign extension of (signed) Java bytes is usually a nuisance, but + // we exploit it here to more easily obtain the sign of bytes read. + // Instead of cleaning up the sign extension bits by masking eagerly, + // we delay until we find the final (positive) byte, when we clear all + // accumulated bits with one xor. We depend on javac to constant fold. + fastpath: + { + int tempPos = pos; + + if (bufferSize == tempPos) { + break fastpath; + } + + final byte[] buffer = this.buffer; + long x; + int y; + if ((y = buffer[tempPos++]) >= 0) { + pos = tempPos; + return y; + } else if (bufferSize - tempPos < 9) { + break fastpath; + } else if ((y ^= (buffer[tempPos++] << 7)) < 0) { + x = y ^ (~0 << 7); + } else if ((y ^= (buffer[tempPos++] << 14)) >= 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14)); + } else if ((y ^= (buffer[tempPos++] << 21)) < 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); + } else if ((x = y ^ ((long) buffer[tempPos++] << 28)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); + } else if ((x ^= ((long) buffer[tempPos++] << 35)) < 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); + } else if ((x ^= ((long) buffer[tempPos++] << 42)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); + } else if ((x ^= ((long) buffer[tempPos++] << 49)) < 0L) { + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49); + } else { + x ^= ((long) buffer[tempPos++] << 56); + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49) + ^ (~0L << 56); + if (x < 0L) { + if (buffer[tempPos++] < 0L) { + break fastpath; // Will throw malformedVarint() + } + } + } + pos = tempPos; + return x; + } + return readRawVarint64SlowPath(); + } + + @Override + long readRawVarint64SlowPath() throws IOException { + long result = 0; + for (int shift = 0; shift < 64; shift += 7) { + final byte b = readRawByte(); + result |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + @Override + public int readRawLittleEndian32() throws IOException { + int tempPos = pos; + + if (bufferSize - tempPos < FIXED32_SIZE) { + refillBuffer(FIXED32_SIZE); + tempPos = pos; + } + + final byte[] buffer = this.buffer; + pos = tempPos + FIXED32_SIZE; + return ((buffer[tempPos] & 0xff) + | ((buffer[tempPos + 1] & 0xff) << 8) + | ((buffer[tempPos + 2] & 0xff) << 16) + | ((buffer[tempPos + 3] & 0xff) << 24)); + } + + @Override + public long readRawLittleEndian64() throws IOException { + int tempPos = pos; + + if (bufferSize - tempPos < FIXED64_SIZE) { + refillBuffer(FIXED64_SIZE); + tempPos = pos; + } + + final byte[] buffer = this.buffer; + pos = tempPos + FIXED64_SIZE; + return (((buffer[tempPos] & 0xffL)) + | ((buffer[tempPos + 1] & 0xffL) << 8) + | ((buffer[tempPos + 2] & 0xffL) << 16) + | ((buffer[tempPos + 3] & 0xffL) << 24) + | ((buffer[tempPos + 4] & 0xffL) << 32) + | ((buffer[tempPos + 5] & 0xffL) << 40) + | ((buffer[tempPos + 6] & 0xffL) << 48) + | ((buffer[tempPos + 7] & 0xffL) << 56)); + } + + // ----------------------------------------------------------------- + + @Override + public void enableAliasing(boolean enabled) { + // TODO: Ideally we should throw here. Do nothing for backward compatibility. + } + + @Override + public void resetSizeCounter() { + totalBytesRetired = -pos; + } + + @Override + public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { + if (byteLimit < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + byteLimit += totalBytesRetired + pos; + final int oldLimit = currentLimit; + if (byteLimit > oldLimit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + currentLimit = byteLimit; + + recomputeBufferSizeAfterLimit(); + + return oldLimit; + } + + private void recomputeBufferSizeAfterLimit() { + bufferSize += bufferSizeAfterLimit; + final int bufferEnd = totalBytesRetired + bufferSize; + if (bufferEnd > currentLimit) { + // Limit is in current buffer. + bufferSizeAfterLimit = bufferEnd - currentLimit; + bufferSize -= bufferSizeAfterLimit; + } else { + bufferSizeAfterLimit = 0; + } + } + + @Override + public void popLimit(final int oldLimit) { + currentLimit = oldLimit; + recomputeBufferSizeAfterLimit(); + } + + @Override + public int getBytesUntilLimit() { + if (currentLimit == Integer.MAX_VALUE) { + return -1; + } + + final int currentAbsolutePosition = totalBytesRetired + pos; + return currentLimit - currentAbsolutePosition; + } + + @Override + public boolean isAtEnd() throws IOException { + return pos == bufferSize && !tryRefillBuffer(1); + } + + @Override + public int getTotalBytesRead() { + return totalBytesRetired + pos; + } + + private interface RefillCallback { + void onRefill(); + } + + private RefillCallback refillCallback = null; + + /** + * Reads more bytes from the input, making at least {@code n} bytes available in the buffer. + * Caller must ensure that the requested space is not yet available, and that the requested + * space is less than BUFFER_SIZE. + * + * @throws InvalidProtocolBufferException The end of the stream or the current limit was + * reached. + */ + private void refillBuffer(int n) throws IOException { + if (!tryRefillBuffer(n)) { + // We have to distinguish the exception between sizeLimitExceeded and truncatedMessage. So + // we just throw an sizeLimitExceeded exception here if it exceeds the sizeLimit + if (n > sizeLimit - totalBytesRetired - pos) { + throw InvalidProtocolBufferException.sizeLimitExceeded(); + } else { + throw InvalidProtocolBufferException.truncatedMessage(); + } + } + } + + /** + * Tries to read more bytes from the input, making at least {@code n} bytes available in the + * buffer. Caller must ensure that the requested space is not yet available, and that the + * requested space is less than BUFFER_SIZE. + * + * @return {@code true} If the bytes could be made available; {@code false} 1. Current at the + * end of the stream 2. The current limit was reached 3. The total size limit was reached + */ + private boolean tryRefillBuffer(int n) throws IOException { + if (pos + n <= bufferSize) { + throw new IllegalStateException( + "refillBuffer() called when " + n + " bytes were already available in buffer"); + } + + // Check whether the size of total message needs to read is bigger than the size limit. + // We shouldn't throw an exception here as isAtEnd() function needs to get this function's + // return as the result. + if (n > sizeLimit - totalBytesRetired - pos) { + return false; + } + + // Shouldn't throw the exception here either. + if (totalBytesRetired + pos + n > currentLimit) { + // Oops, we hit a limit. + return false; + } + + if (refillCallback != null) { + refillCallback.onRefill(); + } + + int tempPos = pos; + if (tempPos > 0) { + if (bufferSize > tempPos) { + System.arraycopy(buffer, tempPos, buffer, 0, bufferSize - tempPos); + } + totalBytesRetired += tempPos; + bufferSize -= tempPos; + pos = 0; + } + + // Here we should refill the buffer as many bytes as possible. + int bytesRead = + read( + input, + buffer, + bufferSize, + Math.min( + // the size of allocated but unused bytes in the buffer + buffer.length - bufferSize, + // do not exceed the total bytes limit + sizeLimit - totalBytesRetired - bufferSize)); + if (bytesRead == 0 || bytesRead < -1 || bytesRead > buffer.length) { + throw new IllegalStateException( + input.getClass() + + "#read(byte[]) returned invalid result: " + + bytesRead + + "\nThe InputStream implementation is buggy."); + } + if (bytesRead > 0) { + bufferSize += bytesRead; + recomputeBufferSizeAfterLimit(); + return (bufferSize >= n) ? true : tryRefillBuffer(n); + } + + return false; + } + + @Override + public byte readRawByte() throws IOException { + if (pos == bufferSize) { + refillBuffer(1); + } + return buffer[pos++]; + } + + @Override + public byte[] readRawBytes(final int size) throws IOException { + final int tempPos = pos; + if (size <= (bufferSize - tempPos) && size > 0) { + pos = tempPos + size; + return Arrays.copyOfRange(buffer, tempPos, tempPos + size); + } else { + // TODO: Do we want to protect from malicious input streams here? + return readRawBytesSlowPath(size, /* ensureNoLeakedReferences= */ false); + } + } + + /** + * Exactly like readRawBytes, but caller must have already checked the fast path: (size <= + * (bufferSize - pos) && size > 0) + * + * If ensureNoLeakedReferences is true, the value is guaranteed to have not escaped to + * untrusted code. + */ + private byte[] readRawBytesSlowPath( + final int size, boolean ensureNoLeakedReferences) throws IOException { + // Attempt to read the data in one byte array when it's safe to do. + byte[] result = readRawBytesSlowPathOneChunk(size); + if (result != null) { + return ensureNoLeakedReferences ? result.clone() : result; + } + + final int originalBufferPos = pos; + final int bufferedBytes = bufferSize - pos; + + // Mark the current buffer consumed. + totalBytesRetired += bufferSize; + pos = 0; + bufferSize = 0; + + // Determine the number of bytes we need to read from the input stream. + int sizeLeft = size - bufferedBytes; + + // The size is very large. For security reasons we read them in small + // chunks. + List chunks = readRawBytesSlowPathRemainingChunks(sizeLeft); + + // OK, got everything. Now concatenate it all into one buffer. + final byte[] bytes = new byte[size]; + + // Start by copying the leftover bytes from this.buffer. + System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); + + // And now all the chunks. + int tempPos = bufferedBytes; + for (final byte[] chunk : chunks) { + System.arraycopy(chunk, 0, bytes, tempPos, chunk.length); + tempPos += chunk.length; + } + + // Done. + return bytes; + } + + /** + * Attempts to read the data in one byte array when it's safe to do. Returns null if the size to + * read is too large and needs to be allocated in smaller chunks for security reasons. + * + *

Returns a byte[] that may have escaped to user code via InputStream APIs. + */ + private byte[] readRawBytesSlowPathOneChunk(final int size) throws IOException { + if (size == 0) { + return Internal.EMPTY_BYTE_ARRAY; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + + // Integer-overflow-conscious check that the message size so far has not exceeded sizeLimit. + int currentMessageSize = totalBytesRetired + pos + size; + if (currentMessageSize - sizeLimit > 0) { + throw InvalidProtocolBufferException.sizeLimitExceeded(); + } + + // Verify that the message size so far has not exceeded currentLimit. + if (currentMessageSize > currentLimit) { + // Read to the end of the stream anyway. + skipRawBytes(currentLimit - totalBytesRetired - pos); + throw InvalidProtocolBufferException.truncatedMessage(); + } + + final int bufferedBytes = bufferSize - pos; + // Determine the number of bytes we need to read from the input stream. + int sizeLeft = size - bufferedBytes; + // TODO: Consider using a value larger than DEFAULT_BUFFER_SIZE. + if (sizeLeft < DEFAULT_BUFFER_SIZE || sizeLeft <= available(input)) { + // Either the bytes we need are known to be available, or the required buffer is + // within an allowed threshold - go ahead and allocate the buffer now. + final byte[] bytes = new byte[size]; + + // Copy all of the buffered bytes to the result buffer. + System.arraycopy(buffer, pos, bytes, 0, bufferedBytes); + totalBytesRetired += bufferSize; + pos = 0; + bufferSize = 0; + + // Fill the remaining bytes from the input stream. + int tempPos = bufferedBytes; + while (tempPos < bytes.length) { + int n = read(input, bytes, tempPos, size - tempPos); + if (n == -1) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + totalBytesRetired += n; + tempPos += n; + } + + return bytes; + } + + return null; + } + + /** + * Reads the remaining data in small chunks from the input stream. + * + * Returns a byte[] that may have escaped to user code via InputStream APIs. + */ + private List readRawBytesSlowPathRemainingChunks(int sizeLeft) throws IOException { + // The size is very large. For security reasons, we can't allocate the + // entire byte array yet. The size comes directly from the input, so a + // maliciously-crafted message could provide a bogus very large size in + // order to trick the app into allocating a lot of memory. We avoid this + // by allocating and reading only a small chunk at a time, so that the + // malicious message must actually *be* extremely large to cause + // problems. Meanwhile, we limit the allowed size of a message elsewhere. + final List chunks = new ArrayList<>(); + + while (sizeLeft > 0) { + // TODO: Consider using a value larger than DEFAULT_BUFFER_SIZE. + final byte[] chunk = new byte[Math.min(sizeLeft, DEFAULT_BUFFER_SIZE)]; + int tempPos = 0; + while (tempPos < chunk.length) { + final int n = input.read(chunk, tempPos, chunk.length - tempPos); + if (n == -1) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + totalBytesRetired += n; + tempPos += n; + } + sizeLeft -= chunk.length; + chunks.add(chunk); + } + + return chunks; + } + + /** + * Like readBytes, but caller must have already checked the fast path: (size <= (bufferSize - + * pos) && size > 0 || size == 0) + */ + private ByteString readBytesSlowPath(final int size) throws IOException { + final byte[] result = readRawBytesSlowPathOneChunk(size); + if (result != null) { + // We must copy as the byte array was handed off to the InputStream and a malicious + // implementation could retain a reference. + return ByteString.copyFrom(result); + } + + final int originalBufferPos = pos; + final int bufferedBytes = bufferSize - pos; + + // Mark the current buffer consumed. + totalBytesRetired += bufferSize; + pos = 0; + bufferSize = 0; + + // Determine the number of bytes we need to read from the input stream. + int sizeLeft = size - bufferedBytes; + + // The size is very large. For security reasons we read them in small + // chunks. + List chunks = readRawBytesSlowPathRemainingChunks(sizeLeft); + + // OK, got everything. Now concatenate it all into one buffer. + final byte[] bytes = new byte[size]; + + // Start by copying the leftover bytes from this.buffer. + System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); + + // And now all the chunks. + int tempPos = bufferedBytes; + for (final byte[] chunk : chunks) { + System.arraycopy(chunk, 0, bytes, tempPos, chunk.length); + tempPos += chunk.length; + } + + return ByteString.wrap(bytes); + } + + @Override + public void skipRawBytes(final int size) throws IOException { + if (size <= (bufferSize - pos) && size >= 0) { + // We have all the bytes we need already. + pos += size; + } else { + skipRawBytesSlowPath(size); + } + } + + /** + * Exactly like skipRawBytes, but caller must have already checked the fast path: (size <= + * (bufferSize - pos) && size >= 0) + */ + private void skipRawBytesSlowPath(final int size) throws IOException { + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + + if (totalBytesRetired + pos + size > currentLimit) { + // Read to the end of the stream anyway. + skipRawBytes(currentLimit - totalBytesRetired - pos); + // Then fail. + throw InvalidProtocolBufferException.truncatedMessage(); + } + + int totalSkipped = 0; + if (refillCallback == null) { + // Skipping more bytes than are in the buffer. First skip what we have. + totalBytesRetired += pos; + totalSkipped = bufferSize - pos; + bufferSize = 0; + pos = 0; + + try { + while (totalSkipped < size) { + int toSkip = size - totalSkipped; + long skipped = skip(input, toSkip); + if (skipped < 0 || skipped > toSkip) { + throw new IllegalStateException( + input.getClass() + + "#skip returned invalid result: " + + skipped + + "\nThe InputStream implementation is buggy."); + } else if (skipped == 0) { + // The API contract of skip() permits an inputstream to skip zero bytes for any reason + // it wants. In particular, ByteArrayInputStream will just return zero over and over + // when it's at the end of its input. In order to actually confirm that we've hit the + // end of input, we need to issue a read call via the other path. + break; + } + totalSkipped += (int) skipped; + } + } finally { + totalBytesRetired += totalSkipped; + recomputeBufferSizeAfterLimit(); + } + } + if (totalSkipped < size) { + // Skipping more bytes than are in the buffer. First skip what we have. + int tempPos = bufferSize - pos; + pos = bufferSize; + + // Keep refilling the buffer until we get to the point we wanted to skip to. + // This has the side effect of ensuring the limits are updated correctly. + refillBuffer(1); + while (size - tempPos > bufferSize) { + tempPos += bufferSize; + pos = bufferSize; + refillBuffer(1); + } + + pos = size - tempPos; + } + } +} \ No newline at end of file diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/UnsafeDirectNioDecoder.java b/protobuf-sdk/src/main/java/com/google/protobuf/UnsafeDirectNioDecoder.java new file mode 100644 index 0000000..5f14d17 --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/UnsafeDirectNioDecoder.java @@ -0,0 +1,737 @@ +package com.google.protobuf; + +import java.io.IOException; +import java.nio.Buffer; +import java.nio.ByteBuffer; + +import static com.google.protobuf.Internal.*; +import static com.google.protobuf.WireFormat.*; +import static com.google.protobuf.WireFormat.FIXED64_SIZE; + +/** + * A {@link CodedInputStream} implementation that uses a backing direct ByteBuffer as the input. + * Requires the use of {@code sun.misc.Unsafe} to perform fast reads on the buffer. + */ +final class UnsafeDirectNioDecoder extends CodedInputStream { + /** The direct buffer that is backing this stream. */ + private final ByteBuffer buffer; + + /** + * If {@code true}, indicates that the buffer is backing a {@link ByteString} and is therefore + * considered to be an immutable input source. + */ + private final boolean immutable; + + /** The unsafe address of the content of {@link #buffer}. */ + private final long address; + + /** The unsafe address of the current read limit of the buffer. */ + private long limit; + + /** The unsafe address of the current read position of the buffer. */ + private long pos; + + /** The unsafe address of the starting read position. */ + private long startPos; + + /** The amount of available data in the buffer beyond {@link #limit}. */ + private int bufferSizeAfterLimit; + + /** The last tag that was read from this stream. */ + private int lastTag; + + /** + * If {@code true}, indicates that calls to read {@link ByteString} or {@code byte[]} + * may return slices of the underlying buffer, rather than copies. + */ + private boolean enableAliasing; + + /** The absolute position of the end of the current message. */ + private int currentLimit = Integer.MAX_VALUE; + + static boolean isSupported() { + return UnsafeUtil.hasUnsafeByteBufferOperations(); + } + + UnsafeDirectNioDecoder(ByteBuffer buffer, boolean immutable) { + this.buffer = buffer; + address = UnsafeUtil.addressOffset(buffer); + limit = address + buffer.limit(); + pos = address + buffer.position(); + startPos = pos; + this.immutable = immutable; + } + + @Override + public int readTag() throws IOException { + if (isAtEnd()) { + lastTag = 0; + return 0; + } + + lastTag = readRawVarint32(); + if (WireFormat.getTagFieldNumber(lastTag) == 0) { + // If we actually read zero (or any tag number corresponding to field + // number zero), that's not a valid tag. + throw InvalidProtocolBufferException.invalidTag(); + } + return lastTag; + } + + @Override + public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { + if (lastTag != value) { + throw InvalidProtocolBufferException.invalidEndTag(); + } + } + + @Override + public int getLastTag() { + return lastTag; + } + + @Override + public boolean skipField(final int tag) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + skipRawVarint(); + return true; + case WireFormat.WIRETYPE_FIXED64: + skipRawBytes(FIXED64_SIZE); + return true; + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + skipRawBytes(readRawVarint32()); + return true; + case WireFormat.WIRETYPE_START_GROUP: + skipMessage(); + checkLastTagWas( + WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); + return true; + case WireFormat.WIRETYPE_END_GROUP: + return false; + case WireFormat.WIRETYPE_FIXED32: + skipRawBytes(FIXED32_SIZE); + return true; + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + @Override + public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + { + long value = readInt64(); + output.writeUInt32NoTag(tag); + output.writeUInt64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_FIXED64: + { + long value = readRawLittleEndian64(); + output.writeUInt32NoTag(tag); + output.writeFixed64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + { + ByteString value = readBytes(); + output.writeUInt32NoTag(tag); + output.writeBytesNoTag(value); + return true; + } + case WireFormat.WIRETYPE_START_GROUP: + { + output.writeUInt32NoTag(tag); + skipMessage(output); + int endtag = + WireFormat.makeTag( + WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); + checkLastTagWas(endtag); + output.writeUInt32NoTag(endtag); + return true; + } + case WireFormat.WIRETYPE_END_GROUP: + { + return false; + } + case WireFormat.WIRETYPE_FIXED32: + { + int value = readRawLittleEndian32(); + output.writeUInt32NoTag(tag); + output.writeFixed32NoTag(value); + return true; + } + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + // ----------------------------------------------------------------- + + @Override + public double readDouble() throws IOException { + return Double.longBitsToDouble(readRawLittleEndian64()); + } + + @Override + public float readFloat() throws IOException { + return Float.intBitsToFloat(readRawLittleEndian32()); + } + + @Override + public long readUInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public long readInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public int readInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public long readFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readFixed32() throws IOException { + return readRawLittleEndian32(); + } + + @Override + public boolean readBool() throws IOException { + return readRawVarint64() != 0; + } + + @Override + public String readString() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= remaining()) { + // TODO: Is there a way to avoid this copy? + // TODO: It might be possible to share the optimized loop with + // readStringRequireUtf8 by implementing Java replacement logic there. + // The same as readBytes' logic + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(pos, bytes, 0, size); + String result = new String(bytes, UTF_8); + pos += size; + return result; + } + + if (size == 0) { + return ""; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public String readStringRequireUtf8() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= remaining()) { + final int bufferPos = bufferPos(pos); + String result = Utf8.decodeUtf8(buffer, bufferPos, size); + pos += size; + return result; + } + + if (size == 0) { + return ""; + } + if (size <= 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public void readGroup( + final int fieldNumber, + final MessageLite.Builder builder, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + checkRecursionLimit(); + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + } + + @Override + public T readGroup( + final int fieldNumber, + final Parser parser, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + checkRecursionLimit(); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + return result; + } + + @Deprecated + @Override + public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) + throws IOException { + readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry()); + } + + @Override + public void readMessage( + final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) + throws IOException { + final int length = readRawVarint32(); + checkRecursionLimit(); + final int oldLimit = pushLimit(length); + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + if (getBytesUntilLimit() != 0) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + popLimit(oldLimit); + } + + @Override + public T readMessage( + final Parser parser, final ExtensionRegistryLite extensionRegistry) throws IOException { + int length = readRawVarint32(); + checkRecursionLimit(); + final int oldLimit = pushLimit(length); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + if (getBytesUntilLimit() != 0) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + popLimit(oldLimit); + return result; + } + + @Override + public ByteString readBytes() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= remaining()) { + if (immutable && enableAliasing) { + final ByteBuffer result = slice(pos, pos + size); + pos += size; + return ByteString.wrap(result); + } else { + // Use UnsafeUtil to copy the memory to bytes instead of using ByteBuffer ways. + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(pos, bytes, 0, size); + pos += size; + return ByteString.wrap(bytes); + } + } + + if (size == 0) { + return ByteString.EMPTY; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public byte[] readByteArray() throws IOException { + return readRawBytes(readRawVarint32()); + } + + @Override + public ByteBuffer readByteBuffer() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= remaining()) { + // "Immutable" implies that buffer is backing a ByteString. + // Disallow slicing in this case to prevent the caller from modifying the contents + // of the ByteString. + if (!immutable && enableAliasing) { + final ByteBuffer result = slice(pos, pos + size); + pos += size; + return result; + } else { + // The same as readBytes' logic + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(pos, bytes, 0, size); + pos += size; + return ByteBuffer.wrap(bytes); + } + // TODO: Investigate making the ByteBuffer be made read-only + } + + if (size == 0) { + return EMPTY_BYTE_BUFFER; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public int readUInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public int readEnum() throws IOException { + return readRawVarint32(); + } + + @Override + public int readSFixed32() throws IOException { + return readRawLittleEndian32(); + } + + @Override + public long readSFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readSInt32() throws IOException { + return decodeZigZag32(readRawVarint32()); + } + + @Override + public long readSInt64() throws IOException { + return decodeZigZag64(readRawVarint64()); + } + + // ================================================================= + + @Override + public int readRawVarint32() throws IOException { + // See implementation notes for readRawVarint64 + fastpath: + { + long tempPos = pos; + + if (limit == tempPos) { + break fastpath; + } + + int x; + if ((x = UnsafeUtil.getByte(tempPos++)) >= 0) { + pos = tempPos; + return x; + } else if (limit - tempPos < 9) { + break fastpath; + } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) { + x ^= (~0 << 7); + } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) { + x ^= (~0 << 7) ^ (~0 << 14); + } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) { + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); + } else { + int y = UnsafeUtil.getByte(tempPos++); + x ^= y << 28; + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); + if (y < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0) { + break fastpath; // Will throw malformedVarint() + } + } + pos = tempPos; + return x; + } + return (int) readRawVarint64SlowPath(); + } + + private void skipRawVarint() throws IOException { + if (remaining() >= MAX_VARINT_SIZE) { + skipRawVarintFastPath(); + } else { + skipRawVarintSlowPath(); + } + } + + private void skipRawVarintFastPath() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (UnsafeUtil.getByte(pos++) >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + private void skipRawVarintSlowPath() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (readRawByte() >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + @Override + public long readRawVarint64() throws IOException { + // Implementation notes: + // + // Optimized for one-byte values, expected to be common. + // The particular code below was selected from various candidates + // empirically, by winning VarintBenchmark. + // + // Sign extension of (signed) Java bytes is usually a nuisance, but + // we exploit it here to more easily obtain the sign of bytes read. + // Instead of cleaning up the sign extension bits by masking eagerly, + // we delay until we find the final (positive) byte, when we clear all + // accumulated bits with one xor. We depend on javac to constant fold. + fastpath: + { + long tempPos = pos; + + if (limit == tempPos) { + break fastpath; + } + + long x; + int y; + if ((y = UnsafeUtil.getByte(tempPos++)) >= 0) { + pos = tempPos; + return y; + } else if (limit - tempPos < 9) { + break fastpath; + } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) { + x = y ^ (~0 << 7); + } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14)); + } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); + } else if ((x = y ^ ((long) UnsafeUtil.getByte(tempPos++) << 28)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); + } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 35)) < 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); + } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 42)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); + } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 49)) < 0L) { + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49); + } else { + x ^= ((long) UnsafeUtil.getByte(tempPos++) << 56); + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49) + ^ (~0L << 56); + if (x < 0L) { + if (UnsafeUtil.getByte(tempPos++) < 0L) { + break fastpath; // Will throw malformedVarint() + } + } + } + pos = tempPos; + return x; + } + return readRawVarint64SlowPath(); + } + + @Override + long readRawVarint64SlowPath() throws IOException { + long result = 0; + for (int shift = 0; shift < 64; shift += 7) { + final byte b = readRawByte(); + result |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + @Override + public int readRawLittleEndian32() throws IOException { + long tempPos = pos; + + if (limit - tempPos < FIXED32_SIZE) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + + pos = tempPos + FIXED32_SIZE; + return ((UnsafeUtil.getByte(tempPos) & 0xff) + | ((UnsafeUtil.getByte(tempPos + 1) & 0xff) << 8) + | ((UnsafeUtil.getByte(tempPos + 2) & 0xff) << 16) + | ((UnsafeUtil.getByte(tempPos + 3) & 0xff) << 24)); + } + + @Override + public long readRawLittleEndian64() throws IOException { + long tempPos = pos; + + if (limit - tempPos < FIXED64_SIZE) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + + pos = tempPos + FIXED64_SIZE; + return ((UnsafeUtil.getByte(tempPos) & 0xffL) + | ((UnsafeUtil.getByte(tempPos + 1) & 0xffL) << 8) + | ((UnsafeUtil.getByte(tempPos + 2) & 0xffL) << 16) + | ((UnsafeUtil.getByte(tempPos + 3) & 0xffL) << 24) + | ((UnsafeUtil.getByte(tempPos + 4) & 0xffL) << 32) + | ((UnsafeUtil.getByte(tempPos + 5) & 0xffL) << 40) + | ((UnsafeUtil.getByte(tempPos + 6) & 0xffL) << 48) + | ((UnsafeUtil.getByte(tempPos + 7) & 0xffL) << 56)); + } + + @Override + public void enableAliasing(boolean enabled) { + this.enableAliasing = enabled; + } + + @Override + public void resetSizeCounter() { + startPos = pos; + } + + @Override + public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { + if (byteLimit < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + byteLimit += getTotalBytesRead(); + final int oldLimit = currentLimit; + if (byteLimit > oldLimit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + currentLimit = byteLimit; + + recomputeBufferSizeAfterLimit(); + + return oldLimit; + } + + @Override + public void popLimit(final int oldLimit) { + currentLimit = oldLimit; + recomputeBufferSizeAfterLimit(); + } + + @Override + public int getBytesUntilLimit() { + if (currentLimit == Integer.MAX_VALUE) { + return -1; + } + + return currentLimit - getTotalBytesRead(); + } + + @Override + public boolean isAtEnd() throws IOException { + return pos == limit; + } + + @Override + public int getTotalBytesRead() { + return (int) (pos - startPos); + } + + @Override + public byte readRawByte() throws IOException { + if (pos == limit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + return UnsafeUtil.getByte(pos++); + } + + @Override + public byte[] readRawBytes(final int length) throws IOException { + if (length >= 0 && length <= remaining()) { + byte[] bytes = new byte[length]; + slice(pos, pos + length).get(bytes); + pos += length; + return bytes; + } + + if (length <= 0) { + if (length == 0) { + return EMPTY_BYTE_ARRAY; + } else { + throw InvalidProtocolBufferException.negativeSize(); + } + } + + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public void skipRawBytes(final int length) throws IOException { + if (length >= 0 && length <= remaining()) { + // We have all the bytes we need already. + pos += length; + return; + } + + if (length < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + private void recomputeBufferSizeAfterLimit() { + limit += bufferSizeAfterLimit; + final int bufferEnd = (int) (limit - startPos); + if (bufferEnd > currentLimit) { + // Limit is in current buffer. + bufferSizeAfterLimit = bufferEnd - currentLimit; + limit -= bufferSizeAfterLimit; + } else { + bufferSizeAfterLimit = 0; + } + } + + private int remaining() { + return (int) (limit - pos); + } + + private int bufferPos(long pos) { + return (int) (pos - address); + } + + private ByteBuffer slice(long begin, long end) throws IOException { + int prevPos = buffer.position(); + int prevLimit = buffer.limit(); + // View ByteBuffer as Buffer to avoid cross-Java version issues. + // See https://issues.apache.org/jira/browse/MRESOLVER-85 + Buffer asBuffer = buffer; + try { + asBuffer.position(bufferPos(begin)); + asBuffer.limit(bufferPos(end)); + return buffer.slice(); + } catch (IllegalArgumentException e) { + InvalidProtocolBufferException ex = InvalidProtocolBufferException.truncatedMessage(); + ex.initCause(e); + throw ex; + } finally { + asBuffer.position(prevPos); + asBuffer.limit(prevLimit); + } + } +} \ No newline at end of file diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/UnsafeDirectNioEncoder.java b/protobuf-sdk/src/main/java/com/google/protobuf/UnsafeDirectNioEncoder.java new file mode 100644 index 0000000..07ef277 --- /dev/null +++ b/protobuf-sdk/src/main/java/com/google/protobuf/UnsafeDirectNioEncoder.java @@ -0,0 +1,369 @@ +package com.google.protobuf; + +import java.io.IOException; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import static com.google.protobuf.WireFormat.*; + +/** + * A {@link CodedOutputStream} that writes directly to a direct {@link ByteBuffer} using {@code + * sun.misc.Unsafe}. + */ +final class UnsafeDirectNioEncoder extends CodedOutputStream { + private final ByteBuffer originalBuffer; + private final ByteBuffer buffer; + private final long address; + private final long initialPosition; + private final long limit; + private final long oneVarintLimit; + private long position; + + UnsafeDirectNioEncoder(ByteBuffer buffer) { + this.originalBuffer = buffer; + this.buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + address = UnsafeUtil.addressOffset(buffer); + initialPosition = address + buffer.position(); + limit = address + buffer.limit(); + oneVarintLimit = limit - MAX_VARINT_SIZE; + position = initialPosition; + } + + static boolean isSupported() { + return UnsafeUtil.hasUnsafeByteBufferOperations(); + } + + @Override + public void writeTag(int fieldNumber, int wireType) throws IOException { + writeUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); + } + + @Override + public void writeInt32(int fieldNumber, int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeInt32NoTag(value); + } + + @Override + public void writeUInt32(int fieldNumber, int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeUInt32NoTag(value); + } + + @Override + public void writeFixed32(int fieldNumber, int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); + writeFixed32NoTag(value); + } + + @Override + public void writeUInt64(int fieldNumber, long value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeUInt64NoTag(value); + } + + @Override + public void writeFixed64(int fieldNumber, long value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); + writeFixed64NoTag(value); + } + + @Override + public void writeBool(int fieldNumber, boolean value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + write((byte) (value ? 1 : 0)); + } + + @Override + public void writeString(int fieldNumber, String value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeStringNoTag(value); + } + + @Override + public void writeBytes(int fieldNumber, ByteString value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeBytesNoTag(value); + } + + @Override + public void writeByteArray(int fieldNumber, byte[] value) throws IOException { + writeByteArray(fieldNumber, value, 0, value.length); + } + + @Override + public void writeByteArray(int fieldNumber, byte[] value, int offset, int length) + throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeByteArrayNoTag(value, offset, length); + } + + @Override + public void writeByteBuffer(int fieldNumber, ByteBuffer value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeUInt32NoTag(value.capacity()); + writeRawBytes(value); + } + + @Override + public void writeMessage(int fieldNumber, MessageLite value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeMessageNoTag(value); + } + + @Override + void writeMessage(int fieldNumber, MessageLite value, Schema schema) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeMessageNoTag(value, schema); + } + + @Override + public void writeMessageSetExtension(int fieldNumber, MessageLite value) throws IOException { + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); + writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); + writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value); + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); + } + + @Override + public void writeRawMessageSetExtension(int fieldNumber, ByteString value) throws IOException { + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); + writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); + writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value); + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); + } + + @Override + public void writeMessageNoTag(MessageLite value) throws IOException { + writeUInt32NoTag(value.getSerializedSize()); + value.writeTo(this); + } + + @Override + void writeMessageNoTag(MessageLite value, Schema schema) throws IOException { + writeUInt32NoTag(((AbstractMessageLite) value).getSerializedSize(schema)); + schema.writeTo(value, wrapper); + } + + @Override + public void write(byte value) throws IOException { + if (position >= limit) { + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, 1)); + } + UnsafeUtil.putByte(position++, value); + } + + @Override + public void writeBytesNoTag(ByteString value) throws IOException { + writeUInt32NoTag(value.size()); + value.writeTo(this); + } + + @Override + public void writeByteArrayNoTag(byte[] value, int offset, int length) throws IOException { + writeUInt32NoTag(length); + write(value, offset, length); + } + + @Override + public void writeRawBytes(ByteBuffer value) throws IOException { + if (value.hasArray()) { + write(value.array(), value.arrayOffset(), value.capacity()); + } else { + ByteBuffer duplicated = value.duplicate(); + Java8Compatibility.clear(duplicated); + write(duplicated); + } + } + + @Override + public void writeInt32NoTag(int value) throws IOException { + if (value >= 0) { + writeUInt32NoTag(value); + } else { + // Must sign-extend. + writeUInt64NoTag(value); + } + } + + @Override + public void writeUInt32NoTag(int value) throws IOException { + if (position <= oneVarintLimit) { + // Optimization to avoid bounds checks on each iteration. + while (true) { + if ((value & ~0x7F) == 0) { + UnsafeUtil.putByte(position++, (byte) value); + return; + } else { + UnsafeUtil.putByte(position++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + } + } else { + while (position < limit) { + if ((value & ~0x7F) == 0) { + UnsafeUtil.putByte(position++, (byte) value); + return; + } else { + UnsafeUtil.putByte(position++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + } + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, 1)); + } + } + + @Override + public void writeFixed32NoTag(int value) throws IOException { + buffer.putInt(bufferPos(position), value); + position += FIXED32_SIZE; + } + + @Override + public void writeUInt64NoTag(long value) throws IOException { + if (position <= oneVarintLimit) { + // Optimization to avoid bounds checks on each iteration. + while (true) { + if ((value & ~0x7FL) == 0) { + UnsafeUtil.putByte(position++, (byte) value); + return; + } else { + UnsafeUtil.putByte(position++, (byte) (((int) value & 0x7F) | 0x80)); + value >>>= 7; + } + } + } else { + while (position < limit) { + if ((value & ~0x7FL) == 0) { + UnsafeUtil.putByte(position++, (byte) value); + return; + } else { + UnsafeUtil.putByte(position++, (byte) (((int) value & 0x7F) | 0x80)); + value >>>= 7; + } + } + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, 1)); + } + } + + @Override + public void writeFixed64NoTag(long value) throws IOException { + buffer.putLong(bufferPos(position), value); + position += FIXED64_SIZE; + } + + @Override + public void write(byte[] value, int offset, int length) throws IOException { + if (value == null + || offset < 0 + || length < 0 + || (value.length - length) < offset + || (limit - length) < position) { + if (value == null) { + throw new NullPointerException("value"); + } + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, length)); + } + + UnsafeUtil.copyMemory(value, offset, position, length); + position += length; + } + + @Override + public void writeLazy(byte[] value, int offset, int length) throws IOException { + write(value, offset, length); + } + + @Override + public void write(ByteBuffer value) throws IOException { + try { + int length = value.remaining(); + repositionBuffer(position); + buffer.put(value); + position += length; + } catch (BufferOverflowException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void writeLazy(ByteBuffer value) throws IOException { + write(value); + } + + @Override + public void writeStringNoTag(String value) throws IOException { + long prevPos = position; + try { + // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), + // and at most 3 times of it. We take advantage of this in both branches below. + int maxEncodedSize = value.length() * Utf8.MAX_BYTES_PER_CHAR; + int maxLengthVarIntSize = computeUInt32SizeNoTag(maxEncodedSize); + int minLengthVarIntSize = computeUInt32SizeNoTag(value.length()); + if (minLengthVarIntSize == maxLengthVarIntSize) { + // Save the current position and increment past the length field. We'll come back + // and write the length field after the encoding is complete. + int stringStart = bufferPos(position) + minLengthVarIntSize; + Java8Compatibility.position(buffer, stringStart); + + // Encode the string. + Utf8.encodeUtf8(value, buffer); + + // Write the length and advance the position. + int length = buffer.position() - stringStart; + writeUInt32NoTag(length); + position += length; + } else { + // Calculate and write the encoded length. + int length = Utf8.encodedLength(value); + writeUInt32NoTag(length); + + // Write the string and advance the position. + repositionBuffer(position); + Utf8.encodeUtf8(value, buffer); + position += length; + } + } catch (Utf8.UnpairedSurrogateException e) { + // Roll back the change and convert to an IOException. + position = prevPos; + repositionBuffer(position); + + // TODO: We should throw an IOException here instead. + inefficientWriteStringNoTag(value, e); + } catch (IllegalArgumentException e) { + // Thrown by buffer.position() if out of range. + throw new OutOfSpaceException(e); + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void flush() { + // Update the position of the original buffer. + Java8Compatibility.position(originalBuffer, bufferPos(position)); + } + + @Override + public int spaceLeft() { + return (int) (limit - position); + } + + @Override + public int getTotalBytesWritten() { + return (int) (position - initialPosition); + } + + private void repositionBuffer(long pos) { + Java8Compatibility.position(buffer, bufferPos(pos)); + } + + private int bufferPos(long pos) { + return (int) (pos - address); + } +} \ No newline at end of file diff --git a/protobuf-sdk/src/main/java/com/google/protobuf/java_features.proto b/protobuf-sdk/src/main/java/com/google/protobuf/java_features.proto deleted file mode 100644 index d02d067..0000000 --- a/protobuf-sdk/src/main/java/com/google/protobuf/java_features.proto +++ /dev/null @@ -1,52 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google Inc. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -syntax = "proto2"; - -package pb; - -import "google/protobuf/descriptor.proto"; - -option java_package = "com.google.protobuf"; -option java_outer_classname = "JavaFeaturesProto"; - -extend google.protobuf.FeatureSet { - optional JavaFeatures java = 1001; -} - -message JavaFeatures { - // Whether or not to treat an enum field as closed. This option is only - // applicable to enum fields, and will be removed in the future. It is - // consistent with the legacy behavior of using proto3 enum types for proto2 - // fields. - optional bool legacy_closed_enum = 1 [ - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE, - edition_defaults = { edition: EDITION_PROTO2, value: "true" }, - edition_defaults = { edition: EDITION_PROTO3, value: "false" } - ]; - - // The UTF8 validation strategy to use. See go/editions-utf8-validation for - // more information on this feature. - enum Utf8Validation { - // Invalid default, which should never be used. - UTF8_VALIDATION_UNKNOWN = 0; - // Respect the UTF8 validation behavior specified by the global - // utf8_validation feature. - DEFAULT = 1; - // Verifies UTF8 validity overriding the global utf8_validation - // feature. This represents the legacy java_string_check_utf8 option. - VERIFY = 2; - } - optional Utf8Validation utf8_validation = 2 [ - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE, - edition_defaults = { edition: EDITION_PROTO2, value: "DEFAULT" } - ]; -}