From 4f87f0d25b75e045f55cc09c8a9085c1cd7cb238 Mon Sep 17 00:00:00 2001 From: Jeff Davidson Date: Fri, 8 May 2015 14:10:11 -0700 Subject: Add a flag to use offset/length with byte arrays. This is an advanced option that most users won't want/need. However, it can greatly optimize flows where we want to reuse byte[] buffers from other locations without having to first copy the contents into a new array of the exact correct size. Bug: 20636336 Change-Id: Ia8d0af82e952858f9571f84110da621da776619c --- java/pom.xml | 6 ++ .../protobuf/nano/CodedInputByteBufferNano.java | 14 ++++ .../protobuf/nano/CodedOutputByteBufferNano.java | 32 ++++++++ .../test/java/com/google/protobuf/NanoTest.java | 23 ++++++ .../compiler/javanano/javanano_generator.cc | 10 +++ .../protobuf/compiler/javanano/javanano_params.h | 22 +++++- .../compiler/javanano/javanano_primitive_field.cc | 86 ++++++++++++++++++---- .../compiler/javanano/javanano_primitive_field.h | 2 + .../unittest_bytes_offset_length_nano.proto | 39 ++++++++++ 9 files changed, 217 insertions(+), 17 deletions(-) create mode 100644 src/google/protobuf/unittest_bytes_offset_length_nano.proto diff --git a/java/pom.xml b/java/pom.xml index f73e874..ec890c3 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -231,6 +231,12 @@ + + + + + + target/generated-test-sources diff --git a/java/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java b/java/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java index df7fee0..7cfbf8a 100644 --- a/java/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java +++ b/java/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java @@ -543,6 +543,20 @@ public final class CodedInputByteBufferNano { } /** + * Get current (absolute) position in buffer. + */ + public int getAbsolutePosition() { + return bufferPos; + } + + /** + * Return the raw underlying data in the buffer, directly. + */ + public byte[] getBuffer() { + return buffer; + } + + /** * Retrieves a subset of data in the buffer. The returned array is not backed by the original * buffer array. * diff --git a/java/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java b/java/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java index 4d0b48e..304c042 100644 --- a/java/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java +++ b/java/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java @@ -174,6 +174,14 @@ public final class CodedOutputByteBufferNano { writeBytesNoTag(value); } + /** Write a {@code bytes} field, including tag, to the stream. */ + public void writeBytes(final int fieldNumber, final byte[] value, + final int offset, final int length) + throws IOException { + writeTag(fieldNumber, WireFormatNano.WIRETYPE_LENGTH_DELIMITED); + writeBytesNoTag(value, offset, length); + } + /** Write a {@code uint32} field, including tag, to the stream. */ public void writeUInt32(final int fieldNumber, final int value) throws IOException { @@ -517,6 +525,13 @@ public final class CodedOutputByteBufferNano { writeRawBytes(value); } + /** Write a {@code bytes} field to the stream. */ + public void writeBytesNoTag(final byte[] value, final int offset, final int length) + throws IOException { + writeRawVarint32(length); + writeRawBytes(value, offset, length); + } + /** Write a {@code uint32} field to the stream. */ public void writeUInt32NoTag(final int value) throws IOException { writeRawVarint32(value); @@ -658,6 +673,15 @@ public final class CodedOutputByteBufferNano { /** * Compute the number of bytes that would be needed to encode a + * {@code bytes} field of the given length, including tag. + */ + public static int computeBytesSize(final int fieldNumber, + final int length) { + return computeTagSize(fieldNumber) + computeBytesSizeNoTag(length); + } + + /** + * Compute the number of bytes that would be needed to encode a * {@code uint32} field, including tag. */ public static int computeUInt32Size(final int fieldNumber, final int value) { @@ -838,6 +862,14 @@ public final class CodedOutputByteBufferNano { /** * Compute the number of bytes that would be needed to encode a + * {@code bytes} field of the given length. + */ + public static int computeBytesSizeNoTag(final int length) { + return computeRawVarint32Size(length) + length; + } + + /** + * Compute the number of bytes that would be needed to encode a * {@code uint32} field. */ public static int computeUInt32SizeNoTag(final int value) { diff --git a/java/src/test/java/com/google/protobuf/NanoTest.java b/java/src/test/java/com/google/protobuf/NanoTest.java index ef1572a..9f0b3c1 100644 --- a/java/src/test/java/com/google/protobuf/NanoTest.java +++ b/java/src/test/java/com/google/protobuf/NanoTest.java @@ -30,6 +30,7 @@ package com.google.protobuf; +import com.google.protobuf.nano.BytesOffsetLengthTestNanoOuterClass.BytesOffsetLengthTestNano; import com.google.protobuf.nano.CodedInputByteBufferNano; import com.google.protobuf.nano.CodedOutputByteBufferNano; import com.google.protobuf.nano.EnumClassNanoMultiple; @@ -3852,6 +3853,28 @@ public class NanoTest extends TestCase { assertFalse(clone.equals(anotherMessage)); } + public void testBytesOffsetLength() throws Exception { + BytesOffsetLengthTestNano msg = new BytesOffsetLengthTestNano(); + msg.fooBuffer = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + msg.fooOffset = 2; + msg.fooLength = 3; + msg.barBuffer = msg.fooBuffer; + msg.barOffset = 7; + msg.barLength = 1; + + byte[] bytes = MessageNano.toByteArray(msg); + // Two tags + two lengths + the arrays + assertEquals("Unexpected size of encoded proto", 8, bytes.length); + + msg = BytesOffsetLengthTestNano.parseFrom(bytes); + byte[] foo = new byte[msg.fooLength]; + System.arraycopy(msg.fooBuffer, msg.fooOffset, foo, 0, msg.fooLength); + assertTrue("foo was not deserialized correctly", Arrays.equals(new byte[] { 2, 3, 4 }, foo)); + byte[] bar = new byte[msg.barLength]; + System.arraycopy(msg.barBuffer, msg.barOffset, bar, 0, msg.barLength); + assertTrue("bar was not deserialized correctly", Arrays.equals(new byte[] { 7 }, bar)); + } + private void assertHasWireData(MessageNano message, boolean expected) { byte[] bytes = MessageNano.toByteArray(message); int wireLength = bytes.length; diff --git a/src/google/protobuf/compiler/javanano/javanano_generator.cc b/src/google/protobuf/compiler/javanano/javanano_generator.cc index 99ebe12..e842791 100644 --- a/src/google/protobuf/compiler/javanano/javanano_generator.cc +++ b/src/google/protobuf/compiler/javanano/javanano_generator.cc @@ -158,6 +158,8 @@ bool JavaNanoGenerator::Generate(const FileDescriptor* file, params.set_generate_intdefs(option_value == "true"); } else if (option_name == "generate_clear") { params.set_generate_clear(option_value == "true"); + } else if (option_name == "bytes_offset_length") { + params.set_bytes_offset_length(option_value == "true"); } else { *error = "Ignore unknown javanano generator option: " + option_name; } @@ -175,6 +177,14 @@ bool JavaNanoGenerator::Generate(const FileDescriptor* file, return false; } + // Theoretically possible, but not implemented. + if (params.bytes_offset_length() + && (params.optional_field_accessors() || params.generate_equals())) { + error->assign("bytes_offset_length=true cannot be used in conjunction" + " with optional_field_style=accessors or generate_equals=true"); + return false; + } + // ----------------------------------------------------------------- FileGenerator file_generator(file, params); diff --git a/src/google/protobuf/compiler/javanano/javanano_params.h b/src/google/protobuf/compiler/javanano/javanano_params.h index e3b4bb9..482c6c2 100644 --- a/src/google/protobuf/compiler/javanano/javanano_params.h +++ b/src/google/protobuf/compiler/javanano/javanano_params.h @@ -68,6 +68,7 @@ class Params { bool generate_clear_; bool generate_clone_; bool generate_intdefs_; + bool bytes_offset_length_; public: Params(const string & base_name) : @@ -85,7 +86,8 @@ class Params { reftypes_primitive_enums_(false), generate_clear_(true), generate_clone_(false), - generate_intdefs_(false) { + generate_intdefs_(false), + bytes_offset_length_(false) { } const string& base_name() const { @@ -249,6 +251,24 @@ class Params { bool generate_intdefs() const { return generate_intdefs_; } + + // An advanced setting which uses buffer/offset/length tuples for each + // non-repeated bytes field, instead of a byte array which is serialized + // directly. + // The field is considered present iff the offset is not equal to the default + // value of -1; the value of the buffer has no relevance otherwise. + // In serialization, the [fieldName]Buffer array will be serialized starting + // at [fieldName]Offset and with length [fieldName]Length. + // In deserialization, the underlying byte array will be the same instance + // backing the underlying CodedInputByteBufferNano for all bytes fields, with + // appropriate offsets and lengths. + // Use with caution! This feature comes with no SLA. + void set_bytes_offset_length(bool value) { + bytes_offset_length_ = value; + } + bool bytes_offset_length() const { + return bytes_offset_length_; + } }; } // namespace javanano diff --git a/src/google/protobuf/compiler/javanano/javanano_primitive_field.cc b/src/google/protobuf/compiler/javanano/javanano_primitive_field.cc index bda52c6..7a1655e 100644 --- a/src/google/protobuf/compiler/javanano/javanano_primitive_field.cc +++ b/src/google/protobuf/compiler/javanano/javanano_primitive_field.cc @@ -288,8 +288,16 @@ GenerateMembers(io::Printer* printer, bool lazy_init) const { } } - printer->Print(variables_, - "public $type$ $name$;\n"); + JavaType java_type = GetJavaType(descriptor_); + if (java_type == JAVATYPE_BYTES && params_.bytes_offset_length()) { + printer->Print(variables_, + "public $type$ $name$Buffer;\n" + "public int $name$Offset;\n" + "public int $name$Length;\n"); + } else { + printer->Print(variables_, + "public $type$ $name$;\n"); + } if (params_.generate_has()) { printer->Print(variables_, @@ -299,8 +307,16 @@ GenerateMembers(io::Printer* printer, bool lazy_init) const { void PrimitiveFieldGenerator:: GenerateClearCode(io::Printer* printer) const { - printer->Print(variables_, - "$name$ = $default_copy_if_needed$;\n"); + JavaType java_type = GetJavaType(descriptor_); + if (java_type == JAVATYPE_BYTES && params_.bytes_offset_length()) { + printer->Print(variables_, + "$name$Buffer = $default_copy_if_needed$;\n" + "$name$Offset = -1;\n" + "$name$Length = 0;\n"); + } else { + printer->Print(variables_, + "$name$ = $default_copy_if_needed$;\n"); + } if (params_.generate_has()) { printer->Print(variables_, @@ -310,8 +326,17 @@ GenerateClearCode(io::Printer* printer) const { void PrimitiveFieldGenerator:: GenerateMergingCode(io::Printer* printer) const { - printer->Print(variables_, - "this.$name$ = input.read$capitalized_type$();\n"); + JavaType java_type = GetJavaType(descriptor_); + if (java_type == JAVATYPE_BYTES && params_.bytes_offset_length()) { + printer->Print(variables_, + "this.$name$Buffer = input.getBuffer();\n" + "this.$name$Length = input.readRawVarint32();\n" + "this.$name$Offset = input.getAbsolutePosition();\n" + "input.skipRawBytes(this.$name$Length);\n"); + } else { + printer->Print(variables_, + "this.$name$ = input.read$capitalized_type$();\n"); + } if (params_.generate_has()) { printer->Print(variables_, @@ -336,7 +361,10 @@ GenerateSerializationConditional(io::Printer* printer) const { "if ("); } JavaType java_type = GetJavaType(descriptor_); - if (IsArrayType(java_type)) { + if (java_type == JAVATYPE_BYTES && params_.bytes_offset_length()) { + printer->Print(variables_, + "this.$name$Offset != -1) {\n"); + } else if (IsArrayType(java_type)) { printer->Print(variables_, "!java.util.Arrays.equals(this.$name$, $default$)) {\n"); } else if (IsReferenceType(java_type)) { @@ -360,28 +388,53 @@ void PrimitiveFieldGenerator:: GenerateSerializationCode(io::Printer* printer) const { if (descriptor_->is_required() && !params_.generate_has()) { // Always serialize a required field if we don't have the 'has' signal. - printer->Print(variables_, - "output.write$capitalized_type$($number$, this.$name$);\n"); + GenerateWriteCode(printer); } else { GenerateSerializationConditional(printer); + printer->Indent(); + GenerateWriteCode(printer); + printer->Outdent(); + printer->Print("}\n"); + } +} + +void PrimitiveFieldGenerator:: +GenerateWriteCode(io::Printer* printer) const { + JavaType java_type = GetJavaType(descriptor_); + if (java_type == JAVATYPE_BYTES && params_.bytes_offset_length()) { printer->Print(variables_, - " output.write$capitalized_type$($number$, this.$name$);\n" - "}\n"); + "output.write$capitalized_type$($number$, this.$name$Buffer,\n" + " this.$name$Offset, this.$name$Length);\n"); + } else { + printer->Print(variables_, + "output.write$capitalized_type$($number$, this.$name$);\n"); } } void PrimitiveFieldGenerator:: GenerateSerializedSizeCode(io::Printer* printer) const { if (descriptor_->is_required() && !params_.generate_has()) { + GenerateComputeSizeCode(printer); + } else { + GenerateSerializationConditional(printer); + printer->Indent(); + GenerateComputeSizeCode(printer); + printer->Outdent(); + printer->Print("}\n"); + } +} + +void PrimitiveFieldGenerator:: +GenerateComputeSizeCode(io::Printer* printer) const { + JavaType java_type = GetJavaType(descriptor_); + if (java_type == JAVATYPE_BYTES && params_.bytes_offset_length()) { printer->Print(variables_, "size += com.google.protobuf.nano.CodedOutputByteBufferNano\n" - " .compute$capitalized_type$Size($number$, this.$name$);\n"); + " .compute$capitalized_type$Size($number$, this.$name$Length);\n"); } else { - GenerateSerializationConditional(printer); printer->Print(variables_, - " size += com.google.protobuf.nano.CodedOutputByteBufferNano\n" - " .compute$capitalized_type$Size($number$, this.$name$);\n" - "}\n"); + "size += com.google.protobuf.nano.CodedOutputByteBufferNano\n" + " .compute$capitalized_type$Size($number$, this.$name$);\n"); } } @@ -587,6 +640,7 @@ GenerateMembers(io::Printer* printer, bool lazy_init) const { void AccessorPrimitiveFieldGenerator:: GenerateClearCode(io::Printer* printer) const { + printer->Print(variables_, "$name$_ = $default_copy_if_needed$;\n"); } diff --git a/src/google/protobuf/compiler/javanano/javanano_primitive_field.h b/src/google/protobuf/compiler/javanano/javanano_primitive_field.h index 5ace0de..daa712e 100644 --- a/src/google/protobuf/compiler/javanano/javanano_primitive_field.h +++ b/src/google/protobuf/compiler/javanano/javanano_primitive_field.h @@ -63,6 +63,8 @@ class PrimitiveFieldGenerator : public FieldGenerator { private: void GenerateSerializationConditional(io::Printer* printer) const; + void GenerateWriteCode(io::Printer* printer) const; + void GenerateComputeSizeCode(io::Printer* printer) const; const FieldDescriptor* descriptor_; map variables_; diff --git a/src/google/protobuf/unittest_bytes_offset_length_nano.proto b/src/google/protobuf/unittest_bytes_offset_length_nano.proto new file mode 100644 index 0000000..629ead1 --- /dev/null +++ b/src/google/protobuf/unittest_bytes_offset_length_nano.proto @@ -0,0 +1,39 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2015 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package protobuf_unittest; + +option java_package = "com.google.protobuf.nano"; +option java_outer_classname = "BytesOffsetLengthTestNanoOuterClass"; + +message BytesOffsetLengthTestNano { + optional bytes foo = 1; + optional bytes bar = 2; +} -- cgit v1.1