/**
* Copyright 2015 StreamSets Inc.
*
* Licensed under the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.streamsets.pipeline.lib.util;
import com.google.protobuf.ByteString;
import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage;
import com.google.protobuf.ExtensionRegistry;
import com.google.protobuf.UnknownFieldSet;
import com.streamsets.pipeline.api.Field;
import com.streamsets.pipeline.api.Record;
import com.streamsets.pipeline.sdk.RecordCreator;
import org.junit.Assert;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Utility class used by protobuf related tests.
* Supplies test protobuf data and compares expected and actual outputs based on the supplied data
*/
public class ProtobufTestUtil {
public static ExtensionRegistry createExtensionRegistry(Map<String, Set<Descriptors.FieldDescriptor>> extensionMap) {
ExtensionRegistry extensionRegistry = ExtensionRegistry.newInstance();
for(Map.Entry<String, Set<Descriptors.FieldDescriptor>> e : extensionMap.entrySet()) {
Set<Descriptors.FieldDescriptor> value = e.getValue();
for (Descriptors.FieldDescriptor f : value) {
extensionRegistry.add(f);
}
}
return extensionRegistry;
}
public static List<DynamicMessage> getMessages(
Descriptors.Descriptor md,
ExtensionRegistry extensionRegistry,
byte[] data
) throws Exception {
DynamicMessage.Builder builder = DynamicMessage.newBuilder(md);
ByteArrayInputStream in = new ByteArrayInputStream(data);
List<DynamicMessage> messages = new ArrayList<>();
while (builder.mergeDelimitedFrom(in, extensionRegistry)) {
messages.add(builder.build());
builder.clear();
}
return messages;
}
public static byte[] getProtoBufData() throws IOException {
ByteArrayOutputStream bOut = new ByteArrayOutputStream();
for(int i = 0; i <10; i++) {
getSingleProtobufData(bOut, i);
}
bOut.flush();
return bOut.toByteArray();
}
public static void getSingleProtobufData(OutputStream bOut, int i) throws IOException {
PersonProto.Person johnThePerson = PersonProto.Person.newBuilder()
.setId(i)
.setName("John Doe" + i)
.setExtension(ExtensionsProto.PersonExtension.NestedPersonExtension.residenceAddress, "SJ")
.setUnknownFields(UnknownFieldsUtil.getPersonUnknownFields())
.addEmail("jdoe" + i + "@example.com")
.addPhone(
PersonProto.Person.PhoneNumber.newBuilder()
.setNumber("555-4321")
.setType(PersonProto.Person.PhoneType.HOME))
.addPhone(
PersonProto.Person.PhoneNumber.newBuilder()
.setNumber("666-4321")) // default type is HOME
.build();
EmployeeProto.Employee.Builder employee;
if(i % 2 ==0) {
EngineerProto.Engineer johnTheEng = EngineerProto.Engineer.newBuilder()
.setEmployeeId(String.valueOf(i))
.setDepName("r&d")
.setPerson(johnThePerson)
.setExtension(ExtensionsProto.EngineerExtension.factoryAddress, "South SF")
.build();
employee = EmployeeProto.Employee.newBuilder()
.setEngineer(johnTheEng)
.setExtension(ExtensionsProto.stringField, "SF");
} else {
ExecutiveProto.Executive johnTheExec = ExecutiveProto.Executive.newBuilder()
.setEmployeeId(String.valueOf(i))
.setPerson(johnThePerson)
.setExtension(ExtensionsProto.ExecutiveExtension.officeAddress, "SOMA")
.build();
employee = EmployeeProto.Employee.newBuilder()
.setExec(johnTheExec)
.setExtension(ExtensionsProto.stringField, "SF")
.setExtension(ExtensionsProto.boolField, true)
.setExtension(ExtensionsProto.intField, 4375)
.setExtension(ExtensionsProto.doubleField, 23423.4234)
.setExtension(ExtensionsProto.floatField, 22.22f)
.setExtension(ExtensionsProto.bytesField, ByteString.copyFromUtf8("SanFrancisco"));
}
employee.setUnknownFields(UnknownFieldsUtil.getEmployeeUnknownFields());
employee.build().writeDelimitedTo(bOut);
bOut.flush();
}
public static void checkProtobufRecords(Field field, int i) {
// root field is map containing name, id, email, phone
// name, id, email are primitive fields
Map<String, Field> valueAsMap = field.getValueAsMap();
Assert.assertNotNull(valueAsMap);
if(i %2 == 0) {
// get the nested Person object
Assert.assertTrue(valueAsMap.containsKey("engineer"));
valueAsMap = valueAsMap.get("engineer").getValueAsMap();
Assert.assertNotNull(valueAsMap);
} else {
Assert.assertTrue(valueAsMap.containsKey("exec"));
valueAsMap = valueAsMap.get("exec").getValueAsMap();
Assert.assertNotNull(valueAsMap);
}
Assert.assertTrue(valueAsMap.containsKey("employeeId"));
Assert.assertEquals(String.valueOf(i), valueAsMap.get("employeeId").getValueAsString());
if(i % 2 == 0) {
Assert.assertTrue(valueAsMap.containsKey("depName"));
Assert.assertEquals("r&d", valueAsMap.get("depName").getValueAsString());
Assert.assertTrue(valueAsMap.containsKey("depid"));
Assert.assertEquals(0, valueAsMap.get("depid").getValue());
}
// check the person nested object
Assert.assertTrue(valueAsMap.containsKey("person"));
valueAsMap = valueAsMap.get("person").getValueAsMap();
Assert.assertNotNull(valueAsMap);
Assert.assertTrue(valueAsMap.containsKey("name"));
Assert.assertEquals("John Doe" + i, valueAsMap.get("name").getValueAsString());
Assert.assertTrue(valueAsMap.containsKey("name"));
Assert.assertEquals("John Doe" + i, valueAsMap.get("name").getValueAsString());
Assert.assertTrue(valueAsMap.containsKey("id"));
Assert.assertEquals(i, valueAsMap.get("id").getValueAsInteger());
Assert.assertTrue(valueAsMap.containsKey("email"));
List<Field> email = valueAsMap.get("email").getValueAsList();
Assert.assertEquals(1, email.size());
Assert.assertEquals("jdoe" + i + "@example.com", email.get(0).getValueAsString());
// phone is a repeated message [List of Map expected]
Assert.assertTrue(valueAsMap.containsKey("phone"));
List<Field> phone = valueAsMap.get("phone").getValueAsList();
Assert.assertEquals(2, phone.size());
valueAsMap = phone.get(0).getValueAsMap();
Assert.assertEquals(2, valueAsMap.size());
// map contains 2 keys number and type of type String
Assert.assertTrue(valueAsMap.containsKey("number"));
Assert.assertEquals("555-4321", valueAsMap.get("number").getValueAsString());
Assert.assertTrue(valueAsMap.containsKey("type"));
Assert.assertEquals("HOME", valueAsMap.get("type").getValueAsString());
valueAsMap = phone.get(1).getValueAsMap();
Assert.assertEquals(2, valueAsMap.size());
Assert.assertTrue(valueAsMap.containsKey("number"));
Assert.assertEquals("666-4321", valueAsMap.get("number").getValueAsString());
Assert.assertTrue(valueAsMap.containsKey("type"));
Assert.assertEquals("HOME", valueAsMap.get("type").getValueAsString());
}
public static void checkProtobufRecordsForExtensions(Field field, int i) {
// The following extensions are expected as fields
// extension stringField in every employee
// extensions boolField, intField, longField, doubleField, floatField, bytesField in employee when i is odd [executive]
// factoryAddress in engineer object
// officeAddress in executive object
// residenceAddress in Person object
Map<String, Field> valueAsMap = field.getValueAsMap();
Assert.assertNotNull(valueAsMap);
Assert.assertTrue(valueAsMap.containsKey("stringField"));
Assert.assertEquals("SF", valueAsMap.get("stringField").getValueAsString());
if(i %2 != 0) {
Assert.assertTrue(valueAsMap.containsKey("boolField"));
Assert.assertEquals(true, valueAsMap.get("boolField").getValue());
Assert.assertTrue(valueAsMap.containsKey("intField"));
Assert.assertEquals(4375, valueAsMap.get("intField").getValue());
Assert.assertTrue(valueAsMap.containsKey("doubleField"));
Assert.assertEquals(23423.4234, valueAsMap.get("doubleField").getValue());
Assert.assertTrue(valueAsMap.containsKey("floatField"));
Assert.assertEquals(22.22f, valueAsMap.get("floatField").getValue());
Assert.assertTrue(valueAsMap.containsKey("bytesField"));
Assert.assertTrue(Arrays.equals("SanFrancisco".getBytes(), valueAsMap.get("bytesField").getValueAsByteArray()));
}
if(i % 2 == 0) {
//engineer extension
Assert.assertTrue(valueAsMap.containsKey("engineer"));
valueAsMap = valueAsMap.get("engineer").getValueAsMap();
Assert.assertNotNull(valueAsMap);
Assert.assertTrue(valueAsMap.containsKey("factoryAddress"));
Assert.assertEquals("South SF", valueAsMap.get("factoryAddress").getValue());
} else {
//executive extension
Assert.assertTrue(valueAsMap.containsKey("exec"));
valueAsMap = valueAsMap.get("exec").getValueAsMap();
Assert.assertNotNull(valueAsMap);
Assert.assertTrue(valueAsMap.containsKey("officeAddress"));
Assert.assertEquals("SOMA", valueAsMap.get("officeAddress").getValue());
}
// Person extension
valueAsMap = valueAsMap.get("person").getValueAsMap();
Assert.assertTrue(valueAsMap.containsKey("residenceAddress"));
Assert.assertEquals("SJ", valueAsMap.get("residenceAddress").getValueAsString());
}
public static void checkRecordForUnknownFields(Record record, int i) throws IOException {
// unknown fields are expected in paths for person and employee
String attribute = record.getHeader().getAttribute(ProtobufTypeUtil.PROTOBUF_UNKNOWN_FIELDS_PREFIX + "/");
UnknownFieldSet.Builder builder = UnknownFieldSet.newBuilder();
builder.mergeDelimitedFrom(new ByteArrayInputStream(org.apache.commons.codec.binary.Base64.decodeBase64(attribute.getBytes())));
UnknownFieldSet unknownFieldSet = builder.build();
UnknownFieldsUtil.checkEmployeeUnknownFields(unknownFieldSet);
if(i%2 == 0) {
attribute = record.getHeader().getAttribute(ProtobufTypeUtil.PROTOBUF_UNKNOWN_FIELDS_PREFIX + "/engineer/person");
} else {
attribute = record.getHeader().getAttribute(ProtobufTypeUtil.PROTOBUF_UNKNOWN_FIELDS_PREFIX + "/exec/person");
}
builder = UnknownFieldSet.newBuilder();
builder.mergeDelimitedFrom(new ByteArrayInputStream(org.apache.commons.codec.binary.Base64.decodeBase64(attribute.getBytes())));
unknownFieldSet = builder.build();
UnknownFieldsUtil.checkPersonUnknownFields(unknownFieldSet);
}
public static void compareProtoRecords(List<Record> records, int offset) throws IOException {
for(int i = 0; i < records.size(); i++) {
int dataSuffix = offset + i;
Map<String, Field> valueAsMap = records.get(i).get().getValueAsMap();
Assert.assertNotNull(valueAsMap);
checkProtobufRecords(records.get(i).get(), dataSuffix);
checkRecordForUnknownFields(records.get(i), dataSuffix);
}
}
// generator related
public static List<Record> getProtobufRecords() throws IOException {
List<Record> records = new ArrayList<>();
for(int i = 0; i < 10; i++) {
Record r = RecordCreator.create();
// create Person field
Field name = Field.create("John Doe" + i);
Field id = Field.create(i);
Field email = Field.create("jdoe" + i + "@example.com");
Field emailList = Field.create(Arrays.asList(email));
Field number = Field.create("555-4321");
Field type = Field.create("HOME");
Map<String, Field> phone = new HashMap<>();
phone.put("number", number);
phone.put("type", type);
Field phoneField = Field.create(phone);
Field phoneList = Field.create(Arrays.asList(phoneField));
Map<String, Field> person = new HashMap<>();
person.put("name", name);
person.put("id", id);
person.put("email", emailList);
person.put("phone", phoneList);
person.put("residenceAddress", Field.create("SJ"));
Field personField = Field.create(person);
Map<String, Field> employee = new HashMap<>();
if (i % 2 == 0) {
// create Engineer field
Map<String, Field> engineer = new HashMap<>();
engineer.put("employeeId", Field.create(String.valueOf(i)));
engineer.put("depName", Field.create("r&d"));
engineer.put("person", personField);
engineer.put("factoryAddress", Field.create("South SF"));
Field engineerField = Field.create(engineer);
employee.put("engineer", engineerField);
employee.put("exec", Field.create(Field.Type.MAP, null));
employee.put("stringField", Field.create("SF"));
} else {
// create Exec field
Map<String, Field> exec = new HashMap<>();
exec.put("employeeId", Field.create(String.valueOf(i)));
exec.put("person", personField);
exec.put("officeAddress", Field.create("SOMA"));
Field exedcField = Field.create(exec);
employee.put("exec", exedcField);
employee.put("engineer", Field.create(Field.Type.MAP, null));
employee.put("stringField", Field.create("SF"));
employee.put("boolField", Field.create(true));
employee.put("doubleField", Field.create(23423.4234));
employee.put("intField", Field.create(4375));
employee.put("floatField", Field.create(22.22));
employee.put("bytesField", Field.create("SanFrancisco".getBytes()));
}
r.set(Field.create(employee));
byte[] bytes = UnknownFieldsUtil.getBytes(UnknownFieldsUtil.getPersonUnknownFields());
if(i%2 == 0) {
r.getHeader().setAttribute(ProtobufTypeUtil.PROTOBUF_UNKNOWN_FIELDS_PREFIX + "/engineer/person", new String(bytes));
} else {
r.getHeader().setAttribute(ProtobufTypeUtil.PROTOBUF_UNKNOWN_FIELDS_PREFIX + "/exec/person", new String(bytes));
}
bytes = UnknownFieldsUtil.getBytes(UnknownFieldsUtil.getEmployeeUnknownFields());
r.getHeader().setAttribute(ProtobufTypeUtil.PROTOBUF_UNKNOWN_FIELDS_PREFIX + "/", new String(bytes));
records.add(r);
}
return records;
}
public static void checkProtobufDataFields(byte[] bytes) throws IOException {
ByteArrayInputStream bIn = new ByteArrayInputStream(bytes);
EmployeeProto.Employee.Builder builder = EmployeeProto.Employee.newBuilder();
EmployeeProto.Employee employee;
for(int i = 0; i < 10; i++) {
builder.mergeDelimitedFrom(bIn);
employee = builder.build();
PersonProto.Person person;
if( i % 2 == 0) {
EngineerProto.Engineer engineer = employee.getEngineer();
Assert.assertNotNull(engineer);
Assert.assertEquals(String.valueOf(i), engineer.getEmployeeId());
Assert.assertEquals("r&d", engineer.getDepName());
Assert.assertNotNull(engineer.getDepid());
person = engineer.getPerson();
} else {
ExecutiveProto.Executive exec = employee.getExec();
Assert.assertNotNull(exec);
Assert.assertEquals(String.valueOf(i), exec.getEmployeeId());
person = exec.getPerson();
}
Assert.assertNotNull(person);
Assert.assertEquals("John Doe" + i, person.getName());
Assert.assertEquals(i, person.getId());
List<PersonProto.Person.PhoneNumber> phoneList = person.getPhoneList();
Assert.assertEquals(1, phoneList.size());
Assert.assertEquals("555-4321", phoneList.get(0).getNumber());
Assert.assertEquals(PersonProto.Person.PhoneType.HOME, phoneList.get(0).getType());
List<String> emailList = person.getEmailList();
Assert.assertEquals(1, emailList.size());
Assert.assertEquals("jdoe" + i + "@example.com", emailList.get(0));
builder.clear();
}
}
public static void checkProtobufDataExtensions(byte[] bytes) throws IOException {
ExtensionRegistry extensionRegistry = ExtensionRegistry.newInstance();
ExtensionsProto.registerAllExtensions(extensionRegistry);
ByteArrayInputStream bIn = new ByteArrayInputStream(bytes);
EmployeeProto.Employee.Builder builder = EmployeeProto.Employee.newBuilder();
EmployeeProto.Employee employee;
for(int i = 0; i < 10; i++) {
builder.mergeDelimitedFrom(bIn, extensionRegistry);
employee = builder.build();
PersonProto.Person person;
if( i % 2 == 0) {
Assert.assertEquals("SF", employee.getExtension(ExtensionsProto.stringField));
Assert.assertEquals(true, employee.getExtension(ExtensionsProto.boolField));
Assert.assertEquals(43243, (int)employee.getExtension(ExtensionsProto.intField));
Assert.assertEquals(2343254354L, (Object) employee.getExtension(ExtensionsProto.longField));
Assert.assertEquals(3534.234, (Object) employee.getExtension(ExtensionsProto.doubleField));
Assert.assertEquals(343.34f, (Object) employee.getExtension(ExtensionsProto.floatField));
Assert.assertEquals(ByteString.copyFromUtf8("NewYork"), employee.getExtension(ExtensionsProto.bytesField));
EngineerProto.Engineer engineer = employee.getEngineer();
Assert.assertNotNull(engineer);
Assert.assertEquals("South SF", engineer.getExtension(ExtensionsProto.EngineerExtension.factoryAddress));
person = engineer.getPerson();
} else {
Assert.assertEquals("SF", employee.getExtension(ExtensionsProto.stringField));
Assert.assertEquals(true, employee.getExtension(ExtensionsProto.boolField));
Assert.assertEquals(4375, (int)employee.getExtension(ExtensionsProto.intField));
Assert.assertEquals(2343254354L, (Object) employee.getExtension(ExtensionsProto.longField));
Assert.assertEquals(23423.4234, (Object) employee.getExtension(ExtensionsProto.doubleField));
Assert.assertEquals(22.22f, (Object) employee.getExtension(ExtensionsProto.floatField));
Assert.assertEquals(ByteString.copyFromUtf8("SanFrancisco"), employee.getExtension(ExtensionsProto.bytesField));
ExecutiveProto.Executive exec = employee.getExec();
Assert.assertNotNull(exec);
Assert.assertEquals("SOMA", exec.getExtension(ExtensionsProto.ExecutiveExtension.officeAddress));
person = exec.getPerson();
}
Assert.assertNotNull(person);
Assert.assertEquals(
"SJ",
person.getExtension(ExtensionsProto.PersonExtension.NestedPersonExtension.residenceAddress)
);
builder.clear();
}
}
public static void checkProtobufDataUnknownFields(byte[] bytes) throws IOException {
ExtensionRegistry extensionRegistry = ExtensionRegistry.newInstance();
ExtensionsProto.registerAllExtensions(extensionRegistry);
ByteArrayInputStream bIn = new ByteArrayInputStream(bytes);
EmployeeProto.Employee.Builder builder = EmployeeProto.Employee.newBuilder();
EmployeeProto.Employee employee;
for(int i = 0; i < 10; i++) {
builder.mergeDelimitedFrom(bIn, extensionRegistry);
employee = builder.build();
PersonProto.Person person;
UnknownFieldsUtil.checkEmployeeUnknownFields(employee.getUnknownFields());
if( i % 2 == 0) {
EngineerProto.Engineer engineer = employee.getEngineer();
Assert.assertNotNull(engineer);
person = engineer.getPerson();
} else {
ExecutiveProto.Executive exec = employee.getExec();
Assert.assertNotNull(exec);
person = exec.getPerson();
}
Assert.assertNotNull(person);
UnknownFieldsUtil.checkPersonUnknownFields(person.getUnknownFields());
builder.clear();
}
}
/**
* Test utility class that supplies and checks for unknown field sets
*/
public static class UnknownFieldsUtil {
public static void checkPersonUnknownFields(UnknownFieldSet unknownFieldSet) {
Map<Integer, UnknownFieldSet.Field> integerFieldMap = unknownFieldSet.asMap();
Assert.assertEquals(2, integerFieldMap.size());
Assert.assertTrue(integerFieldMap.containsKey(123));
Assert.assertEquals(1, integerFieldMap.get(123).getFixed32List().size());
Assert.assertEquals(1234, (int)integerFieldMap.get(123).getFixed32List().get(0));
Assert.assertTrue(integerFieldMap.containsKey(234));
Assert.assertEquals(1, integerFieldMap.get(234).getFixed64List().size());
Assert.assertEquals(12345678, (long)integerFieldMap.get(234).getFixed64List().get(0));
}
public static void checkEmployeeUnknownFields(UnknownFieldSet unknownFieldSet) {
Map<Integer, UnknownFieldSet.Field> integerFieldMap = unknownFieldSet.asMap();
Assert.assertEquals(2, integerFieldMap.size());
Assert.assertTrue(integerFieldMap.containsKey(345));
Assert.assertEquals(1, integerFieldMap.get(345).getLengthDelimitedList().size());
Assert.assertEquals(
integerFieldMap.get(345).getLengthDelimitedList().get(0),
ByteString.copyFromUtf8("Hello San FRancisco!")
);
Assert.assertTrue(integerFieldMap.containsKey(456));
Assert.assertEquals(1, integerFieldMap.get(456).getVarintList().size());
Assert.assertEquals(123456789, (long)integerFieldMap.get(456).getVarintList().get(0));
}
public static UnknownFieldSet getPersonUnknownFields() throws IOException {
// add unknown fields
UnknownFieldSet.Field unknownIntField = UnknownFieldSet.Field.newBuilder()
.addFixed32(1234)
.build();
UnknownFieldSet.Field unknownLongField = UnknownFieldSet.Field.newBuilder()
.addFixed64(12345678)
.build();
UnknownFieldSet unknownFieldSet = UnknownFieldSet.newBuilder()
.addField(123, unknownIntField)
.addField(234, unknownLongField)
.build();
return unknownFieldSet;
}
public static UnknownFieldSet getEmployeeUnknownFields() {
// add unknown fields
UnknownFieldSet.Field unknownStringField = UnknownFieldSet.Field.newBuilder()
.addLengthDelimited(ByteString.copyFromUtf8("Hello San FRancisco!"))
.build();
UnknownFieldSet.Field unknownVarIntField = UnknownFieldSet.Field.newBuilder()
.addVarint(123456789)
.build();
UnknownFieldSet employeeUnknownFields = UnknownFieldSet.newBuilder()
.addField(345, unknownStringField)
.addField(456, unknownVarIntField)
.build();
return employeeUnknownFields;
}
public static byte[] getBytes(UnknownFieldSet unknownFieldSet) throws IOException {
ByteArrayOutputStream bOut = new ByteArrayOutputStream();
unknownFieldSet.writeDelimitedTo(bOut);
bOut.flush();
bOut.close();
return org.apache.commons.codec.binary.Base64.encodeBase64(bOut.toByteArray());
}
}
}