@Test public void testDatastoreToGcs_EntityToJson_noTransform() throws Exception { DoFnTester<Entity, String> fnTester = DoFnTester.of(EntityToJson.newBuilder() .setJsTransformPath(StaticValueProvider.of(null)) .setJsTransformFunctionName(StaticValueProvider.of(null)) .build()); Builder entityBuilder = Entity.newBuilder(); JsonFormat.parser().usingTypeRegistry( TypeRegistry.newBuilder() .add(Entity.getDescriptor()) .build()) .merge(mEntityJson, entityBuilder); Entity entity = entityBuilder.build(); List<String> entityJsonOutputs = fnTester.processBundle(entity); Assert.assertEquals(mEntityJson, entityJsonOutputs.get(0)); }
@Test public void testDatastoreToGcs_EntityToJson_withTransform() throws Exception { DoFnTester<Entity, String> fnTester = DoFnTester.of(EntityToJson.newBuilder() .setJsTransformPath(StaticValueProvider.of(jsTransformPath)) .setJsTransformFunctionName(StaticValueProvider.of("transform")) .build()); Builder entityBuilder = Entity.newBuilder(); JsonFormat.parser().usingTypeRegistry( TypeRegistry.newBuilder() .add(Entity.getDescriptor()) .build()) .merge(mEntityJson, entityBuilder); Entity entity = entityBuilder.build(); List<String> entityJsonOutputs = fnTester.processBundle(entity); Assert.assertEquals(mTransformedEntityJson, entityJsonOutputs.get(0)); }
@Test public void testGcsToDatastore_EntityToJson_noTransform() throws Exception { DoFnTester<String, Entity> fnTester = DoFnTester.of(JsonToEntity.newBuilder() .setJsTransformPath(StaticValueProvider.of(null)) .setJsTransformFunctionName(StaticValueProvider.of(null)) .build()); List<Entity> output = fnTester.processBundle(mEntityJson); Entity outputEntity = output.get(0); Printer printer = JsonFormat.printer() .omittingInsignificantWhitespace() .usingTypeRegistry( TypeRegistry.newBuilder() .add(Entity.getDescriptor()) .build()); Assert.assertEquals(mEntityJson, printer.print(outputEntity)); }
@Test public void testDatastoreToBq_EntityToTableRow_notransform() throws Exception, IOException { DoFnTester<Entity, TableRow> fnTester = DoFnTester.of(EntityToTableRow.newBuilder() .setStrictCast(StaticValueProvider.of(true)) .setTableSchemaJson(StaticValueProvider.of(mTableSchemaJson)) .setJsTransformFunctionName(StaticValueProvider.of(null)) .setJsTransformPath(StaticValueProvider.of(null)) .build()); Builder entityBuilder = Entity.newBuilder(); JsonFormat.parser().usingTypeRegistry( TypeRegistry.newBuilder() .add(Entity.getDescriptor()) .build()) .merge(mEntityJson, entityBuilder); Entity entity = entityBuilder.build(); List<TableRow> tableRows = fnTester.processBundle(entity); TableRow tr = tableRows.get(0); Assert.assertEquals(1, tableRows.size()); Assert.assertEquals("key(Drawing, '31ce830e-91d0-405e-855a-abe416cadc1f')", tr.get("__key__")); Assert.assertEquals("79a1d9d9-e255-427a-9b09-f45157e97790", tr.get("canvasId")); }
private JsonNode getDefaultRequestJSON(Message msg) throws JsonParseException, IOException { Message.Builder o2 = DefaultCustomPredictRequest.newBuilder(); TypeRegistry registry = TypeRegistry.newBuilder().add(o2.getDescriptorForType()).build(); JsonFormat.Printer jPrinter = JsonFormat.printer(); String result = jPrinter.usingTypeRegistry(registry).print(msg); ObjectMapper mapper = new ObjectMapper(); JsonFactory factory = mapper.getFactory(); JsonParser parser = factory.createParser(result); JsonNode jNode = mapper.readTree(parser); if (jNode.has(PredictionBusinessServiceImpl.REQUEST_CUSTOM_DATA_FIELD)) { JsonNode values = jNode.get(PredictionBusinessServiceImpl.REQUEST_CUSTOM_DATA_FIELD).get("values"); ((ObjectNode) jNode).set(PredictionBusinessServiceImpl.REQUEST_CUSTOM_DATA_FIELD, values); } return jNode; }
public ClassificationReply getPredictReplyFromJson(String client,JsonNode json) { RPCConfig config = services.get(client); try { TypeRegistry registry = null; if (config != null && config.replyClass != null && json.has(PredictionBusinessServiceImpl.REPLY_CUSTOM_DATA_FIELD)) { if (!json.get(PredictionBusinessServiceImpl.REPLY_CUSTOM_DATA_FIELD).has("@type")) ((ObjectNode) json.get(PredictionBusinessServiceImpl.REPLY_CUSTOM_DATA_FIELD)).put("@type", "type.googleapis.com/" + config.replyClass.getName()); Method m = config.replyBuilder; Message.Builder o = (Message.Builder) m.invoke(null); registry = TypeRegistry.newBuilder().add(o.getDescriptorForType()).build(); } ClassificationReply.Builder builder = ClassificationReply.newBuilder(); JsonFormat.Parser jFormatter = JsonFormat.parser(); if (registry != null) jFormatter = jFormatter.usingTypeRegistry(registry); jFormatter.merge(json.toString(), builder); ClassificationReply reply = builder.build(); return reply; } catch (Exception e) { logger.error("Failed to convert json "+json.toString()+" to PredictReply",e); return null; } }
private ClassificationRequest getPredictRequestWithCustomDefaultFromJSON(JsonNode json) throws InvalidProtocolBufferException { ObjectMapper mapper = new ObjectMapper(); ObjectNode data = mapper.createObjectNode(); data.put("@type", "type.googleapis.com/" + DefaultCustomPredictRequest.class.getName()); data.put("values", json.get(PredictionBusinessServiceImpl.REQUEST_CUSTOM_DATA_FIELD)); ((ObjectNode) json).put(PredictionBusinessServiceImpl.REQUEST_CUSTOM_DATA_FIELD, data); Message.Builder o = DefaultCustomPredictRequest.newBuilder(); TypeRegistry registry = TypeRegistry.newBuilder().add(o.getDescriptorForType()).build(); ClassificationRequest.Builder builder = ClassificationRequest.newBuilder(); JsonFormat.Parser jFormatter = JsonFormat.parser(); if (registry != null) jFormatter = jFormatter.usingTypeRegistry(registry); jFormatter.merge(json.toString(), builder); ClassificationRequest request = builder.build(); return request; }
private JsonFormat.Printer getJsonPrinter() { if (mJsonPrinter == null) { TypeRegistry typeRegistry = TypeRegistry.newBuilder() .add(Entity.getDescriptor()) .build(); mJsonPrinter = JsonFormat.printer() .usingTypeRegistry(typeRegistry) .omittingInsignificantWhitespace(); } return mJsonPrinter; }
private JsonFormat.Parser getJsonParser() { if (mJsonParser == null) { TypeRegistry typeRegistry = TypeRegistry.newBuilder() .add(Entity.getDescriptor()) .build(); mJsonParser = JsonFormat.parser() .usingTypeRegistry(typeRegistry); } return mJsonParser; }
private TypeRegistry.Builder addPlatformExtensions(TypeRegistry.Builder registryBuilder) { ExtensionRegistry extensions = ToolProtoUtil.getStandardPlatformExtensions(); for (String extendedType : EXTENDED_ELEMENTS) { for (ExtensionRegistry.ExtensionInfo info : extensions.getAllImmutableExtensionsByExtendedType(extendedType)) { if (null != info.defaultInstance) { registryBuilder.add(info.defaultInstance.getDescriptorForType()); } } } return registryBuilder; }
public static JsonFormat.Printer createPrinter() { TypeRegistry typeRegistry = TypeRegistry.newBuilder() .add(Entity.getDescriptor()) .build(); return JsonFormat.printer() .usingTypeRegistry(typeRegistry) .omittingInsignificantWhitespace(); }
public static JsonFormat.Parser createParser() { TypeRegistry typeRegistry = TypeRegistry.newBuilder() .add(Entity.getDescriptor()) .build(); return JsonFormat.parser() .usingTypeRegistry(typeRegistry); }
private JsonNode getJSONFromMethod(Method m,Message msg,String fieldname) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, JsonParseException, IOException { Message.Builder o2 = (Message.Builder) m.invoke(null); TypeRegistry registry = TypeRegistry.newBuilder().add(o2.getDescriptorForType()).build(); JsonFormat.Printer jPrinter = JsonFormat.printer(); String result = jPrinter.usingTypeRegistry(registry).print(msg); ObjectMapper mapper = new ObjectMapper(); JsonFactory factory = mapper.getFactory(); JsonParser parser = factory.createParser(result); JsonNode jNode = mapper.readTree(parser); if (jNode.has(fieldname) && jNode.get(fieldname).has("@type")) ((ObjectNode) jNode.get(fieldname)).remove("@type"); return jNode; }
@Test public void testParseFromJSON() throws InvalidProtocolBufferException { String json = "{\"meta\":{\"modelName\":\"some-name\"},\"custom\":{\"@type\":\"type.googleapis.com/io.seldon.api.rpc.example.CustomPredictReply\",\"data\":\"some custom data\"}}"; ClassificationReply.Builder builder = ClassificationReply.newBuilder(); CustomPredictReply.Builder customBuilder = CustomPredictReply.newBuilder(); TypeRegistry registry = TypeRegistry.newBuilder().add(customBuilder.getDescriptorForType()).build(); JsonFormat.Parser jFormatter = JsonFormat.parser().usingTypeRegistry(registry); jFormatter.merge(json, builder); ClassificationReply reply = builder.build(); System.out.println(reply); }
@Test public void testParseFromJSONDefault() throws InvalidProtocolBufferException { String json = "{\"data\":{\"@type\":\"type.googleapis.com/io.seldon.api.rpc.DefaultCustomPredictRequest\",\"values\":[1.2,2.1]}}"; ClassificationRequest.Builder builder = ClassificationRequest.newBuilder(); DefaultCustomPredictRequest.Builder customBuilder = DefaultCustomPredictRequest.newBuilder(); TypeRegistry registry = TypeRegistry.newBuilder().add(customBuilder.getDescriptorForType()).build(); JsonFormat.Parser jFormatter = JsonFormat.parser().usingTypeRegistry(registry); jFormatter.merge(json, builder); ClassificationRequest request = builder.build(); System.out.println(request); }
public static void dump(File src) throws Exception { try(FileInputStream f = new FileInputStream(src)) { Data data = Data.parseFrom(f); JsonFormat.Printer printer = JsonFormat.printer().usingTypeRegistry(TypeRegistry.getEmptyTypeRegistry()); System.out.println(printer.print(data)); } }
private void assertRoundTripEquals(Message message, TypeRegistry registry) throws Exception { JsonFormat.Printer printer = JsonFormat.printer().usingTypeRegistry(registry); JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(registry); Message.Builder builder = message.newBuilderForType(); parser.merge(printer.print(message), builder); Message parsedMessage = builder.build(); assertEquals(message.toString(), parsedMessage.toString()); }
public void testEmptyWrapperTypesInAny() throws Exception { JsonFormat.TypeRegistry registry = JsonFormat.TypeRegistry.newBuilder().add(TestAllTypes.getDescriptor()).build(); JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(registry); Any.Builder builder = Any.newBuilder(); parser.merge( "{\n" + " \"@type\": \"type.googleapis.com/google.protobuf.BoolValue\",\n" + " \"value\": false\n" + "}\n", builder); Any any = builder.build(); assertEquals(0, any.getValue().size()); }
public void run() throws Exception { typeRegistry = TypeRegistry.newBuilder().add( TestMessagesProto3.TestAllTypesProto3.getDescriptor()).build(); while (doTestIo()) { this.testCount++; } System.err.println("ConformanceJava: received EOF from test runner after " + this.testCount + " tests"); }
@Nullable private String valueToString(Value v) throws IOException { if (strictCast() && !v.getValueTypeCase().equals(ValueTypeCase.STRING_VALUE)) { return null; } switch (v.getValueTypeCase()) { case STRING_VALUE: return v.getStringValue(); case INTEGER_VALUE: return Long.toString(v.getIntegerValue()); case DOUBLE_VALUE: return Double.toString(v.getDoubleValue()); case BOOLEAN_VALUE: return Boolean.toString(v.getBooleanValue()); case TIMESTAMP_VALUE: return valueToTimestamp(v); case NULL_VALUE: return null; case BLOB_VALUE: return valueToBytes(v); case ARRAY_VALUE: ArrayList<String> arr = new ArrayList<>(); for (Value av: v.getArrayValue().getValuesList()) { arr.add(valueToString(av)); } return Transport.getJsonFactory().toString(arr); case ENTITY_VALUE: try { return JsonFormat.printer() .usingTypeRegistry(TypeRegistry.newBuilder() .add(Entity.getDescriptor()) .build()) .omittingInsignificantWhitespace() .print(v.getEntityValue()); } catch (InvalidProtocolBufferException e) { e.printStackTrace(); return null; } case GEO_POINT_VALUE: return valueToGeopoint(v); case KEY_VALUE: return keyToString(v.getKeyValue()); default: throw new IllegalArgumentException( "ValueType Case not handled: " + v.getValueTypeCase()); } }
private void generateOutputFiles(Service serviceConfig) throws FileNotFoundException, IOException { // Create normalized service proto, in binary form. if (!Strings.isNullOrEmpty(options.get(BIN_OUT))) { File outFileBinaryServiceConfig = new File(options.get(BIN_OUT)); OutputStream normalizedOut = new FileOutputStream(outFileBinaryServiceConfig); serviceConfig.writeTo(normalizedOut); normalizedOut.close(); } // Create normalized service proto, in text form. if (!Strings.isNullOrEmpty(options.get(TXT_OUT))) { File outFileTxtServiceConfig = new File(options.get(TXT_OUT)); try (PrintWriter textPrintWriter = new PrintWriter(outFileTxtServiceConfig, StandardCharsets.UTF_8.name())) { TextFormat.print(serviceConfig, textPrintWriter); } } // Create normalized service proto, in json form. if (!Strings.isNullOrEmpty(options.get(JSON_OUT))) { File outFileJsonServiceConfig = new File(options.get(JSON_OUT)); TypeRegistry registry = addPlatformExtensions(TypeRegistry.newBuilder()) .add(Service.getDescriptor()) .add(com.google.protobuf.BoolValue.getDescriptor()) .add(com.google.protobuf.BytesValue.getDescriptor()) .add(com.google.protobuf.DoubleValue.getDescriptor()) .add(com.google.protobuf.FloatValue.getDescriptor()) .add(com.google.protobuf.Int32Value.getDescriptor()) .add(com.google.protobuf.Int64Value.getDescriptor()) .add(com.google.protobuf.StringValue.getDescriptor()) .add(com.google.protobuf.UInt32Value.getDescriptor()) .add(com.google.protobuf.UInt64Value.getDescriptor()) .build(); JsonFormat.Printer jsonPrinter = JsonFormat.printer().usingTypeRegistry(registry); try (PrintWriter jsonPrintWriter = new PrintWriter(outFileJsonServiceConfig, StandardCharsets.UTF_8.name())) { jsonPrinter.appendTo(serviceConfig, jsonPrintWriter); } } }
private void assertRoundTripEquals(Message message) throws Exception { assertRoundTripEquals(message, TypeRegistry.getEmptyTypeRegistry()); }
public void testAnyInMaps() throws Exception { JsonFormat.TypeRegistry registry = JsonFormat.TypeRegistry.newBuilder().add(TestAllTypes.getDescriptor()).build(); JsonFormat.Printer printer = JsonFormat.printer().usingTypeRegistry(registry); TestAny.Builder testAny = TestAny.newBuilder(); testAny.putAnyMap("int32_wrapper", Any.pack(Int32Value.newBuilder().setValue(123).build())); testAny.putAnyMap("int64_wrapper", Any.pack(Int64Value.newBuilder().setValue(456).build())); testAny.putAnyMap("timestamp", Any.pack(Timestamps.parse("1969-12-31T23:59:59Z"))); testAny.putAnyMap("duration", Any.pack(Durations.parse("12345.1s"))); testAny.putAnyMap("field_mask", Any.pack(FieldMaskUtil.fromString("foo.bar,baz"))); Value numberValue = Value.newBuilder().setNumberValue(1.125).build(); Struct.Builder struct = Struct.newBuilder(); struct.putFields("number", numberValue); testAny.putAnyMap("struct", Any.pack(struct.build())); Value nullValue = Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build(); testAny.putAnyMap( "list_value", Any.pack(ListValue.newBuilder().addValues(numberValue).addValues(nullValue).build())); testAny.putAnyMap("number_value", Any.pack(numberValue)); testAny.putAnyMap("any_value_number", Any.pack(Any.pack(numberValue))); testAny.putAnyMap("any_value_default", Any.pack(Any.getDefaultInstance())); testAny.putAnyMap("default", Any.getDefaultInstance()); assertEquals( "{\n" + " \"anyMap\": {\n" + " \"int32_wrapper\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Int32Value\",\n" + " \"value\": 123\n" + " },\n" + " \"int64_wrapper\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Int64Value\",\n" + " \"value\": \"456\"\n" + " },\n" + " \"timestamp\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Timestamp\",\n" + " \"value\": \"1969-12-31T23:59:59Z\"\n" + " },\n" + " \"duration\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n" + " \"value\": \"12345.100s\"\n" + " },\n" + " \"field_mask\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.FieldMask\",\n" + " \"value\": \"foo.bar,baz\"\n" + " },\n" + " \"struct\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Struct\",\n" + " \"value\": {\n" + " \"number\": 1.125\n" + " }\n" + " },\n" + " \"list_value\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.ListValue\",\n" + " \"value\": [1.125, null]\n" + " },\n" + " \"number_value\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Value\",\n" + " \"value\": 1.125\n" + " },\n" + " \"any_value_number\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Any\",\n" + " \"value\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Value\",\n" + " \"value\": 1.125\n" + " }\n" + " },\n" + " \"any_value_default\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Any\",\n" + " \"value\": {}\n" + " },\n" + " \"default\": {}\n" + " }\n" + "}", printer.print(testAny.build())); assertRoundTripEquals(testAny.build(), registry); }