@Override public void doMerge(JsonParser parser, int currentDepth, Message.Builder messageBuilder) throws IOException { Value.Builder builder = (Value.Builder) messageBuilder; JsonToken token = parser.currentToken(); if (token.isBoolean()) { builder.setBoolValue(ParseSupport.parseBool(parser)); } else if (token.isNumeric()) { builder.setNumberValue(ParseSupport.parseDouble(parser)); } else if (token == JsonToken.VALUE_NULL) { builder.setNullValue(NullValue.NULL_VALUE); } else if (token.isScalarValue()) { builder.setStringValue(ParseSupport.parseString(parser)); } else if (token == JsonToken.START_OBJECT) { Struct.Builder structBuilder = builder.getStructValueBuilder(); StructMarshaller.INSTANCE.mergeValue(parser, currentDepth + 1, structBuilder); } else if (token == JsonToken.START_ARRAY) { ListValue.Builder listValueBuilder = builder.getListValueBuilder(); ListValueMarshaller.INSTANCE.mergeValue(parser, currentDepth + 1, listValueBuilder); } else { throw new IllegalStateException("Unexpected json data: " + parser.getText()); } }
@Test public void anyInMaps() throws Exception { TestAny.Builder testAny = TestAny.newBuilder(); testAny.putAnyMap("int32_wrapper", Any.pack(Int32Value.newBuilder().setValue(123).build())); testAny.putAnyMap("int64_wrapper", Any.pack(Int64Value.newBuilder().setValue(456).build())); testAny.putAnyMap("timestamp", Any.pack(Timestamps.parse("1969-12-31T23:59:59Z"))); testAny.putAnyMap("duration", Any.pack(Durations.parse("12345.1s"))); testAny.putAnyMap("field_mask", Any.pack(FieldMaskUtil.fromString("foo.bar,baz"))); Value numberValue = Value.newBuilder().setNumberValue(1.125).build(); Struct.Builder struct = Struct.newBuilder(); struct.putFields("number", numberValue); testAny.putAnyMap("struct", Any.pack(struct.build())); Value nullValue = Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build(); testAny.putAnyMap( "list_value", Any.pack(ListValue.newBuilder().addValues(numberValue).addValues(nullValue).build())); testAny.putAnyMap("number_value", Any.pack(numberValue)); testAny.putAnyMap("any_value_number", Any.pack(Any.pack(numberValue))); testAny.putAnyMap("any_value_default", Any.pack(Any.getDefaultInstance())); testAny.putAnyMap("default", Any.getDefaultInstance()); assertMatchesUpstream(testAny.build(), TestAllTypes.getDefaultInstance()); }
private static LogEntry.Builder newTestLogEntry(String name, int responseCode) { Value.Builder vb = Value.newBuilder(); Map<String, Value> values = Maps.newHashMap(); values.put("api_key", vb.setStringValue(TEST_API_KEY).build()); values.put("http_method", vb.setStringValue("GET").build()); values.put("timestamp", vb.setNumberValue(TEST_CLOCK.currentTimeMillis()).build()); values.put("http_response_code", vb.setNumberValue(responseCode).build()); values.put("response_size", vb.setNumberValue(TEST_SIZE).build()); values.put("request_size", vb.setNumberValue(TEST_SIZE).build()); if (responseCode >= 400) { values.put("error_cause", vb.setStringValue("internal").build()); } return LogEntry .newBuilder() .setStructPayload(Struct.newBuilder().putAllFields(values)) .setName(name) .setSeverity(responseCode >= 400 ? LogSeverity.ERROR : LogSeverity.INFO) .setTimestamp(REALLY_EARLY); }
@Test public void testPrepareJob() throws Exception { PrepareJobResponse response = stub.prepare( PrepareJobRequest.newBuilder() .setPipelineOptions(Struct.getDefaultInstance()) .setPipeline(Pipeline.getDefaultInstance()) .setJobName("myJobName") .build()); ApiServiceDescriptor stagingEndpoint = response.getArtifactStagingEndpoint(); ArtifactServiceStager stager = ArtifactServiceStager.overChannel( InProcessChannelBuilder.forName(stagingEndpoint.getUrl()).build()); File foo = writeTempFile("foo", "foo, bar, baz".getBytes()); File bar = writeTempFile("spam", "spam, ham, eggs".getBytes()); stager.stage(ImmutableList.<File>of(foo, bar)); List<byte[]> tempDirFiles = readFlattendFiles(runnerTemp.getRoot()); assertThat( tempDirFiles, hasItems( arrayEquals(Files.readAllBytes(foo.toPath())), arrayEquals(Files.readAllBytes(bar.toPath())))); // TODO: 'run' the job with some sort of noop backend, to verify state is cleaned up. }
@Override public void doMerge(JsonParser parser, int currentDepth, Message.Builder messageBuilder) throws IOException { Struct.Builder builder = (Struct.Builder) messageBuilder; while ((parser.nextValue()) != JsonToken.END_OBJECT) { builder.putFields( parser.getCurrentName(), ValueMarshaller.INSTANCE.readValue(parser, currentDepth + 1)); } }
@Override protected void doWrite(Struct message, JsonGenerator gen) throws IOException { for (Map.Entry<String, Value> entry : message.getFieldsMap().entrySet()) { gen.writeFieldName(entry.getKey()); ValueMarshaller.INSTANCE.writeValue(entry.getValue(), gen); } }
@Test public void struct() throws Exception { // Build a struct with all possible values. TestStruct.Builder builder = TestStruct.newBuilder(); Struct.Builder structBuilder = builder.getStructValueBuilder(); structBuilder.putFields("null_value", Value.newBuilder().setNullValueValue(0).build()); structBuilder.putFields("number_value", Value.newBuilder().setNumberValue(1.25).build()); structBuilder.putFields("string_value", Value.newBuilder().setStringValue("hello").build()); Struct.Builder subStructBuilder = Struct.newBuilder(); subStructBuilder.putFields("number_value", Value.newBuilder().setNumberValue(1234).build()); structBuilder.putFields( "struct_value", Value.newBuilder().setStructValue(subStructBuilder.build()).build()); ListValue.Builder listBuilder = ListValue.newBuilder(); listBuilder.addValues(Value.newBuilder().setNumberValue(1.125).build()); listBuilder.addValues(Value.newBuilder().setNullValueValue(0).build()); structBuilder.putFields( "list_value", Value.newBuilder().setListValue(listBuilder.build()).build()); TestStruct message = builder.build(); assertMatchesUpstream(message); builder = TestStruct.newBuilder(); builder.setValue(Value.newBuilder().setNullValueValue(0).build()); message = builder.build(); assertMatchesUpstream(message); builder = TestStruct.newBuilder(); listBuilder = builder.getListValueBuilder(); listBuilder.addValues(Value.newBuilder().setNumberValue(31831.125).build()); listBuilder.addValues(Value.newBuilder().setNullValueValue(0).build()); message = builder.build(); assertMatchesUpstream(message); }
/** * Creates additional types (Value, Struct and ListValue) to be added to the Service config. * TODO (guptasu): Fix this hack. Find a better way to add the predefined types. * TODO (guptasu): Add them only when required and not in all cases. */ static Iterable<Type> createAdditionalServiceTypes() { Map<String, DescriptorProto> additionalMessages = Maps.newHashMap(); additionalMessages.put(Struct.getDescriptor().getFullName(), Struct.getDescriptor().toProto()); additionalMessages.put(Value.getDescriptor().getFullName(), Value.getDescriptor().toProto()); additionalMessages.put(ListValue.getDescriptor().getFullName(), ListValue.getDescriptor().toProto()); additionalMessages.put(Empty.getDescriptor().getFullName(), Empty.getDescriptor().toProto()); additionalMessages.put(Int32Value.getDescriptor().getFullName(), Int32Value.getDescriptor().toProto()); additionalMessages.put(DoubleValue.getDescriptor().getFullName(), DoubleValue.getDescriptor().toProto()); additionalMessages.put(BoolValue.getDescriptor().getFullName(), BoolValue.getDescriptor().toProto()); additionalMessages.put(StringValue.getDescriptor().getFullName(), StringValue.getDescriptor().toProto()); for (Descriptor descriptor : Struct.getDescriptor().getNestedTypes()) { additionalMessages.put(descriptor.getFullName(), descriptor.toProto()); } // TODO (guptasu): Remove this hard coding. Without this, creation of Model from Service throws. // Needs investigation. String fileName = "struct.proto"; List<Type> additionalTypes = Lists.newArrayList(); for (String typeName : additionalMessages.keySet()) { additionalTypes.add(TypesBuilderFromDescriptor.createType(typeName, additionalMessages.get(typeName), fileName)); } return additionalTypes; }
/** Converts the provided {@link PipelineOptions} to a {@link Struct}. */ public static Struct toProto(PipelineOptions options) { Struct.Builder builder = Struct.newBuilder(); try { // The JSON format of a Protobuf Struct is the JSON object that is equivalent to that struct // (with values encoded in a standard json-codeable manner). See Beam PR 3719 for more. JsonFormat.parser().merge(MAPPER.writeValueAsString(options), builder); return builder.build(); } catch (IOException e) { throw new RuntimeException(e); } }
@Test public void testToFromProto() throws Exception { options.getOptionsId(); Struct originalStruct = PipelineOptionsTranslation.toProto(options); PipelineOptions deserializedStruct = PipelineOptionsTranslation.fromProto(originalStruct); Struct reserializedStruct = PipelineOptionsTranslation.toProto(deserializedStruct); assertThat(reserializedStruct.getFieldsMap(), equalTo(originalStruct.getFieldsMap())); }
@Test public void customSettingsRetained() throws Exception { TestOptions options = PipelineOptionsFactory.as(TestOptions.class); options.setExample(23); Struct serialized = PipelineOptionsTranslation.toProto(options); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat(deserialized.as(TestOptions.class).getExample(), equalTo(23)); }
@Test public void ignoredSettingsNotSerialized() throws Exception { TestUnserializableOptions opts = PipelineOptionsFactory.as(TestUnserializableOptions.class); opts.setUnserializable(new Object()); Struct serialized = PipelineOptionsTranslation.toProto(opts); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat( deserialized.as(TestUnserializableOptions.class).getUnserializable(), is(nullValue())); }
@Test public void defaultsRestored() throws Exception { Struct serialized = PipelineOptionsTranslation.toProto(PipelineOptionsFactory.as(TestDefaultOptions.class)); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat(deserialized.as(TestDefaultOptions.class).getDefault(), equalTo(19)); }
StructMarshaller() { super(Struct.getDefaultInstance()); }
@Test public void anyFields() throws Exception { TestAllTypes content = TestAllTypes.newBuilder().setOptionalInt32(1234).build(); TestAny message = TestAny.newBuilder().setAnyValue(Any.pack(content)).build(); assertMatchesUpstream(message, TestAllTypes.getDefaultInstance()); TestAny messageWithDefaultAnyValue = TestAny.newBuilder().setAnyValue(Any.getDefaultInstance()).build(); assertMatchesUpstream(messageWithDefaultAnyValue); // Well-known types have a special formatting when embedded in Any. // // 1. Any in Any. Any anyMessage = Any.pack(Any.pack(content)); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); // 2. Wrappers in Any. anyMessage = Any.pack(Int32Value.newBuilder().setValue(12345).build()); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); anyMessage = Any.pack(UInt32Value.newBuilder().setValue(12345).build()); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); anyMessage = Any.pack(Int64Value.newBuilder().setValue(12345).build()); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); anyMessage = Any.pack(UInt64Value.newBuilder().setValue(12345).build()); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); anyMessage = Any.pack(FloatValue.newBuilder().setValue(12345).build()); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); anyMessage = Any.pack(DoubleValue.newBuilder().setValue(12345).build()); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); anyMessage = Any.pack(BoolValue.newBuilder().setValue(true).build()); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); anyMessage = Any.pack(StringValue.newBuilder().setValue("Hello").build()); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); anyMessage = Any.pack(BytesValue.newBuilder().setValue(ByteString.copyFrom(new byte[] {1, 2})).build()); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); // 3. Timestamp in Any. anyMessage = Any.pack(Timestamps.parse("1969-12-31T23:59:59Z")); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); // 4. Duration in Any anyMessage = Any.pack(Durations.parse("12345.10s")); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); // 5. FieldMask in Any anyMessage = Any.pack(FieldMaskUtil.fromString("foo.bar,baz")); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); // 6. Struct in Any Struct.Builder structBuilder = Struct.newBuilder(); structBuilder.putFields("number", Value.newBuilder().setNumberValue(1.125).build()); anyMessage = Any.pack(structBuilder.build()); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); // 7. Value (number type) in Any Value.Builder valueBuilder = Value.newBuilder(); valueBuilder.setNumberValue(1); anyMessage = Any.pack(valueBuilder.build()); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); // 8. Value (null type) in Any anyMessage = Any.pack(Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build()); assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance()); }
/** * Make a {@code LogEntry} from the instance. * * @param name the name of log * @param timestampMillis the timestamp of the log in milliseconds * @return the corresponding {@code LogEntry.Builder} */ public LogEntry.Builder asLogEntry(String name, long timestampMillis) { Value.Builder vb = Value.newBuilder(); Map<String, Value> values = Maps.newHashMap(); values.put("http_response_code", vb.setNumberValue(getResponseCode()).build()); values.put("timestamp", vb.setNumberValue(timestampMillis).build()); if (getRequestSize() > 0) { values.put("request_size", vb.setNumberValue(getRequestSize()).build()); } if (getResponseSize() > 0) { values.put("response_size", vb.setNumberValue(getResponseSize()).build()); } if (!Strings.isNullOrEmpty(getMethod())) { values.put("http_method", vb.setStringValue(getMethod()).build()); } if (!Strings.isNullOrEmpty(getApiName())) { values.put("api_name", vb.setStringValue(getApiName()).build()); } if (!Strings.isNullOrEmpty(getApiMethod())) { values.put("api_method", vb.setStringValue(getApiMethod()).build()); } if (!Strings.isNullOrEmpty(getApiKey())) { values.put("api_key", vb.setStringValue(getApiKey()).build()); } if (!Strings.isNullOrEmpty(getProducerProjectId())) { values.put("producer_project_id", vb.setStringValue(getProducerProjectId()).build()); } if (!Strings.isNullOrEmpty(getReferer())) { values.put("referer", vb.setStringValue(getReferer()).build()); } if (!Strings.isNullOrEmpty(getLocation())) { values.put("location", vb.setStringValue(getLocation()).build()); } if (!Strings.isNullOrEmpty(getLogMessage())) { values.put("log_message", vb.setStringValue(getLogMessage()).build()); } if (!Strings.isNullOrEmpty(getUrl())) { values.put("url", vb.setStringValue(getUrl()).build()); } LogSeverity severity = LogSeverity.INFO; if (getResponseCode() >= 400) { values.put("error_cause", vb.setStringValue(getErrorCause().name()).build()); severity = LogSeverity.ERROR; } Struct.Builder theStruct = Struct.newBuilder().putAllFields(values); return LogEntry .newBuilder() .setTimestamp(Timestamps.fromEpoch(timestampMillis)) .setStructPayload(theStruct) .setName(name) .setSeverity(severity); }
/** Converts the provided {@link Struct} into {@link PipelineOptions}. */ public static PipelineOptions fromProto(Struct protoOptions) throws IOException { return MAPPER.readValue(JsonFormat.printer().print(protoOptions), PipelineOptions.class); }
public static void run(Pipeline p, Struct options, Path stagingLocation) throws Exception { // Validate that the pipeline is well-formed. PipelineTranslation.fromProto(p); throw new UnsupportedOperationException("Not implemented"); }
public void testStruct() throws Exception { // Build a struct with all possible values. TestStruct.Builder builder = TestStruct.newBuilder(); Struct.Builder structBuilder = builder.getStructValueBuilder(); structBuilder.putFields("null_value", Value.newBuilder().setNullValueValue(0).build()); structBuilder.putFields("number_value", Value.newBuilder().setNumberValue(1.25).build()); structBuilder.putFields("string_value", Value.newBuilder().setStringValue("hello").build()); Struct.Builder subStructBuilder = Struct.newBuilder(); subStructBuilder.putFields("number_value", Value.newBuilder().setNumberValue(1234).build()); structBuilder.putFields( "struct_value", Value.newBuilder().setStructValue(subStructBuilder.build()).build()); ListValue.Builder listBuilder = ListValue.newBuilder(); listBuilder.addValues(Value.newBuilder().setNumberValue(1.125).build()); listBuilder.addValues(Value.newBuilder().setNullValueValue(0).build()); structBuilder.putFields( "list_value", Value.newBuilder().setListValue(listBuilder.build()).build()); TestStruct message = builder.build(); assertEquals( "{\n" + " \"structValue\": {\n" + " \"null_value\": null,\n" + " \"number_value\": 1.25,\n" + " \"string_value\": \"hello\",\n" + " \"struct_value\": {\n" + " \"number_value\": 1234.0\n" + " },\n" + " \"list_value\": [1.125, null]\n" + " }\n" + "}", toJsonString(message)); assertRoundTripEquals(message); builder = TestStruct.newBuilder(); builder.setValue(Value.newBuilder().setNullValueValue(0).build()); message = builder.build(); assertEquals("{\n" + " \"value\": null\n" + "}", toJsonString(message)); assertRoundTripEquals(message); builder = TestStruct.newBuilder(); listBuilder = builder.getListValueBuilder(); listBuilder.addValues(Value.newBuilder().setNumberValue(31831.125).build()); listBuilder.addValues(Value.newBuilder().setNullValueValue(0).build()); message = builder.build(); assertEquals("{\n" + " \"listValue\": [31831.125, null]\n" + "}", toJsonString(message)); assertRoundTripEquals(message); }
public void testAnyInMaps() throws Exception { JsonFormat.TypeRegistry registry = JsonFormat.TypeRegistry.newBuilder().add(TestAllTypes.getDescriptor()).build(); JsonFormat.Printer printer = JsonFormat.printer().usingTypeRegistry(registry); TestAny.Builder testAny = TestAny.newBuilder(); testAny.putAnyMap("int32_wrapper", Any.pack(Int32Value.newBuilder().setValue(123).build())); testAny.putAnyMap("int64_wrapper", Any.pack(Int64Value.newBuilder().setValue(456).build())); testAny.putAnyMap("timestamp", Any.pack(Timestamps.parse("1969-12-31T23:59:59Z"))); testAny.putAnyMap("duration", Any.pack(Durations.parse("12345.1s"))); testAny.putAnyMap("field_mask", Any.pack(FieldMaskUtil.fromString("foo.bar,baz"))); Value numberValue = Value.newBuilder().setNumberValue(1.125).build(); Struct.Builder struct = Struct.newBuilder(); struct.putFields("number", numberValue); testAny.putAnyMap("struct", Any.pack(struct.build())); Value nullValue = Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build(); testAny.putAnyMap( "list_value", Any.pack(ListValue.newBuilder().addValues(numberValue).addValues(nullValue).build())); testAny.putAnyMap("number_value", Any.pack(numberValue)); testAny.putAnyMap("any_value_number", Any.pack(Any.pack(numberValue))); testAny.putAnyMap("any_value_default", Any.pack(Any.getDefaultInstance())); testAny.putAnyMap("default", Any.getDefaultInstance()); assertEquals( "{\n" + " \"anyMap\": {\n" + " \"int32_wrapper\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Int32Value\",\n" + " \"value\": 123\n" + " },\n" + " \"int64_wrapper\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Int64Value\",\n" + " \"value\": \"456\"\n" + " },\n" + " \"timestamp\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Timestamp\",\n" + " \"value\": \"1969-12-31T23:59:59Z\"\n" + " },\n" + " \"duration\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n" + " \"value\": \"12345.100s\"\n" + " },\n" + " \"field_mask\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.FieldMask\",\n" + " \"value\": \"foo.bar,baz\"\n" + " },\n" + " \"struct\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Struct\",\n" + " \"value\": {\n" + " \"number\": 1.125\n" + " }\n" + " },\n" + " \"list_value\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.ListValue\",\n" + " \"value\": [1.125, null]\n" + " },\n" + " \"number_value\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Value\",\n" + " \"value\": 1.125\n" + " },\n" + " \"any_value_number\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Any\",\n" + " \"value\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Value\",\n" + " \"value\": 1.125\n" + " }\n" + " },\n" + " \"any_value_default\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Any\",\n" + " \"value\": {}\n" + " },\n" + " \"default\": {}\n" + " }\n" + "}", printer.print(testAny.build())); assertRoundTripEquals(testAny.build(), registry); }
abstract Struct getOptions();
abstract Builder setOptions(Struct options);