public static JDBCType jdbcType(Type type) { if (type.equals(BooleanType.BOOLEAN)) { return JDBCType.BOOLEAN; } if (type.equals(BigintType.BIGINT) || type.equals(TimestampType.TIMESTAMP)) { return JDBCType.BIGINT; } if (type.equals(DoubleType.DOUBLE)) { return JDBCType.DOUBLE; } if (type.equals(DateType.DATE)) { return JDBCType.INTEGER; } if (type.equals(VarcharType.VARCHAR)) { return JDBCType.VARBINARY; } return null; }
private static ColumnStats doComputeColumnStats(OrcReader orcReader, long columnId, Type type) throws IOException { int columnIndex = columnIndex(orcReader.getColumnNames(), columnId); OrcRecordReader reader = orcReader.createRecordReader(ImmutableMap.of(columnIndex, type), OrcPredicate.TRUE, UTC, new AggregatedMemoryContext()); if (type.equals(BooleanType.BOOLEAN)) { return indexBoolean(type, reader, columnIndex, columnId); } if (type.equals(BigintType.BIGINT) || type.equals(DateType.DATE) || type.equals(TimestampType.TIMESTAMP)) { return indexLong(type, reader, columnIndex, columnId); } if (type.equals(DoubleType.DOUBLE)) { return indexDouble(type, reader, columnIndex, columnId); } if (type.equals(VarcharType.VARCHAR)) { return indexString(type, reader, columnIndex, columnId); } return null; }
@Override public Slice getSlice(int field) { checkState(!closed, "cursor is closed"); try { Type type = getType(field); if (type.equals(VarcharType.VARCHAR)) { return utf8Slice(resultSet.getString(field + 1)); } if (type.equals(VarbinaryType.VARBINARY)) { return wrappedBuffer(resultSet.getBytes(field + 1)); } throw new PrestoException(INTERNAL_ERROR, "Unhandled type for slice: " + type.getTypeSignature()); } catch (SQLException e) { throw handleSqlException(e); } }
@NotNull public static String serializeSessionProperty(Type type, Object value) { if (value == null) { throw new PrestoException(INVALID_SESSION_PROPERTY, "Session property can not be null"); } if (BooleanType.BOOLEAN.equals(type)) { return value.toString(); } if (BigintType.BIGINT.equals(type)) { return value.toString(); } if (DoubleType.DOUBLE.equals(type)) { return value.toString(); } if (VarcharType.VARCHAR.equals(type)) { return value.toString(); } if (type instanceof ArrayType || type instanceof MapType) { return getJsonCodecForType(type).toJson(value); } throw new PrestoException(INVALID_SESSION_PROPERTY, format("Session property type %s is not supported", type)); }
@NotNull private static Object deserializeSessionProperty(Type type, String value) { if (value == null) { throw new PrestoException(INVALID_SESSION_PROPERTY, "Session property can not be null"); } if (VarcharType.VARCHAR.equals(type)) { return value; } if (BooleanType.BOOLEAN.equals(type)) { return Boolean.valueOf(value); } if (BigintType.BIGINT.equals(type)) { return Long.valueOf(value); } if (DoubleType.DOUBLE.equals(type)) { return Double.valueOf(value); } if (type instanceof ArrayType || type instanceof MapType) { return getJsonCodecForType(type).fromJson(value); } throw new PrestoException(INVALID_SESSION_PROPERTY, format("Session property type %s is not supported", type)); }
private static <T> JsonCodec<T> getJsonCodecForType(Type type) { if (VarcharType.VARCHAR.equals(type)) { return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(String.class); } if (BooleanType.BOOLEAN.equals(type)) { return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(Boolean.class); } if (BigintType.BIGINT.equals(type)) { return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(Long.class); } if (DoubleType.DOUBLE.equals(type)) { return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(Double.class); } if (type instanceof ArrayType) { Type elementType = ((ArrayType) type).getElementType(); return (JsonCodec<T>) JSON_CODEC_FACTORY.listJsonCodec(getJsonCodecForType(elementType)); } if (type instanceof MapType) { Type keyType = ((MapType) type).getKeyType(); Type valueType = ((MapType) type).getValueType(); return (JsonCodec<T>) JSON_CODEC_FACTORY.mapJsonCodec(getMapKeyType(keyType), getJsonCodecForType(valueType)); } throw new PrestoException(INVALID_SESSION_PROPERTY, format("Session property type %s is not supported", type)); }
private static Class<?> getMapKeyType(Type type) { if (VarcharType.VARCHAR.equals(type)) { return String.class; } if (BooleanType.BOOLEAN.equals(type)) { return Boolean.class; } if (BigintType.BIGINT.equals(type)) { return Long.class; } if (DoubleType.DOUBLE.equals(type)) { return Double.class; } throw new PrestoException(INVALID_SESSION_PROPERTY, format("Session property map key type %s is not supported", type)); }
@Test public void testVarianceStateSerialization() { StateCompiler compiler = new StateCompiler(); AccumulatorStateFactory<VarianceState> factory = compiler.generateStateFactory(VarianceState.class); AccumulatorStateSerializer<VarianceState> serializer = compiler.generateStateSerializer(VarianceState.class); VarianceState singleState = factory.createSingleState(); VarianceState deserializedState = factory.createSingleState(); singleState.setMean(1); singleState.setCount(2); singleState.setM2(3); BlockBuilder builder = VarcharType.VARCHAR.createBlockBuilder(new BlockBuilderStatus(), 1); serializer.serialize(singleState, builder); Block block = builder.build(); serializer.deserialize(block, 0, deserializedState); assertEquals(deserializedState.getCount(), singleState.getCount()); assertEquals(deserializedState.getMean(), singleState.getMean()); assertEquals(deserializedState.getM2(), singleState.getM2()); }
@Test public void testComplexSerialization() { StateCompiler compiler = new StateCompiler(); AccumulatorStateFactory<TestComplexState> factory = compiler.generateStateFactory(TestComplexState.class); AccumulatorStateSerializer<TestComplexState> serializer = compiler.generateStateSerializer(TestComplexState.class); TestComplexState singleState = factory.createSingleState(); TestComplexState deserializedState = factory.createSingleState(); singleState.setBoolean(true); singleState.setLong(1); singleState.setDouble(2.0); singleState.setByte((byte) 3); BlockBuilder builder = VarcharType.VARCHAR.createBlockBuilder(new BlockBuilderStatus(), 1); serializer.serialize(singleState, builder); Block block = builder.build(); serializer.deserialize(block, 0, deserializedState); assertEquals(deserializedState.getBoolean(), singleState.getBoolean()); assertEquals(deserializedState.getLong(), singleState.getLong()); assertEquals(deserializedState.getDouble(), singleState.getDouble()); assertEquals(deserializedState.getByte(), singleState.getByte()); }
@Test public void testNonExistent() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", VarcharType.VARCHAR, "very/deep/varchar", null, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", BigintType.BIGINT, "no_bigint", null, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", DoubleType.DOUBLE, "double/is_missing", null, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BooleanType.BOOLEAN, "hello", null, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkIsNull(providers, row1); checkIsNull(providers, row2); checkIsNull(providers, row3); checkIsNull(providers, row4); }
@Test public void testStringNumber() throws Exception { byte[] json = "{\"a_number\":481516,\"a_string\":\"2342\"}".getBytes(StandardCharsets.UTF_8); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", VarcharType.VARCHAR, "a_number", null, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", BigintType.BIGINT, "a_number", null, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", VarcharType.VARCHAR, "a_string", null, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", null, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkValue(providers, row1, "481516"); checkValue(providers, row2, 481516); checkValue(providers, row3, "2342"); checkValue(providers, row4, 2342); }
@Test public void testFixedWithString() { String str = "Ich bin zwei Oeltanks"; byte[] row = str.getBytes(StandardCharsets.UTF_8); RawRowDecoder rowDecoder = new RawRowDecoder(); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", VarcharType.VARCHAR, null, null, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "0", null, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", VarcharType.VARCHAR, "0:4", null, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", VarcharType.VARCHAR, "5:8", null, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(row, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkValue(providers, row1, str); checkValue(providers, row2, str); // these only work for single byte encodings... checkValue(providers, row3, str.substring(0, 4)); checkValue(providers, row4, str.substring(5, 8)); }
@Test public void testNulls() { String csv = ",,,"; CsvRowDecoder rowDecoder = new CsvRowDecoder(); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", VarcharType.VARCHAR, "0", null, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", BigintType.BIGINT, "1", null, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", DoubleType.DOUBLE, "2", null, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BooleanType.BOOLEAN, "3", null, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(csv.getBytes(StandardCharsets.UTF_8), null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkValue(providers, row1, ""); checkValue(providers, row2, 0); checkValue(providers, row3, 0.0d); checkValue(providers, row4, false); }
@Test public void testNonExistent() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "very/deep/varchar", null, null, false, false); KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "no_bigint", null, null, false, false); KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", DoubleType.DOUBLE, "double/is_missing", null, null, false, false); KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BooleanType.BOOLEAN, "hello", null, null, false, false); List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4); Set<KinesisFieldValueProvider> providers = new HashSet<>(); boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); assertTrue(valid); assertEquals(providers.size(), columns.size()); DecoderTestUtil.checkIsNull(providers, row1); DecoderTestUtil.checkIsNull(providers, row2); DecoderTestUtil.checkIsNull(providers, row3); DecoderTestUtil.checkIsNull(providers, row4); }
@Test public void testStringNumber() throws Exception { byte[] json = "{\"a_number\":481516,\"a_string\":\"2342\"}".getBytes(StandardCharsets.UTF_8); JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "a_number", null, null, false, false); KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "a_number", null, null, false, false); KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "a_string", null, null, false, false); KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", null, null, false, false); List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4); Set<KinesisFieldValueProvider> providers = new HashSet<>(); boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); assertTrue(valid); assertEquals(providers.size(), columns.size()); DecoderTestUtil.checkValue(providers, row1, "481516"); DecoderTestUtil.checkValue(providers, row2, 481516); DecoderTestUtil.checkValue(providers, row3, "2342"); DecoderTestUtil.checkValue(providers, row4, 2342); }
@Test public void testFixedWithString() { String str = "Ich bin zwei Oeltanks"; byte[] row = str.getBytes(StandardCharsets.UTF_8); RawKinesisRowDecoder rowDecoder = new RawKinesisRowDecoder(); KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, null, null, null, false, false); KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "0", null, null, false, false); KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "0:4", null, null, false, false); KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", VarcharType.VARCHAR, "5:8", null, null, false, false); List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4); Set<KinesisFieldValueProvider> providers = new HashSet<>(); boolean valid = rowDecoder.decodeRow(row, providers, columns, buildMap(columns)); assertTrue(valid); assertEquals(providers.size(), columns.size()); DecoderTestUtil.checkValue(providers, row1, str); DecoderTestUtil.checkValue(providers, row2, str); // these only work for single byte encodings... DecoderTestUtil.checkValue(providers, row3, str.substring(0, 4)); DecoderTestUtil.checkValue(providers, row4, str.substring(5, 8)); }
@JsonCreator public PRTable( @JsonProperty(value = "name", required = true) String name, @JsonProperty(value = "columns", required = true) List<RiakColumn> columns, @JsonProperty(value = "comment", required = false) String comment, @JsonProperty(value = "subtables", required = false) List<PRSubTable> subtables) { checkArgument(!isNullOrEmpty(name), "name is null or is empty"); this.name = checkNotNull(name, "name is null"); this.columns = ImmutableList.copyOf(checkNotNull(columns, "columns is null")); this.comment = Optional.ofNullable(comment); this.subtables = Optional.ofNullable(ImmutableList.copyOf(subtables)); for (RiakColumn column : this.columns) { if (column.getPkey() && column.getType() == VarcharType.VARCHAR && this.pkey == null) { this.pkey = column.getName(); } } if (pkey == null) { log.warn("Primary Key is not defined in columns effectively. Some queries become slow."); } // TODO: verify all subtable definitions here }
@JsonCreator public PRSubTable( @JsonProperty(value = "name", required = true) String name, @JsonProperty(value = "columns", required = true) List<RiakColumn> columns, @JsonProperty("path") String path) { checkArgument(!isNullOrEmpty(name), "name is null or is empty"); this.name = checkNotNull(name, "name is null"); this.path = checkNotNull(path, "Subtable can't be empty in subtable"); // should check whether valid JSONPath this.columns = ImmutableList.copyOf(checkNotNull(columns, "columns is null")); int index = 2; for (RiakColumn column : this.columns) { if (column.getPkey() && column.getType() == VarcharType.VARCHAR && this.pkey == null) { this.pkey = column.getName(); } } }
@Test public void testSerialization() throws IOException { String connectorId = "riak foo bar"; String columnName = "DevClownVanColummmmm"; RiakColumn column = new RiakColumn(columnName, VarcharType.VARCHAR, "comment, ...", true, false); RiakColumnHandle c = new RiakColumnHandle(connectorId, column, 54); ObjectMapper om = new ObjectMapper(); String s = COLUMN_CODEC.toJson(c); RiakColumnHandle c2 = COLUMN_CODEC.fromJson(s); assert(c.equals(c2)); assert(c.getColumn().getType() == VarcharType.VARCHAR); assert(c2.getColumn().getType() == VarcharType.VARCHAR); }
@Test public void test() { VarcharType varcharType = VarcharType.createVarcharType(100); System.out.println(varcharType.getDisplayName()); System.out.println(varcharType.toString()); TypeSignature varcharSig = VarcharType.getParametrizedVarcharSignature("100"); System.out.println(varcharSig.toString()); }
private static StorageType toStorageType(Type type) { Class<?> javaType = type.getJavaType(); if (javaType == boolean.class) { return StorageType.BOOLEAN; } if (javaType == long.class) { return StorageType.LONG; } if (javaType == double.class) { return StorageType.DOUBLE; } if (javaType == Slice.class) { if (type.equals(VarcharType.VARCHAR)) { return StorageType.STRING; } if (type.equals(VarbinaryType.VARBINARY)) { return StorageType.BYTES; } } if (isArrayType(type)) { return arrayOf(toStorageType(type.getTypeParameters().get(0))); } if (isMapType(type)) { return mapOf(toStorageType(type.getTypeParameters().get(0)), toStorageType(type.getTypeParameters().get(1))); } throw new PrestoException(NOT_SUPPORTED, "No storage type for type: " + type); }
@Test public void testString() throws Exception { InternalAggregationFunction stringAgg = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("checksum", AGGREGATE, VARBINARY, VARCHAR)); Block block = createStringsBlock("a", "a", null, "b", "c"); assertAggregation(stringAgg, 1.0, expectedChecksum(VarcharType.VARCHAR, block), block); }
@Test public void testLearnLibSvm() throws Exception { Type mapType = typeManager.getParameterizedType("map", ImmutableList.of(parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.DOUBLE)), ImmutableList.of()); InternalAggregationFunction aggregation = new AggregationCompiler(typeManager).generateAggregationFunction(LearnLibSvmClassifierAggregation.class, ClassifierType.BIGINT_CLASSIFIER, ImmutableList.of(BigintType.BIGINT, mapType, VarcharType.VARCHAR)); assertLearnClassifer(aggregation.bind(ImmutableList.of(0, 1, 2), Optional.empty(), Optional.empty(), 1.0).createAccumulator()); }
private static Page getPage() throws JsonProcessingException { Type mapType = typeManager.getParameterizedType("map", ImmutableList.of(parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.DOUBLE)), ImmutableList.of()); int datapoints = 100; RowPageBuilder builder = RowPageBuilder.rowPageBuilder(BigintType.BIGINT, mapType, VarcharType.VARCHAR); Random rand = new Random(0); for (int i = 0; i < datapoints; i++) { long label = rand.nextDouble() < 0.5 ? 0 : 1; builder.row(label, mapSliceOf(BigintType.BIGINT, DoubleType.DOUBLE, 0, label + rand.nextGaussian()), "C=1"); } return builder.build(); }
public static ObjectInspector getRowColumnInspector(Type type) { if (type.equals(BooleanType.BOOLEAN)) { return writableBooleanObjectInspector; } if (type.equals(BigintType.BIGINT)) { return writableLongObjectInspector; } if (type.equals(DoubleType.DOUBLE)) { return writableDoubleObjectInspector; } if (type.equals(VarcharType.VARCHAR)) { return writableStringObjectInspector; } if (type.equals(VarbinaryType.VARBINARY)) { return writableBinaryObjectInspector; } if (type.equals(DateType.DATE)) { return writableDateObjectInspector; } if (type.equals(TimestampType.TIMESTAMP)) { return writableTimestampObjectInspector; } if (isArrayType(type) || isMapType(type) || isRowType(type)) { return getJavaObjectInspector(type); } throw new IllegalArgumentException("unsupported type: " + type); }
@Test public void testNullValues() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", ISO8601JsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", ISO8601JsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", ISO8601JsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row6 = new DecoderTestColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", ISO8601JsonFieldDecoder.NAME, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); // sanity checks checkIsNull(providers, row1); checkIsNull(providers, row2); checkIsNull(providers, row3); checkIsNull(providers, row4); checkIsNull(providers, row5); checkIsNull(providers, row6); }
@Test public void testSimple() throws Exception { byte[] json = ByteStreams.toByteArray(TestJsonDecoder.class.getResourceAsStream("/decoder/json/message.json")); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", VarcharType.VARCHAR, "source", null, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "user/screen_name", null, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", BigintType.BIGINT, "id", null, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "user/statuses_count", null, null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", BooleanType.BOOLEAN, "user/geo_enabled", null, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkValue(providers, row1, "<a href=\"http://twitterfeed.com\" rel=\"nofollow\">twitterfeed</a>"); checkValue(providers, row2, "EKentuckyNews"); checkValue(providers, row3, 493857959588286460L); checkValue(providers, row4, 7630); checkValue(providers, row5, true); }
@Test public void testNullValues() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", SecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", SecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", SecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row6 = new DecoderTestColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", SecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); // sanity checks checkIsNull(providers, row1); checkIsNull(providers, row2); checkIsNull(providers, row3); checkIsNull(providers, row4); checkIsNull(providers, row5); checkIsNull(providers, row6); }
@Test public void testNullValues() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", MillisecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", MillisecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", MillisecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row6 = new DecoderTestColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", MillisecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); // sanity checks checkIsNull(providers, row1); checkIsNull(providers, row2); checkIsNull(providers, row3); checkIsNull(providers, row4); checkIsNull(providers, row5); checkIsNull(providers, row6); }
@Test public void testNullValues() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", RFC2822JsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", RFC2822JsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", RFC2822JsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row6 = new DecoderTestColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", RFC2822JsonFieldDecoder.NAME, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, map(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); // sanity checks checkIsNull(providers, row1); checkIsNull(providers, row2); checkIsNull(providers, row3); checkIsNull(providers, row4); checkIsNull(providers, row5); checkIsNull(providers, row6); }
@Test public void testSimple() { ByteBuffer buf = ByteBuffer.allocate(100); buf.putLong(4815162342L); // 0 - 7 buf.putInt(12345678); // 8 - 11 buf.putShort((short) 4567); // 12 - 13 buf.put((byte) 123); // 14 buf.put("Ich bin zwei Oeltanks".getBytes(StandardCharsets.UTF_8)); // 15+ byte[] row = new byte[buf.position()]; System.arraycopy(buf.array(), 0, row, 0, buf.position()); RawRowDecoder rowDecoder = new RawRowDecoder(); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", BigintType.BIGINT, "0", "LONG", null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", BigintType.BIGINT, "8", "INT", null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", BigintType.BIGINT, "12", "SHORT", null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "14", "BYTE", null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", VarcharType.VARCHAR, "15", null, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(row, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkValue(providers, row1, 4815162342L); checkValue(providers, row2, 12345678); checkValue(providers, row3, 4567); checkValue(providers, row4, 123); checkValue(providers, row5, "Ich bin zwei Oeltanks"); }
@SuppressWarnings("NumericCastThatLosesPrecision") @Test public void testFloatStuff() { ByteBuffer buf = ByteBuffer.allocate(100); buf.putDouble(Math.PI); buf.putFloat((float) Math.E); buf.putDouble(Math.E); byte[] row = new byte[buf.position()]; System.arraycopy(buf.array(), 0, row, 0, buf.position()); RawRowDecoder rowDecoder = new RawRowDecoder(); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", VarcharType.VARCHAR, null, "DOUBLE", null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "8", "FLOAT", null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(row, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkValue(providers, row1, Math.PI); checkValue(providers, row2, Math.E); }
@Test public void testSimple() { String csv = "\"row 1\",row2,\"row3\",100,\"200\",300,4.5"; CsvRowDecoder rowDecoder = new CsvRowDecoder(); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", VarcharType.VARCHAR, "0", null, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "1", null, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", VarcharType.VARCHAR, "2", null, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "3", null, null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", BigintType.BIGINT, "4", null, null, false, false, false); DecoderTestColumnHandle row6 = new DecoderTestColumnHandle("", 5, "row6", BigintType.BIGINT, "5", null, null, false, false, false); DecoderTestColumnHandle row7 = new DecoderTestColumnHandle("", 6, "row7", DoubleType.DOUBLE, "6", null, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6, row7); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(csv.getBytes(StandardCharsets.UTF_8), null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkValue(providers, row1, "row 1"); checkValue(providers, row2, "row2"); checkValue(providers, row3, "row3"); checkValue(providers, row4, 100); checkValue(providers, row5, 200); checkValue(providers, row6, 300); checkValue(providers, row7, 4.5d); }
private Type getType(TableType columnType) { switch (columnType) { case BOOLEAN: return BooleanType.BOOLEAN; case NUMBER: return DoubleType.DOUBLE; case STRING: return VarcharType.VARCHAR; default: throw new PrestoException(SpreadsheetErrorCode.INTERNAL_ERROR, "Not Supported [" + columnType + "]"); } }
public static Map.Entry<SchemaTableName, KinesisStreamDescription> createSimpleJsonStreamDescription(String streamName, SchemaTableName schemaTableName) { // Format: {"id" : 1324, "name" : "some string"} ArrayList<KinesisStreamFieldDescription> fieldList = new ArrayList<KinesisStreamFieldDescription>(); fieldList.add(new KinesisStreamFieldDescription("id", BigintType.BIGINT, "id", "comment", null, null, false)); fieldList.add(new KinesisStreamFieldDescription("name", VarcharType.VARCHAR, "name", "comment", null, null, false)); KinesisStreamFieldGroup grp = new KinesisStreamFieldGroup("json", fieldList); KinesisStreamDescription desc = new KinesisStreamDescription(schemaTableName.getTableName(), schemaTableName.getSchemaName(), streamName, grp); return new AbstractMap.SimpleImmutableEntry<>(schemaTableName, desc); }
@Test public void testSimple() throws Exception { byte[] json = ByteStreams.toByteArray(TestJsonDecoder.class.getResourceAsStream("/decoder/json/message.json")); JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "source", null, null, false, false); KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "user/screen_name", null, null, false, false); KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "id", null, null, false, false); KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "user/statuses_count", null, null, false, false); KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", BooleanType.BOOLEAN, "user/geo_enabled", null, null, false, false); List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5); Set<KinesisFieldValueProvider> providers = new HashSet<>(); boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); assertTrue(valid); assertEquals(providers.size(), columns.size()); DecoderTestUtil.checkValue(providers, row1, "<a href=\"http://twitterfeed.com\" rel=\"nofollow\">twitterfeed</a>"); DecoderTestUtil.checkValue(providers, row2, "EKentuckyNews"); DecoderTestUtil.checkValue(providers, row3, 493857959588286460L); DecoderTestUtil.checkValue(providers, row4, 7630); DecoderTestUtil.checkValue(providers, row5, true); }
@Test public void testNullValues() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false); KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false); KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); Set<KinesisFieldValueProvider> providers = new HashSet<>(); boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); assertTrue(valid); assertEquals(providers.size(), columns.size()); // sanity checks DecoderTestUtil.checkIsNull(providers, row1); DecoderTestUtil.checkIsNull(providers, row2); DecoderTestUtil.checkIsNull(providers, row3); DecoderTestUtil.checkIsNull(providers, row4); DecoderTestUtil.checkIsNull(providers, row5); DecoderTestUtil.checkIsNull(providers, row6); }
@Test public void testNullValues() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false); KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false); KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); Set<KinesisFieldValueProvider> providers = new HashSet<>(); boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); assertTrue(valid); assertEquals(providers.size(), columns.size()); // sanity checks DecoderTestUtil.checkIsNull(providers, row1); DecoderTestUtil.checkIsNull(providers, row2); DecoderTestUtil.checkIsNull(providers, row3); DecoderTestUtil.checkIsNull(providers, row4); DecoderTestUtil.checkIsNull(providers, row5); DecoderTestUtil.checkIsNull(providers, row6); }
@Test public void testNullValues() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false); KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false); KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false); KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false); KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false); KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false); List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); Set<KinesisFieldValueProvider> providers = new HashSet<>(); boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); assertTrue(valid); assertEquals(providers.size(), columns.size()); // sanity checks DecoderTestUtil.checkIsNull(providers, row1); DecoderTestUtil.checkIsNull(providers, row2); DecoderTestUtil.checkIsNull(providers, row3); DecoderTestUtil.checkIsNull(providers, row4); DecoderTestUtil.checkIsNull(providers, row5); DecoderTestUtil.checkIsNull(providers, row6); }