@Test public void testTopicHasData() throws Exception { MaterializedResult result = queryRunner.execute("SELECT count(1) from " + topicName); MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) .row(0) .build(); assertEquals(result, expected); int count = 1000; createMessages(topicName, count); result = queryRunner.execute("SELECT count(1) from " + topicName); expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) .row(count) .build(); assertEquals(result, expected); }
public static JDBCType jdbcType(Type type) { if (type.equals(BooleanType.BOOLEAN)) { return JDBCType.BOOLEAN; } if (type.equals(BigintType.BIGINT) || type.equals(TimestampType.TIMESTAMP)) { return JDBCType.BIGINT; } if (type.equals(DoubleType.DOUBLE)) { return JDBCType.DOUBLE; } if (type.equals(DateType.DATE)) { return JDBCType.INTEGER; } if (type.equals(VarcharType.VARCHAR)) { return JDBCType.VARBINARY; } return null; }
private static ColumnStats doComputeColumnStats(OrcReader orcReader, long columnId, Type type) throws IOException { int columnIndex = columnIndex(orcReader.getColumnNames(), columnId); OrcRecordReader reader = orcReader.createRecordReader(ImmutableMap.of(columnIndex, type), OrcPredicate.TRUE, UTC, new AggregatedMemoryContext()); if (type.equals(BooleanType.BOOLEAN)) { return indexBoolean(type, reader, columnIndex, columnId); } if (type.equals(BigintType.BIGINT) || type.equals(DateType.DATE) || type.equals(TimestampType.TIMESTAMP)) { return indexLong(type, reader, columnIndex, columnId); } if (type.equals(DoubleType.DOUBLE)) { return indexDouble(type, reader, columnIndex, columnId); } if (type.equals(VarcharType.VARCHAR)) { return indexString(type, reader, columnIndex, columnId); } return null; }
@Test public void testTableHasData() throws Exception { MaterializedResult result = queryRunner.execute("SELECT count(1) from " + tableName); MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) .row(0) .build(); assertEquals(result, expected); int count = 1000; populateData(count); result = queryRunner.execute("SELECT count(1) from " + tableName); expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) .row(count) .build(); assertEquals(result, expected); }
@NotNull public static String serializeSessionProperty(Type type, Object value) { if (value == null) { throw new PrestoException(INVALID_SESSION_PROPERTY, "Session property can not be null"); } if (BooleanType.BOOLEAN.equals(type)) { return value.toString(); } if (BigintType.BIGINT.equals(type)) { return value.toString(); } if (DoubleType.DOUBLE.equals(type)) { return value.toString(); } if (VarcharType.VARCHAR.equals(type)) { return value.toString(); } if (type instanceof ArrayType || type instanceof MapType) { return getJsonCodecForType(type).toJson(value); } throw new PrestoException(INVALID_SESSION_PROPERTY, format("Session property type %s is not supported", type)); }
@NotNull private static Object deserializeSessionProperty(Type type, String value) { if (value == null) { throw new PrestoException(INVALID_SESSION_PROPERTY, "Session property can not be null"); } if (VarcharType.VARCHAR.equals(type)) { return value; } if (BooleanType.BOOLEAN.equals(type)) { return Boolean.valueOf(value); } if (BigintType.BIGINT.equals(type)) { return Long.valueOf(value); } if (DoubleType.DOUBLE.equals(type)) { return Double.valueOf(value); } if (type instanceof ArrayType || type instanceof MapType) { return getJsonCodecForType(type).fromJson(value); } throw new PrestoException(INVALID_SESSION_PROPERTY, format("Session property type %s is not supported", type)); }
private static <T> JsonCodec<T> getJsonCodecForType(Type type) { if (VarcharType.VARCHAR.equals(type)) { return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(String.class); } if (BooleanType.BOOLEAN.equals(type)) { return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(Boolean.class); } if (BigintType.BIGINT.equals(type)) { return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(Long.class); } if (DoubleType.DOUBLE.equals(type)) { return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(Double.class); } if (type instanceof ArrayType) { Type elementType = ((ArrayType) type).getElementType(); return (JsonCodec<T>) JSON_CODEC_FACTORY.listJsonCodec(getJsonCodecForType(elementType)); } if (type instanceof MapType) { Type keyType = ((MapType) type).getKeyType(); Type valueType = ((MapType) type).getValueType(); return (JsonCodec<T>) JSON_CODEC_FACTORY.mapJsonCodec(getMapKeyType(keyType), getJsonCodecForType(valueType)); } throw new PrestoException(INVALID_SESSION_PROPERTY, format("Session property type %s is not supported", type)); }
private static Class<?> getMapKeyType(Type type) { if (VarcharType.VARCHAR.equals(type)) { return String.class; } if (BooleanType.BOOLEAN.equals(type)) { return Boolean.class; } if (BigintType.BIGINT.equals(type)) { return Long.class; } if (DoubleType.DOUBLE.equals(type)) { return Double.class; } throw new PrestoException(INVALID_SESSION_PROPERTY, format("Session property map key type %s is not supported", type)); }
@Override protected RelationType visitUnnest(Unnest node, AnalysisContext context) { ImmutableList.Builder<Field> outputFields = ImmutableList.builder(); for (Expression expression : node.getExpressions()) { ExpressionAnalysis expressionAnalysis = analyzeExpression(expression, context.getLateralTupleDescriptor(), context); Type expressionType = expressionAnalysis.getType(expression); if (expressionType instanceof ArrayType) { outputFields.add(Field.newUnqualified(Optional.empty(), ((ArrayType) expressionType).getElementType())); } else if (expressionType instanceof MapType) { outputFields.add(Field.newUnqualified(Optional.empty(), ((MapType) expressionType).getKeyType())); outputFields.add(Field.newUnqualified(Optional.empty(), ((MapType) expressionType).getValueType())); } else { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Cannot unnest type: " + expressionType); } } if (node.isWithOrdinality()) { outputFields.add(Field.newUnqualified(Optional.empty(), BigintType.BIGINT)); } RelationType descriptor = new RelationType(outputFields.build()); analysis.setOutputDescriptor(node, descriptor); return descriptor; }
@Test public void testNonExistent() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", VarcharType.VARCHAR, "very/deep/varchar", null, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", BigintType.BIGINT, "no_bigint", null, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", DoubleType.DOUBLE, "double/is_missing", null, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BooleanType.BOOLEAN, "hello", null, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkIsNull(providers, row1); checkIsNull(providers, row2); checkIsNull(providers, row3); checkIsNull(providers, row4); }
@Test public void testStringNumber() throws Exception { byte[] json = "{\"a_number\":481516,\"a_string\":\"2342\"}".getBytes(StandardCharsets.UTF_8); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", VarcharType.VARCHAR, "a_number", null, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", BigintType.BIGINT, "a_number", null, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", VarcharType.VARCHAR, "a_string", null, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", null, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkValue(providers, row1, "481516"); checkValue(providers, row2, 481516); checkValue(providers, row3, "2342"); checkValue(providers, row4, 2342); }
@Test public void testNulls() { String csv = ",,,"; CsvRowDecoder rowDecoder = new CsvRowDecoder(); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", VarcharType.VARCHAR, "0", null, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", BigintType.BIGINT, "1", null, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", DoubleType.DOUBLE, "2", null, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BooleanType.BOOLEAN, "3", null, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(csv.getBytes(StandardCharsets.UTF_8), null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkValue(providers, row1, ""); checkValue(providers, row2, 0); checkValue(providers, row3, 0.0d); checkValue(providers, row4, false); }
@Test public void testStreamHasData() throws Exception { MaterializedResult result = queryRunner.execute("Select count(1) from " + dummyStreamName); MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) .row(0) .build(); assertEquals(result.getRowCount(), expected.getRowCount()); int count = 500; createDummyMessages(dummyStreamName, count); result = queryRunner.execute("SELECT count(1) from " + dummyStreamName); expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) .row(count) .build(); assertEquals(result.getRowCount(), expected.getRowCount()); log.info("Completed second test (select counts)"); }
@Test public void testNonExistent() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "very/deep/varchar", null, null, false, false); KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "no_bigint", null, null, false, false); KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", DoubleType.DOUBLE, "double/is_missing", null, null, false, false); KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BooleanType.BOOLEAN, "hello", null, null, false, false); List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4); Set<KinesisFieldValueProvider> providers = new HashSet<>(); boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); assertTrue(valid); assertEquals(providers.size(), columns.size()); DecoderTestUtil.checkIsNull(providers, row1); DecoderTestUtil.checkIsNull(providers, row2); DecoderTestUtil.checkIsNull(providers, row3); DecoderTestUtil.checkIsNull(providers, row4); }
@Test public void testStringNumber() throws Exception { byte[] json = "{\"a_number\":481516,\"a_string\":\"2342\"}".getBytes(StandardCharsets.UTF_8); JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "a_number", null, null, false, false); KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "a_number", null, null, false, false); KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "a_string", null, null, false, false); KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", null, null, false, false); List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4); Set<KinesisFieldValueProvider> providers = new HashSet<>(); boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); assertTrue(valid); assertEquals(providers.size(), columns.size()); DecoderTestUtil.checkValue(providers, row1, "481516"); DecoderTestUtil.checkValue(providers, row2, 481516); DecoderTestUtil.checkValue(providers, row3, "2342"); DecoderTestUtil.checkValue(providers, row4, 2342); }
@Test public void testStreamHasData() throws Exception { MaterializedResult result = queryRunner.execute("Select count(1) from " + streamName); MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) .row(0) .build(); assertEquals(result, expected); int count = 500; createMessages(streamName, count); result = queryRunner.execute("SELECT count(1) from " + streamName); expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) .row(count) .build(); assertEquals(result, expected); }
@Override public ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName) { return new ConnectorTableMetadata( schemaTableName, ImmutableList.of( new ColumnMetadata("number", BigintType.BIGINT), new ColumnMetadata("state", createUnboundedVarcharType()), new ColumnMetadata("user", createUnboundedVarcharType()), new ColumnMetadata("title", createUnboundedVarcharType()))); }
@Override public ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName) { return new ConnectorTableMetadata( schemaTableName, ImmutableList.of( new ColumnMetadata("id", createUnboundedVarcharType()), new ColumnMetadata("text", createUnboundedVarcharType()), new ColumnMetadata("retweet_count", BigintType.BIGINT), new ColumnMetadata("user_name", createUnboundedVarcharType()), new ColumnMetadata("user_screen_name", createUnboundedVarcharType()))); }
@Test public void testManySegments() throws Exception { MaterializedResult result = queryRunner.execute("SELECT count(_message) from " + topicName); MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) .row(100000) .build(); assertEquals(result, expected); }
@Test public void testGetTableHandle() throws Exception { JmxTableHandle handle = metadata.getTableHandle(SESSION, RUNTIME_TABLE); assertEquals(handle.getConnectorId(), "test"); assertEquals(handle.getObjectName(), RUNTIME_OBJECT); List<JmxColumnHandle> columns = handle.getColumns(); assertTrue(columns.contains(new JmxColumnHandle("test", "node", VARCHAR))); assertTrue(columns.contains(new JmxColumnHandle("test", "Name", VARCHAR))); assertTrue(columns.contains(new JmxColumnHandle("test", "StartTime", BigintType.BIGINT))); }
private List<String> toConjuncts(List<JdbcColumnHandle> columns, TupleDomain<ColumnHandle> tupleDomain) { ImmutableList.Builder<String> builder = ImmutableList.builder(); for (JdbcColumnHandle column : columns) { Type type = column.getColumnType(); if (type.equals(BigintType.BIGINT) || type.equals(DoubleType.DOUBLE) || type.equals(BooleanType.BOOLEAN)) { Domain domain = tupleDomain.getDomains().get().get(column); if (domain != null) { builder.add(toPredicate(column.getColumnName(), domain)); } } } return builder.build(); }
@Override public long getLong(int field) { checkState(!closed, "cursor is closed"); try { Type type = getType(field); if (type.equals(BigintType.BIGINT)) { return resultSet.getLong(field + 1); } if (type.equals(DateType.DATE)) { // JDBC returns a date using a timestamp at midnight in the JVM timezone long localMillis = resultSet.getDate(field + 1).getTime(); // Convert it to a midnight in UTC long utcMillis = ISOChronology.getInstance().getZone().getMillisKeepLocal(UTC, localMillis); // convert to days return TimeUnit.MILLISECONDS.toDays(utcMillis); } if (type.equals(TimeType.TIME)) { Time time = resultSet.getTime(field + 1); return UTC_CHRONOLOGY.millisOfDay().get(time.getTime()); } if (type.equals(TimestampType.TIMESTAMP)) { Timestamp timestamp = resultSet.getTimestamp(field + 1); return timestamp.getTime(); } throw new PrestoException(INTERNAL_ERROR, "Unhandled type for long: " + type.getTypeSignature()); } catch (SQLException e) { throw handleSqlException(e); } }
@Override public InternalAggregationFunction specialize(Map<String, Type> types, int arity, TypeManager typeManager, FunctionRegistry functionRegistry) { Type keyType = types.get("K"); Type valueType = BigintType.BIGINT; return generateAggregation(keyType, valueType); }
private void generateGetCurrentJoinPosition(ClassDefinition classDefinition, CallSiteBinder callSiteBinder, FieldDefinition lookupSourceField, FieldDefinition probePageField, Optional<Integer> probeHashChannel, FieldDefinition probeHashBlockField, FieldDefinition positionField) { MethodDefinition method = classDefinition.declareMethod( a(PUBLIC), "getCurrentJoinPosition", type(long.class)); Variable thisVariable = method.getThis(); BytecodeBlock body = method.getBody() .append(new IfStatement() .condition(thisVariable.invoke("currentRowContainsNull", boolean.class)) .ifTrue(constantLong(-1).ret())); BytecodeExpression position = thisVariable.getField(positionField); BytecodeExpression page = thisVariable.getField(probePageField); BytecodeExpression probeHashBlock = thisVariable.getField(probeHashBlockField); if (probeHashChannel.isPresent()) { body.append(thisVariable.getField(lookupSourceField).invoke("getJoinPosition", long.class, position, page, constantType(callSiteBinder, BigintType.BIGINT).invoke("getLong", long.class, probeHashBlock, position) .cast(int.class))) .retLong(); } else { body.append(thisVariable.getField(lookupSourceField).invoke("getJoinPosition", long.class, position, page)).retLong(); } }
@VisibleForTesting static SwitchGenerationCase checkSwitchGenerationCase(Type type, List<RowExpression> values) { if (values.size() > 32) { // 32 is chosen because // * SET_CONTAINS performs worst when smaller than but close to power of 2 // * Benchmark shows performance of SET_CONTAINS is better at 50, but similar at 25. return SwitchGenerationCase.SET_CONTAINS; } if (!(type instanceof BigintType || type instanceof DateType)) { return SwitchGenerationCase.HASH_SWITCH; } for (RowExpression expression : values) { // For non-constant expressions, they will be added to the default case in the generated switch code. They do not affect any of // the cases other than the default one. Therefore, it's okay to skip them when choosing between DIRECT_SWITCH and HASH_SWITCH. // Same argument applies for nulls. if (!(expression instanceof ConstantExpression)) { continue; } Object constant = ((ConstantExpression) expression).getValue(); if (constant == null) { continue; } long longConstant = (Long) constant; if (longConstant < Integer.MIN_VALUE || longConstant > Integer.MAX_VALUE) { return SwitchGenerationCase.HASH_SWITCH; } } return SwitchGenerationCase.DIRECT_SWITCH; }
public List<Type> getTypes() { if (hashEnabled) { return ImmutableList.copyOf(Iterables.concat(types, ImmutableList.of(BigintType.BIGINT))); } return types; }
@Test public void testLong() throws Exception { InternalAggregationFunction longAgg = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("checksum", AGGREGATE, VARBINARY, BIGINT)); Block block = createLongsBlock(null, 1L, 2L, 100L, null, Long.MAX_VALUE, Long.MIN_VALUE); assertAggregation(longAgg, 1.0, expectedChecksum(BigintType.BIGINT, block), block); }
@Test public void testArray() throws Exception { ArrayType arrayType = new ArrayType(BigintType.BIGINT); InternalAggregationFunction stringAgg = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("checksum", AGGREGATE, VarbinaryType.VARBINARY.getTypeSignature(), arrayType.getTypeSignature())); Block block = createArrayBigintBlock(asList(null, asList(1L, 2L), asList(3L, 4L), asList(5L, 6L))); assertAggregation(stringAgg, 1.0, expectedChecksum(arrayType, block), block); }
@Test public void testUnique() throws Exception { SymbolAllocator allocator = new SymbolAllocator(); Set<Symbol> symbols = ImmutableSet.<Symbol>builder() .add(allocator.newSymbol("foo_1_0", BigintType.BIGINT)) .add(allocator.newSymbol("foo", BigintType.BIGINT)) .add(allocator.newSymbol("foo", BigintType.BIGINT)) .add(allocator.newSymbol("foo", BigintType.BIGINT)) .build(); assertEquals(symbols.size(), 4); }
private static Block featuresHelper(double... features) { BlockBuilder blockBuilder = new VariableWidthBlockBuilder(new BlockBuilderStatus(), features.length, 8 + 8); for (int i = 0; i < features.length; i++) { BigintType.BIGINT.writeLong(blockBuilder, i); DoubleType.DOUBLE.writeDouble(blockBuilder, features[i]); } return blockBuilder.build(); }
@Test public void testLearn() throws Exception { Type mapType = typeManager.getParameterizedType("map", ImmutableList.of(parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.DOUBLE)), ImmutableList.of()); InternalAggregationFunction aggregation = new AggregationCompiler(typeManager).generateAggregationFunction(LearnClassifierAggregation.class, ClassifierType.BIGINT_CLASSIFIER, ImmutableList.of(BigintType.BIGINT, mapType)); assertLearnClassifer(aggregation.bind(ImmutableList.of(0, 1), Optional.empty(), Optional.empty(), 1.0).createAccumulator()); }
@Test public void testLearnLibSvm() throws Exception { Type mapType = typeManager.getParameterizedType("map", ImmutableList.of(parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.DOUBLE)), ImmutableList.of()); InternalAggregationFunction aggregation = new AggregationCompiler(typeManager).generateAggregationFunction(LearnLibSvmClassifierAggregation.class, ClassifierType.BIGINT_CLASSIFIER, ImmutableList.of(BigintType.BIGINT, mapType, VarcharType.VARCHAR)); assertLearnClassifer(aggregation.bind(ImmutableList.of(0, 1, 2), Optional.empty(), Optional.empty(), 1.0).createAccumulator()); }
private static Page getPage() throws JsonProcessingException { Type mapType = typeManager.getParameterizedType("map", ImmutableList.of(parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.DOUBLE)), ImmutableList.of()); int datapoints = 100; RowPageBuilder builder = RowPageBuilder.rowPageBuilder(BigintType.BIGINT, mapType, VarcharType.VARCHAR); Random rand = new Random(0); for (int i = 0; i < datapoints; i++) { long label = rand.nextDouble() < 0.5 ? 0 : 1; builder.row(label, mapSliceOf(BigintType.BIGINT, DoubleType.DOUBLE, 0, label + rand.nextGaussian()), "C=1"); } return builder.build(); }
public static ObjectInspector getRowColumnInspector(Type type) { if (type.equals(BooleanType.BOOLEAN)) { return writableBooleanObjectInspector; } if (type.equals(BigintType.BIGINT)) { return writableLongObjectInspector; } if (type.equals(DoubleType.DOUBLE)) { return writableDoubleObjectInspector; } if (type.equals(VarcharType.VARCHAR)) { return writableStringObjectInspector; } if (type.equals(VarbinaryType.VARBINARY)) { return writableBinaryObjectInspector; } if (type.equals(DateType.DATE)) { return writableDateObjectInspector; } if (type.equals(TimestampType.TIMESTAMP)) { return writableTimestampObjectInspector; } if (isArrayType(type) || isMapType(type) || isRowType(type)) { return getJavaObjectInspector(type); } throw new IllegalArgumentException("unsupported type: " + type); }
@Test public void testNullValues() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", ISO8601JsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", ISO8601JsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", ISO8601JsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row6 = new DecoderTestColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", ISO8601JsonFieldDecoder.NAME, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); // sanity checks checkIsNull(providers, row1); checkIsNull(providers, row2); checkIsNull(providers, row3); checkIsNull(providers, row4); checkIsNull(providers, row5); checkIsNull(providers, row6); }
@Test public void testSimple() throws Exception { byte[] json = ByteStreams.toByteArray(TestJsonDecoder.class.getResourceAsStream("/decoder/json/message.json")); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", VarcharType.VARCHAR, "source", null, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "user/screen_name", null, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", BigintType.BIGINT, "id", null, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "user/statuses_count", null, null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", BooleanType.BOOLEAN, "user/geo_enabled", null, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkValue(providers, row1, "<a href=\"http://twitterfeed.com\" rel=\"nofollow\">twitterfeed</a>"); checkValue(providers, row2, "EKentuckyNews"); checkValue(providers, row3, 493857959588286460L); checkValue(providers, row4, 7630); checkValue(providers, row5, true); }
@Test public void testNullValues() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", SecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", SecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", SecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row6 = new DecoderTestColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", SecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); // sanity checks checkIsNull(providers, row1); checkIsNull(providers, row2); checkIsNull(providers, row3); checkIsNull(providers, row4); checkIsNull(providers, row5); checkIsNull(providers, row6); }
@Test public void testNullValues() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", MillisecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", MillisecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", MillisecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row6 = new DecoderTestColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", MillisecondsSinceEpochJsonFieldDecoder.NAME, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); // sanity checks checkIsNull(providers, row1); checkIsNull(providers, row2); checkIsNull(providers, row3); checkIsNull(providers, row4); checkIsNull(providers, row5); checkIsNull(providers, row6); }
@Test public void testNullValues() throws Exception { byte[] json = "{}".getBytes(StandardCharsets.UTF_8); JsonRowDecoder rowDecoder = new JsonRowDecoder(PROVIDER.get()); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", RFC2822JsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", RFC2822JsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", RFC2822JsonFieldDecoder.NAME, null, false, false, false); DecoderTestColumnHandle row6 = new DecoderTestColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", RFC2822JsonFieldDecoder.NAME, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(json, null, providers, columns, map(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); // sanity checks checkIsNull(providers, row1); checkIsNull(providers, row2); checkIsNull(providers, row3); checkIsNull(providers, row4); checkIsNull(providers, row5); checkIsNull(providers, row6); }
@Test public void testSimple() { ByteBuffer buf = ByteBuffer.allocate(100); buf.putLong(4815162342L); // 0 - 7 buf.putInt(12345678); // 8 - 11 buf.putShort((short) 4567); // 12 - 13 buf.put((byte) 123); // 14 buf.put("Ich bin zwei Oeltanks".getBytes(StandardCharsets.UTF_8)); // 15+ byte[] row = new byte[buf.position()]; System.arraycopy(buf.array(), 0, row, 0, buf.position()); RawRowDecoder rowDecoder = new RawRowDecoder(); DecoderTestColumnHandle row1 = new DecoderTestColumnHandle("", 0, "row1", BigintType.BIGINT, "0", "LONG", null, false, false, false); DecoderTestColumnHandle row2 = new DecoderTestColumnHandle("", 1, "row2", BigintType.BIGINT, "8", "INT", null, false, false, false); DecoderTestColumnHandle row3 = new DecoderTestColumnHandle("", 2, "row3", BigintType.BIGINT, "12", "SHORT", null, false, false, false); DecoderTestColumnHandle row4 = new DecoderTestColumnHandle("", 3, "row4", BigintType.BIGINT, "14", "BYTE", null, false, false, false); DecoderTestColumnHandle row5 = new DecoderTestColumnHandle("", 4, "row5", VarcharType.VARCHAR, "15", null, null, false, false, false); List<DecoderColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5); Set<FieldValueProvider> providers = new HashSet<>(); boolean corrupt = rowDecoder.decodeRow(row, null, providers, columns, buildMap(columns)); assertFalse(corrupt); assertEquals(providers.size(), columns.size()); checkValue(providers, row1, 4815162342L); checkValue(providers, row2, 12345678); checkValue(providers, row3, 4567); checkValue(providers, row4, 123); checkValue(providers, row5, "Ich bin zwei Oeltanks"); }