@Test public void testCompactDictionaryBlocks() throws Exception { Slice[] expectedValues = createExpectedValues(10); BlockBuilder blockBuilder = BIGINT.createBlockBuilder(new BlockBuilderStatus(), expectedValues.length); for (Slice expectedValue : expectedValues) { BIGINT.writeLong(blockBuilder, expectedValue.length()); } Block lengthsDictionary = blockBuilder.build(); // Create 2 dictionary blocks with the same source id DictionaryId commonSourceId = randomDictionaryId(); DictionaryBlock commonSourceIdBlock1 = createDictionaryBlock(expectedValues, 100, commonSourceId); DictionaryBlock commonSourceIdBlock2 = new DictionaryBlock(commonSourceIdBlock1.getPositionCount(), lengthsDictionary, commonSourceIdBlock1.getIds(), commonSourceId); // Create block with a different source id DictionaryBlock randomSourceIdBlock = createDictionaryBlock(expectedValues, 100, randomDictionaryId()); Page page = new Page(commonSourceIdBlock1, randomSourceIdBlock, commonSourceIdBlock2); page.compact(); // Blocks that had the same source id before compacting page should have the same source id after compacting page assertNotEquals(((DictionaryBlock) page.getBlock(0)).getDictionarySourceId(), ((DictionaryBlock) page.getBlock(1)).getDictionarySourceId()); assertEquals(((DictionaryBlock) page.getBlock(0)).getDictionarySourceId(), ((DictionaryBlock) page.getBlock(2)).getDictionarySourceId()); }
public static Block createTestBlock() { BlockBuilder blockBuilder = INTERVAL_DAY_TIME.createBlockBuilder(new BlockBuilderStatus(), 15); INTERVAL_DAY_TIME.writeLong(blockBuilder, 1111); INTERVAL_DAY_TIME.writeLong(blockBuilder, 1111); INTERVAL_DAY_TIME.writeLong(blockBuilder, 1111); INTERVAL_DAY_TIME.writeLong(blockBuilder, 2222); INTERVAL_DAY_TIME.writeLong(blockBuilder, 2222); INTERVAL_DAY_TIME.writeLong(blockBuilder, 2222); INTERVAL_DAY_TIME.writeLong(blockBuilder, 2222); INTERVAL_DAY_TIME.writeLong(blockBuilder, 2222); INTERVAL_DAY_TIME.writeLong(blockBuilder, 3333); INTERVAL_DAY_TIME.writeLong(blockBuilder, 3333); INTERVAL_DAY_TIME.writeLong(blockBuilder, 4444); return blockBuilder.build(); }
@Test public void testPrimitiveLongSerialization() { StateCompiler compiler = new StateCompiler(); AccumulatorStateFactory<LongState> factory = compiler.generateStateFactory(LongState.class); AccumulatorStateSerializer<LongState> serializer = compiler.generateStateSerializer(LongState.class); LongState state = factory.createSingleState(); LongState deserializedState = factory.createSingleState(); state.setLong(2); BlockBuilder builder = BIGINT.createBlockBuilder(new BlockBuilderStatus(), 1); serializer.serialize(state, builder); Block block = builder.build(); assertEquals(BIGINT.getLong(block, 0), state.getLong()); serializer.deserialize(block, 0, deserializedState); assertEquals(deserializedState.getLong(), state.getLong()); }
@Test public void testVarianceStateSerialization() { StateCompiler compiler = new StateCompiler(); AccumulatorStateFactory<VarianceState> factory = compiler.generateStateFactory(VarianceState.class); AccumulatorStateSerializer<VarianceState> serializer = compiler.generateStateSerializer(VarianceState.class); VarianceState singleState = factory.createSingleState(); VarianceState deserializedState = factory.createSingleState(); singleState.setMean(1); singleState.setCount(2); singleState.setM2(3); BlockBuilder builder = VarcharType.VARCHAR.createBlockBuilder(new BlockBuilderStatus(), 1); serializer.serialize(singleState, builder); Block block = builder.build(); serializer.deserialize(block, 0, deserializedState); assertEquals(deserializedState.getCount(), singleState.getCount()); assertEquals(deserializedState.getMean(), singleState.getMean()); assertEquals(deserializedState.getM2(), singleState.getM2()); }
private static Block createChannel(int positionCount, int arraySize, ArrayType arrayType) { BlockBuilder blockBuilder = arrayType.createBlockBuilder(new BlockBuilderStatus(), positionCount); for (int position = 0; position < positionCount; position++) { BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); for (int i = 0; i < arraySize; i++) { if (arrayType.getElementType().getJavaType() == long.class) { arrayType.getElementType().writeLong(entryBuilder, ThreadLocalRandom.current().nextLong()); } else if (arrayType.getElementType().equals(VARCHAR)) { arrayType.getElementType().writeSlice(entryBuilder, Slices.utf8Slice("test_string")); } else { throw new UnsupportedOperationException(); } } blockBuilder.closeEntry(); } return blockBuilder.build(); }
@Test public void testPrimitiveBooleanSerialization() { StateCompiler compiler = new StateCompiler(); AccumulatorStateFactory<BooleanState> factory = compiler.generateStateFactory(BooleanState.class); AccumulatorStateSerializer<BooleanState> serializer = compiler.generateStateSerializer(BooleanState.class); BooleanState state = factory.createSingleState(); BooleanState deserializedState = factory.createSingleState(); state.setBoolean(true); BlockBuilder builder = BooleanType.BOOLEAN.createBlockBuilder(new BlockBuilderStatus(), 1); serializer.serialize(state, builder); Block block = builder.build(); serializer.deserialize(block, 0, deserializedState); assertEquals(deserializedState.getBoolean(), state.getBoolean()); }
public static Block createStringArraysBlock(Iterable<? extends Iterable<String>> values) { ArrayType arrayType = new ArrayType(VARCHAR); BlockBuilder builder = arrayType.createBlockBuilder(new BlockBuilderStatus(), 100); for (Iterable<String> value : values) { if (value == null) { builder.appendNull(); } else { arrayType.writeObject(builder, createStringsBlock(value)); } } return builder.build(); }
public static void output(ArrayType outputType, MinMaxByNState state, BlockBuilder out) { TypedKeyValueHeap heap = state.getTypedKeyValueHeap(); if (heap == null || heap.isEmpty()) { out.appendNull(); return; } Type elementType = outputType.getElementType(); BlockBuilder arrayBlockBuilder = out.beginBlockEntry(); BlockBuilder reversedBlockBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), heap.getCapacity()); long startSize = heap.getEstimatedSize(); heap.popAll(reversedBlockBuilder); state.addMemoryUsage(heap.getEstimatedSize() - startSize); for (int i = reversedBlockBuilder.getPositionCount() - 1; i >= 0; i--) { elementType.appendTo(reversedBlockBuilder, i, arrayBlockBuilder); } out.closeEntry(); }
public TypedSet(Type elementType, int expectedSize) { checkArgument(expectedSize > 0, "expectedSize must be > 0"); this.elementType = requireNonNull(elementType, "elementType must not be null"); this.elementBlock = elementType.createBlockBuilder(new BlockBuilderStatus(), expectedSize); int hashSize = arraySize(expectedSize, FILL_RATIO); this.maxFill = calculateMaxFill(hashSize); this.hashMask = hashSize - 1; blockPositionByHash.ensureCapacity(hashSize); for (int i = 0; i < hashSize; i++) { blockPositionByHash.set(i, EMPTY_SLOT); } this.containsNullElement = false; }
public static Block createTestBlock() { BlockBuilder blockBuilder = INTERVAL_YEAR_MONTH.createBlockBuilder(new BlockBuilderStatus(), 15); INTERVAL_YEAR_MONTH.writeLong(blockBuilder, 1111); INTERVAL_YEAR_MONTH.writeLong(blockBuilder, 1111); INTERVAL_YEAR_MONTH.writeLong(blockBuilder, 1111); INTERVAL_YEAR_MONTH.writeLong(blockBuilder, 2222); INTERVAL_YEAR_MONTH.writeLong(blockBuilder, 2222); INTERVAL_YEAR_MONTH.writeLong(blockBuilder, 2222); INTERVAL_YEAR_MONTH.writeLong(blockBuilder, 2222); INTERVAL_YEAR_MONTH.writeLong(blockBuilder, 2222); INTERVAL_YEAR_MONTH.writeLong(blockBuilder, 3333); INTERVAL_YEAR_MONTH.writeLong(blockBuilder, 3333); INTERVAL_YEAR_MONTH.writeLong(blockBuilder, 4444); return blockBuilder.build(); }
public Block markDistinctRows(Page page) { GroupByIdBlock ids = groupByHash.getGroupIds(page); BlockBuilder blockBuilder = BOOLEAN.createBlockBuilder(new BlockBuilderStatus(), ids.getPositionCount()); for (int i = 0; i < ids.getPositionCount(); i++) { if (ids.getGroupId(i) == nextDistinctId) { BOOLEAN.writeBoolean(blockBuilder, true); nextDistinctId++; } else { BOOLEAN.writeBoolean(blockBuilder, false); } } return blockBuilder.build(); }
@Test public void testEvaluateClassifierPredictions() throws Exception { TypeRegistry typeRegistry = new TypeRegistry(); typeRegistry.addParametricType(new ClassifierParametricType()); typeRegistry.addType(RegressorType.REGRESSOR); typeRegistry.addType(ModelType.MODEL); metadata.addFunctions(new MLFunctionFactory(typeRegistry).listFunctions()); InternalAggregationFunction aggregation = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("evaluate_classifier_predictions", AGGREGATE, StandardTypes.VARCHAR, StandardTypes.BIGINT, StandardTypes.BIGINT)); Accumulator accumulator = aggregation.bind(ImmutableList.of(0, 1), Optional.empty(), Optional.empty(), 1.0).createAccumulator(); accumulator.addInput(getPage()); BlockBuilder finalOut = accumulator.getFinalType().createBlockBuilder(new BlockBuilderStatus(), 1); accumulator.evaluateFinal(finalOut); Block block = finalOut.build(); String output = VARCHAR.getSlice(block, 0).toStringUtf8(); List<String> parts = ImmutableList.copyOf(Splitter.on('\n').omitEmptyStrings().split(output)); assertEquals(parts.size(), 7, output); assertEquals(parts.get(0), "Accuracy: 1/2 (50.00%)"); }
public static Block mapConcat(Type keyType, Type valueType, Block leftMap, Block rightMap) { TypedSet typedSet = new TypedSet(keyType, rightMap.getPositionCount()); BlockBuilder blockBuilder = new InterleavedBlockBuilder(ImmutableList.of(keyType, valueType), new BlockBuilderStatus(), leftMap.getPositionCount() + rightMap.getPositionCount()); for (int i = 0; i < rightMap.getPositionCount(); i += 2) { typedSet.add(rightMap, i); keyType.appendTo(rightMap, i, blockBuilder); valueType.appendTo(rightMap, i + 1, blockBuilder); } for (int i = 0; i < leftMap.getPositionCount(); i += 2) { if (!typedSet.contains(leftMap, i)) { keyType.appendTo(leftMap, i, blockBuilder); valueType.appendTo(leftMap, i + 1, blockBuilder); } } return blockBuilder.build(); }
public static Block createTestBlock() { BlockBuilder blockBuilder = BOOLEAN.createBlockBuilder(new BlockBuilderStatus(), 15); BOOLEAN.writeBoolean(blockBuilder, true); BOOLEAN.writeBoolean(blockBuilder, true); BOOLEAN.writeBoolean(blockBuilder, true); BOOLEAN.writeBoolean(blockBuilder, false); BOOLEAN.writeBoolean(blockBuilder, false); BOOLEAN.writeBoolean(blockBuilder, false); BOOLEAN.writeBoolean(blockBuilder, false); BOOLEAN.writeBoolean(blockBuilder, false); BOOLEAN.writeBoolean(blockBuilder, true); BOOLEAN.writeBoolean(blockBuilder, true); BOOLEAN.writeBoolean(blockBuilder, false); return blockBuilder.build(); }
@Override public void start() { if (builder == null) { if (nullBuilder == null || (nullBuilder.getPositionCount() >= NULL_BUILDER_POSITIONS_THRESHOLD && nullBuilder.getSizeInBytes() >= NULL_BUILDER_SIZE_IN_BYTES_THRESHOLD)) { nullBuilder = rowType.createBlockBuilder(new BlockBuilderStatus(), NULL_BUILDER_POSITIONS_THRESHOLD); } currentEntryBuilder = nullBuilder.beginBlockEntry(); } else { while (builder.getPositionCount() < fieldIndex) { builder.appendNull(); } currentEntryBuilder = builder.beginBlockEntry(); } for (BlockConverter converter : converters) { converter.beforeValue(currentEntryBuilder); } }
public static Block createTestBlock() { BlockBuilder blockBuilder = TIMESTAMP_WITH_TIME_ZONE.createBlockBuilder(new BlockBuilderStatus(), 15); TIMESTAMP_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(1111, getTimeZoneKeyForOffset(0))); TIMESTAMP_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(1111, getTimeZoneKeyForOffset(1))); TIMESTAMP_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(1111, getTimeZoneKeyForOffset(2))); TIMESTAMP_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(2222, getTimeZoneKeyForOffset(3))); TIMESTAMP_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(2222, getTimeZoneKeyForOffset(4))); TIMESTAMP_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(2222, getTimeZoneKeyForOffset(5))); TIMESTAMP_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(2222, getTimeZoneKeyForOffset(6))); TIMESTAMP_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(2222, getTimeZoneKeyForOffset(7))); TIMESTAMP_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(3333, getTimeZoneKeyForOffset(8))); TIMESTAMP_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(3333, getTimeZoneKeyForOffset(9))); TIMESTAMP_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(4444, getTimeZoneKeyForOffset(10))); return blockBuilder.build(); }
public static Block createArrayBigintBlock(Iterable<? extends Iterable<Long>> values) { ArrayType arrayType = new ArrayType(BIGINT); BlockBuilder builder = arrayType.createBlockBuilder(new BlockBuilderStatus(), 100); for (Iterable<Long> value : values) { if (value == null) { builder.appendNull(); } else { arrayType.writeObject(builder, createLongsBlock(value)); } } return builder.build(); }
public static Block createTestBlock() { BlockBuilder blockBuilder = DOUBLE.createBlockBuilder(new BlockBuilderStatus(), 15); DOUBLE.writeDouble(blockBuilder, 11.11); DOUBLE.writeDouble(blockBuilder, 11.11); DOUBLE.writeDouble(blockBuilder, 11.11); DOUBLE.writeDouble(blockBuilder, 22.22); DOUBLE.writeDouble(blockBuilder, 22.22); DOUBLE.writeDouble(blockBuilder, 22.22); DOUBLE.writeDouble(blockBuilder, 22.22); DOUBLE.writeDouble(blockBuilder, 22.22); DOUBLE.writeDouble(blockBuilder, 33.33); DOUBLE.writeDouble(blockBuilder, 33.33); DOUBLE.writeDouble(blockBuilder, 44.44); return blockBuilder.build(); }
public static Block createTestBlock() { BlockBuilder blockBuilder = COLOR.createBlockBuilder(new BlockBuilderStatus(), 15); COLOR.writeLong(blockBuilder, rgb(1, 1, 1)); COLOR.writeLong(blockBuilder, rgb(1, 1, 1)); COLOR.writeLong(blockBuilder, rgb(1, 1, 1)); COLOR.writeLong(blockBuilder, rgb(2, 2, 2)); COLOR.writeLong(blockBuilder, rgb(2, 2, 2)); COLOR.writeLong(blockBuilder, rgb(2, 2, 2)); COLOR.writeLong(blockBuilder, rgb(2, 2, 2)); COLOR.writeLong(blockBuilder, rgb(2, 2, 2)); COLOR.writeLong(blockBuilder, rgb(3, 3, 3)); COLOR.writeLong(blockBuilder, rgb(3, 3, 3)); COLOR.writeLong(blockBuilder, rgb(4, 4, 4)); return blockBuilder.build(); }
@Test public void testBigintSerializedSize() { BlockBuilder builder = BIGINT.createBlockBuilder(new BlockBuilderStatus(), 5); // empty page Page page = new Page(builder.build()); int pageSize = serializedSize(ImmutableList.of(BIGINT), page); assertEquals(pageSize, 35); // page overhead // page with one value BIGINT.writeLong(builder, 123); page = new Page(builder.build()); int firstValueSize = serializedSize(ImmutableList.of(BIGINT), page) - pageSize; assertEquals(firstValueSize, 9); // value size + value overhead // page with two values BIGINT.writeLong(builder, 456); page = new Page(builder.build()); int secondValueSize = serializedSize(ImmutableList.of(BIGINT), page) - (pageSize + firstValueSize); assertEquals(secondValueSize, 8); // value size (value overhead is shared with previous value) }
public static Block createTestBlock() { BlockBuilder blockBuilder = TIME_WITH_TIME_ZONE.createBlockBuilder(new BlockBuilderStatus(), 15); TIME_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(1111, getTimeZoneKeyForOffset(0))); TIME_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(1111, getTimeZoneKeyForOffset(1))); TIME_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(1111, getTimeZoneKeyForOffset(2))); TIME_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(2222, getTimeZoneKeyForOffset(3))); TIME_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(2222, getTimeZoneKeyForOffset(4))); TIME_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(2222, getTimeZoneKeyForOffset(5))); TIME_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(2222, getTimeZoneKeyForOffset(6))); TIME_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(2222, getTimeZoneKeyForOffset(7))); TIME_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(3333, getTimeZoneKeyForOffset(8))); TIME_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(3333, getTimeZoneKeyForOffset(9))); TIME_WITH_TIME_ZONE.writeLong(blockBuilder, packDateTimeWithZone(4444, getTimeZoneKeyForOffset(10))); return blockBuilder.build(); }
private static Object mapKeyToObject(ConnectorSession session, String jsonKey, Type type) { BlockBuilder blockBuilder; if (type instanceof FixedWidthType) { blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), 1); } else { blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), 1, jsonKey.length()); } if (type.getJavaType() == boolean.class) { type.writeBoolean(blockBuilder, Boolean.parseBoolean(jsonKey)); } else if (type.getJavaType() == long.class) { type.writeLong(blockBuilder, Long.parseLong(jsonKey)); } else if (type.getJavaType() == double.class) { type.writeDouble(blockBuilder, Double.parseDouble(jsonKey)); } else if (type.getJavaType() == Slice.class) { type.writeSlice(blockBuilder, Slices.utf8Slice(jsonKey)); } return type.getObjectValue(session, blockBuilder.build(), 0); }
private Block toBlock(Object value) { BlockBuilder blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), 1); Class<?> javaType = type.getJavaType(); if (value == null) { blockBuilder.appendNull(); } else if (javaType == boolean.class) { type.writeBoolean(blockBuilder, (Boolean) value); } else if (javaType == long.class) { type.writeLong(blockBuilder, (Long) value); } else if (javaType == double.class) { type.writeDouble(blockBuilder, (Double) value); } else if (javaType == Slice.class) { Slice slice = (Slice) value; type.writeSlice(blockBuilder, slice, 0, slice.length()); } else { type.writeObject(blockBuilder, value); } return blockBuilder.build(); }
@Test public void testRowHistograms() throws Exception { RowType innerRowType = new RowType(ImmutableList.of(BIGINT, DOUBLE), Optional.of(ImmutableList.of("f1", "f2"))); MapType mapType = new MapType(innerRowType, BIGINT); InternalAggregationFunction aggregationFunction = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), innerRowType.getTypeSignature().toString())); BlockBuilder builder = innerRowType.createBlockBuilder(new BlockBuilderStatus(), 3); innerRowType.writeObject(builder, testRowBigintBigint(1L, 1.0)); innerRowType.writeObject(builder, testRowBigintBigint(2L, 2.0)); innerRowType.writeObject(builder, testRowBigintBigint(3L, 3.0)); assertAggregation( aggregationFunction, 1.0, ImmutableMap.of(ImmutableList.of(1L, 1.0), 1L, ImmutableList.of(2L, 2.0), 1L, ImmutableList.of(3L, 3.0), 1L), builder.build()); }
private void test(IntStream keyInputStream, Stream<String> valueInputStream, BlockComparator comparator, Iterator<String> outputIterator) { BlockBuilder keysBlockBuilder = BIGINT.createBlockBuilder(new BlockBuilderStatus(), INPUT_SIZE); BlockBuilder valuesBlockBuilder = VARCHAR.createBlockBuilder(new BlockBuilderStatus(), INPUT_SIZE); keyInputStream.forEach(x -> BIGINT.writeLong(keysBlockBuilder, x)); valueInputStream.forEach(x -> VARCHAR.writeString(valuesBlockBuilder, x)); TypedKeyValueHeap heap = new TypedKeyValueHeap(comparator, BIGINT, VARCHAR, OUTPUT_SIZE); heap.addAll(keysBlockBuilder, valuesBlockBuilder); BlockBuilder resultBlockBuilder = VARCHAR.createBlockBuilder(new BlockBuilderStatus(), OUTPUT_SIZE); heap.popAll(resultBlockBuilder); Block resultBlock = resultBlockBuilder.build(); assertEquals(resultBlock.getPositionCount(), OUTPUT_SIZE); for (int i = 0; i < OUTPUT_SIZE; i++) { assertEquals(VARCHAR.getSlice(resultBlock, i).toStringUtf8(), outputIterator.next()); } }
private static Block serializeList(Type type, BlockBuilder builder, Object object) { List<?> list = (List) object; if (list == null) { requireNonNull(builder, "parent builder is null").appendNull(); return null; } List<Type> typeParameters = type.getTypeParameters(); checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter"); Type elementType = typeParameters.get(0); BlockBuilder currentBuilder; if (builder != null) { currentBuilder = builder.beginBlockEntry(); } else { currentBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), list.size()); } for (Object element : list) { serializeObject(elementType, currentBuilder, element); } if (builder != null) { builder.closeEntry(); return null; } else { Block resultBlock = currentBuilder.build(); return resultBlock; } }
private static Block serializeMap(Type type, BlockBuilder builder, Object object) { Map<?, ?> map = (Map) object; if (map == null) { requireNonNull(builder, "parent builder is null").appendNull(); return null; } List<Type> typeParameters = type.getTypeParameters(); checkArgument(typeParameters.size() == 2, "map must have exactly 2 type parameter"); Type keyType = typeParameters.get(0); Type valueType = typeParameters.get(1); BlockBuilder currentBuilder; if (builder != null) { currentBuilder = builder.beginBlockEntry(); } else { currentBuilder = new InterleavedBlockBuilder(typeParameters, new BlockBuilderStatus(), map.size()); } for (Map.Entry<?, ?> entry : map.entrySet()) { // Hive skips map entries with null keys if (entry.getKey() != null) { serializeObject(keyType, currentBuilder, entry.getKey()); serializeObject(valueType, currentBuilder, entry.getValue()); } } if (builder != null) { builder.closeEntry(); return null; } else { Block resultBlock = currentBuilder.build(); return resultBlock; } }
@SqlType("array(double)") public static Block geohash_decode(@SqlType(StandardTypes.VARCHAR) Slice geohash) { BlockBuilder blockBuilder = DOUBLE.createBlockBuilder(new BlockBuilderStatus(), 2); LatLong coordinates = GeoHash.decodeHash(geohash.toStringUtf8()); DOUBLE.writeDouble(blockBuilder, coordinates.getLat()); DOUBLE.writeDouble(blockBuilder, coordinates.getLon()); return blockBuilder.build(); }
private static Block toBlock(double[] poly) { FixedWidthBlockBuilder blockBuilder = new FixedWidthBlockBuilder(8, new BlockBuilderStatus(), poly.length); for (double d : poly) { blockBuilder.writeLong(Double.doubleToLongBits(d)); blockBuilder.closeEntry(); } return blockBuilder.build(); }
private Block createZeroBlock(Type type, int rowsCount, Slice constantSlice) { checkArgument(SUPPORTED_TYPES.contains(type), "Unsupported type [%s]", type); BlockBuilder builder; if (type instanceof FixedWidthType) { builder = type.createBlockBuilder(new BlockBuilderStatus(), rowsCount); } else { builder = type.createBlockBuilder(new BlockBuilderStatus(), rowsCount, constantSlice.length()); } for (int i = 0; i < rowsCount; i++) { Class<?> javaType = type.getJavaType(); if (javaType == boolean.class) { type.writeBoolean(builder, false); } else if (javaType == long.class) { type.writeLong(builder, 0); } else if (javaType == double.class) { type.writeDouble(builder, 0.0); } else if (javaType == Slice.class) { type.writeSlice(builder, constantSlice, 0, constantSlice.length()); } else { throw new UnsupportedOperationException("Unknown javaType: " + javaType.getName()); } } return builder.build(); }
public static Block arrayBlockOf(Type elementType, Object... values) { BlockBuilder blockBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), 1024); for (Object value : values) { appendToBlockBuilder(elementType, value, blockBuilder); } return blockBuilder.build(); }
public static Block mapBlockOf(Type keyType, Type valueType, Object key, Object value) { BlockBuilder blockBuilder = new InterleavedBlockBuilder(ImmutableList.of(keyType, valueType), new BlockBuilderStatus(), 1024); appendToBlockBuilder(keyType, key, blockBuilder); appendToBlockBuilder(valueType, value, blockBuilder); return blockBuilder.build(); }
public static Block rowBlockOf(List<Type> parameterTypes, Object... values) { InterleavedBlockBuilder blockBuilder = new InterleavedBlockBuilder(parameterTypes, new BlockBuilderStatus(), 1024); for (int i = 0; i < values.length; i++) { appendToBlockBuilder(parameterTypes.get(i), values[i], blockBuilder); } return blockBuilder.build(); }
private static Block serializeList(Type type, BlockBuilder builder, Object object, ListObjectInspector inspector) { List<?> list = inspector.getList(object); if (list == null) { requireNonNull(builder, "parent builder is null").appendNull(); return null; } List<Type> typeParameters = type.getTypeParameters(); checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter"); Type elementType = typeParameters.get(0); ObjectInspector elementInspector = inspector.getListElementObjectInspector(); BlockBuilder currentBuilder; if (builder != null) { currentBuilder = builder.beginBlockEntry(); } else { currentBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), list.size()); } for (Object element : list) { serializeObject(elementType, currentBuilder, element, elementInspector); } if (builder != null) { builder.closeEntry(); return null; } else { Block resultBlock = currentBuilder.build(); return resultBlock; } }
@Test public void testStackRepresentation() throws Exception { Block array = arrayBlockOf(BIGINT, 1L, 2L); Block actualBlock = mapBlockOf(DOUBLE, new ArrayType(BIGINT), ImmutableMap.of(1.0, array)); DynamicSliceOutput actualSliceOutput = new DynamicSliceOutput(100); writeBlock(actualSliceOutput, actualBlock); Block expectedBlock = new InterleavedBlockBuilder(ImmutableList.<Type>of(DOUBLE, new ArrayType(BIGINT)), new BlockBuilderStatus(), 3) .writeDouble(1.0) .closeEntry() .writeObject( BIGINT .createBlockBuilder(new BlockBuilderStatus(), 1) .writeLong(1L) .closeEntry() .writeLong(2L) .closeEntry() .build() ) .closeEntry() .build(); DynamicSliceOutput expectedSliceOutput = new DynamicSliceOutput(100); writeBlock(expectedSliceOutput, expectedBlock); assertEquals(actualSliceOutput.slice(), expectedSliceOutput.slice()); }
public static Block createTestBlock(Type arrayType) { BlockBuilder blockBuilder = arrayType.createBlockBuilder(new BlockBuilderStatus(), 4); arrayType.writeObject(blockBuilder, arrayBlockOf(BIGINT, 1, 2)); arrayType.writeObject(blockBuilder, arrayBlockOf(BIGINT, 1, 2, 3)); arrayType.writeObject(blockBuilder, arrayBlockOf(BIGINT, 1, 2, 3)); arrayType.writeObject(blockBuilder, arrayBlockOf(BIGINT, 100, 200, 300)); return blockBuilder.build(); }
@Override public Block[] getSequenceBlocks(int start, int length) { BlockBuilder blockBuilder = VARBINARY.createBlockBuilder(new BlockBuilderStatus(), length); for (int i = 0; i < length; i++) { VARBINARY.writeSlice(blockBuilder, Slices.wrappedBuffer(Ints.toByteArray(i))); } return new Block[] {blockBuilder.build()}; }
public static Block mapBlockOf(Type keyType, Type valueType, Map<?, ?> value) { BlockBuilder blockBuilder = new InterleavedBlockBuilder(ImmutableList.of(keyType, valueType), new BlockBuilderStatus(), value.size() * 2); for (Map.Entry<?, ?> entry : value.entrySet()) { appendToBlockBuilder(keyType, entry.getKey(), blockBuilder); appendToBlockBuilder(valueType, entry.getValue(), blockBuilder); } return blockBuilder.build(); }
public static void input(Type type, ArrayAggregationState state, Block value, int position) { BlockBuilder blockBuilder = state.getBlockBuilder(); if (blockBuilder == null) { blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), 100); state.setBlockBuilder(blockBuilder); } long startSize = blockBuilder.getRetainedSizeInBytes(); type.appendTo(value, position, blockBuilder); state.addMemoryUsage(blockBuilder.getRetainedSizeInBytes() - startSize); }
public KeyValuePairs(Type keyType, Type valueType, boolean isMultiValue) { this.keyType = requireNonNull(keyType, "keyType is null"); this.valueType = requireNonNull(valueType, "valueType is null"); this.keySet = new TypedSet(keyType, EXPECTED_ENTRIES); keyBlockBuilder = this.keyType.createBlockBuilder(new BlockBuilderStatus(), EXPECTED_ENTRIES, expectedValueSize(keyType, EXPECTED_ENTRY_SIZE)); valueBlockBuilder = this.valueType.createBlockBuilder(new BlockBuilderStatus(), EXPECTED_ENTRIES, expectedValueSize(valueType, EXPECTED_ENTRY_SIZE)); this.isMultiValue = isMultiValue; }