private static <F, T extends Comparable<T>> Domain createDomain(Type type, boolean hasNullValue, RangeStatistics<F> rangeStatistics, Function<F, T> function) { F min = rangeStatistics.getMin(); F max = rangeStatistics.getMax(); if (min != null && max != null) { return Domain.create(ValueSet.ofRanges(Range.range(type, function.apply(min), true, function.apply(max), true)), hasNullValue); } if (max != null) { return Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(type, function.apply(max))), hasNullValue); } if (min != null) { return Domain.create(ValueSet.ofRanges(Range.greaterThanOrEqual(type, function.apply(min))), hasNullValue); } return Domain.create(ValueSet.all(type), hasNullValue); }
@Test public void testBoolean() throws Exception { assertEquals(getDomain(BOOLEAN, 0, null), none(BOOLEAN)); assertEquals(getDomain(BOOLEAN, 10, null), all(BOOLEAN)); assertEquals(getDomain(BOOLEAN, 0, booleanColumnStats(null, null)), none(BOOLEAN)); assertEquals(getDomain(BOOLEAN, 0, booleanColumnStats(0L, null)), none(BOOLEAN)); assertEquals(getDomain(BOOLEAN, 0, booleanColumnStats(0L, 0L)), none(BOOLEAN)); assertEquals(getDomain(BOOLEAN, 10, booleanColumnStats(0L, 0L)), onlyNull(BOOLEAN)); assertEquals(getDomain(BOOLEAN, 10, booleanColumnStats(10L, null)), notNull(BOOLEAN)); assertEquals(getDomain(BOOLEAN, 10, booleanColumnStats(10L, 10L)), singleValue(BOOLEAN, true)); assertEquals(getDomain(BOOLEAN, 10, booleanColumnStats(10L, 0L)), singleValue(BOOLEAN, false)); assertEquals(getDomain(BOOLEAN, 20, booleanColumnStats(10L, 5L)), all(BOOLEAN)); assertEquals(getDomain(BOOLEAN, 20, booleanColumnStats(10L, 10L)), create(ValueSet.ofRanges(Range.equal(BOOLEAN, true)), true)); assertEquals(getDomain(BOOLEAN, 20, booleanColumnStats(10L, 0L)), create(ValueSet.ofRanges(Range.equal(BOOLEAN, false)), true)); }
private static Domain extractOrderableDomain(ComparisonExpression.Type comparisonType, Type type, Object value, boolean complement) { checkArgument(value != null); switch (comparisonType) { case EQUAL: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.equal(type, value)), complement), false); case GREATER_THAN: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.greaterThan(type, value)), complement), false); case GREATER_THAN_OR_EQUAL: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.greaterThanOrEqual(type, value)), complement), false); case LESS_THAN: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.lessThan(type, value)), complement), false); case LESS_THAN_OR_EQUAL: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.lessThanOrEqual(type, value)), complement), false); case NOT_EQUAL: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.lessThan(type, value), Range.greaterThan(type, value)), complement), false); case IS_DISTINCT_FROM: // Need to potential complement the whole domain for IS_DISTINCT_FROM since it is null-aware return complementIfNecessary(Domain.create(ValueSet.ofRanges(Range.lessThan(type, value), Range.greaterThan(type, value)), true), complement); default: throw new AssertionError("Unhandled type: " + comparisonType); } }
@Test public void testRoundTrip() throws Exception { TupleDomain<Symbol> tupleDomain = withColumnDomains(ImmutableMap.<Symbol, Domain>builder() .put(A, Domain.singleValue(BIGINT, 1L)) .put(B, Domain.onlyNull(DOUBLE)) .put(C, Domain.notNull(VARCHAR)) .put(D, Domain.singleValue(BOOLEAN, true)) .put(E, Domain.singleValue(BIGINT, 2L)) .put(F, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(DOUBLE, 1.1), Range.equal(DOUBLE, 2.0), Range.range(DOUBLE, 3.0, false, 3.5, true)), true)) .put(G, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(VARCHAR, utf8Slice("2013-01-01")), Range.greaterThan(VARCHAR, utf8Slice("2013-10-01"))), false)) .put(H, Domain.singleValue(TIMESTAMP, TIMESTAMP_VALUE)) .put(I, Domain.singleValue(DATE, DATE_VALUE)) .put(J, Domain.singleValue(COLOR, COLOR_VALUE_1)) .put(K, Domain.notNull(HYPER_LOG_LOG)) .build()); ExtractionResult result = fromPredicate(toPredicate(tupleDomain)); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), tupleDomain); }
@Test public void testFromAndPredicate() throws Exception { Expression originalPredicate = and( and(greaterThan(A, longLiteral(1L)), unprocessableExpression1(A)), and(lessThan(A, longLiteral(5L)), unprocessableExpression2(A))); ExtractionResult result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), and(unprocessableExpression1(A), unprocessableExpression2(A))); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.range(BIGINT, 1L, false, 5L, false)), false)))); // Test complements originalPredicate = not(and( and(greaterThan(A, longLiteral(1L)), unprocessableExpression1(A)), and(lessThan(A, longLiteral(5L)), unprocessableExpression2(A)))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), originalPredicate); assertTrue(result.getTupleDomain().isAll()); originalPredicate = not(and( not(and(greaterThan(A, longLiteral(1L)), unprocessableExpression1(A))), not(and(lessThan(A, longLiteral(5L)), unprocessableExpression2(A))))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), originalPredicate); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.notNull(BIGINT)))); }
@Test public void testFromNotPredicate() throws Exception { Expression originalPredicate = not(and(equal(A, longLiteral(1L)), unprocessableExpression1(A))); ExtractionResult result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), originalPredicate); assertTrue(result.getTupleDomain().isAll()); originalPredicate = not(unprocessableExpression1(A)); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), originalPredicate); assertTrue(result.getTupleDomain().isAll()); originalPredicate = not(TRUE_LITERAL); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertTrue(result.getTupleDomain().isNone()); originalPredicate = not(equal(A, longLiteral(1L))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 1L), Range.greaterThan(BIGINT, 1L)), false)))); }
private static <F, T extends Comparable<T>> Domain createDomain(Type type, boolean hasNullValue, ParquetRangeStatistics<F> rangeStatistics, Function<F, T> function) { F min = rangeStatistics.getMin(); F max = rangeStatistics.getMax(); if (min != null && max != null) { return Domain.create(ValueSet.ofRanges(Range.range(type, function.apply(min), true, function.apply(max), true)), hasNullValue); } if (max != null) { return Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(type, function.apply(max))), hasNullValue); } if (min != null) { return Domain.create(ValueSet.ofRanges(Range.greaterThanOrEqual(type, function.apply(min))), hasNullValue); } return Domain.create(ValueSet.all(type), hasNullValue); }
private static boolean isEqualRange(TupleDomain<Symbol> tupleDomain, Symbol symbol, long upperBound) { if (tupleDomain.isNone()) { return false; } Domain domain = tupleDomain.getDomains().get().get(symbol); return domain.getValues().equals(ValueSet.ofRanges(Range.lessThanOrEqual(domain.getType(), upperBound))); }
private static OptionalInt extractUpperBound(TupleDomain<Symbol> tupleDomain, Symbol symbol) { if (tupleDomain.isNone()) { return OptionalInt.empty(); } Domain rowNumberDomain = tupleDomain.getDomains().get().get(symbol); if (rowNumberDomain == null) { return OptionalInt.empty(); } ValueSet values = rowNumberDomain.getValues(); if (values.isAll() || values.isNone() || values.getRanges().getRangeCount() <= 0) { return OptionalInt.empty(); } Range span = values.getRanges().getSpan(); if (span.getHigh().isUpperUnbounded()) { return OptionalInt.empty(); } verify(rowNumberDomain.getType().equals(BIGINT)); long upperBound = (Long) span.getHigh().getValue(); if (span.getHigh().getBound() == BELOW) { upperBound--; } if (upperBound > Integer.MAX_VALUE) { return OptionalInt.empty(); } return OptionalInt.of(Ints.checkedCast(upperBound)); }
@Test public void testFromBetweenPredicate() throws Exception { Expression originalExpression = between(A, longLiteral(1L), longLiteral(2L)); ExtractionResult result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.range(BIGINT, 1L, true, 2L, true)), false)))); originalExpression = between(A, longLiteral(1L), doubleLiteral(2.1)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.range(BIGINT, 1L, true, 2L, true)), false)))); originalExpression = between(A, longLiteral(1L), nullLiteral()); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertTrue(result.getTupleDomain().isNone()); // Test complements originalExpression = not(between(A, longLiteral(1L), longLiteral(2L))); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 1L), Range.greaterThan(BIGINT, 2L)), false)))); originalExpression = not(between(A, longLiteral(1L), doubleLiteral(2.1))); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 1L), Range.greaterThan(BIGINT, 2L)), false)))); originalExpression = not(between(A, longLiteral(1L), nullLiteral())); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 1L)), false)))); }
@Test public void testExpressionConstantFolding() throws Exception { Expression originalExpression = comparison(GREATER_THAN, reference(L), function("from_hex", stringLiteral("123456"))); ExtractionResult result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); Slice value = Slices.wrappedBuffer(BaseEncoding.base16().decode("123456")); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(L, Domain.create(ValueSet.ofRanges(Range.greaterThan(VARBINARY, value)), false)))); Expression expression = toPredicate(result.getTupleDomain()); assertEquals(expression, comparison(GREATER_THAN, reference(L), varbinaryLiteral(value))); }
public static ShardPredicate create(TupleDomain<RaptorColumnHandle> tupleDomain) { StringJoiner predicate = new StringJoiner(" AND ").setEmptyValue("true"); ImmutableList.Builder<JDBCType> types = ImmutableList.builder(); ImmutableList.Builder<Object> values = ImmutableList.builder(); for (Entry<RaptorColumnHandle, Domain> entry : tupleDomain.getDomains().get().entrySet()) { Domain domain = entry.getValue(); if (domain.isNullAllowed() || domain.isAll()) { continue; } RaptorColumnHandle handle = entry.getKey(); Type type = handle.getColumnType(); JDBCType jdbcType = jdbcType(type); if (jdbcType == null) { continue; } if (handle.isShardUuid()) { // TODO: support multiple shard UUIDs if (domain.isSingleValue()) { predicate.add("shard_uuid = ?"); types.add(jdbcType(type)); Slice uuidSlice = checkType(entry.getValue().getSingleValue(), Slice.class, "value"); values.add(uuidStringToBytes(uuidSlice)); } continue; } if (!domain.getType().isOrderable()) { continue; } Ranges ranges = domain.getValues().getRanges(); // TODO: support multiple ranges if (ranges.getRangeCount() != 1) { continue; } Range range = getOnlyElement(ranges.getOrderedRanges()); Object minValue = null; Object maxValue = null; if (range.isSingleValue()) { minValue = range.getSingleValue(); maxValue = range.getSingleValue(); } else { if (!range.getLow().isLowerUnbounded()) { minValue = range.getLow().getValue(); } if (!range.getHigh().isUpperUnbounded()) { maxValue = range.getHigh().getValue(); } } String min = minColumn(handle.getColumnId()); String max = maxColumn(handle.getColumnId()); if (minValue != null) { predicate.add(format("(%s >= ? OR %s IS NULL)", max, max)); types.add(jdbcType); values.add(minValue); } if (maxValue != null) { predicate.add(format("(%s <= ? OR %s IS NULL)", min, min)); types.add(jdbcType); values.add(maxValue); } } return new ShardPredicate(predicate.toString(), types.build(), values.build()); }
private static Domain createDomain(Range first, Range... ranges) { return Domain.create(ValueSet.ofRanges(first, ranges), false); }
public ShardAssertion between(RaptorColumnHandle column, Type type, Object low, Object high) { return range(column, Range.range(type, low, true, high, true)); }
private String toPredicate(String columnName, Domain domain) { checkArgument(domain.getType().isOrderable(), "Domain type must be orderable"); if (domain.getValues().isNone()) { return domain.isNullAllowed() ? quote(columnName) + " IS NULL" : "FALSE"; } if (domain.getValues().isAll()) { return domain.isNullAllowed() ? "TRUE" : quote(columnName) + " IS NOT NULL"; } List<String> disjuncts = new ArrayList<>(); List<Object> singleValues = new ArrayList<>(); for (Range range : domain.getValues().getRanges().getOrderedRanges()) { checkState(!range.isAll()); // Already checked if (range.isSingleValue()) { singleValues.add(range.getLow().getValue()); } else { List<String> rangeConjuncts = new ArrayList<>(); if (!range.getLow().isLowerUnbounded()) { switch (range.getLow().getBound()) { case ABOVE: rangeConjuncts.add(toPredicate(columnName, ">", range.getLow().getValue())); break; case EXACTLY: rangeConjuncts.add(toPredicate(columnName, ">=", range.getLow().getValue())); break; case BELOW: throw new IllegalArgumentException("Low Marker should never use BELOW bound: " + range); default: throw new AssertionError("Unhandled bound: " + range.getLow().getBound()); } } if (!range.getHigh().isUpperUnbounded()) { switch (range.getHigh().getBound()) { case ABOVE: throw new IllegalArgumentException("High Marker should never use ABOVE bound: " + range); case EXACTLY: rangeConjuncts.add(toPredicate(columnName, "<=", range.getHigh().getValue())); break; case BELOW: rangeConjuncts.add(toPredicate(columnName, "<", range.getHigh().getValue())); break; default: throw new AssertionError("Unhandled bound: " + range.getHigh().getBound()); } } // If rangeConjuncts is null, then the range was ALL, which should already have been checked for checkState(!rangeConjuncts.isEmpty()); disjuncts.add("(" + Joiner.on(" AND ").join(rangeConjuncts) + ")"); } } // Add back all of the possible single values either as an equality or an IN predicate if (singleValues.size() == 1) { disjuncts.add(toPredicate(columnName, "=", getOnlyElement(singleValues))); } else if (singleValues.size() > 1) { disjuncts.add(quote(columnName) + " IN (" + Joiner.on(",").join(transform(singleValues, QueryBuilder::encode)) + ")"); } // Add nullability disjuncts checkState(!disjuncts.isEmpty()); if (domain.isNullAllowed()) { disjuncts.add(quote(columnName) + " IS NULL"); } return "(" + Joiner.on(" OR ").join(disjuncts) + ")"; }
@Test public void testTupleDomain() throws Exception { // single value getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.singleValue(VARCHAR, utf8Slice("foo"))) )); // multiple values (string) getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.union(ImmutableList.of(Domain.singleValue(VARCHAR, utf8Slice("foo")), Domain.singleValue(VARCHAR, utf8Slice("bar"))))) )); // inequality (string) getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(ValueSet.ofRanges(Range.greaterThan(VARCHAR, utf8Slice("foo"))), false)) )); getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(ValueSet.ofRanges(Range.greaterThan(VARCHAR, utf8Slice("foo"))), false)) )); getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(VARCHAR, utf8Slice("foo"))), false)) )); getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(ValueSet.ofRanges(Range.lessThan(VARCHAR, utf8Slice("foo"))), false)) )); // is null getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.onlyNull(VARCHAR)) )); // not null getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.notNull(VARCHAR)) )); // specific value or null getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.union(ImmutableList.of(Domain.singleValue(VARCHAR, utf8Slice("foo")), Domain.onlyNull(VARCHAR)))) )); getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(ValueSet.ofRanges(Range.range(VARCHAR, utf8Slice("bar"), true, utf8Slice("foo"), true)), false)) )); getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(ValueSet.ofRanges( Range.range(VARCHAR, utf8Slice("bar"), true, utf8Slice("foo"), true), Range.range(VARCHAR, utf8Slice("hello"), false, utf8Slice("world"), false)), false )) )); getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>of( textColumn, Domain.create(ValueSet.ofRanges( Range.range(VARCHAR, utf8Slice("bar"), true, utf8Slice("foo"), true), Range.range(VARCHAR, utf8Slice("hello"), false, utf8Slice("world"), false), Range.equal(VARCHAR, utf8Slice("apple")), Range.equal(VARCHAR, utf8Slice("banana")), Range.equal(VARCHAR, utf8Slice("zoo"))), false ), valueColumn, Domain.create(ValueSet.ofRanges( Range.range(BIGINT, 1L, true, 5L, true), Range.range(BIGINT, 10L, false, 20L, false)), true ) ) )); }
private static List<Expression> extractDisjuncts(Type type, Ranges ranges, QualifiedNameReference reference) { List<Expression> disjuncts = new ArrayList<>(); List<Expression> singleValues = new ArrayList<>(); for (Range range : ranges.getOrderedRanges()) { checkState(!range.isAll()); // Already checked if (range.isSingleValue()) { singleValues.add(toExpression(range.getSingleValue(), type)); } else if (isBetween(range)) { // Specialize the range with BETWEEN expression if possible b/c it is currently more efficient disjuncts.add(new BetweenPredicate(reference, toExpression(range.getLow().getValue(), type), toExpression(range.getHigh().getValue(), type))); } else { List<Expression> rangeConjuncts = new ArrayList<>(); if (!range.getLow().isLowerUnbounded()) { switch (range.getLow().getBound()) { case ABOVE: rangeConjuncts.add(new ComparisonExpression(GREATER_THAN, reference, toExpression(range.getLow().getValue(), type))); break; case EXACTLY: rangeConjuncts.add(new ComparisonExpression(GREATER_THAN_OR_EQUAL, reference, toExpression(range.getLow().getValue(), type))); break; case BELOW: throw new IllegalStateException("Low Marker should never use BELOW bound: " + range); default: throw new AssertionError("Unhandled bound: " + range.getLow().getBound()); } } if (!range.getHigh().isUpperUnbounded()) { switch (range.getHigh().getBound()) { case ABOVE: throw new IllegalStateException("High Marker should never use ABOVE bound: " + range); case EXACTLY: rangeConjuncts.add(new ComparisonExpression(LESS_THAN_OR_EQUAL, reference, toExpression(range.getHigh().getValue(), type))); break; case BELOW: rangeConjuncts.add(new ComparisonExpression(LESS_THAN, reference, toExpression(range.getHigh().getValue(), type))); break; default: throw new AssertionError("Unhandled bound: " + range.getHigh().getBound()); } } // If rangeConjuncts is null, then the range was ALL, which should already have been checked for checkState(!rangeConjuncts.isEmpty()); disjuncts.add(combineConjuncts(rangeConjuncts)); } } // Add back all of the possible single values either as an equality or an IN predicate if (singleValues.size() == 1) { disjuncts.add(new ComparisonExpression(EQUAL, reference, getOnlyElement(singleValues))); } else if (singleValues.size() > 1) { disjuncts.add(new InPredicate(reference, new InListExpression(singleValues))); } return disjuncts; }
private static boolean isBetween(Range range) { return !range.getLow().isLowerUnbounded() && range.getLow().getBound() == Marker.Bound.EXACTLY && !range.getHigh().isUpperUnbounded() && range.getHigh().getBound() == Marker.Bound.EXACTLY; }
@Test public void testToPredicate() throws Exception { TupleDomain<Symbol> tupleDomain; tupleDomain = withColumnDomains(ImmutableMap.of(A, Domain.notNull(BIGINT))); assertEquals(toPredicate(tupleDomain), isNotNull(A)); tupleDomain = withColumnDomains(ImmutableMap.of(A, Domain.onlyNull(BIGINT))); assertEquals(toPredicate(tupleDomain), isNull(A)); tupleDomain = withColumnDomains(ImmutableMap.of(A, Domain.none(BIGINT))); assertEquals(toPredicate(tupleDomain), FALSE_LITERAL); tupleDomain = withColumnDomains(ImmutableMap.of(A, Domain.all(BIGINT))); assertEquals(toPredicate(tupleDomain), TRUE_LITERAL); tupleDomain = withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.greaterThan(BIGINT, 1L)), false))); assertEquals(toPredicate(tupleDomain), greaterThan(A, longLiteral(1L))); tupleDomain = withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.greaterThanOrEqual(BIGINT, 1L)), false))); assertEquals(toPredicate(tupleDomain), greaterThanOrEqual(A, longLiteral(1L))); tupleDomain = withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 1L)), false))); assertEquals(toPredicate(tupleDomain), lessThan(A, longLiteral(1L))); tupleDomain = withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.range(BIGINT, 0L, false, 1L, true)), false))); assertEquals(toPredicate(tupleDomain), and(greaterThan(A, longLiteral(0L)), lessThanOrEqual(A, longLiteral(1L)))); tupleDomain = withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(BIGINT, 1L)), false))); assertEquals(toPredicate(tupleDomain), lessThanOrEqual(A, longLiteral(1L))); tupleDomain = withColumnDomains(ImmutableMap.of(A, Domain.singleValue(BIGINT, 1L))); assertEquals(toPredicate(tupleDomain), equal(A, longLiteral(1L))); tupleDomain = withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), false))); assertEquals(toPredicate(tupleDomain), in(A, ImmutableList.of(1L, 2L))); tupleDomain = withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 1L)), true))); assertEquals(toPredicate(tupleDomain), or(lessThan(A, longLiteral(1L)), isNull(A))); tupleDomain = withColumnDomains(ImmutableMap.of(J, Domain.create(ValueSet.of(COLOR, COLOR_VALUE_1), true))); assertEquals(toPredicate(tupleDomain), or(equal(J, colorLiteral(COLOR_VALUE_1)), isNull(J))); tupleDomain = withColumnDomains(ImmutableMap.of(J, Domain.create(ValueSet.of(COLOR, COLOR_VALUE_1).complement(), true))); assertEquals(toPredicate(tupleDomain), or(not(equal(J, colorLiteral(COLOR_VALUE_1))), isNull(J))); tupleDomain = withColumnDomains(ImmutableMap.of(K, Domain.onlyNull(HYPER_LOG_LOG))); assertEquals(toPredicate(tupleDomain), isNull(K)); tupleDomain = withColumnDomains(ImmutableMap.of(K, Domain.notNull(HYPER_LOG_LOG))); assertEquals(toPredicate(tupleDomain), isNotNull(K)); }
@Test public void testFromOrPredicate() throws Exception { Expression originalPredicate = or( and(greaterThan(A, longLiteral(1L)), unprocessableExpression1(A)), and(lessThan(A, longLiteral(5L)), unprocessableExpression2(A))); ExtractionResult result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), originalPredicate); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.notNull(BIGINT)))); originalPredicate = or( and(equal(A, longLiteral(1L)), unprocessableExpression1(A)), and(equal(A, longLiteral(2L)), unprocessableExpression2(A))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), originalPredicate); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), false)))); // Same unprocessableExpression means that we can do more extraction // If both sides are operating on the same single symbol originalPredicate = or( and(equal(A, longLiteral(1L)), unprocessableExpression1(A)), and(equal(A, longLiteral(2L)), unprocessableExpression1(A))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), unprocessableExpression1(A)); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), false)))); // And not if they have different symbols originalPredicate = or( and(equal(A, longLiteral(1L)), unprocessableExpression1(A)), and(equal(B, doubleLiteral(2.0)), unprocessableExpression1(A))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), originalPredicate); assertTrue(result.getTupleDomain().isAll()); // We can make another optimization if one side is the super set of the other side originalPredicate = or( and(greaterThan(A, longLiteral(1L)), greaterThan(B, doubleLiteral(1.0)), unprocessableExpression1(A)), and(greaterThan(A, longLiteral(2L)), greaterThan(B, doubleLiteral(2.0)), unprocessableExpression1(A))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), unprocessableExpression1(A)); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of( A, Domain.create(ValueSet.ofRanges(Range.greaterThan(BIGINT, 1L)), false), B, Domain.create(ValueSet.ofRanges(Range.greaterThan(DOUBLE, 1.0)), false)))); // We can't make those inferences if the unprocessableExpressions are non-deterministic originalPredicate = or( and(equal(A, longLiteral(1L)), randPredicate(A)), and(equal(A, longLiteral(2L)), randPredicate(A))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), originalPredicate); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), false)))); // Test complements originalPredicate = not(or( and(greaterThan(A, longLiteral(1L)), unprocessableExpression1(A)), and(lessThan(A, longLiteral(5L)), unprocessableExpression2(A)))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), and( not(and(greaterThan(A, longLiteral(1L)), unprocessableExpression1(A))), not(and(lessThan(A, longLiteral(5L)), unprocessableExpression2(A))))); assertTrue(result.getTupleDomain().isAll()); originalPredicate = not(or( not(and(greaterThan(A, longLiteral(1L)), unprocessableExpression1(A))), not(and(lessThan(A, longLiteral(5L)), unprocessableExpression2(A))))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), and(unprocessableExpression1(A), unprocessableExpression2(A))); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.range(BIGINT, 1L, false, 5L, false)), false)))); }
@Test public void testFromFlippedBasicComparisons() throws Exception { // Test out the extraction of all basic comparisons where the reference literal ordering is flipped ComparisonExpression originalExpression = comparison(GREATER_THAN, longLiteral(2L), reference(A)); ExtractionResult result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 2L)), false)))); originalExpression = comparison(GREATER_THAN_OR_EQUAL, longLiteral(2L), reference(A)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(BIGINT, 2L)), false)))); originalExpression = comparison(LESS_THAN, longLiteral(2L), reference(A)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.greaterThan(BIGINT, 2L)), false)))); originalExpression = comparison(LESS_THAN_OR_EQUAL, longLiteral(2L), reference(A)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.greaterThanOrEqual(BIGINT, 2L)), false)))); originalExpression = comparison(EQUAL, longLiteral(2L), reference(A)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 2L)), false)))); originalExpression = comparison(EQUAL, colorLiteral(COLOR_VALUE_1), reference(J)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(J, Domain.create(ValueSet.of(COLOR, COLOR_VALUE_1), false)))); originalExpression = comparison(NOT_EQUAL, longLiteral(2L), reference(A)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 2L), Range.greaterThan(BIGINT, 2L)), false)))); originalExpression = comparison(NOT_EQUAL, colorLiteral(COLOR_VALUE_1), reference(J)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(J, Domain.create(ValueSet.of(COLOR, COLOR_VALUE_1).complement(), false)))); originalExpression = comparison(IS_DISTINCT_FROM, longLiteral(2L), reference(A)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 2L), Range.greaterThan(BIGINT, 2L)), true)))); originalExpression = comparison(IS_DISTINCT_FROM, colorLiteral(COLOR_VALUE_1), reference(J)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(J, Domain.create(ValueSet.of(COLOR, COLOR_VALUE_1).complement(), true)))); originalExpression = comparison(IS_DISTINCT_FROM, nullLiteral(), reference(A)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.notNull(BIGINT)))); }
@Test public void testFromInPredicate() throws Exception { Expression originalExpression = in(A, ImmutableList.of(1L)); ExtractionResult result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.singleValue(BIGINT, 1L)))); originalExpression = in(J, ImmutableList.of(colorLiteral(COLOR_VALUE_1))); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(J, Domain.singleValue(COLOR, COLOR_VALUE_1)))); originalExpression = in(A, ImmutableList.of(1L, 2L)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), false)))); originalExpression = in(J, ImmutableList.of(colorLiteral(COLOR_VALUE_1), colorLiteral(COLOR_VALUE_2))); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(J, Domain.create(ValueSet.of(COLOR, COLOR_VALUE_1, COLOR_VALUE_2), false)))); originalExpression = not(in(A, ImmutableList.of(1L, 2L))); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 1L), Range.range(BIGINT, 1L, false, 2L, false), Range.greaterThan(BIGINT, 2L)), false)))); originalExpression = not(in(J, ImmutableList.of(colorLiteral(COLOR_VALUE_1), colorLiteral(COLOR_VALUE_2)))); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(J, Domain.create(ValueSet.of(COLOR, COLOR_VALUE_1, COLOR_VALUE_2).complement(), false)))); // TODO update domain translator to properly handle cast // originalExpression = in(A, Arrays.asList(1L, 2L, (Expression) null)); // result = fromPredicate(originalExpression, TYPES, COLUMN_HANDLES); // assertEquals(result.getRemainingExpression(), TRUE_LITERAL); // assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(ACH, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), false)))); // // originalExpression = not(in(A, Arrays.asList(1L, 2L, (Expression) null))); // result = fromPredicate(originalExpression, TYPES, COLUMN_HANDLES); // assertEquals(result.getRemainingExpression(), TRUE_LITERAL); // assertTrue(result.getTupleDomain().isNone()); // // originalExpression = in(A, Arrays.asList((Expression) null)); // result = fromPredicate(originalExpression, TYPES, COLUMN_HANDLES); // assertEquals(result.getRemainingExpression(), TRUE_LITERAL); // assertTrue(result.getTupleDomain().isNone()); // // originalExpression = not(in(A, Arrays.asList((Expression) null))); // result = fromPredicate(originalExpression, TYPES, COLUMN_HANDLES); // assertEquals(result.getRemainingExpression(), TRUE_LITERAL); // assertTrue(result.getTupleDomain().isNone()); }
private static Set<List<Object>> getPartitionKeysSet(CassandraTable table, TupleDomain<ColumnHandle> tupleDomain) { ImmutableList.Builder<Set<Object>> partitionColumnValues = ImmutableList.builder(); for (CassandraColumnHandle columnHandle : table.getPartitionKeyColumns()) { Domain domain = tupleDomain.getDomains().get().get(columnHandle); // if there is no constraint on a partition key, return an empty set if (domain == null) { return ImmutableSet.of(); } // todo does cassandra allow null partition keys? if (domain.isNullAllowed()) { return ImmutableSet.of(); } Set<Object> values = domain.getValues().getValuesProcessor().transform( ranges -> { ImmutableSet.Builder<Object> columnValues = ImmutableSet.builder(); for (Range range : ranges.getOrderedRanges()) { // if the range is not a single value, we can not perform partition pruning if (!range.isSingleValue()) { return ImmutableSet.of(); } Object value = range.getSingleValue(); CassandraType valueType = columnHandle.getCassandraType(); columnValues.add(valueType.validatePartitionKey(value)); } return columnValues.build(); }, discreteValues -> { if (discreteValues.isWhiteList()) { return ImmutableSet.copyOf(discreteValues.getValues()); } return ImmutableSet.of(); }, allOrNone -> ImmutableSet.of()); partitionColumnValues.add(values); } return Sets.cartesianProduct(partitionColumnValues.build()); }
private void doMetadataDelete(HiveStorageFormat storageFormat, SchemaTableName tableName) throws Exception { // creating the table doCreateEmptyTable(tableName, storageFormat, CREATE_TABLE_COLUMNS_PARTITIONED); // verify table directory is empty Set<String> initialFiles = listAllDataFiles(tableName.getSchemaName(), tableName.getTableName()); assertTrue(initialFiles.isEmpty()); MaterializedResult.Builder expectedResultBuilder = MaterializedResult.resultBuilder(SESSION, CREATE_TABLE_PARTITIONED_DATA.getTypes()); ConnectorTableHandle tableHandle = getTableHandle(tableName); insertData(tableHandle, CREATE_TABLE_PARTITIONED_DATA, newSession()); expectedResultBuilder.rows(CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows()); // verify partitions were created List<String> partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available")); assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream() .map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)) .collect(toList())); // verify table directory is not empty Set<String> filesAfterInsert = listAllDataFiles(tableName.getSchemaName(), tableName.getTableName()); assertFalse(filesAfterInsert.isEmpty()); // verify the data ConnectorSession session = newSession(); List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); MaterializedResult result = readTable(tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEqualsIgnoreOrder(result.getMaterializedRows(), expectedResultBuilder.build().getMaterializedRows()); // get ds column handle Map<String, HiveColumnHandle> columnHandleMap = columnHandles.stream() .map(columnHandle -> (HiveColumnHandle) columnHandle) .collect(Collectors.toMap(HiveColumnHandle::getName, Function.identity())); HiveColumnHandle dsColumnHandle = columnHandleMap.get("ds"); int dsColumnOrdinalPosition = columnHandles.indexOf(dsColumnHandle); // delete ds=2015-07-03 session = newSession(); TupleDomain<ColumnHandle> tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(dsColumnHandle, NullableValue.of(VARCHAR, utf8Slice("2015-07-03")))); Constraint<ColumnHandle> constraint = new Constraint<>(tupleDomain, convertToPredicate(tupleDomain)); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, constraint, Optional.empty()); ConnectorTableLayoutHandle tableLayoutHandle = Iterables.getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); metadata.metadataDelete(session, tableHandle, tableLayoutHandle); // verify the data session = newSession(); ImmutableList<MaterializedRow> expectedRows = expectedResultBuilder.build().getMaterializedRows().stream() .filter(row -> !"2015-07-03".equals(row.getField(dsColumnOrdinalPosition))) .collect(ImmutableCollectors.toImmutableList()); MaterializedResult actualAfterDelete = readTable(tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEqualsIgnoreOrder(actualAfterDelete.getMaterializedRows(), expectedRows); // delete ds=2015-07-01 and 2015-07-02 session = newSession(); TupleDomain<ColumnHandle> tupleDomain2 = TupleDomain.withColumnDomains( ImmutableMap.of(dsColumnHandle, Domain.create(ValueSet.ofRanges(Range.range(VARCHAR, utf8Slice("2015-07-01"), true, utf8Slice("2015-07-02"), true)), false))); Constraint<ColumnHandle> constraint2 = new Constraint<>(tupleDomain2, convertToPredicate(tupleDomain2)); List<ConnectorTableLayoutResult> tableLayoutResults2 = metadata.getTableLayouts(session, tableHandle, constraint2, Optional.empty()); ConnectorTableLayoutHandle tableLayoutHandle2 = Iterables.getOnlyElement(tableLayoutResults2).getTableLayout().getHandle(); metadata.metadataDelete(session, tableHandle, tableLayoutHandle2); // verify the data session = newSession(); MaterializedResult actualAfterDelete2 = readTable(tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEqualsIgnoreOrder(actualAfterDelete2.getMaterializedRows(), ImmutableList.of()); // verify table directory is empty Set<String> filesAfterDelete = listAllDataFiles(tableName.getSchemaName(), tableName.getTableName()); assertTrue(filesAfterDelete.isEmpty()); }