private static com.netflix.iceberg.expressions.Expression filter( Operation op, String name, int value, Function<Integer, Long> startTsMicros) { switch (op) { case LT: return predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value))); case LT_EQ: return predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value + 1))); case GT: return predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value + 1))); case GT_EQ: return predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value))); case EQ: return and( predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value))), predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value + 1))) ); case NOT_EQ: return or( predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value + 1))), predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value))) ); case IN: case NOT_IN: default: throw new IllegalArgumentException("Cannot convert operation to year filter: " + op); } }
static <T> UnboundPredicate<T> truncateDecimal( String name, BoundPredicate<BigDecimal> pred, Transform<BigDecimal, T> transform) { BigDecimal boundary = pred.literal().value(); switch (pred.op()) { case LT: // adjust closed and then transform ltEq BigDecimal minusOne = new BigDecimal( boundary.unscaledValue().subtract(BigInteger.ONE), boundary.scale()); return predicate(Expression.Operation.LT_EQ, name, transform.apply(minusOne)); case LT_EQ: return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary)); case GT: // adjust closed and then transform gtEq BigDecimal plusOne = new BigDecimal( boundary.unscaledValue().add(BigInteger.ONE), boundary.scale()); return predicate(Expression.Operation.GT_EQ, name, transform.apply(plusOne)); case GT_EQ: return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary)); case EQ: return predicate(pred.op(), name, transform.apply(boundary)); default: return null; } }
static <T> UnboundPredicate<T> truncateLong( String name, BoundPredicate<Long> pred, Transform<Long, T> transform) { long boundary = pred.literal().value(); switch (pred.op()) { case LT: // adjust closed and then transform ltEq return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary - 1L)); case LT_EQ: return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary)); case GT: // adjust closed and then transform gtEq return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary + 1L)); case GT_EQ: return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary)); case EQ: return predicate(pred.op(), name, transform.apply(boundary)); default: return null; } }
static <T> UnboundPredicate<T> truncateInteger( String name, BoundPredicate<Integer> pred, Transform<Integer, T> transform) { int boundary = pred.literal().value(); switch (pred.op()) { case LT: // adjust closed and then transform ltEq return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary - 1)); case LT_EQ: return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary)); case GT: // adjust closed and then transform gtEq return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary + 1)); case GT_EQ: return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary)); case EQ: return predicate(pred.op(), name, transform.apply(boundary)); default: return null; } }
static <S, T> UnboundPredicate<T> truncateArray( String name, BoundPredicate<S> pred, Transform<S, T> transform) { S boundary = pred.literal().value(); switch (pred.op()) { case LT: case LT_EQ: return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary)); case GT: case GT_EQ: return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary)); case EQ: return predicate(Expression.Operation.EQ, name, transform.apply(boundary)); // case IN: // TODO // return Expressions.predicate(Operation.IN, name, transform.apply(boundary)); default: return null; } } }
@Override public UnboundPredicate<Integer> project(String name, BoundPredicate<Long> pred) { if (pred.op() == NOT_NULL || pred.op() == IS_NULL) { return Expressions.predicate(pred.op(), name); } return ProjectionUtil.truncateLong(name, pred, this); }
@Override public UnboundPredicate<Integer> project(String name, BoundPredicate<Integer> pred) { if (pred.op() == NOT_NULL || pred.op() == IS_NULL) { return Expressions.predicate(pred.op(), name); } return ProjectionUtil.truncateInteger(name, pred, this); }
@Override public UnboundPredicate<ByteBuffer> project(String name, BoundPredicate<ByteBuffer> pred) { if (pred.op() == NOT_NULL || pred.op() == IS_NULL) { return Expressions.predicate(pred.op(), name); } return ProjectionUtil.truncateArray(name, pred, this); }
@Override public UnboundPredicate<Long> project(String name, BoundPredicate<Long> pred) { if (pred.op() == NOT_NULL || pred.op() == IS_NULL) { return Expressions.predicate(pred.op(), name); } return ProjectionUtil.truncateLong(name, pred, this); }
@Override public UnboundPredicate<CharSequence> project(String name, BoundPredicate<CharSequence> pred) { if (pred.op() == NOT_NULL || pred.op() == IS_NULL) { return Expressions.predicate(pred.op(), name); } return ProjectionUtil.truncateArray(name, pred, this); }
@Override public UnboundPredicate<Integer> project(String name, BoundPredicate<Integer> pred) { if (pred.op() == NOT_NULL || pred.op() == IS_NULL) { return Expressions.predicate(pred.op(), name); } return ProjectionUtil.truncateInteger(name, pred, this); }
@Override public UnboundPredicate<BigDecimal> project(String name, BoundPredicate<BigDecimal> pred) { if (pred.op() == NOT_NULL || pred.op() == IS_NULL) { return Expressions.predicate(pred.op(), name); } return ProjectionUtil.truncateDecimal(name, pred, this); }
@Override public UnboundPredicate<Integer> projectStrict(String name, BoundPredicate<T> predicate) { switch (predicate.op()) { case NOT_EQ: // TODO: need to translate not(eq(...)) into notEq in expressions return Expressions.predicate(predicate.op(), name, apply(predicate.literal().value())); // case NOT_IN: // return null; default: // no strict projection for comparison or equality return null; } }
outImage = apply(out); if (inImage != outImage) { return Expressions.predicate(LT_EQ, name, inImage); } else { return Expressions.predicate(LT, name, inImage); outImage = apply(out); if (inImage != outImage) { return Expressions.predicate(LT_EQ, name, inImage); } else { return Expressions.predicate(LT, name, inImage);
@Override public UnboundPredicate<Integer> project(String name, BoundPredicate<T> predicate) { switch (predicate.op()) { case EQ: return Expressions.predicate( predicate.op(), name, apply(predicate.literal().value())); // case IN: // return Expressions.predicate(); default: // comparison predicates can't be projected, notEq can't be projected // TODO: small ranges can be projected. // for example, (x > 0) and (x < 3) can be turned into in({1, 2}) and projected. return null; } }
private void removeTimeFilters(List<Expression> expressions, Expression expression) { if (expression.op() == Operation.AND) { And and = (And) expression; removeTimeFilters(expressions, and.left()); removeTimeFilters(expressions, and.right()); return; } else if (expression instanceof UnboundPredicate) { UnboundPredicate pred = (UnboundPredicate) expression; NamedReference ref = (NamedReference) pred.ref(); Literal<?> lit = pred.literal(); if (TIMESTAMP_NAMES.contains(ref.name())) { Literal<Long> tsLiteral = lit.to(Types.TimestampType.withoutZone()); long millis = toMillis(tsLiteral.value()); addTimestampFilter(Expressions.predicate(pred.op(), "timestamp_ms", millis)); return; } } expressions.add(expression); }
private static com.netflix.iceberg.expressions.Expression convert(Operation op, Expression left, Expression right) { Pair<Transform, String> attrPair = null; Operation leftOperation = null; Literal lit = null; if (right instanceof Literal) { lit = (Literal) right; attrPair = convertAttr(left); leftOperation = op; } else if (left instanceof Literal) { lit = (Literal) left; attrPair = convertAttr(right); leftOperation = op.flipLR(); } if (attrPair != null) { switch (attrPair.first()) { case IDENTITY: return predicate(leftOperation, attrPair.second(), valueFromSpark(lit)); case YEAR: return filter(leftOperation, attrPair.second(), (int) lit.value(), SparkExpressions::yearToTimestampMicros); case DAY: return filter(leftOperation, attrPair.second(), (int) lit.value(), SparkExpressions::dayToTimestampMicros); default: } } return null; }
if (unary.child() instanceof Attribute) { Attribute attr = (Attribute) unary.child(); return predicate(op, attr.name());