How to use

SparkExpressions

in

com.netflix.iceberg.spark

@Override
public org.apache.spark.sql.catalyst.expressions.Expression[] pushCatalystFilters(
  org.apache.spark.sql.catalyst.expressions.Expression[] filters) {
 this.tasks = null; // invalidate cached tasks, if present
 List<Expression> expressions = Lists.newArrayListWithExpectedSize(filters.length);
 List<org.apache.spark.sql.catalyst.expressions.Expression> pushed =
   Lists.newArrayListWithExpectedSize(filters.length);
 for (org.apache.spark.sql.catalyst.expressions.Expression filter : filters) {
  Expression expr = SparkExpressions.convert(filter);
  if (expr != null) {
   expressions.add(expr);
   pushed.add(filter);
  }
 }
 this.filterExpressions = expressions;
 this.pushedExprs = pushed.toArray(new org.apache.spark.sql.catalyst.expressions.Expression[0]);
 // invalidate the schema that will be projected
 this.schema = null;
 this.type = null;
 // Spark doesn't support residuals per task, so return all filters
 // to get Spark to handle record-level filtering
 return filters;
}

private static List<Object> convertLiterals(List<Expression> values) {
 List<Object> converted = Lists.newArrayListWithExpectedSize(values.size());
 for (Expression value : values) {
  if (value instanceof Literal) {
   Literal lit = (Literal) value;
   converted.add(valueFromSpark(lit));
  } else {
   return null;
  }
 }
 return converted;
}

case NOT_EQ:
 BinaryExpression binary = (BinaryExpression) expr;
 return convert(op, binary.left(), binary.right());
case NOT:
 com.netflix.iceberg.expressions.Expression child = convert(((Not) expr).child());
 if (child != null) {
  return not(child);
case AND:
 And andExpr = (And) expr;
 com.netflix.iceberg.expressions.Expression andLeft = convert(andExpr.left());
 com.netflix.iceberg.expressions.Expression andRight = convert(andExpr.right());
 if (andLeft != null && andRight != null) {
  return and(convert(andExpr.left()), convert(andExpr.right()));
 com.netflix.iceberg.expressions.Expression orLeft = convert(orExpr.left());
 com.netflix.iceberg.expressions.Expression orRight = convert(orExpr.right());
 if (orLeft != null && orRight != null) {
  return or(orLeft, orRight);
 if (expr instanceof In) {
  In inExpr = (In) expr;
  List<Object> literals = convertLiterals(seqAsJavaListConverter(inExpr.list()).asJava());
  if (literals != null) {
   return convertIn(inExpr.value(), literals);
  } else {
  return convertIn(inExpr.child(), literals);

private static com.netflix.iceberg.expressions.Expression convert(Operation op,
                                 Expression left,
                                 Expression right) {
 Pair<Transform, String> attrPair = null;
 Operation leftOperation = null;
 Literal lit = null;
 if (right instanceof Literal) {
  lit = (Literal) right;
  attrPair = convertAttr(left);
  leftOperation = op;
 } else if (left instanceof Literal) {
  lit = (Literal) left;
  attrPair = convertAttr(right);
  leftOperation = op.flipLR();
 }
 if (attrPair != null) {
  switch (attrPair.first()) {
   case IDENTITY:
    return predicate(leftOperation, attrPair.second(), valueFromSpark(lit));
   case YEAR:
    return filter(leftOperation, attrPair.second(), (int) lit.value(),
      SparkExpressions::yearToTimestampMicros);
   case DAY:
    return filter(leftOperation, attrPair.second(), (int) lit.value(),
      SparkExpressions::dayToTimestampMicros);
   default:
  }
 }
 return null;
}

private static com.netflix.iceberg.expressions.Expression filter(
  Operation op, String name, int value, Function<Integer, Long> startTsMicros) {
 switch (op) {
  case LT:
   return predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value)));
  case LT_EQ:
   return predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value + 1)));
  case GT:
   return predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value + 1)));
  case GT_EQ:
   return predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value)));
  case EQ:
   return and(
     predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value))),
     predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value + 1)))
   );
  case NOT_EQ:
   return or(
     predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value + 1))),
     predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value)))
   );
  case IN:
  case NOT_IN:
  default:
   throw new IllegalArgumentException("Cannot convert operation to year filter: " + op);
 }
}

private void pushFilters(DataSourceReader reader,
             com.netflix.iceberg.expressions.Expression... filters) {
 Expression[] expressions = new Expression[filters.length];
 for (int i = 0; i < filters.length; i += 1) {
  expressions[i] = SparkExpressions.convert(filters[i], SCHEMA);
 }
 pushFilters(reader, expressions);
}