private static List<PartitionFilter> addUpperFilters(String[] fields, int position, PartitionKey keyUpper, List<PartitionFilter> filters, Supplier<PartitionFilter.Builder> supplier) { if (position >= fields.length) { return filters; } String fieldName = fields[position]; int upper = (Integer) keyUpper.getField(fieldName); // only add the filter if this condition is satisfiable. For example, not for hour<0 or month<1 if (isSatisfiableUpperBound(fieldName, upper)) { filters.add(supplier.get().addRangeCondition(fieldName, null, upper).build()); } return addUpperFilters(fields, position + 1, keyUpper, filters, nextSupplier(supplier, fieldName, upper)); }
filters.add(supplier.get().addValueCondition(fieldName, lower).build()); } else { filters.add(supplier.get().addRangeCondition(fieldName, lower, upper).build()); filters.add(supplier.get().addValueCondition(fieldName, lower + 1).build()); } else if (lower + 2 < upper) { filters.add(supplier.get().addRangeCondition(fieldName, lower + 1, upper).build());
private static List<PartitionFilter> addUpperFilters(String[] fields, int position, PartitionKey keyUpper, List<PartitionFilter> filters, Supplier<PartitionFilter.Builder> supplier) { if (position >= fields.length) { return filters; } String fieldName = fields[position]; int upper = (Integer) keyUpper.getField(fieldName); // only add the filter if this condition is satisfiable. For example, not for hour<0 or month<1 if (isSatisfiableUpperBound(fieldName, upper)) { filters.add(supplier.get().addRangeCondition(fieldName, null, upper).build()); } return addUpperFilters(fields, position + 1, keyUpper, filters, nextSupplier(supplier, fieldName, upper)); }
.addRangeCondition("type", "x", "z") .build(); .addRangeCondition("time", null, 160000L) .build();
filters.add(supplier.get().addValueCondition(fieldName, lower).build()); } else { filters.add(supplier.get().addRangeCondition(fieldName, lower, upper).build()); filters.add(supplier.get().addValueCondition(fieldName, lower + 1).build()); } else if (lower + 2 < upper) { filters.add(supplier.get().addRangeCondition(fieldName, lower + 1, upper).build());
.addRangeCondition("month", 4, 7) .addValueCondition("market", "asia") .addRangeCondition("duration", 60 * minute, 90 * minute) .build();
final String path9 = "9:42"; final PartitionFilter filter9 = PartitionFilter.builder().addRangeCondition("hour", 9, null).build();
final PartitionFilter partitionFilter = PartitionFilter.builder().addRangeCondition("i", 1, 7).build(); final Predicate<PartitionDetail> predicate = new Predicate<PartitionDetail>() { @Override
List<Partition> consumedPartitions = Lists.newArrayList(); final PartitionFilter partitionFilter = PartitionFilter.builder().addRangeCondition("i", 1, 7).build(); final Predicate<PartitionDetail> predicate = new Predicate<PartitionDetail>() { @Override
@Test public void testSetGetInputPartitionFilter() throws Exception { Map<String, String> arguments = new HashMap<>(); PartitionFilter filter = PartitionFilter.builder() .addRangeCondition("i", 30, 40) .addValueCondition("l", 17L) .addValueCondition("s", "x") .build(); PartitionedFileSetArguments.setInputPartitionFilter(arguments, filter); Assert.assertEquals(filter, PartitionedFileSetArguments.getInputPartitionFilter(arguments)); arguments = new HashMap<>(); filter = PartitionFilter.builder() .addRangeCondition("i", 30, 40) .addValueCondition("s", "x") .build(); PartitionedFileSetArguments.setInputPartitionFilter(arguments, filter); Assert.assertEquals(filter, PartitionedFileSetArguments.getInputPartitionFilter(arguments)); arguments = new HashMap<>(); filter = PartitionFilter.ALWAYS_MATCH; PartitionedFileSetArguments.setInputPartitionFilter(arguments, filter); Assert.assertEquals(filter, PartitionedFileSetArguments.getInputPartitionFilter(arguments)); }
inputArgs, PartitionFilter.builder().addRangeCondition("x", "na", "nx").build()); Map<String, String> outputArgs = new HashMap<>(); PartitionKey outputKey = PartitionKey.builder().addStringField("x", "xx").build();
private static List<PartitionFilter> addLowerFilters(String[] fields, int position, PartitionKey keyLower, List<PartitionFilter> filters, Supplier<PartitionFilter.Builder> supplier) { if (position >= fields.length) { return filters; } String fieldName = fields[position]; int lower = (Integer) keyLower.getField(fieldName); // if this field is at the finest granularity (minutes), we must include its value in the range // otherwise we exclude it from the range and descend into the next finer granularity with a value // constraints on the current field name. For example: // - for hour:15/minute:10, we add a filter for hour>=16 (excluding 15) and descent for hour=15 // - now the remaining field is minute:10, we add a filter for minute>=10 (including 10) int lowerBound = position == fields.length - 1 ? lower : lower + 1; // only add the filter if this condition is satisfiable. For example, not for hour>=24 or month>=13 if (isSatisfiableLowerBound(fieldName, lowerBound)) { filters.add(supplier.get().addRangeCondition(fieldName, lowerBound, null).build()); } return addLowerFilters(fields, position + 1, keyLower, filters, nextSupplier(supplier, fieldName, lower)); }
private static List<PartitionFilter> addLowerFilters(String[] fields, int position, PartitionKey keyLower, List<PartitionFilter> filters, Supplier<PartitionFilter.Builder> supplier) { if (position >= fields.length) { return filters; } String fieldName = fields[position]; int lower = (Integer) keyLower.getField(fieldName); // if this field is at the finest granularity (minutes), we must include its value in the range // otherwise we exclude it from the range and descend into the next finer granularity with a value // constraints on the current field name. For example: // - for hour:15/minute:10, we add a filter for hour>=16 (excluding 15) and descent for hour=15 // - now the remaining field is minute:10, we add a filter for minute>=10 (including 10) int lowerBound = position == fields.length - 1 ? lower : lower + 1; // only add the filter if this condition is satisfiable. For example, not for hour>=24 or month>=13 if (isSatisfiableLowerBound(fieldName, lowerBound)) { filters.add(supplier.get().addRangeCondition(fieldName, lowerBound, null).build()); } return addLowerFilters(fields, position + 1, keyLower, filters, nextSupplier(supplier, fieldName, lower)); }
@Test(expected = IllegalArgumentException.class) public void testIncompatibleMatch() { PartitionFilter filter = PartitionFilter.builder() .addValueCondition("year", 2012) .addRangeCondition("month", 4, 7) .addValueCondition("market", "asia") .build(); // field of incompatible type filter.match(PartitionKey .builder() .addField("month", "january") .addField("market", "latin") .addField("year", 2012) .build()); }
@Test(expected = IllegalArgumentException.class) public void testBuilderDuplicateField() { PartitionFilter.builder() .addValueCondition("x", 10) .addValueCondition("y", 10L) .addRangeCondition("x", 14, 15).build(); }
@Test(expected = IllegalArgumentException.class) public void testBuilderEmptyField2() { PartitionFilter.builder().addRangeCondition("", 1, 2).build(); }
public void deleteMatchingPartitionsByTime(long upperLimit) throws IOException { if (upperLimit > 0 && upperLimit < Long.MAX_VALUE) { PartitionFilter filter = PartitionFilter.builder().addRangeCondition(SNAPSHOT_FIELD, null, upperLimit).build(); Set<PartitionDetail> partitions = files.getPartitions(filter); for (PartitionDetail partition : partitions) { files.dropPartition(partition.getPartitionKey()); } } }
@Test public void testBuilderNullRange() { PartitionFilter filter = PartitionFilter.builder() .addValueCondition("a", 1) .<Long>addRangeCondition("x", null, null) .build(); Assert.assertEquals(1, filter.getConditions().size()); // only the one for "a" Assert.assertNull(filter.getCondition("x")); }
private static <T extends Comparable<T>> PartitionFilter.Builder addCondition(PartitionFilter.Builder builder, String field, T[] condition) { return condition.length == 1 ? builder.addValueCondition(field, condition[0]) : builder.addRangeCondition(field, condition[0], condition[1]); }
@Test(expected = IllegalArgumentException.class) public void testBuilderNullField2() { PartitionFilter.builder().addRangeCondition(null, 1, 2).build(); }