/** * If a partition filter was specified, return the partition keys of all partitions * matching the filter. Otherwise return null. */ @Nullable protected Collection<PartitionKey> computeInputKeys() { PartitionFilter filter; try { filter = PartitionedFileSetArguments.getInputPartitionFilter(runtimeArguments); } catch (Exception e) { throw new DataSetException("Partition filter must be correctly specified in arguments."); } if (filter == null) { return null; } return getPartitionPaths(filter); // never returns null }
/** * If a partition filter was specified, return the partition keys of all partitions * matching the filter. Otherwise return null. */ @Nullable protected Collection<PartitionKey> computeInputKeys() { PartitionFilter filter; try { filter = PartitionedFileSetArguments.getInputPartitionFilter(runtimeArguments); } catch (Exception e) { throw new DataSetException("Partition filter must be correctly specified in arguments."); } if (filter == null) { return null; } return getPartitionPaths(filter); // never returns null }
@Override public void initialize(MapReduceTaskContext<byte[], Put> context) throws Exception { InputContext inputContext = context.getInputContext(); Preconditions.checkArgument(PARTITIONED.equals(inputContext.getInputName())); Preconditions.checkArgument(inputContext instanceof PartitionedFileSetInputContext); this.pfsInputcontext = (PartitionedFileSetInputContext) inputContext; Preconditions.checkNotNull(pfsInputcontext.getInputPartitionKey()); Preconditions.checkArgument( pfsInputcontext.getInputPartitionKeys().contains(pfsInputcontext.getInputPartitionKey()) ); Map<String, String> dsArguments = RuntimeArguments.extractScope(Scope.DATASET, PARTITIONED, context.getRuntimeArguments()); PartitionFilter inputPartitionFilter = PartitionedFileSetArguments.getInputPartitionFilter(dsArguments); Preconditions.checkNotNull(inputPartitionFilter); // verify that the partition matches the partition filter for this MapReduce Preconditions.checkArgument(inputPartitionFilter.match(pfsInputcontext.getInputPartitionKey())); }
@Test public void testSetGetInputPartitionFilter() throws Exception { Map<String, String> arguments = new HashMap<>(); PartitionFilter filter = PartitionFilter.builder() .addRangeCondition("i", 30, 40) .addValueCondition("l", 17L) .addValueCondition("s", "x") .build(); PartitionedFileSetArguments.setInputPartitionFilter(arguments, filter); Assert.assertEquals(filter, PartitionedFileSetArguments.getInputPartitionFilter(arguments)); arguments = new HashMap<>(); filter = PartitionFilter.builder() .addRangeCondition("i", 30, 40) .addValueCondition("s", "x") .build(); PartitionedFileSetArguments.setInputPartitionFilter(arguments, filter); Assert.assertEquals(filter, PartitionedFileSetArguments.getInputPartitionFilter(arguments)); arguments = new HashMap<>(); filter = PartitionFilter.ALWAYS_MATCH; PartitionedFileSetArguments.setInputPartitionFilter(arguments, filter); Assert.assertEquals(filter, PartitionedFileSetArguments.getInputPartitionFilter(arguments)); }