private InputFormat<E, Void> getDelegateInputFormatForPartition(Dataset<E> dataset, String partitionDir, Configuration conf) { if (!(dataset instanceof FileSystemDataset)) { throw new UnsupportedOperationException("Partitions only supported for " + "FileSystemDataset. Dataset: " + dataset); } FileSystemDataset<E> fsDataset = (FileSystemDataset<E>) dataset; LOG.debug("Getting delegate input format for dataset {} with partition directory {}", dataset, partitionDir); PartitionKey key = fsDataset.keyFromDirectory(new Path(partitionDir)); LOG.debug("Partition key: {}", key); if (key != null) { PartitionedDataset<E> partition = fsDataset.getPartition(key, false); LOG.debug("Partition: {}", partition); return getDelegateInputFormat(partition, conf); } throw new DatasetException("Cannot find partition " + partitionDir); }
@Override public void setConf(Configuration configuration) { conf = configuration; View<E> view = load(configuration); String partitionDir = conf.get(KITE_PARTITION_DIR); if (view.getDataset().getDescriptor().isPartitioned() && partitionDir != null) { delegate = getDelegateInputFormatForPartition(view.getDataset(), partitionDir, conf); } else { delegate = getDelegateInputFormat(view, conf); } }
@Override public void setConf(Configuration configuration) { conf = configuration; View<E> view = load(configuration); String partitionDir = conf.get(KITE_PARTITION_DIR); if (view.getDataset().getDescriptor().isPartitioned() && partitionDir != null) { delegate = getDelegateInputFormatForPartition(view.getDataset(), partitionDir, conf); } else { delegate = getDelegateInputFormat(view, conf); } }
private InputFormat<E, Void> getDelegateInputFormatForPartition(Dataset<E> dataset, String partitionDir, Configuration conf) { if (!(dataset instanceof FileSystemDataset)) { throw new UnsupportedOperationException("Partitions only supported for " + "FileSystemDataset. Dataset: " + dataset); } FileSystemDataset<E> fsDataset = (FileSystemDataset<E>) dataset; LOG.debug("Getting delegate input format for dataset {} with partition directory {}", dataset, partitionDir); PartitionKey key = fsDataset.keyFromDirectory(new Path(partitionDir)); LOG.debug("Partition key: {}", key); if (key != null) { PartitionedDataset<E> partition = fsDataset.getPartition(key, false); LOG.debug("Partition: {}", partition); return getDelegateInputFormat(partition, conf); } throw new DatasetException("Cannot find partition " + partitionDir); }