PathIterator pathIterator() { if (dataset.getDescriptor().isPartitioned()) { return new PathIterator(fs, root, partitionIterator()); } else { return new PathIterator(fs, root, null); } }
@Override public boolean hasNext() { if (files != null && files.hasNext()) { return true; } else { return advance(); } }
@Override public Path next() { if (!hasNext()) { throw new NoSuchElementException(); } return files.next(); }
FileSystemDataset<Object> inDataset = (FileSystemDataset) repo.create( "import", "inputformat", inDescriptor); Iterator<Path> iter = inDataset.pathIterator().iterator(); Preconditions.checkArgument(iter.hasNext(), "Data path has no data files: " + source);
@SuppressWarnings("unchecked") // See https://github.com/Parquet/parquet-mr/issues/106 private void openNextReader() { if (Formats.PARQUET.equals(descriptor.getFormat())) { this.reader = new ParquetFileSystemDatasetReader(fileSystem, filesIter.next(), accessor.getReadSchema(), accessor.getType()); } else if (Formats.JSON.equals(descriptor.getFormat())) { this.reader = new JSONFileReader<E>( fileSystem, filesIter.next(), accessor); } else if (Formats.CSV.equals(descriptor.getFormat())) { this.reader = new CSVFileReader<E>(fileSystem, filesIter.next(), descriptor, accessor); } else if (Formats.INPUTFORMAT.equals(descriptor.getFormat())) { this.reader = new InputFormatReader(fileSystem, filesIter.next(), descriptor); } else { this.reader = new FileSystemDatasetReader<E>(fileSystem, filesIter.next(), accessor.getReadSchema(), accessor.getType()); } reader.initialize(); this.readerIterator = Iterators.filter(reader, constraints.toEntityPredicate( (pathIter != null ? pathIter.getStorageKey() : null), accessor)); }
(FileSystemDataset) repo.create("import", "json", jsonDescriptor); Iterator<Path> iter = jsonDataset.pathIterator().iterator(); Preconditions.checkArgument(iter.hasNext(), "JSON path has no data files: " + source);
return new PathIterator() { private int index = 0;
(FileSystemDataset) repo.create("default", "csv", csvDescriptor); Iterator<Path> iter = csvDataset.pathIterator().iterator(); Preconditions.checkArgument(iter.hasNext(), "CSV path has no data files: " + source);