@Override public void connect() throws FileBasedHelperException { this.fileBasedHelper.connect(); }
/** * Downloads a file from the source * * @param file * is the file to download * @return an iterator over the file * TODO Add support for different file formats besides text e.g. avro iterator, byte iterator, json iterator. */ public Iterator<D> downloadFile(String file) throws IOException { return this.fileDownloader.downloadFile(file); }
@Override public InputStream getFileStream(String path) throws FileBasedHelperException { return this.fileBasedHelper.getFileStream(path); }
@Override public Iterator<String> downloadFile(String filePath) throws IOException { Preconditions.checkArgument(this.token != null); try { log.info("downloading file: " + filePath); InputStream inputStream = this.fileBasedExtractor.getCloser().register(this.fileBasedExtractor.getFsHelper().getFileStream(filePath)); return new RecordIterator(inputStream, this.token, this.charset); } catch (FileBasedHelperException e) { throw new IOException("Exception when trying to download file " + filePath, e); } }
public void testReadRecordWithEmptyFiles() throws DataRecordException, IOException, FileBasedHelperException { String file1 = "file1.txt"; String file2 = "file2.txt"; String file3 = "file3.txt"; WorkUnitState state = new WorkUnitState(); state.setProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL, Joiner.on(",").join(file1, file2, file3)); FileBasedHelper fsHelper = Mockito.mock(FileBasedHelper.class); Mockito.when(fsHelper.getFileStream(file1)).thenReturn(IOUtils.toInputStream("")); Mockito.when(fsHelper.getFileStream(file2)).thenReturn(IOUtils.toInputStream("")); Mockito.when(fsHelper.getFileStream(file3)).thenReturn(IOUtils.toInputStream("")); FileBasedExtractor<String, String> extractor = new DummyFileBasedExtractor<String, String>(state, fsHelper); Assert.assertEquals(getNumRecords(extractor), 0); }
@Override public List<String> getcurrentFsSnapshot(State state) { List<String> results = Lists.newArrayList(); String path = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY); try { LOGGER.info("Running ls command with input " + path); results = this.fsHelper.ls(path); } catch (FileBasedHelperException e) { LOGGER.error("Not able to run ls command due to " + e.getMessage() + " will not pull any files", e); } return results; } }
private void incrementBytesReadCounter() { try { this.counters.inc(CounterNames.FileBytesRead, this.fsHelper.getFileSize(this.currentFile)); } catch (FileBasedHelperException e) { LOG.info("Unable to get file size. Will skip increment to bytes counter " + e.getMessage()); LOG.debug(e.getMessage(), e); } catch (UnsupportedOperationException e) { LOG.info("Unable to get file size. Will skip increment to bytes counter " + e.getMessage()); LOG.debug(e.getMessage(), e); } } }
@Override public void initFileSystemHelper(State state) throws FileBasedHelperException { this.fsHelper = new AvroFsHelper(state); this.fsHelper.connect(); }
@Override public String next() { return this.hasNext() ? this.scanner.next() : null; }
@Override public void close() throws IOException { this.fileBasedHelper.close(); }
@Override public List<String> ls(String path) throws FileBasedHelperException { return this.fileBasedHelper.ls(path); }
@Override public void shutdown(SourceState state) { if (this.fsHelper != null) { log.info("Shutting down the FileSystemHelper connection"); try { this.fsHelper.close(); } catch (IOException e) { log.error("Unable to shutdown FileSystemHelper", e); } } }
@Override public void close() throws IOException { try { this.fsHelper.close(); } catch (IOException e) { LOG.error("Could not successfully close file system helper due to error: " + e.getMessage(), e); } }
public void testReadRecordWithNoFiles() throws DataRecordException, IOException { WorkUnitState state = new WorkUnitState(); state.setProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL, ""); FileBasedHelper fsHelper = Mockito.mock(FileBasedHelper.class); FileBasedExtractor<String, String> extractor = new DummyFileBasedExtractor<String, String>(state, fsHelper); Assert.assertEquals(getNumRecords(extractor), 0); }
@Override public Extractor<String, String> getExtractor(WorkUnitState state) throws IOException { return new DummyExtractor(); } }
public void testReadRecordWithNonEmptyFiles() throws DataRecordException, IOException, FileBasedHelperException { String file1 = "file1.txt"; String file2 = "file2.txt"; String file3 = "file3.txt"; WorkUnitState state = new WorkUnitState(); state.setProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL, Joiner.on(",").join(file1, file2, file3)); FileBasedHelper fsHelper = Mockito.mock(FileBasedHelper.class); Mockito.when(fsHelper.getFileStream(file1)).thenReturn(IOUtils.toInputStream("record1 \n record2")); Mockito.when(fsHelper.getFileStream(file2)).thenReturn(IOUtils.toInputStream("record3 \n record4")); Mockito.when(fsHelper.getFileStream(file3)).thenReturn(IOUtils.toInputStream("record5 \n record6 \n record7")); FileBasedExtractor<String, String> extractor = new DummyFileBasedExtractor<String, String>(state, fsHelper); Assert.assertEquals(getNumRecords(extractor), 7); }
@Override public void initFileSystemHelper(State state) throws FileBasedHelperException { this.fsHelper = new AvroFsHelper(state); this.fsHelper.connect(); }
public void testReadRecordWithEmptyAndNonEmptyFiles() throws DataRecordException, IOException, FileBasedHelperException { String file1 = "file1.txt"; String file2 = "file2.txt"; String file3 = "file3.txt"; WorkUnitState state = new WorkUnitState(); state.setProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL, Joiner.on(",").join(file1, file2, file3)); FileBasedHelper fsHelper = Mockito.mock(FileBasedHelper.class); Mockito.when(fsHelper.getFileStream(file1)).thenReturn(IOUtils.toInputStream("record1 \n record2")); Mockito.when(fsHelper.getFileStream(file2)).thenReturn(IOUtils.toInputStream("")); Mockito.when(fsHelper.getFileStream(file3)).thenReturn(IOUtils.toInputStream("record3 \n record4 \n record5")); FileBasedExtractor<String, String> extractor = new DummyFileBasedExtractor<String, String>(state, fsHelper); Assert.assertEquals(getNumRecords(extractor), 5); }
@Override public void initFileSystemHelper(State state) throws FileBasedHelperException { this.fsHelper = new SftpFsHelper(state); this.fsHelper.connect(); } }
@Override public void initFileSystemHelper(State state) throws FileBasedHelperException { this.fsHelper = new HadoopFsHelper(state); this.fsHelper.connect(); }