@Override public Iterator<String> downloadFile(String filePath) throws IOException { Preconditions.checkArgument(this.token != null); try { log.info("downloading file: " + filePath); InputStream inputStream = this.fileBasedExtractor.getCloser().register(this.fileBasedExtractor.getFsHelper().getFileStream(filePath)); return new RecordIterator(inputStream, this.token, this.charset); } catch (FileBasedHelperException e) { throw new IOException("Exception when trying to download file " + filePath, e); } }
@Test public void testRecordIterator() throws UnsupportedEncodingException { String charset = "UTF-8"; String delimiter = "\n\r"; String record1 = "record1"; String record2 = "record2\n"; String record3 = "record3\r"; InputStream inputStream = new ByteArrayInputStream(Joiner.on(delimiter).join(record1, record2, record3).getBytes(charset)); TokenizedFileDownloader.RecordIterator recordIterator = new TokenizedFileDownloader.RecordIterator(inputStream, delimiter, charset); Assert.assertTrue(recordIterator.hasNext()); Assert.assertEquals(recordIterator.next(), record1); Assert.assertTrue(recordIterator.hasNext()); Assert.assertEquals(recordIterator.next(), record2); Assert.assertTrue(recordIterator.hasNext()); Assert.assertEquals(recordIterator.next(), record3); Assert.assertFalse(recordIterator.hasNext()); } }
@Override public Iterator<String> downloadFile(String filePath) throws IOException { Preconditions.checkArgument(this.token != null); try { log.info("downloading file: " + filePath); InputStream inputStream = this.fileBasedExtractor.getCloser().register(this.fileBasedExtractor.getFsHelper().getFileStream(filePath)); return new RecordIterator(inputStream, this.token, this.charset); } catch (FileBasedHelperException e) { throw new IOException("Exception when trying to download file " + filePath, e); } }