@Override public List<String> getcurrentFsSnapshot(State state) { List<String> results = Lists.newArrayList(); String path = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY); try { LOGGER.info("Running ls command with input " + path); results = this.fsHelper.ls(path); } catch (FileBasedHelperException e) { LOGGER.error("Not able to run ls command due to " + e.getMessage() + " will not pull any files", e); } return results; } }
/** * Provide list of files snapshot where snap shot is consist of list of file ID with modified time. * Folder ID and file ID are all optional where missing folder id represent search from root folder where * missing file ID represents all files will be included on current and subfolder. * * {@inheritDoc} * @see org.apache.gobblin.source.extractor.filebased.FileBasedSource#getcurrentFsSnapshot(org.apache.gobblin.configuration.State) */ @Override public List<String> getcurrentFsSnapshot(State state) { List<String> results = new ArrayList<>(); String folderId = state.getProp(SOURCE_FILEBASED_DATA_DIRECTORY, ""); try { LOG.info("Running ls with folderId: " + folderId); List<String> fileIds = this.fsHelper.ls(folderId); for (String fileId : fileIds) { results.add(fileId + splitPattern + this.fsHelper.getFileMTime(fileId)); } } catch (FileBasedHelperException e) { throw new RuntimeException("Failed to retrieve list of file IDs for folderID: " + folderId, e); } return results; }
/** * This method is responsible for connecting to the source and taking * a snapshot of the folder where the data is present, it then returns * a list of the files in String format * @param state is used to connect to the source * @return a list of file name or paths present on the external data * directory */ public List<String> getcurrentFsSnapshot(State state) { List<String> results = new ArrayList<>(); String path = getLsPattern(state); try { log.info("Running ls command with input " + path); results = this.fsHelper.ls(path); for (int i = 0; i < results.size(); i++) { URI uri = new URI(results.get(i)); String filePath = uri.toString(); if (!uri.isAbsolute()) { File file = new File(state.getProp(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY), uri.toString()); filePath = file.getAbsolutePath(); } results.set(i, filePath + this.splitPattern + this.fsHelper.getFileMTime(filePath)); } } catch (FileBasedHelperException | URISyntaxException e) { log.error("Not able to fetch the filename/file modified time to " + e.getMessage() + " will not pull any files", e); } return results; }
@Override public List<String> getcurrentFsSnapshot(State state) { List<String> results = Lists.newArrayList(); String path = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY); try { LOGGER.info("Running ls command with input " + path); results = this.fsHelper.ls(path); } catch (FileBasedHelperException e) { LOGGER.error("Not able to run ls command due to " + e.getMessage() + " will not pull any files", e); } return results; } }
/** * Provide list of files snapshot where snap shot is consist of list of file ID with modified time. * Folder ID and file ID are all optional where missing folder id represent search from root folder where * missing file ID represents all files will be included on current and subfolder. * * {@inheritDoc} * @see org.apache.gobblin.source.extractor.filebased.FileBasedSource#getcurrentFsSnapshot(org.apache.gobblin.configuration.State) */ @Override public List<String> getcurrentFsSnapshot(State state) { List<String> results = new ArrayList<>(); String folderId = state.getProp(SOURCE_FILEBASED_DATA_DIRECTORY, ""); try { LOG.info("Running ls with folderId: " + folderId); List<String> fileIds = this.fsHelper.ls(folderId); for (String fileId : fileIds) { results.add(fileId + splitPattern + this.fsHelper.getFileMTime(fileId)); } } catch (FileBasedHelperException e) { throw new RuntimeException("Failed to retrieve list of file IDs for folderID: " + folderId, e); } return results; }
/** * This method is responsible for connecting to the source and taking * a snapshot of the folder where the data is present, it then returns * a list of the files in String format * @param state is used to connect to the source * @return a list of file name or paths present on the external data * directory */ public List<String> getcurrentFsSnapshot(State state) { List<String> results = new ArrayList<>(); String path = getLsPattern(state); try { log.info("Running ls command with input " + path); results = this.fsHelper.ls(path); for (int i = 0; i < results.size(); i++) { URI uri = new URI(results.get(i)); String filePath = uri.toString(); if (!uri.isAbsolute()) { File file = new File(state.getProp(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY), uri.toString()); filePath = file.getAbsolutePath(); } results.set(i, filePath + this.splitPattern + this.fsHelper.getFileMTime(filePath)); } } catch (FileBasedHelperException | URISyntaxException e) { log.error("Not able to fetch the filename/file modified time to " + e.getMessage() + " will not pull any files", e); } return results; }