Tabnine Logo
FileBasedHelperException
Code IndexAdd Tabnine to your IDE (free)

How to use
FileBasedHelperException
in
org.apache.gobblin.source.extractor.filebased

Best Java code snippets using org.apache.gobblin.source.extractor.filebased.FileBasedHelperException (Showing top 20 results out of 315)

origin: apache/incubator-gobblin

 @Override
 public long getFileMTime(String fileId) throws FileBasedHelperException {
  Preconditions.checkNotNull(fileId, "fileId is required");
  Path p = new Path(fileId);

  try {
   FileStatus status = fileSystem.getFileStatus(p);
   return status.getModificationTime();
  } catch (IOException e) {
   throw new FileBasedHelperException("Failed to retrieve getModificationTime on path: " + p + " , fileId: " + fileId, e);
  }
 }
}
origin: apache/incubator-gobblin

 private void incrementBytesReadCounter() {
  try {
   this.counters.inc(CounterNames.FileBytesRead, this.fsHelper.getFileSize(this.currentFile));
  } catch (FileBasedHelperException e) {
   LOG.info("Unable to get file size. Will skip increment to bytes counter " + e.getMessage());
   LOG.debug(e.getMessage(), e);
  } catch (UnsupportedOperationException e) {
   LOG.info("Unable to get file size. Will skip increment to bytes counter " + e.getMessage());
   LOG.debug(e.getMessage(), e);
  }
 }
}
origin: apache/incubator-gobblin

@Override
public long getFileSize(String fileId) throws FileBasedHelperException {
 Preconditions.checkNotNull(fileId, "fileId is required");
 Path p = new Path(fileId);
 try {
  FileStatus status = fileSystem.getFileStatus(p);
  return status.getLen();
 } catch (IOException e) {
  throw new FileBasedHelperException("Failed to get metadata on " + fileId, e);
 }
}
origin: apache/incubator-gobblin

 @Override
 public List<String> getcurrentFsSnapshot(State state) {
  List<String> results = Lists.newArrayList();
  String path = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY);

  try {
   LOGGER.info("Running ls command with input " + path);
   results = this.fsHelper.ls(path);
  } catch (FileBasedHelperException e) {
   LOGGER.error("Not able to run ls command due to " + e.getMessage() + " will not pull any files", e);
  }
  return results;
 }
}
origin: apache/incubator-gobblin

@Override
public InputStream getFileStream(String fileId) throws FileBasedHelperException {
 Preconditions.checkNotNull(fileId, "fileId is required");
 Path p = new Path(fileId);
 try {
  if (bufferSizeByte.isPresent()) {
   return fileSystem.open(p, bufferSizeByte.get());
  }
  return fileSystem.open(p);
 } catch (IOException e) {
  throw new FileBasedHelperException("Failed to open files stream on path: " + p + " , fileId: " + fileId, e);
 }
}
origin: apache/incubator-gobblin

 @SuppressWarnings("unchecked")
 public Iterator<D> downloadFile(String file) throws IOException {

  log.info("Beginning to download gzip compressed file: " + file);

  try {
   InputStream inputStream =
     this.fileBasedExtractor.getCloser().register(this.fileBasedExtractor.getFsHelper().getFileStream(file));
   Iterator<D> fileItr = (Iterator<D>) IOUtils.lineIterator(new GZIPInputStream(inputStream),
     ConfigurationKeys.DEFAULT_CHARSET_ENCODING);
   if (this.fileBasedExtractor.isShouldSkipFirstRecord() && fileItr.hasNext()) {
    fileItr.next();
   }
   return fileItr;
  } catch (FileBasedHelperException e) {
   throw new IOException("Exception while downloading file " + file + " with message " + e.getMessage(), e);
  }
 }
}
origin: apache/incubator-gobblin

@Override
public void connect() throws FileBasedHelperException {
 String uri = this.state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI);
 try {
  if (Strings.isNullOrEmpty(uri)) {
   throw new FileBasedHelperException(ConfigurationKeys.SOURCE_FILEBASED_FS_URI + " has not been configured");
  }
  this.createFileSystem(uri);
 } catch (IOException e) {
  throw new FileBasedHelperException("Cannot connect to given URI " + uri + " due to " + e.getMessage(), e);
 } catch (URISyntaxException e) {
  throw new FileBasedHelperException("Malformed uri " + uri + " due to " + e.getMessage(), e);
 } catch (InterruptedException e) {
  throw new FileBasedHelperException("Interrupted exception is caught when getting the proxy file system", e);
 }
}
origin: apache/incubator-gobblin

 @SuppressWarnings("unchecked")
 public Iterator<D> downloadFile(String file) throws IOException {

  log.info("Beginning to download file: " + file);

  try {
   InputStream inputStream =
     this.fileBasedExtractor.getCloser().register(this.fileBasedExtractor.getFsHelper().getFileStream(file));
   Iterator<D> fileItr = (Iterator<D>) IOUtils.lineIterator(inputStream, ConfigurationKeys.DEFAULT_CHARSET_ENCODING);
   if (this.fileBasedExtractor.isShouldSkipFirstRecord() && fileItr.hasNext()) {
    fileItr.next();
   }
   return fileItr;
  } catch (FileBasedHelperException e) {
   throw new IOException("Exception while downloading file " + file + " with message " + e.getMessage(), e);
  }
 }
}
origin: apache/incubator-gobblin

 @Override
 public long getFileSize(String filePath) throws FileBasedHelperException {
  try {
   return this.getFileSystem().getFileStatus(new Path(filePath)).getLen();
  } catch (IOException e) {
   throw new FileBasedHelperException(
     String.format("Failed to get size for file at path %s due to error %s", filePath, e.getMessage()), e);
  }
 }
}
origin: org.apache.gobblin/gobblin-core

 private void incrementBytesReadCounter() {
  try {
   this.counters.inc(CounterNames.FileBytesRead, this.fsHelper.getFileSize(this.currentFile));
  } catch (FileBasedHelperException e) {
   LOG.info("Unable to get file size. Will skip increment to bytes counter " + e.getMessage());
   LOG.debug(e.getMessage(), e);
  } catch (UnsupportedOperationException e) {
   LOG.info("Unable to get file size. Will skip increment to bytes counter " + e.getMessage());
   LOG.debug(e.getMessage(), e);
  }
 }
}
origin: apache/incubator-gobblin

@Override
public long getFileMTime(String filePath) throws FileBasedHelperException {
 try {
  return this.getFileSystem().getFileStatus(new Path(filePath)).getModificationTime();
 } catch (IOException e) {
  throw new FileBasedHelperException(String
    .format("Failed to get last modified time for file at path %s due to error %s", filePath, e.getMessage()), e);
 }
}
origin: org.apache.gobblin/gobblin-core

 @Override
 public List<String> getcurrentFsSnapshot(State state) {
  List<String> results = Lists.newArrayList();
  String path = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY);

  try {
   LOGGER.info("Running ls command with input " + path);
   results = this.fsHelper.ls(path);
  } catch (FileBasedHelperException e) {
   LOGGER.error("Not able to run ls command due to " + e.getMessage() + " will not pull any files", e);
  }
  return results;
 }
}
origin: apache/incubator-gobblin

@Override
public long getFileSize(String filePath) throws FileBasedHelperException {
 try {
  return this.getFileSystem().getFileStatus(new Path(filePath)).getLen();
 } catch (IOException e) {
  throw new FileBasedHelperException(
    String.format("Failed to get size for file at path %s due to error %s", filePath, e.getMessage()), e);
 }
}
origin: org.apache.gobblin/gobblin-core

 @SuppressWarnings("unchecked")
 public Iterator<D> downloadFile(String file) throws IOException {

  log.info("Beginning to download file: " + file);

  try {
   InputStream inputStream =
     this.fileBasedExtractor.getCloser().register(this.fileBasedExtractor.getFsHelper().getFileStream(file));
   Iterator<D> fileItr = (Iterator<D>) IOUtils.lineIterator(inputStream, ConfigurationKeys.DEFAULT_CHARSET_ENCODING);
   if (this.fileBasedExtractor.isShouldSkipFirstRecord() && fileItr.hasNext()) {
    fileItr.next();
   }
   return fileItr;
  } catch (FileBasedHelperException e) {
   throw new IOException("Exception while downloading file " + file + " with message " + e.getMessage(), e);
  }
 }
}
origin: apache/incubator-gobblin

@Override
public List<String> ls(String path) throws FileBasedHelperException {
 List<String> results = new ArrayList<>();
 try {
  lsr(new Path(path), results);
 } catch (IOException e) {
  throw new FileBasedHelperException("Cannot do ls on path " + path + " due to " + e.getMessage(), e);
 }
 return results;
}
origin: org.apache.gobblin/gobblin-core

 @SuppressWarnings("unchecked")
 public Iterator<D> downloadFile(String file) throws IOException {

  log.info("Beginning to download gzip compressed file: " + file);

  try {
   InputStream inputStream =
     this.fileBasedExtractor.getCloser().register(this.fileBasedExtractor.getFsHelper().getFileStream(file));
   Iterator<D> fileItr = (Iterator<D>) IOUtils.lineIterator(new GZIPInputStream(inputStream),
     ConfigurationKeys.DEFAULT_CHARSET_ENCODING);
   if (this.fileBasedExtractor.isShouldSkipFirstRecord() && fileItr.hasNext()) {
    fileItr.next();
   }
   return fileItr;
  } catch (FileBasedHelperException e) {
   throw new IOException("Exception while downloading file " + file + " with message " + e.getMessage(), e);
  }
 }
}
origin: apache/incubator-gobblin

/**
 * List files under folder ID recursively. Folder won't be included in the result. If there's no files under folder ID, it returns empty list.
 * If folder ID is not defined, it will provide files under root directory.
 * {@inheritDoc}
 * @see org.apache.gobblin.source.extractor.filebased.FileBasedHelper#ls(java.lang.String)
 */
@Override
public List<String> ls(String folderId) throws FileBasedHelperException {
 List<String> result = new ArrayList<>();
 if (StringUtils.isEmpty(folderId)) {
  folderId = "/";
 }
 Path p = new Path(folderId);
 FileStatus[] statusList = null;
 try {
  statusList = fileSystem.listStatus(p);
 } catch (FileNotFoundException e) {
  return result;
 } catch (IOException e) {
  throw new FileBasedHelperException("Falied to list status on path " + p + ", folderID: " + folderId, e);
 }
 for (FileStatus status : statusList) {
  if (status.isDirectory()) {
   result.addAll(ls(GoogleDriveFileSystem.toFileId(status.getPath())));
  } else {
   result.add(GoogleDriveFileSystem.toFileId(status.getPath()));
  }
 }
 return result;
}
origin: apache/incubator-gobblin

@Override
public List<String> ls(String path) throws FileBasedHelperException {
 try {
  List<String> list = new ArrayList<>();
  ChannelSftp channel = getSftpChannel();
  Vector<LsEntry> vector = channel.ls(path);
  for (LsEntry entry : vector) {
   list.add(entry.getFilename());
  }
  channel.disconnect();
  return list;
 } catch (SftpException e) {
  throw new FileBasedHelperException("Cannot execute ls command on sftp connection", e);
 }
}
origin: apache/incubator-gobblin

/**
 * Returns an {@link DataFileReader} to the specified avro file.
 * <p>
 * Note: It is the caller's responsibility to close the returned {@link DataFileReader}.
 * </p>
 *
 * @param file The path to the avro file to open.
 * @return A {@link DataFileReader} for the specified avro file.
 * @throws FileBasedHelperException if there is a problem opening the {@link InputStream} for the specified file.
 */
public DataFileReader<GenericRecord> getAvroFile(String file) throws FileBasedHelperException {
 try {
  if (!this.getFileSystem().exists(new Path(file))) {
   LOGGER.warn(file + " does not exist.");
   return null;
  }
  if (this.getState().getPropAsBoolean(ConfigurationKeys.SHOULD_FS_PROXY_AS_USER,
    ConfigurationKeys.DEFAULT_SHOULD_FS_PROXY_AS_USER)) {
   return new DataFileReader<>(new ProxyFsInput(new Path(file), this.getFileSystem()),
     new GenericDatumReader<GenericRecord>());
  }
  return new DataFileReader<>(new FsInput(new Path(file), this.getFileSystem().getConf()),
    new GenericDatumReader<GenericRecord>());
 } catch (IOException e) {
  throw new FileBasedHelperException("Failed to open avro file " + file + " due to error " + e.getMessage(), e);
 }
}
origin: apache/incubator-gobblin

public Schema getAvroSchema(String file) throws FileBasedHelperException {
 DataFileReader<GenericRecord> dfr = null;
 try {
  if (this.getState().getPropAsBoolean(ConfigurationKeys.SHOULD_FS_PROXY_AS_USER,
    ConfigurationKeys.DEFAULT_SHOULD_FS_PROXY_AS_USER)) {
   dfr = new DataFileReader<>(new ProxyFsInput(new Path(file), this.getFileSystem()),
     new GenericDatumReader<GenericRecord>());
  } else {
   dfr = new DataFileReader<>(new FsInput(new Path(file), this.getFileSystem().getConf()),
     new GenericDatumReader<GenericRecord>());
  }
  return dfr.getSchema();
 } catch (IOException e) {
  throw new FileBasedHelperException("Failed to open avro file " + file + " due to error " + e.getMessage(), e);
 } finally {
  if (dfr != null) {
   try {
    dfr.close();
   } catch (IOException e) {
    LOGGER.error("Failed to close avro file " + file, e);
   }
  }
 }
}
org.apache.gobblin.source.extractor.filebasedFileBasedHelperException

Most used methods

  • <init>
  • getMessage

Popular in Java

  • Making http post requests using okhttp
  • getSharedPreferences (Context)
  • findViewById (Activity)
  • putExtra (Intent)
  • Selector (java.nio.channels)
    A controller for the selection of SelectableChannel objects. Selectable channels can be registered w
  • Deque (java.util)
    A linear collection that supports element insertion and removal at both ends. The name deque is shor
  • Queue (java.util)
    A collection designed for holding elements prior to processing. Besides basic java.util.Collection o
  • Set (java.util)
    A Set is a data structure which does not allow duplicate elements.
  • Annotation (javassist.bytecode.annotation)
    The annotation structure.An instance of this class is returned bygetAnnotations() in AnnotationsAttr
  • Base64 (org.apache.commons.codec.binary)
    Provides Base64 encoding and decoding as defined by RFC 2045.This class implements section 6.8. Base
  • Best IntelliJ plugins
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now