private static byte[] encodeUtf8(String str) { if (str == null) { return new byte[0]; } try { return str.getBytes("UTF-8"); } catch (UnsupportedEncodingException e) { throw new DatasetIOException("[FATAL] Cannot decode UTF-8", e); } }
@Test public void testMissingHDFSAuthority() { try { Datasets.load("dataset:hdfs:/tmp/data/ns/test", Object.class); Assert.fail("Shouldn't be able to connect to HDFS"); } catch (DatasetIOException e) { Assert.assertTrue("Should have helpful error message", e.getMessage().contains("make sure the default hdfs URI is configured")); } }
private static FileSystem fsForPath(Path dataPath, Configuration conf) { try { return dataPath.getFileSystem(conf); } catch (IOException ex) { throw new DatasetIOException( "Cannot get FileSystem for descriptor", ex); } }
static FileSystem fsForPath(Configuration conf, Path path) { try { return path.getFileSystem(conf); } catch (IOException ex) { throw new DatasetIOException("Cannot access FileSystem for uri:" + path, ex); } }
static FileSystem fsForPath(Configuration conf, Path path) { try { return path.getFileSystem(conf); } catch (IOException ex) { throw new DatasetIOException("Cannot access FileSystem for uri:" + path, ex); } }
public HiveExternalMetadataProvider(Configuration conf, Path rootDirectory) { super(conf); Preconditions.checkNotNull(rootDirectory, "Root cannot be null"); try { this.rootFileSystem = rootDirectory.getFileSystem(conf); this.rootDirectory = rootFileSystem.makeQualified(rootDirectory); } catch (IOException ex) { throw new DatasetIOException("Could not get FileSystem for root path", ex); } }
private boolean advance() { try { next = reader.readNext(); } catch (IOException ex) { throw new DatasetIOException("Could not read record", ex); } return (next != null); }
private void advance() { try { this.next = reader.read(); } catch (EOFException e) { this.next = null; } catch (IOException e) { this.state = ReaderWriterState.ERROR; throw new DatasetIOException("Unable to read next record from: " + path, e); } }
public HiveExternalMetadataProvider(Configuration conf, Path rootDirectory) { super(conf); Preconditions.checkNotNull(rootDirectory, "Root cannot be null"); try { this.rootFileSystem = rootDirectory.getFileSystem(conf); this.rootDirectory = rootFileSystem.makeQualified(rootDirectory); } catch (IOException ex) { throw new DatasetIOException("Could not get FileSystem for root path", ex); } }
private Schema loadSchema(Path schemaPath) { Schema schema = null; InputStream inputStream = null; boolean threw = true; try { inputStream = rootFileSystem.open(schemaPath); schema = new Schema.Parser().parse(inputStream); threw = false; } catch (IOException e) { throw new DatasetIOException( "Unable to load schema file:" + schemaPath, e); } finally { try { Closeables.close(inputStream, threw); } catch (IOException e) { throw new DatasetIOException("Cannot close", e); } } return schema; }
@Override public void close() { if (!state.equals(ReaderWriterState.OPEN)) { return; } LOG.debug("Closing reader on path:{}", path); iterator = null; try { incoming.close(); } catch (IOException e) { throw new DatasetIOException("Unable to close reader path:" + path, e); } state = ReaderWriterState.CLOSED; }
@Override public long getSize() { long size = 0; for (Iterator<Path> i = dirIterator(); i.hasNext(); ) { Path dir = i.next(); try { for (FileStatus st : fs.listStatus(dir)) { size += st.getLen(); } } catch (IOException e) { throw new DatasetIOException("Cannot find size of " + dir, e); } } return size; }
@Override public long getSize() { long size = 0; for (Iterator<Path> i = dirIterator(); i.hasNext(); ) { Path dir = i.next(); try { for (FileStatus st : fileSystem.listStatus(dir)) { size += st.getLen(); } } catch (IOException e) { throw new DatasetIOException("Cannot find size of " + dir, e); } } return size; }
/** * Returns the path of the newest schema file, or null if none exists. */ private Path newestFile() { try { FileStatus[] statuses = rootFileSystem.listStatus(schemaDirectory); // No schema files exist, so return null; if (statuses.length == 0) { return null; } // Sort the schema files and return the newest one. Arrays.sort(statuses, new FileNameComparator()); return statuses[statuses.length - 1].getPath(); } catch (IOException e) { throw new DatasetIOException("Unable to list schema files.", e); } }
/** * Create a signal for the specified constraints. * * @param viewConstraints The constraints to create a signal for. * * @throws DatasetException if the signal could not be created. */ public void signalReady(Constraints viewConstraints) { try { rootFileSystem.mkdirs(signalDirectory); } catch (IOException e) { throw new DatasetIOException("Unable to create signal manager directory: " + signalDirectory, e); } String normalizedConstraints = getNormalizedConstraints(viewConstraints); Path signalPath = new Path(signalDirectory, normalizedConstraints); try{ // create the output stream to overwrite the current contents, if the directory or file // exists it will be overwritten to get a new timestamp FSDataOutputStream os = rootFileSystem.create(signalPath, true); os.close(); } catch (IOException e) { throw new DatasetIOException("Could not access signal path: " + signalPath, e); } }
/** * Precondition-style static validation that a dataset exists * * @param fs A FileSystem where the metadata should be stored * @param location The Path where the metadata should be stored * @throws org.kitesdk.data.DatasetNotFoundException if the descriptor location is missing * @throws org.kitesdk.data.DatasetIOException if any IOException is thrown */ private static void checkExists(FileSystem fs, Path location) { try { if (!fs.exists(location)) { throw new DatasetNotFoundException( "Descriptor location does not exist: " + location); } } catch (IOException ex) { throw new DatasetIOException( "Cannot access descriptor location: " + location, ex); } }
/** * Returns the closest directory for the given {@code path}. * * @param fs a {@link FileSystem} to search * @param path a {@link Path} to resolve * @return the closest directory to {@link Path} */ private static Path getDirectory(FileSystem fs, Path path) { try { if (!fs.exists(path) || fs.isDirectory(path)) { return path; } else { return path.getParent(); } } catch (IOException e) { throw new DatasetIOException("Cannot access path: " + path, e); } } }
public FileSystemMetadataProvider(Configuration conf, Path rootDirectory) { Preconditions.checkNotNull(conf, "Configuration cannot be null"); Preconditions.checkNotNull(rootDirectory, "Root directory cannot be null"); this.conf = conf; try { this.rootFileSystem = rootDirectory.getFileSystem(conf); this.rootDirectory = rootFileSystem.makeQualified(rootDirectory); } catch (IOException ex) { throw new DatasetIOException("Cannot get FileSystem for root path", ex); } }
public static <T> T parse(String json, Class<T> returnType) { ObjectMapper mapper = new ObjectMapper(); try { return mapper.readValue(json, returnType); } catch (JsonParseException e) { throw new ValidationException("Invalid JSON", e); } catch (JsonMappingException e) { throw new ValidationException("Invalid JSON", e); } catch (IOException e) { throw new DatasetIOException("Cannot initialize JSON parser", e); } }
@Override public void partitionAdded(String namespace, String name, String path) { Path partitionPath = new Path(pathForDataset(namespace, name), path); try { rootFileSystem.mkdirs(partitionPath); } catch (IOException ex) { throw new DatasetIOException( "Unable to create partition directory " + partitionPath, ex); } super.partitionAdded(namespace, name, path); }