private Path[] convertDefaults(Path[] psrcs) { Path[] psrcsNew = new Path[psrcs.length]; for (int i = 0; i < psrcs.length; i++) { psrcsNew[i] = convertToDefaultPath(psrcs[i]); } return psrcsNew; }
public static <T extends HoodieRecordPayload, R extends IndexedRecord> HoodieStorageWriter<R> getStorageWriter( String commitTime, Path path, HoodieTable<T> hoodieTable, HoodieWriteConfig config, Schema schema) throws IOException { //TODO - based on the metadata choose the implementation of HoodieStorageWriter // Currently only parquet is supported return newParquetStorageWriter(commitTime, path, config, schema, hoodieTable); }
public boolean canWrite() { return fs.getBytesWritten(file) < maxFileSize; }
@Override public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { final Path translatedPath = convertToDefaultPath(f); return wrapOutputStream(f, fileSystem .create(translatedPath, permission, overwrite, bufferSize, replication, blockSize, progress)); }
@Override public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException { return convertToHoodiePath(fileSystem .startLocalOutput(convertToDefaultPath(fsOutputFile), convertToDefaultPath(tmpLocalFile))); }
@Override public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs, Path dst) throws IOException { fileSystem .copyFromLocalFile(delSrc, overwrite, convertDefaults(srcs), convertToDefaultPath(dst)); }
private Path[] convertDefaults(Path[] psrcs) { Path[] psrcsNew = new Path[psrcs.length]; for (int i = 0; i < psrcs.length; i++) { psrcsNew[i] = convertToDefaultPath(psrcs[i]); } return psrcsNew; }
public static <T extends HoodieRecordPayload, R extends IndexedRecord> HoodieStorageWriter<R> getStorageWriter( String commitTime, Path path, HoodieTable<T> hoodieTable, HoodieWriteConfig config, Schema schema) throws IOException { //TODO - based on the metadata choose the implementation of HoodieStorageWriter // Currently only parquet is supported return newParquetStorageWriter(commitTime, path, config, schema, hoodieTable); }
public boolean canWrite() { return fs.getBytesWritten(file) < maxFileSize; }
@Override public FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress, Options.ChecksumOpt checksumOpt) throws IOException { return fileSystem .create(convertToDefaultPath(f), permission, flags, bufferSize, replication, blockSize, progress, checksumOpt); }
@Override public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, Progressable progress) throws IOException { return fileSystem.create(convertToDefaultPath(f), overwrite, bufferSize, progress); }
@Override public FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return fileSystem .create(convertToDefaultPath(f), permission, flags, bufferSize, replication, blockSize, progress); }
@Override public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException { fileSystem .copyFromLocalFile(delSrc, overwrite, convertToDefaultPath(src), convertToDefaultPath(dst)); }
@Override public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException { fileSystem.completeLocalOutput(convertToDefaultPath(fsOutputFile), convertToDefaultPath(tmpLocalFile)); }
@Override public void copyToLocalFile(boolean delSrc, Path src, Path dst, boolean useRawLocalFileSystem) throws IOException { fileSystem.copyToLocalFile(delSrc, convertToDefaultPath(src), convertToDefaultPath(dst), useRawLocalFileSystem); }
@Override public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException { return fileSystem.append(convertToDefaultPath(f), bufferSize, progress); }
@Override public FSDataOutputStream createNonRecursive(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return fileSystem .createNonRecursive(convertToDefaultPath(f), overwrite, bufferSize, replication, blockSize, progress); }
@Override public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return fileSystem .createNonRecursive(convertToDefaultPath(f), permission, overwrite, bufferSize, replication, blockSize, progress); }
@Override public FSDataOutputStream createNonRecursive(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return fileSystem .createNonRecursive(convertToDefaultPath(f), overwrite, bufferSize, replication, blockSize, progress); }
@Override public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return fileSystem .createNonRecursive(convertToDefaultPath(f), permission, overwrite, bufferSize, replication, blockSize, progress); }