@Override public void open(int taskNumber, int numTasks) throws IOException { super.open(taskNumber, numTasks); try { this.charset = Charset.forName(charsetName); } catch (IllegalCharsetNameException e) { throw new IOException("The charset " + charsetName + " is not valid.", e); } catch (UnsupportedCharsetException e) { throw new IOException("The charset " + charsetName + " is not supported.", e); } }
@Override public void close() throws IOException { if (wrt != null) { this.wrt.close(); } super.close(); }
@Override protected String getDirectoryFileName(int taskNumber) { return super.getDirectoryFileName(taskNumber) + ".avro"; }
final Path path = getOutputFilePath(); final FileSystem fs = path.getFileSystem(); final WriteMode writeMode = getWriteMode(); final OutputDirectoryMode outDirMode = getOutputDirectoryMode();
private void initializeIOFormatClasses(Configuration configuration) { // TODO: That we still have to call something like this is a crime against humanity FileOutputFormat.initDefaultsFromConfiguration(configuration); }
@Override public void configure(Configuration parameters) { super.configure(parameters); // read own parameters this.blockSize = parameters.getLong(BLOCK_SIZE_PARAMETER_KEY, NATIVE_BLOCK_SIZE); if (this.blockSize < 1 && this.blockSize != NATIVE_BLOCK_SIZE) { throw new IllegalArgumentException("The block size parameter must be set and larger than 0."); } if (this.blockSize > Integer.MAX_VALUE) { throw new UnsupportedOperationException("Currently only block size up to Integer.MAX_VALUE are supported"); } }
final Path path = getOutputFilePath(); final FileSystem fs = path.getFileSystem(); final WriteMode writeMode = getWriteMode(); final OutputDirectoryMode outDirMode = getOutputDirectoryMode();
private void initializeIOFormatClasses(Configuration configuration) { // TODO: That we still have to call something like this is a crime against humanity FileOutputFormat.initDefaultsFromConfiguration(configuration); }
@Override public void configure(Configuration parameters) { super.configure(parameters); // read own parameters this.blockSize = parameters.getLong(BLOCK_SIZE_PARAMETER_KEY, NATIVE_BLOCK_SIZE); if (this.blockSize < 1 && this.blockSize != NATIVE_BLOCK_SIZE) { throw new IllegalArgumentException("The block size parameter must be set and larger than 0."); } if (this.blockSize > Integer.MAX_VALUE) { throw new UnsupportedOperationException("Currently only block size up to Integer.MAX_VALUE are supported"); } }
@Override public void open(int taskNumber, int numTasks) throws IOException { super.open(taskNumber, numTasks); this.wrt = this.charsetName == null ? new OutputStreamWriter(new BufferedOutputStream(this.stream, 4096)) : new OutputStreamWriter(new BufferedOutputStream(this.stream, 4096), this.charsetName); }
@Override public void close() throws IOException { if (wrt != null) { this.wrt.flush(); this.wrt.close(); } super.close(); }
@Override protected String getDirectoryFileName(int taskNumber) { if(testFileName) { return "fancy-"+(taskNumber+1)+"-"+taskNumber+".avro"; } else { return super.getDirectoryFileName(taskNumber); } } }
final Path path = getOutputFilePath(); final FileSystem fs = path.getFileSystem(); final WriteMode writeMode = getWriteMode(); final OutputDirectoryMode outDirMode = getOutputDirectoryMode();
private void initializeIOFormatClasses(Configuration configuration) { // TODO: That we still have to call something like this is a crime against humanity FileOutputFormat.initDefaultsFromConfiguration(configuration); }
@Override public void configure(Configuration parameters) { super.configure(parameters); // read own parameters this.blockSize = parameters.getLong(BLOCK_SIZE_PARAMETER_KEY, NATIVE_BLOCK_SIZE); if (this.blockSize < 1 && this.blockSize != NATIVE_BLOCK_SIZE) { throw new IllegalArgumentException("The block size parameter must be set and larger than 0."); } if (this.blockSize > Integer.MAX_VALUE) { throw new UnsupportedOperationException("Currently only block size up to Integer.MAX_VALUE are supported"); } }
@Override public void open(int taskNumber, int numTasks) throws IOException { super.open(taskNumber, numTasks); this.wrt = this.charsetName == null ? new OutputStreamWriter(new BufferedOutputStream(this.stream, 4096)) : new OutputStreamWriter(new BufferedOutputStream(this.stream, 4096), this.charsetName); }
@Override public void close() throws IOException { try { DataOutputViewStreamWrapper o = this.outView; if (o != null) { o.close(); } } finally { super.close(); } }
this.actualFilePath = (numTasks > 1 || outputDirectoryMode == OutputDirectoryMode.ALWAYS) ? p.suffix("/" + getDirectoryFileName(taskNumber)) : p;
@Override public void open(int taskNumber, int numTasks) throws IOException { super.open(taskNumber, numTasks); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.outputFilePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockBasedOutput = new BlockBasedOutput(this.stream, (int) blockSize); this.outView = new DataOutputViewStreamWrapper(this.blockBasedOutput); }
@Override public void close() throws IOException { dataFileWriter.flush(); dataFileWriter.close(); super.close(); } }