public CsvInputFormat(char fieldDelimiter, Class<? extends Value> ... fields) { super(); setFieldDelimiter(fieldDelimiter); setFieldTypes(fields); }
/** * for scala compatible, scala-to-java type conversion always has an object wrapper */ public CollectionDataSource(Object... args) { this(new CollectionInputFormat(), args); }
@Override public void open(int taskNumber, int numTasks) throws IOException { super.open(taskNumber, numTasks); if (this.buffer == null) { this.buffer = new byte[this.bufferSize]; } if (this.targetArray == null) { this.targetArray = new byte[64]; } this.pos = 0; }
@Override public Record nextRecord(Record reuse) throws IOException { // check if read buffer must be filled (less than one record contained) if(this.readBufferFillPos - this.readBufferReadPos < this.recordLength) { // try to fill read buffer if(!this.fillReadBuffer()) { return null; } } // update read buffer read marker this.readBufferReadPos += this.recordLength; return this.readBytes(reuse, readBuffer, (this.readBufferReadPos-this.recordLength)) ? reuse : null; }
@Override public void configure(Configuration parameters) { super.configure(parameters); this.vertexDelimiter = (char) parameters.getInteger(VERTEX_DELIMITER_CHAR, '|'); this.degreeDelimiter = (char) parameters.getInteger(DEGREE_DELIMITER_CHAR, ','); } }
public CsvInputFormat(Class<? extends Value> ... fields) { super(); setFieldTypes(fields); }
@Override public void close() throws IOException { if (wrt != null) { this.wrt.close(); } super.close(); }
public CsvInputFormat(char fieldDelimiter) { super(); setFieldDelimiter(fieldDelimiter); }
public T field(Class<? extends Value> type, int textPosition) { return field(type, textPosition, Float.NEGATIVE_INFINITY); }
private RecordFormatCompilerHints(CompilerHints parent) { copyFrom(parent); }
/** * Creates a configuration builder that can be used to set the input * format's parameters to the config in a fluent fashion. * * @return A config builder for setting parameters. */ public static ConfigBuilder configureRecordFormat(FileDataSink target) { return new ConfigBuilder(target.getParameters()); }
/** * Creates a configuration builder that can be used to set the input format's parameters to the config in a fluent * fashion. * * @return A config builder for setting parameters. */ public static ConfigBuilder configureDelimitedFormat(FileDataSink target) { return new ConfigBuilder(target.getParameters()); }
/** * Creates a configuration builder that can be used to set the input format's parameters to the config in a fluent * fashion. * * @return A config builder for setting parameters. */ public static ConfigBuilder configureRecordFormat(FileDataSource target) { return new ConfigBuilder(target, target.getParameters()); }
/** * {@inheritDoc} * @throws IOException */ @Override public FileBaseStatistics getStatistics(BaseStatistics cachedStats) throws IOException { final FileBaseStatistics stats = super.getStatistics(cachedStats); return stats == null ? null : new FileBaseStatistics(stats.getLastModificationTime(), stats.getTotalInputSize(), this.recordLength); }
/** * Creates a new instance for the given input using the given input format. The contract has the default name. * The input types will be checked. If the input types don't agree, an exception will occur. * * @param args * The input data. It should be a collection, an array or a serializable iterator. * @param name * The given name for the Pact, used in plans, logs and progress messages. */ public CollectionDataSource(String name, Object... args) { this(new CollectionInputFormat(), name, args); }
public void setFieldTypesArray(Class<? extends Value>[] fieldTypes) { setFieldTypes(fieldTypes); }
@Override public void close() throws IOException { if (this.stream != null) { this.stream.write(this.buffer, 0, this.pos); } // close file stream super.close(); }
public CollectionDataSource(Collection<?> args, String name) { this(new CollectionInputFormat(), args, name); }
public CollectionDataSource(String name, Object[][] args) { this(new CollectionInputFormat(), name, args); }
public <T extends Iterator<?>, Serializable> CollectionDataSource(T args, String name) { this(new CollectionInputFormat(), args, name); }