/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }
protected synchronized void open(Path dir, WritableComparator comparator, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { Path dataFile = new Path(dir, DATA_FILE_NAME); Path indexFile = new Path(dir, INDEX_FILE_NAME); // open the data this.data = createDataFileReader(dataFile, conf, options); this.firstPosition = data.getPosition(); if (comparator == null) { Class<? extends WritableComparable> cls; cls = data.getKeyClass().asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else { this.comparator = comparator; } // open the index SequenceFile.Reader.Option[] indexOptions = Options.prependOptions(options, SequenceFile.Reader.file(indexFile)); this.index = new SequenceFile.Reader(conf, indexOptions); }
Options.prependOptions(opts, SequenceFile.Writer.file(dataFile), SequenceFile.Writer.keyClass(keyClass)); Options.prependOptions(opts, SequenceFile.Writer.file(indexFile), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(LongWritable.class),
/** * Create a new Writer with the given options. * @param conf the configuration to use * @param opts the options to create the file with * @return a new Writer * @throws IOException */ public static Writer createWriter(Configuration conf, Writer.Option... opts ) throws IOException { Writer.CompressionOption compressionOption = Options.getOption(Writer.CompressionOption.class, opts); CompressionType kind; if (compressionOption != null) { kind = compressionOption.getValue(); } else { kind = getDefaultCompressionType(conf); opts = Options.prependOptions(opts, Writer.compression(kind)); } switch (kind) { default: case NONE: return new Writer(conf, opts); case RECORD: return new RecordCompressWriter(conf, opts); case BLOCK: return new BlockCompressWriter(conf, opts); } }
@Test public void testAppend() throws Exception { assertArrayEquals("first append", new String[]{"Dr.", "Who", "hi", "there"}, Options.prependOptions(new String[]{"hi", "there"}, "Dr.", "Who")); assertArrayEquals("second append", new String[]{"aa","bb","cc","dd","ee","ff"}, Options.prependOptions(new String[]{"dd", "ee", "ff"}, "aa", "bb", "cc")); }
/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }
/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }
/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }
/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }
/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }
Options.prependOptions(options, SequenceFile.Reader.file(indexFile)); this.index = new SequenceFile.Reader(conf, indexOptions);
@Test public void testAppend() throws Exception { assertArrayEquals("first append", new String[]{"Dr.", "Who", "hi", "there"}, Options.prependOptions(new String[]{"hi", "there"}, "Dr.", "Who")); assertArrayEquals("second append", new String[]{"aa","bb","cc","dd","ee","ff"}, Options.prependOptions(new String[]{"dd", "ee", "ff"}, "aa", "bb", "cc")); }
@Test public void testAppend() throws Exception { assertArrayEquals("first append", new String[]{"Dr.", "Who", "hi", "there"}, Options.prependOptions(new String[]{"hi", "there"}, "Dr.", "Who")); assertArrayEquals("second append", new String[]{"aa","bb","cc","dd","ee","ff"}, Options.prependOptions(new String[]{"dd", "ee", "ff"}, "aa", "bb", "cc")); }
protected synchronized void open(Path dir, WritableComparator comparator, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { Path dataFile = new Path(dir, DATA_FILE_NAME); Path indexFile = new Path(dir, INDEX_FILE_NAME); // open the data this.data = createDataFileReader(dataFile, conf, options); this.firstPosition = data.getPosition(); if (comparator == null) { Class<? extends WritableComparable> cls; cls = data.getKeyClass().asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else { this.comparator = comparator; } // open the index SequenceFile.Reader.Option[] indexOptions = Options.prependOptions(options, SequenceFile.Reader.file(indexFile)); this.index = new SequenceFile.Reader(conf, indexOptions); }
protected synchronized void open(Path dir, WritableComparator comparator, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { Path dataFile = new Path(dir, DATA_FILE_NAME); Path indexFile = new Path(dir, INDEX_FILE_NAME); // open the data this.data = createDataFileReader(dataFile, conf, options); this.firstPosition = data.getPosition(); if (comparator == null) { Class<? extends WritableComparable> cls; cls = data.getKeyClass().asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else { this.comparator = comparator; } // open the index SequenceFile.Reader.Option[] indexOptions = Options.prependOptions(options, SequenceFile.Reader.file(indexFile)); this.index = new SequenceFile.Reader(conf, indexOptions); }
protected synchronized void open(Path dir, WritableComparator comparator, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { Path dataFile = new Path(dir, DATA_FILE_NAME); Path indexFile = new Path(dir, INDEX_FILE_NAME); // open the data this.data = createDataFileReader(dataFile, conf, options); this.firstPosition = data.getPosition(); if (comparator == null) { Class<? extends WritableComparable> cls; cls = data.getKeyClass().asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else { this.comparator = comparator; } // open the index SequenceFile.Reader.Option[] indexOptions = Options.prependOptions(options, SequenceFile.Reader.file(indexFile)); this.index = new SequenceFile.Reader(conf, indexOptions); }
protected synchronized void open(Path dir, WritableComparator comparator, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { Path dataFile = new Path(dir, DATA_FILE_NAME); Path indexFile = new Path(dir, INDEX_FILE_NAME); // open the data this.data = createDataFileReader(dataFile, conf, options); this.firstPosition = data.getPosition(); if (comparator == null) { Class<? extends WritableComparable> cls; cls = data.getKeyClass().asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else { this.comparator = comparator; } // open the index SequenceFile.Reader.Option[] indexOptions = Options.prependOptions(options, SequenceFile.Reader.file(indexFile)); this.index = new SequenceFile.Reader(conf, indexOptions); }
/** * Create a new Writer with the given options. * @param conf the configuration to use * @param opts the options to create the file with * @return a new Writer * @throws IOException */ public static Writer createWriter(Configuration conf, Writer.Option... opts ) throws IOException { Writer.CompressionOption compressionOption = Options.getOption(Writer.CompressionOption.class, opts); CompressionType kind; if (compressionOption != null) { kind = compressionOption.getValue(); } else { kind = getDefaultCompressionType(conf); opts = Options.prependOptions(opts, Writer.compression(kind)); } switch (kind) { default: case NONE: return new Writer(conf, opts); case RECORD: return new RecordCompressWriter(conf, opts); case BLOCK: return new BlockCompressWriter(conf, opts); } }
/** * Create a new Writer with the given options. * @param conf the configuration to use * @param opts the options to create the file with * @return a new Writer * @throws IOException */ public static Writer createWriter(Configuration conf, Writer.Option... opts ) throws IOException { Writer.CompressionOption compressionOption = Options.getOption(Writer.CompressionOption.class, opts); CompressionType kind; if (compressionOption != null) { kind = compressionOption.getValue(); } else { kind = getDefaultCompressionType(conf); opts = Options.prependOptions(opts, Writer.compression(kind)); } switch (kind) { default: case NONE: return new Writer(conf, opts); case RECORD: return new RecordCompressWriter(conf, opts); case BLOCK: return new BlockCompressWriter(conf, opts); } }
/** * Create a new Writer with the given options. * @param conf the configuration to use * @param opts the options to create the file with * @return a new Writer * @throws IOException */ public static Writer createWriter(Configuration conf, Writer.Option... opts ) throws IOException { Writer.CompressionOption compressionOption = Options.getOption(Writer.CompressionOption.class, opts); CompressionType kind; if (compressionOption != null) { kind = compressionOption.getValue(); } else { kind = getDefaultCompressionType(conf); opts = Options.prependOptions(opts, Writer.compression(kind)); } switch (kind) { default: case NONE: return new Writer(conf, opts); case RECORD: return new RecordCompressWriter(conf, opts); case BLOCK: return new BlockCompressWriter(conf, opts); } }