protected WritableComparator(Class<? extends WritableComparable> keyClass, Configuration conf, boolean createInstances) { this.keyClass = keyClass; this.conf = (conf != null) ? conf : new Configuration(); if (createInstances) { key1 = newKey(); key2 = newKey(); buffer = new DataInputBuffer(); } else { key1 = key2 = null; buffer = null; } }
ArrayList<WritableComparable> keyBuilder = new ArrayList<WritableComparable>(1024); while (true) { WritableComparable k = comparator.newKey();
this.comparator = WritableComparator.get(keyClass, conf); this.lastKey = comparator.newKey(); FileSystem fs = dirName.getFileSystem(conf);
nextKey = comparator.newKey();
public WritableComparable newKey(){ return super.newKey(); }
public WritableComparable newKey(){ return super.newKey(); }
public WritableComparable newKey(){ return super.newKey(); }
public WritableComparable newKey(){ return super.newKey(); }
public WritableComparable newKey(){ return super.newKey(); }
protected WritableComparator(Class<? extends WritableComparable> keyClass, Configuration conf, boolean createInstances) { this.keyClass = keyClass; this.conf = (conf != null) ? conf : new Configuration(); if (createInstances) { key1 = newKey(); key2 = newKey(); buffer = new DataInputBuffer(); } else { key1 = key2 = null; buffer = null; } }
protected WritableComparator(Class<? extends WritableComparable> keyClass, Configuration conf, boolean createInstances) { this.keyClass = keyClass; this.conf = (conf != null) ? conf : new Configuration(); if (createInstances) { key1 = newKey(); key2 = newKey(); buffer = new DataInputBuffer(); } else { key1 = key2 = null; buffer = null; } }
protected WritableComparator(Class<? extends WritableComparable> keyClass, Configuration conf, boolean createInstances) { this.keyClass = keyClass; this.conf = (conf != null) ? conf : new Configuration(); if (createInstances) { key1 = newKey(); key2 = newKey(); buffer = new DataInputBuffer(); } else { key1 = key2 = null; buffer = null; } }
protected WritableComparator(Class<? extends WritableComparable> keyClass, Configuration conf, boolean createInstances) { this.keyClass = keyClass; this.conf = (conf != null) ? conf : new Configuration(); if (createInstances) { key1 = newKey(); key2 = newKey(); buffer = new DataInputBuffer(); } else { key1 = key2 = null; buffer = null; } }
/** Create the named map using the named key comparator. */ public Writer(Configuration conf, FileSystem fs, String dirName, WritableComparator comparator, Class valClass, SequenceFile.CompressionType compress, CompressionCodec codec, Progressable progress) throws IOException { this.indexInterval = conf.getInt(INDEX_INTERVAL, this.indexInterval); this.comparator = comparator; this.lastKey = comparator.newKey(); Path dir = new Path(dirName); if (!fs.mkdirs(dir)) { throw new IOException("Mkdirs failed to create directory " + dir.toString()); } Path dataFile = new Path(dir, DATA_FILE_NAME); Path indexFile = new Path(dir, INDEX_FILE_NAME); Class keyClass = comparator.getKeyClass(); this.data = SequenceFile.createWriter (fs, conf, dataFile, keyClass, valClass, compress, codec, progress); this.index = SequenceFile.createWriter (fs, conf, indexFile, keyClass, LongWritable.class, CompressionType.BLOCK, progress); }
/** Create the named map using the named key comparator. */ public Writer(Configuration conf, FileSystem fs, String dirName, WritableComparator comparator, Class valClass, SequenceFile.CompressionType compress, CompressionCodec codec, Progressable progress) throws IOException { this.indexInterval = conf.getInt(INDEX_INTERVAL, this.indexInterval); this.comparator = comparator; this.lastKey = comparator.newKey(); Path dir = new Path(dirName); if (!fs.mkdirs(dir)) { throw new IOException("Mkdirs failed to create directory " + dir.toString()); } Path dataFile = new Path(dir, DATA_FILE_NAME); Path indexFile = new Path(dir, INDEX_FILE_NAME); Class keyClass = comparator.getKeyClass(); this.data = SequenceFile.createWriter (fs, conf, dataFile, keyClass, valClass, compress, codec, progress); this.index = SequenceFile.createWriter (fs, conf, indexFile, keyClass, LongWritable.class, CompressionType.BLOCK, progress); }
private long findPosition(WritableComparable key) throws IOException { readIndex(); // make sure index is read long seekPosition = -1; int seekIndex = binarySearch(key); if (seekIndex < 0) // decode insertion point seekIndex = -seekIndex-2; if (seekIndex == -1) // belongs before first entry seekPosition = firstPosition; // use beginning of file else seekPosition = positions[seekIndex]; // else use index data.get().seek(seekPosition); WritableComparable nextKey = comparator.newKey(); while (data.get().next(nextKey)) { int c = comparator.compare(key, nextKey); if (c < 0) { // at or beyond desired return -1; } else if (c == 0) { return data.get().getPosition(); } } return -1; }
/** * Test that Writable's are configured by Comparator. */ public void testConfigurableWritableComparator() throws Exception { Configuration conf = new Configuration(); conf.set(TEST_WRITABLE_CONFIG_PARAM, TEST_WRITABLE_CONFIG_VALUE); WritableComparator wc = WritableComparator.get(SimpleWritableComparable.class, conf); SimpleWritableComparable key = ((SimpleWritableComparable)wc.newKey()); assertNotNull(wc.getConf()); assertNotNull(key.getConf()); assertEquals(key.getConf().get(TEST_WRITABLE_CONFIG_PARAM), TEST_WRITABLE_CONFIG_VALUE); } }
/** * Test that Writable's are configured by Comparator. */ public void testConfigurableWritableComparator() throws Exception { Configuration conf = new Configuration(); conf.set(TEST_WRITABLE_CONFIG_PARAM, TEST_WRITABLE_CONFIG_VALUE); WritableComparator wc = WritableComparator.get(SimpleWritableComparable.class, conf); SimpleWritableComparable key = ((SimpleWritableComparable)wc.newKey()); assertNotNull(wc.getConf()); assertNotNull(key.getConf()); assertEquals(key.getConf().get(TEST_WRITABLE_CONFIG_PARAM), TEST_WRITABLE_CONFIG_VALUE); } }