/** For backwards compatibility. **/ public static WritableComparator get(Class<? extends WritableComparable> c) { return get(c, null); }
keyClass= (Class<? extends WritableComparable>) keyClassOption.getValue(); this.comparator = WritableComparator.get(keyClass, conf);
/** Create a set naming the element class and compression type. */ public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, SequenceFile.CompressionType compress) throws IOException { this(conf, fs, dirName, WritableComparator.get(keyClass, conf), compress); }
@SuppressWarnings("unchecked") private void open(Path[] inMapFiles, Path outMapFile) throws IOException { inReaders = new Reader[inMapFiles.length]; for (int i = 0; i < inMapFiles.length; i++) { Reader reader = new Reader(inMapFiles[i], conf); if (keyClass == null || valueClass == null) { keyClass = (Class<WritableComparable>) reader.getKeyClass(); valueClass = (Class<Writable>) reader.getValueClass(); } else if (keyClass != reader.getKeyClass() || valueClass != reader.getValueClass()) { throw new HadoopIllegalArgumentException( "Input files cannot be merged as they" + " have different Key and Value classes"); } inReaders[i] = reader; } if (comparator == null) { Class<? extends WritableComparable> cls; cls = keyClass.asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else if (comparator.getKeyClass() != keyClass) { throw new HadoopIllegalArgumentException( "Input files cannot be merged as they" + " have different Key class compared to" + " specified comparator"); } outWriter = new MapFile.Writer(conf, outMapFile, MapFile.Writer.keyClass(keyClass), MapFile.Writer.valueClass(valueClass)); }
/** Sort and merge files containing the named classes. */ public Sorter(FileSystem fs, Class<? extends WritableComparable> keyClass, Class valClass, Configuration conf) { this(fs, WritableComparator.get(keyClass, conf), keyClass, valClass, conf); }
@SuppressWarnings("rawtypes") private int compareKey(final WritableComparator comparators[], final int pos, final WritableComparable key_1, final WritableComparable key_2, final boolean nullsafe) { if (key_1 == null && key_2 == null) { if (nullsafe) { return 0; } else { return -1; } } else if (key_1 == null) { return -1; } else if (key_2 == null) { return 1; } if (comparators[pos] == null) { comparators[pos] = WritableComparator.get(key_1.getClass()); } return comparators[pos].compare(key_1, key_2); }
@SuppressWarnings("rawtypes") private int compareKey(final WritableComparator comparators[], final int pos, final WritableComparable key_1, final WritableComparable key_2, final boolean nullsafe) { if (key_1 == null && key_2 == null) { if (nullsafe) { return 0; } else { return -1; } } else if (key_1 == null) { return -1; } else if (key_2 == null) { return 1; } if (comparators[pos] == null) { comparators[pos] = WritableComparator.get(key_1.getClass()); } return comparators[pos].compare(key_1, key_2); }
private int compareKeys (List<Object> k1, List<Object> k2) { int ret = 0; // join keys have difference sizes? ret = k1.size() - k2.size(); if (ret != 0) { return ret; } for (int i = 0; i < k1.size(); i++) { WritableComparable key_1 = (WritableComparable) k1.get(i); WritableComparable key_2 = (WritableComparable) k2.get(i); if (key_1 == null && key_2 == null) { return nullsafes != null && nullsafes[i] ? 0 : -1; // just return k1 is smaller than k2 } else if (key_1 == null) { return -1; } else if (key_2 == null) { return 1; } ret = WritableComparator.get(key_1.getClass()).compare(key_1, key_2); if(ret != 0) { return ret; } } return ret; }
private int compareKeys (List<Object> k1, List<Object> k2) { int ret = 0; // join keys have difference sizes? ret = k1.size() - k2.size(); if (ret != 0) { return ret; } for (int i = 0; i < k1.size(); i++) { WritableComparable key_1 = (WritableComparable) k1.get(i); WritableComparable key_2 = (WritableComparable) k2.get(i); if (key_1 == null && key_2 == null) { return nullsafes != null && nullsafes[i] ? 0 : -1; // just return k1 is smaller than k2 } else if (key_1 == null) { return -1; } else if (key_2 == null) { return 1; } ret = WritableComparator.get(key_1.getClass()).compare(key_1, key_2); if(ret != 0) { return ret; } } return ret; }
protected synchronized void open(Path dir, WritableComparator comparator, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { Path dataFile = new Path(dir, DATA_FILE_NAME); Path indexFile = new Path(dir, INDEX_FILE_NAME); // open the data this.data = createDataFileReader(dataFile, conf, options); this.firstPosition = data.getPosition(); if (comparator == null) { Class<? extends WritableComparable> cls; cls = data.getKeyClass().asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else { this.comparator = comparator; } // open the index SequenceFile.Reader.Option[] indexOptions = Options.prependOptions(options, SequenceFile.Reader.file(indexFile)); this.index = new SequenceFile.Reader(conf, indexOptions); }
WritableComparator.get((Class<? extends WritableComparable>) exampleObject.getClass()); sortedList.sort(writableComparator);
/* allowNull */ false)); WritableComparator writableComparator = WritableComparator.get((Class<? extends WritableComparable>) exampleObject.getClass());
/** Create the named map for keys of the named class. */ public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress, Progressable progress) throws IOException { this(conf, fs, dirName, WritableComparator.get(keyClass), valClass, compress, progress); }
/** Create a set naming the element class and compression type. */ public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, SequenceFile.CompressionType compress) throws IOException { this(conf, fs, dirName, WritableComparator.get(keyClass, conf), compress); }
/** Create a set naming the element class and compression type. */ public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, SequenceFile.CompressionType compress) throws IOException { this(conf, fs, dirName, WritableComparator.get(keyClass, conf), compress); }
/** Sort and merge files containing the named classes. */ public Sorter(FileSystem fs, Class<? extends WritableComparable> keyClass, Class valClass, Configuration conf) { this(fs, WritableComparator.get(keyClass, conf), keyClass, valClass, conf); }
@SuppressWarnings("unchecked") @Override public void setConf(Configuration conf) { super.setConf(conf); if (conf != null) { JobConf jobConf = new JobConf(conf); comparator = WritableComparator.get(jobConf.getMapOutputKeyClass().asSubclass(WritableComparable.class)); } }
private void sortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile, SequenceFile.Metadata metadata) throws IOException { fs.delete(sortedFile, true); LOG.info("sorting: " + unsortedFile + " to: " + sortedFile); final WritableComparator comparator = WritableComparator.get(RandomDatum.class); SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, comparator, RandomDatum.class, RandomDatum.class, conf, metadata); sorter.sort(new Path[] { unsortedFile }, sortedFile, false); }
private void sortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile, SequenceFile.Metadata metadata) throws IOException { fs.delete(sortedFile, true); LOG.info("sorting: " + unsortedFile + " to: " + sortedFile); final WritableComparator comparator = WritableComparator.get(RandomDatum.class); SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, comparator, RandomDatum.class, RandomDatum.class, conf, metadata); sorter.sort(new Path[] { unsortedFile }, sortedFile, false); }
@Test public void testCompareUnequalWritables() throws Exception { DataOutputBuffer bTrue = writeWritable(new BooleanWritable(true)); DataOutputBuffer bFalse = writeWritable(new BooleanWritable(false)); WritableComparator writableComparator = WritableComparator.get(BooleanWritable.class); assertEquals(0, compare(writableComparator, bTrue, bTrue)); assertEquals(0, compare(writableComparator, bFalse, bFalse)); assertEquals(1, compare(writableComparator, bTrue, bFalse)); assertEquals(-1, compare(writableComparator, bFalse, bTrue)); }