public Writer(Configuration conf, Path dir, SequenceFile.Writer.Option... options) throws IOException { super(conf, dir, options); this.fs = dir.getFileSystem(conf); this.dir = dir; initBloomFilter(conf); }
@Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, WritableComparator comparator, Class valClass) throws IOException { this(conf, new Path(dirName), comparator(comparator), valueClass(valClass)); }
@Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, WritableComparator comparator, Class valClass, CompressionType compress) throws IOException { this(conf, new Path(dirName), comparator(comparator), valueClass(valClass), compression(compress)); }
FileSystem ts = FileSystem.get(conf); String testFileName = TEST_FILE.toString(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, defaultCodec, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultCodec, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD); assertNotNull("testBloomMapFileConstructors error !!!", writer);
FileSystem ts = FileSystem.get(conf); String testFileName = TEST_FILE.toString(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, defaultCodec, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultCodec, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD); assertNotNull("testBloomMapFileConstructors error !!!", writer);
BloomMapFile.Reader reader = null; try { writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), IntWritable.class, Text.class); IntWritable key = new IntWritable(); key.set(i); value.set("00" + i); writer.append(key, value); writer.close();
Path qualifiedDirName = fs.makeQualified(dirName); conf.setInt("io.mapfile.bloom.size", 2048); BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), IntWritable.class, Text.class); IntWritable key = new IntWritable(); key.set(i); value.set("00" + i); writer.append(key, value); writer.close();
BloomMapFile.Reader reader = null; try { writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), IntWritable.class, Text.class); IntWritable key = new IntWritable(); key.set(i); value.set("00" + i); writer.append(key, value); writer.close();
/** * test {@link BloomMapFile.Reader.get()} method */ public void testGetBloomMapFile() { int SIZE = 10; BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); for (int i = 0; i < SIZE; i++) { writer.append(new IntWritable(i), new Text()); } writer.close(); reader = new BloomMapFile.Reader(TEST_FILE, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); for (int i = 0; i < SIZE; i++) { assertNotNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(i), new Text())); } assertNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(SIZE + 5), new Text())); } catch (Exception ex) { fail("unexpect ex in testGetBloomMapFile !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
@SuppressWarnings("deprecation") private void checkMembershipVaryingSizedKeys(String name, List<Text> keys) throws Exception { FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(TEST_DIR); BloomMapFile.Writer writer = null; BloomMapFile.Reader reader = null; try { writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, NullWritable.class); for (Text key : keys) { writer.append(key, NullWritable.get()); } writer.close(); // will check for membership in opposite order of how keys were inserted reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); Collections.reverse(keys); for (Text key : keys) { assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); } reader.close(); fs.delete(qualifiedDirName, true); } finally { IOUtils.cleanup(null, writer, reader); } }
@SuppressWarnings("deprecation") private void checkMembershipVaryingSizedKeys(String name, List<Text> keys) throws Exception { FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(TEST_DIR); BloomMapFile.Writer writer = null; BloomMapFile.Reader reader = null; try { writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, NullWritable.class); for (Text key : keys) { writer.append(key, NullWritable.get()); } writer.close(); // will check for membership in opposite order of how keys were inserted reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); Collections.reverse(keys); for (Text key : keys) { assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); } reader.close(); fs.delete(qualifiedDirName, true); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@link BloomMapFile.Reader.get()} method */ public void testGetBloomMapFile() { int SIZE = 10; BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); for (int i = 0; i < SIZE; i++) { writer.append(new IntWritable(i), new Text()); } writer.close(); reader = new BloomMapFile.Reader(TEST_FILE, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); for (int i = 0; i < SIZE; i++) { assertNotNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(i), new Text())); } assertNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(SIZE + 5), new Text())); } catch (Exception ex) { fail("unexpect ex in testGetBloomMapFile !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@link BloomMapFile.Reader} constructor with * IOException */ public void testIOExceptionInWriterConstructor() { Path dirNameSpy = spy(TEST_FILE); BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); writer.append(new IntWritable(1), new Text("123124142")); writer.close(); when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException()); reader = new BloomMapFile.Reader(dirNameSpy, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); assertNull("testIOExceptionInWriterConstructor error !!!", reader.getBloomFilter()); } catch (Exception ex) { fail("unexpect ex in testIOExceptionInWriterConstructor !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@link BloomMapFile.Reader} constructor with * IOException */ public void testIOExceptionInWriterConstructor() { Path dirNameSpy = spy(TEST_FILE); BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); writer.append(new IntWritable(1), new Text("123124142")); writer.close(); when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException()); reader = new BloomMapFile.Reader(dirNameSpy, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); assertNull("testIOExceptionInWriterConstructor error !!!", reader.getBloomFilter()); } catch (Exception ex) { fail("unexpect ex in testIOExceptionInWriterConstructor !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
@SuppressWarnings("rawtypes") @Override public Closeable createOutputStream(String hdfsPath, HdfsConfiguration configuration) { try { Closeable rout; HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath); Class<? extends WritableComparable> keyWritableClass = configuration.getKeyType().getWritableClass(); Class<? extends WritableComparable> valueWritableClass = configuration.getValueType().getWritableClass(); rout = new BloomMapFile.Writer(hdfsInfo.getConf(), new Path(hdfsPath), org.apache.hadoop.io.MapFile.Writer.keyClass(keyWritableClass), org.apache.hadoop.io.MapFile.Writer.valueClass(valueWritableClass), org.apache.hadoop.io.MapFile.Writer.compression(configuration.getCompressionType(), configuration.getCompressionCodec().getCodec()), org.apache.hadoop.io.MapFile.Writer.progressable(new Progressable() { @Override public void progress() { } })); return rout; } catch (IOException ex) { throw new RuntimeCamelException(ex); } }
private void checkMembershipVaryingSizedKeys(String name, List<Text> keys) throws Exception { Path dirName = new Path(System.getProperty("test.build.data",".") + name + ".bloommapfile"); FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(dirName); BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, NullWritable.class); for (Text key : keys) { writer.append(key, NullWritable.get()); } writer.close(); // will check for membership in the opposite order of how keys were inserted BloomMapFile.Reader reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); Collections.reverse(keys); for (Text key : keys) { assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); } reader.close(); fs.delete(qualifiedDirName, true); }
/** * test {@code BloomMapFile.delete()} method */ public void testDeleteFile() { BloomMapFile.Writer writer = null; try { FileSystem fs = FileSystem.getLocal(conf); writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); assertNotNull("testDeleteFile error !!!", writer); writer.close(); BloomMapFile.delete(fs, TEST_FILE.toString()); } catch (Exception ex) { fail("unexpect ex in testDeleteFile !!!"); } finally { IOUtils.cleanup(null, writer); } }
/** * test {@code BloomMapFile.delete()} method */ public void testDeleteFile() { BloomMapFile.Writer writer = null; try { FileSystem fs = FileSystem.getLocal(conf); writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); assertNotNull("testDeleteFile error !!!", writer); writer.close(); BloomMapFile.delete(fs, TEST_FILE.toString()); } catch (Exception ex) { fail("unexpect ex in testDeleteFile !!!"); } finally { IOUtils.cleanup(null, writer); } }
@Override public long append(HdfsOutputStream hdfsostr, Object key, Object value, TypeConverter typeConverter) { try { Holder<Integer> keySize = new Holder<>(); Writable keyWritable = getWritable(key, typeConverter, keySize); Holder<Integer> valueSize = new Holder<>(); Writable valueWritable = getWritable(value, typeConverter, valueSize); ((BloomMapFile.Writer) hdfsostr.getOut()).append((WritableComparable<?>) keyWritable, valueWritable); return keySize.value + valueSize.value; } catch (Exception ex) { throw new RuntimeCamelException(ex); } }
public Writer(Configuration conf, FileSystem fs, String dirName, WritableComparator comparator, Class valClass, CompressionType compress) throws IOException { super(conf, fs, dirName, comparator, valClass, compress); this.fs = fs; this.dir = new Path(dirName); initBloomFilter(conf); }