@Override public AvroWrapper<Utf8> createKey() { return new AvroWrapper<>(null); }
public AvroWrapper<T> createKey() { return new AvroWrapper<>(null); }
public AvroWrapper<T> createKey() { return new AvroWrapper<>(null); }
@SuppressWarnings({"unchecked"}) public void collect(Object key) throws IOException{ AvroWrapper wrapper = new AvroWrapper(key); writer.write(wrapper, NullWritable.get()); }
public void testProjectionNoreducer(String inputDirectory) throws Exception { JobConf job = new JobConf(); long onel = 1; Schema readerSchema = Schema.create(Schema.Type.STRING); AvroJob.setInputSchema(job, readerSchema); Path inputPath = new Path(inputDirectory + "/myavro2-m-00000.avro"); FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath); FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job); AvroRecordReader<Utf8> recordReader = new AvroRecordReader<>(job, fileSplit); AvroWrapper<Utf8> inputPair = new AvroWrapper<>(null); NullWritable ignore = NullWritable.get(); while (recordReader.next(inputPair, ignore)) { long testl = Long.parseLong(inputPair.datum().toString().split(":")[2].replace("}", "").trim()); Assert.assertEquals(onel, testl); } } }
@SuppressWarnings("deprecation") // Test for a different schema output public void testProjection1(String inputDirectory) throws Exception { JobConf job = new JobConf(); Schema readerSchema = Schema.create(Schema.Type.STRING); AvroJob.setInputSchema(job, readerSchema); Path inputPath = new Path(inputDirectory + "/myavro1-r-00000.avro"); FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath); FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job); AvroWrapper<Utf8> inputPair = new AvroWrapper<>(null); NullWritable ignore = NullWritable.get(); AvroRecordReader<Utf8> recordReader = new AvroRecordReader<>(job, fileSplit); long sumOfCounts = 0; long numOfCounts = 0; while (recordReader.next(inputPair, ignore)) { sumOfCounts += Long.parseLong(inputPair.datum().toString().split(":")[2].replace("}", "").trim()); numOfCounts++; } Assert.assertEquals(numOfCounts, WordCountUtil.COUNTS.size()); long actualSumOfCounts = 0; for (Long count : WordCountUtil.COUNTS.values()) { actualSumOfCounts += count; } Assert.assertEquals(sumOfCounts, actualSumOfCounts); }
@SuppressWarnings("deprecation") // Test for a different schema output public void testProjectionNewMethodsTwo(String inputDirectory) throws Exception { JobConf job = new JobConf(); Schema readerSchema = Schema.create(Schema.Type.STRING); AvroJob.setInputSchema(job, readerSchema); Path inputPath = new Path(inputDirectory + "/testavrofile1-r-00000.avro"); FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath); FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job); AvroWrapper<Utf8> inputPair = new AvroWrapper<>(null); NullWritable ignore = NullWritable.get(); AvroRecordReader<Utf8> recordReader = new AvroRecordReader<>(job, fileSplit); long sumOfCounts = 0; long numOfCounts = 0; while (recordReader.next(inputPair, ignore)) { sumOfCounts += Long.parseLong(inputPair.datum().toString().split(":")[2].replace("}", "").trim()); numOfCounts++; } Assert.assertEquals(numOfCounts, WordCountUtil.COUNTS.size()); long actualSumOfCounts = 0; for (Long count : WordCountUtil.COUNTS.values()) { actualSumOfCounts += count; } Assert.assertEquals(sumOfCounts, actualSumOfCounts); }
public void map(LongWritable key, Text value, OutputCollector<AvroWrapper<Pair<Long,Utf8>>,NullWritable> out, Reporter reporter) throws IOException { out.collect(new AvroWrapper<>(new Pair<>(key.get(), new Utf8(value.toString()))), NullWritable.get()); } }
@Test public void testToString() { String datum = "my string"; AvroWrapper<CharSequence> wrapper = new AvroWrapper<>(datum); assertEquals(datum, wrapper.toString()); } }
public void map(LongWritable key, Text value, OutputCollector<AvroWrapper<Pair<Long,GenericData.Record>>,NullWritable> out, Reporter reporter) throws IOException { GenericData.Record optional_entry = new GenericData.Record(createInnerSchema("optional_field_1")); optional_entry.put("optional_field_1", 0L); GenericData.Array<GenericData.Record> array = new GenericData.Array<>(1, createArraySchema()); array.add(optional_entry); GenericData.Record container = new GenericData.Record(createSchema()); container.put("Optional", array); out.collect(new AvroWrapper<>(new Pair<>(key.get(), container)), NullWritable.get()); } }
AvroWrapper<Pair<Integer, Long>> inputPair = new AvroWrapper<>(null); NullWritable ignore = NullWritable.get();
AvroWrapper<Pair<Integer, Long>> inputPair = new AvroWrapper<>(null); NullWritable ignore = NullWritable.get();
AvroWrapper<Pair<Integer, Long>> inputPair = new AvroWrapper<>(null); NullWritable ignore = NullWritable.get();
private AvroWrapper<K> getWrapper() { if (wrapper == null) { wrapper = new AvroWrapper<K>(); } return wrapper; }
private AvroWrapper<K> getWrapper() { if (wrapper == null) { wrapper = new AvroWrapper<K>(); } return wrapper; }
@SuppressWarnings({"unchecked"}) public void collect(Object key) throws IOException{ AvroWrapper wrapper = new AvroWrapper(key); writer.write(wrapper, NullWritable.get()); }
public void setup(TupleMRContext context, Collector collector) throws IOException, InterruptedException { outputRecord = new Record(getAvroOutputSchema()); wrapper = new AvroWrapper<Record>(); };
public void reduce(Text key, Iterator<Text> values, OutputCollector<AvroWrapper<GenericRecord>, NullWritable> output, Reporter reporter) throws IOException { GenericRecord record = from(key.toString(), schema); AvroWrapper<GenericRecord> wrapper = new AvroWrapper<GenericRecord>(record); output.collect(wrapper, NullWritable.get()); }
@Override public void close() throws IOException { System.out.println(filter); collector.collect( new AvroWrapper<GenericRecord>( AvroBytesRecord.toGenericRecord(filter)), NullWritable.get()); }
public void map(LongWritable key, StockRecord value, OutputCollector<AvroWrapper<Stock>, NullWritable> output, Reporter reporter) throws IOException { output.collect( new AvroWrapper<Stock>(writableToAvro(value)), NullWritable.get()); }