public void testCustomApacheDefaultFormat() { jobConf.set("TsProcessor.default.time.format", "dd/MMM/yyyy:HH:mm:ss Z"); jobConf.set("TsProcessor.default.time.regex", "^(?:[\\d.]+) \\[(\\d{2}/\\w{3}/\\d{4}:\\d{2}:\\d{2}:\\d{2} [-+]\\d{4})\\] .*"); String record = buildSampleApacheRecord(dateWithoutMillis, "dd/MMM/yyyy:HH:mm:ss Z"); doTest(dateWithoutMillis, record); }
public void testDefaultFormat() { String record = buildSampleSimpleRecord(date, "yyyy-MM-dd HH:mm:ss,SSS"); doTest(date, record); }
public void doTest(Date date, String recordData) { ChunkBuilder cb = new ChunkBuilder(); cb.addRecord(recordData.getBytes()); Chunk chunk = cb.getChunk(); chunk.setDataType(DATA_TYPE); chunk.setSource(DATA_SOURCE); ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord> output = new ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord>(); TsProcessor p = new TsProcessor(); p.reset(chunk); p.process(null, chunk, output, Reporter.NULL); ChukwaRecordKey key = buildKey(date, DATA_SOURCE, DATA_TYPE); Map<ChukwaRecordKey, ChukwaRecord> outputData = output.data; assertNotNull("No output data found.", outputData); assertEquals("Output data size not correct.", 1, outputData.size()); ChukwaRecord record = outputData.get(key); assertNotNull("Output record not found.", record); assertEquals("Output record time not correct.", date.getTime(), record.getTime()); assertEquals("Output record body not correct.", recordData, new String(record.getMapFields().get("body").get())); }
public void testParseIllegalRegex() { jobConf.set(TsProcessor.DEFAULT_TIME_REGEX, "("); ChunkBuilder cb = new ChunkBuilder(); cb.addRecord("2012-10-25 00:18:44,818 some sample record data".getBytes()); Chunk chunk = cb.getChunk(); TsProcessor p = new TsProcessor(); p.reset(chunk); ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord> output = new ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord>(); p.process(null, chunk, output, Reporter.NULL); assertEquals("Output data size not correct.", 1, output.data.size()); ChukwaRecordKey key = output.data.keySet().iterator().next(); ChukwaRecord record = output.data.get(key); assertNull("Output should not be error.", record.getValue("cchunkData")); } }
public void testCustomDefaultFormat2() { // this date format produces a date that longer than the format, since z // expands to something like PDT jobConf.set("TsProcessor.default.time.format", "yyyy--MM--dd HH::mm::ss SSS,z"); String record = buildSampleSimpleRecord(date, "yyyy--MM--dd HH::mm::ss SSS,z"); doTest(date, record); }
public void testCustomApacheDataTypeFormat() { jobConf.set("TsProcessor.time.format." + DATA_TYPE, "dd/MMM/yyyy:HH:mm:ss Z"); jobConf.set("TsProcessor.time.regex." + DATA_TYPE, "^(?:[\\d.]+) \\[(\\d{2}/\\w{3}/\\d{4}:\\d{2}:\\d{2}:\\d{2} [-+]\\d{4})\\] .*"); String record = buildSampleApacheRecord(dateWithoutMillis, "dd/MMM/yyyy:HH:mm:ss Z"); doTest(dateWithoutMillis, record); }
public void testCustomDataTypeFormat() { jobConf.set("TsProcessor.time.format." + DATA_TYPE, "yyyy--MM--dd HH::mm::ss SSS"); String record = buildSampleSimpleRecord(date, "yyyy--MM--dd HH::mm::ss SSS"); doTest(date, record); }
public void testCustomDefaultFormat() { jobConf.set("TsProcessor.default.time.format", "yyyy--MM--dd HH::mm::ss SSS"); String record = buildSampleSimpleRecord(date, "yyyy--MM--dd HH::mm::ss SSS"); doTest(date, record); }
public void testCustomDefaultFormatWithCustomDataTypeFormat() { jobConf.set("TsProcessor.default.time.format", "yyyy/MM/dd HH:mm:ss SSS"); jobConf.set("TsProcessor.time.format." + DATA_TYPE, "yyyy--MM--dd HH::mm::ss SSS"); String record = buildSampleSimpleRecord(date, "yyyy--MM--dd HH::mm::ss SSS"); doTest(date, record); }