@Test public void testParquetIncorrectFormat() throws Exception { FileContext.getLocalFSFileContext().delete(new Path(new File(testMeta.dir).getAbsolutePath()), true); List<String> allLines = Lists.newArrayList(); HashSet<String> lines = Sets.newHashSet(); for (int line = 0; line < 5; line++) { lines.add("f0" + "l" + line); } allLines.addAll(lines); File testFile = new File(testMeta.dir, "file0"); FileUtils.write(testFile, StringUtils.join(lines, '\n')); parquetFilePOJOReader.output.setSink(outputSink); parquetFilePOJOReader.setDirectory(testMeta.dir); parquetFilePOJOReader.setParquetSchema(PARQUET_SCHEMA); parquetFilePOJOReader.setParquetToPOJOFieldsMapping( "event_id:event_id_v2:INTEGER,org_id:org_id_v2:STRING,long_id:long_id_v2:LONG,css_file_loaded:css_file_loaded_v2:BOOLEAN,float_val:float_val_v2:FLOAT,double_val:double_val_v2:DOUBLE"); parquetFilePOJOReader.setup(testMeta.context); testMeta.portContext.getAttributes().put(Context.PortContext.TUPLE_CLASS, EventRecordV2.class); parquetFilePOJOReader.output.setup(testMeta.portContext); for (long wid = 0; wid < 2; wid++) { parquetFilePOJOReader.beginWindow(0); parquetFilePOJOReader.emitTuples(); parquetFilePOJOReader.endWindow(); } Assert.assertEquals("number tuples", 0, outputSink.collectedTuples.size()); parquetFilePOJOReader.teardown(); }
public void setup(PortContext context) { jsonParser = new JSONParser(); finder = new JsonKeyFinder(); columnFields = new ArrayList<String>(); columnFieldSetters = Lists.newArrayList(); setPojoClass(context.getValue(Context.PortContext.TUPLE_CLASS)); if (getFieldMappingString() == null) { setFieldInfos(createFieldInfoMap(generateFieldInfoInputs(getPojoClass()))); } else { setFieldInfos(createFieldInfoMap(getFieldMappingString())); } initColumnFieldSetters(getFieldInfos()); initializeActiveFieldSetters(); ListIterator<FieldInfo> itr = fieldInfos.listIterator(); while (itr.hasNext()) { columnFields.add(itr.next().getColumnName()); } finder.setMatchKeyList(columnFields); } };
Attribute.AttributeMap attributes = portContext.getAttributes(); if (attributes != null) { T attr = attributes.get(key);
@Test public void testParquetReadingWithParquetToPojoMapping() throws Exception { FileContext.getLocalFSFileContext().delete(new Path(new File(testMeta.dir).getAbsolutePath()), true); List<EventRecord> data = Lists.newArrayList(); data.add(new EventRecord(1, "cust1", 12321L, true, 12.22f, 12.23)); data.add(new EventRecord(2, "cust2", 12322L, true, 22.22f, 22.23)); data.add(new EventRecord(3, "cust3", 12323L, true, 32.22f, 32.23)); writeParquetFile(PARQUET_SCHEMA, new File(testMeta.dir, "data.parquet"), data); parquetFilePOJOReader.output.setSink(outputSink); parquetFilePOJOReader.setDirectory(testMeta.dir); parquetFilePOJOReader.setParquetSchema(PARQUET_SCHEMA); parquetFilePOJOReader.setParquetToPOJOFieldsMapping( "event_id:event_id_v2:INTEGER,org_id:org_id_v2:STRING,long_id:long_id_v2:LONG,css_file_loaded:css_file_loaded_v2:BOOLEAN,float_val:float_val_v2:FLOAT,double_val:double_val_v2:DOUBLE"); parquetFilePOJOReader.setup(testMeta.context); testMeta.portContext.getAttributes().put(Context.PortContext.TUPLE_CLASS, EventRecordV2.class); parquetFilePOJOReader.output.setup(testMeta.portContext); for (long wid = 0; wid < 2; wid++) { parquetFilePOJOReader.beginWindow(0); parquetFilePOJOReader.emitTuples(); parquetFilePOJOReader.endWindow(); } Assert.assertEquals("number tuples", 3, outputSink.collectedTuples.size()); parquetFilePOJOReader.teardown(); }
@Test public void testParquetEmptyFile() throws Exception { FileContext.getLocalFSFileContext().delete(new Path(new File(testMeta.dir).getAbsolutePath()), true); List<EventRecord> data = Lists.newArrayList(); writeParquetFile(PARQUET_SCHEMA, new File(testMeta.dir, "data.parquet"), data); parquetFilePOJOReader.output.setSink(outputSink); parquetFilePOJOReader.setDirectory(testMeta.dir); parquetFilePOJOReader.setParquetSchema(PARQUET_SCHEMA); parquetFilePOJOReader.setup(testMeta.context); testMeta.portContext.getAttributes().put(Context.PortContext.TUPLE_CLASS, EventRecordV2.class); parquetFilePOJOReader.output.setup(testMeta.portContext); for (long wid = 0; wid < 2; wid++) { parquetFilePOJOReader.beginWindow(0); parquetFilePOJOReader.emitTuples(); parquetFilePOJOReader.endWindow(); } Assert.assertEquals("number tuples", 0, outputSink.collectedTuples.size()); parquetFilePOJOReader.teardown(); }
public void setup(PortContext context) { jsonParser = new JSONParser(); finder = new JsonKeyFinder(); columnFields = new ArrayList<String>(); columnFieldSetters = Lists.newArrayList(); setPojoClass(context.getValue(Context.PortContext.TUPLE_CLASS)); if (getFieldMappingString() == null) { setFieldInfos(createFieldInfoMap(generateFieldInfoInputs(getPojoClass()))); } else { setFieldInfos(createFieldInfoMap(getFieldMappingString())); } initColumnFieldSetters(getFieldInfos()); initializeActiveFieldSetters(); ListIterator<FieldInfo> itr = fieldInfos.listIterator(); while (itr.hasNext()) { columnFields.add(itr.next().getColumnName()); } finder.setMatchKeyList(columnFields); } };
@Handler public void activated(NodeActivationEvent nae) { Node<?> node = nae.getNode(); if (node.context.getValue(OperatorContext.AUTO_RECORD)) { startRecording(null, node, node.getId(), null, 0); } else { for (Map.Entry<String, PortContextPair<InputPort<?>>> entry : node.getPortMappingDescriptor().inputPorts.entrySet()) { if (entry.getValue().context != null && entry.getValue().context.getValue(PortContext.AUTO_RECORD)) { startRecording(null, node, node.getId(), entry.getKey(), 0); } } for (Map.Entry<String, PortContextPair<OutputPort<?>>> entry : node.getPortMappingDescriptor().outputPorts.entrySet()) { if (entry.getValue().context != null && entry.getValue().context.getValue(PortContext.AUTO_RECORD)) { startRecording(null, node, node.getId(), entry.getKey(), 0); } } } }
public void setup(PortContext context) { setPojoClass(context.getValue(Context.PortContext.TUPLE_CLASS)); columnFieldSetters = Lists.newArrayList(); /** * Check if the mapping of Generic record fields to POJO is given, else * use reflection */ if (getGenericRecordToPOJOFieldsMapping() == null) { setFieldInfos(createFieldInfoMap(generateFieldInfoInputs(getPojoClass()))); } else { setFieldInfos(createFieldInfoMap(getGenericRecordToPOJOFieldsMapping())); } initColumnFieldSetters(getFieldInfos()); initializeActiveFieldSetters(); } };
public void setup(PortContext context) { setPojoClass(context.getValue(Context.PortContext.TUPLE_CLASS)); columnFieldSetters = Lists.newArrayList(); /** * Check if the mapping of Generic record fields to POJO is given, else * use reflection */ if (getGenericRecordToPOJOFieldsMapping() == null) { setFieldInfos(createFieldInfoMap(generateFieldInfoInputs(getPojoClass()))); } else { setFieldInfos(createFieldInfoMap(getGenericRecordToPOJOFieldsMapping())); } initColumnFieldSetters(getFieldInfos()); initializeActiveFieldSetters(); } };
public void setup(PortContext context) { inClazz = context.getValue(Context.PortContext.TUPLE_CLASS); }
public void setup(PortContext context) { clazz = context.getValue(Context.PortContext.TUPLE_CLASS); }
@Override public void setup(PortContext context) { cls = context.getValue(Context.PortContext.TUPLE_CLASS); try { parseSchema(); initializeColumnMap(getSchema()); } catch (IOException e) { LOG.error("Exception in parsing schema", e); } }
@Override public void setup(Context.PortContext context) { outputClass = context.getValue(Context.PortContext.TUPLE_CLASS); } };
public void setup(com.datatorrent.api.Context.PortContext context) { pojoType = context.getAttributes().get(Context.PortContext.TUPLE_CLASS); } };
public void setup(PortContext context) { inClazz = context.getValue(Context.PortContext.TUPLE_CLASS); }
public void setup(Context.PortContext context) { clazz = context.getValue(Context.PortContext.TUPLE_CLASS); } };
@Override public void setup(Context.PortContext context) { if (context.getAttributes().contains(Context.PortContext.TUPLE_CLASS)) { pojoClass = context.getAttributes().get(Context.PortContext.TUPLE_CLASS); } }
public void setup(PortContext context) { remainderClazz = context.getValue(Context.PortContext.TUPLE_CLASS); } };
@Override public void setup(Context.PortContext context) { pojoClass = context.getValue(Context.PortContext.TUPLE_CLASS); }
private boolean isInputPortConnectedToDelayOperator(String portName) { Operators.PortContextPair<InputPort<?>> pcPair = descriptor.inputPorts.get(portName); if (pcPair == null || pcPair.context == null) { return false; } return pcPair.context.getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR); }