@Override protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() { // Local properties for MapPartition may not be preserved. SingleInputSemanticProperties origProps = ((SingleInputOperator<?,?,?>) getOperator()).getSemanticProperties(); SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties(); FieldSet readSet = origProps.getReadFields(0); if(readSet != null) { filteredProps.addReadFields(readSet); } return filteredProps; }
/** * Creates SemanticProperties by adding an offset to each input field index of the given SemanticProperties. * * @param props The SemanticProperties to which the offset is added. * @param numInputFields The original number of fields of the input. * @param offset The offset that is added to each input field index. * @return New SemanticProperties with added offset. */ public static SingleInputSemanticProperties addSourceFieldOffset(SingleInputSemanticProperties props, int numInputFields, int offset) { SingleInputSemanticProperties offsetProps = new SingleInputSemanticProperties(); if (props.getReadFields(0) != null) { FieldSet offsetReadFields = new FieldSet(); for (int r : props.getReadFields(0)) { offsetReadFields = offsetReadFields.addField(r + offset); } offsetProps.addReadFields(offsetReadFields); } for (int s = 0; s < numInputFields; s++) { FieldSet targetFields = props.getForwardingTargetFields(0, s); for (int t : targetFields) { offsetProps.addForwardedField(s + offset, t); } } return offsetProps; }
@Test public void testGetReadSet() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); sp.addReadFields(new FieldSet(0, 1)); assertEquals(2, sp.getReadFields(0).size()); assertTrue(sp.getReadFields(0).contains(0)); assertTrue(sp.getReadFields(0).contains(1)); sp.addReadFields(new FieldSet(3)); assertEquals(3, sp.getReadFields(0).size()); assertTrue(sp.getReadFields(0).contains(0)); assertTrue(sp.getReadFields(0).contains(1)); assertTrue(sp.getReadFields(0).contains(3)); }
@Override protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() { // Local properties for GroupReduce may only be preserved on key fields. SingleInputSemanticProperties origProps = getOperator().getSemanticProperties(); SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties(); FieldSet readSet = origProps.getReadFields(0); if(readSet != null) { filteredProps.addReadFields(readSet); } // only add forward field information for key fields if(this.keys != null) { for (int f : this.keys) { FieldSet targets = origProps.getForwardingTargetFields(0, f); for (int t : targets) { filteredProps.addForwardedField(f, t); } } } return filteredProps; }
@Override protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() { // Local properties for GroupCombine may only be preserved on key fields. SingleInputSemanticProperties origProps = ((SingleInputOperator<?,?,?>) getOperator()).getSemanticProperties(); SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties(); FieldSet readSet = origProps.getReadFields(0); if(readSet != null) { filteredProps.addReadFields(readSet); } // only add forward field information for key fields if(this.keys != null) { for (int f : this.keys) { FieldSet targets = origProps.getForwardingTargetFields(0, f); for (int t : targets) { filteredProps.addForwardedField(f, t); } } } return filteredProps; }
@Test(expected = IndexOutOfBoundsException.class) public void testGetReadFieldsInvalidIndex() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); sp.addReadFields(new FieldSet(0, 1)); sp.getReadFields(1); }
@Test public void testForwardedRead() { String[] forwardedFields = { "f0->f0;f1->f2" }; String[] readFields = {"f0; f2"}; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, forwardedFields, null, readFields, threeIntTupleType, fiveIntTupleType); assertTrue(sp.getForwardingTargetFields(0, 0).contains(0)); assertTrue(sp.getForwardingTargetFields(0, 1).contains(2)); assertTrue(sp.getReadFields(0).size() == 2); assertTrue(sp.getReadFields(0).contains(0)); assertTrue(sp.getReadFields(0).contains(2)); }
@Test public void testNonForwardedRead() { String[] nonForwardedFields = { "f1;f2" }; String[] readFields = {"f0; f2"}; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, nonForwardedFields, readFields, threeIntTupleType, threeIntTupleType); assertTrue(sp.getForwardingTargetFields(0, 0).contains(0)); assertTrue(sp.getForwardingTargetFields(0, 1).size() == 0); assertTrue(sp.getForwardingTargetFields(0, 2).size() == 0); assertTrue(sp.getReadFields(0).size() == 2); assertTrue(sp.getReadFields(0).contains(0)); assertTrue(sp.getReadFields(0).contains(2)); }
@Test public void testReadFieldsBasic() { String[] readFields = { "*" }; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, intType, intType); FieldSet fs = sp.getReadFields(0); assertTrue(fs.size() == 1); assertTrue(fs.contains(0)); sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, intType, fiveIntTupleType); fs = sp.getReadFields(0); assertTrue(fs.size() == 1); assertTrue(fs.contains(0)); }
@Test public void testReadFieldsSpaces() { String[] readFields = { " f1 ; f2 " }; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, threeIntTupleType, threeIntTupleType); FieldSet fs = sp.getReadFields(0); assertTrue(fs.size() == 2); assertTrue(fs.contains(2)); assertTrue(fs.contains(1)); }
@Test public void testReadFieldsIndividualStrings() { String[] readFields = { "f1", "f2" }; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, threeIntTupleType, threeIntTupleType); FieldSet fs = sp.getReadFields(0); assertTrue(fs.size() == 2); assertTrue(fs.contains(2)); assertTrue(fs.contains(1)); }
@Test public void testReadFieldsOneString() { String[] readFields = { "f1;f2" }; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, threeIntTupleType, threeIntTupleType); FieldSet fs = sp.getReadFields(0); assertTrue(fs.size() == 2); assertTrue(fs.contains(2)); assertTrue(fs.contains(1)); readFields[0] = "f1;f2;"; sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, threeIntTupleType, threeIntTupleType); fs = sp.getReadFields(0); assertTrue(fs.size() == 2); assertTrue(fs.contains(2)); assertTrue(fs.contains(1)); }
@Test public void testReadFieldsPojoInTuple() { String[] readFields = { "f0; f2.int1; f2.string1" }; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, pojoInTupleType, pojo2Type); FieldSet fs = sp.getReadFields(0); assertTrue(fs.size() == 3); assertTrue(fs.contains(0)); assertTrue(fs.contains(2)); assertTrue(fs.contains(5)); }
@Test public void testUnaryFunctionReadFieldsAnnotation() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L)); input.map(new ReadSetMapper<Tuple3<Long, Long, Long>>()).output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); Plan plan = env.createProgramPlan(); GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next(); MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput(); SingleInputSemanticProperties semantics = mapper.getSemanticProperties(); FieldSet read = semantics.getReadFields(0); assertNotNull(read); assertEquals(2, read.size()); assertTrue(read.contains(0)); assertTrue(read.contains(2)); }
@Test public void testReadFieldsPojo() { String[] readFields = { "int2; string1" }; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, pojoType, threeIntTupleType); FieldSet fs = sp.getReadFields(0); assertTrue(fs.size() == 2); assertTrue(fs.contains(1)); assertTrue(fs.contains(3)); readFields[0] = "*"; sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, pojoType, intType); fs = sp.getReadFields(0); assertTrue(fs.size() == 4); assertTrue(fs.contains(0)); assertTrue(fs.contains(1)); assertTrue(fs.contains(2)); assertTrue(fs.contains(3)); }
@Test public void testReadFieldsNestedTuples() { String[] readFields = { "f0.f1; f0.f2; f2" }; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, nestedTupleType, intType); FieldSet fs = sp.getReadFields(0); assertTrue(fs.size() == 3); assertTrue(fs.contains(1)); assertTrue(fs.contains(2)); assertTrue(fs.contains(4)); readFields[0] = "f0;f1"; sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, nestedTupleType, intType); fs = sp.getReadFields(0); assertTrue(fs.size() == 4); assertTrue(fs.contains(0)); assertTrue(fs.contains(1)); assertTrue(fs.contains(2)); assertTrue(fs.contains(3)); }
@Test public void testReadFieldsNestedTupleWildCard() { String[] readFields = { "*" }; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, nestedTupleType, intType); FieldSet fs = sp.getReadFields(0); assertTrue(fs.size() == 5); assertTrue(fs.contains(0)); assertTrue(fs.contains(1)); assertTrue(fs.contains(2)); assertTrue(fs.contains(3)); assertTrue(fs.contains(4)); readFields[0] = "f0.*;f1"; sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, nestedTupleType, intType); fs = sp.getReadFields(0); assertTrue(fs.size() == 4); assertTrue(fs.contains(0)); assertTrue(fs.contains(1)); assertTrue(fs.contains(2)); assertTrue(fs.contains(3)); }
@Test public void testReadFieldsNestedPojo() { String[] readFields = { "pojo1.int2; string1; pojo1.string1" }; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, nestedPojoType, intType); FieldSet fs = sp.getReadFields(0); assertTrue(fs.size() == 3); assertTrue(fs.contains(2)); assertTrue(fs.contains(4)); assertTrue(fs.contains(5)); readFields[0] = "pojo1.*"; sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, nestedPojoType, intType); fs = sp.getReadFields(0); assertTrue(fs.size() == 4); assertTrue(fs.contains(1)); assertTrue(fs.contains(2)); assertTrue(fs.contains(3)); assertTrue(fs.contains(4)); readFields[0] = "pojo1"; sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, nestedPojoType, intType); fs = sp.getReadFields(0); assertTrue(fs.size() == 4); assertTrue(fs.contains(1)); assertTrue(fs.contains(2)); assertTrue(fs.contains(3)); assertTrue(fs.contains(4)); }
@Override protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() { // Local properties for MapPartition may not be preserved. SingleInputSemanticProperties origProps = ((SingleInputOperator<?,?,?>) getOperator()).getSemanticProperties(); SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties(); FieldSet readSet = origProps.getReadFields(0); if(readSet != null) { filteredProps.addReadFields(readSet); } return filteredProps; }
@Override protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() { // Local properties for MapPartition may not be preserved. SingleInputSemanticProperties origProps = ((SingleInputOperator<?,?,?>) getOperator()).getSemanticProperties(); SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties(); FieldSet readSet = origProps.getReadFields(0); if(readSet != null) { filteredProps.addReadFields(readSet); } return filteredProps; }