@Override @Internal public DualInputSemanticProperties getSemanticProperties() { if (this.udfSemantics == null || analyzedUdfSemantics) { DualInputSemanticProperties props = extractSemanticAnnotationsFromUdf(getFunction().getClass()); if (props != null) { setSemanticProperties(props); } } if (this.udfSemantics == null) { setSemanticProperties(new DualInputSemanticProperties()); } return this.udfSemantics; }
@Test(expected = InvalidSemanticAnnotationException.class) public void testNonForwardedDualInvalidTypes2() { String[] nonForwardedFieldsSecond = { "f1" }; DualInputSemanticProperties dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, null, nonForwardedFieldsSecond, null, null, threeIntTupleType, pojoInTupleType, threeIntTupleType); }
public SemanticProperties getSemanticProperties() { final SemanticProperties sp; if (isBinary) { sp = new DualInputSemanticProperties(); if (returnValue != null) { String[] ff1Array = null;
@Test(expected = InvalidSemanticAnnotationException.class) public void testNonForwardedDualInvalidTypes1() { String[] nonForwardedFieldsFirst = { "f1" }; DualInputSemanticProperties dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, nonForwardedFieldsFirst, null, null, null, fiveIntTupleType, threeIntTupleType, threeIntTupleType); }
@Test(expected = InvalidSemanticAnnotationException.class) public void testForwardedNonForwardedFirstCheck() { String[] forwarded = { "1" }; String[] nonForwarded = { "1" }; SemanticPropUtil.getSemanticPropsDualFromString(new DualInputSemanticProperties(), forwarded, null, nonForwarded, null, null, null, threeIntTupleType, threeIntTupleType, threeIntTupleType); }
@Test(expected = InvalidSemanticAnnotationException.class) public void testForwardedNonForwardedSecondCheck() { String[] forwarded = { "1" }; String[] nonForwarded = { "1" }; SemanticPropUtil.getSemanticPropsDualFromString(new DualInputSemanticProperties(), null, forwarded, null, nonForwarded, null, null, threeIntTupleType, threeIntTupleType, threeIntTupleType); }
DualInputSemanticProperties result = new DualInputSemanticProperties(); getSemanticPropsDualFromString(result, forwardedFirst, forwardedSecond, nonForwardedFirst, nonForwardedSecond, readFirst, readSecond, inType1, inType2, outType);
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class) public void testAddForwardedFieldsTargetTwice2() { DualInputSemanticProperties sp = new DualInputSemanticProperties(); sp.addForwardedField(1, 0, 2); sp.addForwardedField(1, 1, 2); }
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class) public void testAddForwardedFieldsTargetTwice1() { DualInputSemanticProperties sp = new DualInputSemanticProperties(); sp.addForwardedField(0, 0, 2); sp.addForwardedField(0, 1, 2); }
@Override protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() { // Local properties for CoGroup may only be preserved on key fields. DualInputSemanticProperties origProps = ((DualInputOperator<?, ?, ?, ?>) getOperator()).getSemanticProperties(); DualInputSemanticProperties filteredProps = new DualInputSemanticProperties(); FieldSet readSet1 = origProps.getReadFields(0); FieldSet readSet2 = origProps.getReadFields(1); if(readSet1 != null) { filteredProps.addReadFields(0, readSet1); } if(readSet2 != null) { filteredProps.addReadFields(1, readSet2); } // preserve only key fields (first input) for(int f : this.keys1) { FieldSet targets = origProps.getForwardingTargetFields(0, f); for(int t : targets) { filteredProps.addForwardedField(0, f, t); } } // preserve only key fields (second input) for(int f : this.keys2) { FieldSet targets = origProps.getForwardingTargetFields(1, f); for(int t : targets) { filteredProps.addForwardedField(1, f, t); } } return filteredProps; }
int offset1, int offset2) { DualInputSemanticProperties offsetProps = new DualInputSemanticProperties();
public static DualInputSemanticProperties createProjectionPropertiesDual( int[] fields, boolean[] isFromFirst, TypeInformation<?> inType1, TypeInformation<?> inType2) { DualInputSemanticProperties dsp = new DualInputSemanticProperties();
DualInputSemanticProperties sp = new DualInputSemanticProperties(); sp.addForwardedField(0, 0,1); sp.addForwardedField(0, 1,4); assertTrue(sp.getForwardingSourceField(0, 5) < 0); sp = new DualInputSemanticProperties(); sp.addForwardedField(0, 0,0); sp.addForwardedField(0, 0,4); sp = new DualInputSemanticProperties(); sp.addForwardedField(1, 0,1); sp.addForwardedField(1, 1,4); assertTrue(sp.getForwardingSourceField(1, 5) < 0); sp = new DualInputSemanticProperties(); sp.addForwardedField(1, 0,0); sp.addForwardedField(1, 0,4);
String[] readFieldsFirst = {"f1;f2"}; String[] readFieldsSecond = {"f0"}; DualInputSemanticProperties dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, null, null, readFieldsFirst, readFieldsSecond, threeIntTupleType, threeIntTupleType, threeIntTupleType); dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, null, null, readFieldsFirst, readFieldsSecond, nestedTupleType, pojoType, threeIntTupleType); dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, null, null, readFieldsFirst, readFieldsSecond, nestedPojoType, pojoInTupleType, threeIntTupleType); dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, null, null, readFields, readFields, fiveIntTupleType, fiveIntTupleType, threeIntTupleType);
@Test public void testGetReadSet() { // first input DualInputSemanticProperties sp = new DualInputSemanticProperties(); sp.addReadFields(0, new FieldSet(0, 1)); assertEquals(2, sp.getReadFields(0).size()); assertTrue(sp.getReadFields(0).contains(0)); assertTrue(sp.getReadFields(0).contains(1)); sp.addReadFields(0, new FieldSet(3)); assertEquals(3, sp.getReadFields(0).size()); assertTrue(sp.getReadFields(0).contains(0)); assertTrue(sp.getReadFields(0).contains(1)); assertTrue(sp.getReadFields(0).contains(3)); // second input sp = new DualInputSemanticProperties(); sp.addReadFields(1, new FieldSet(0, 1)); assertEquals(2, sp.getReadFields(1).size()); assertTrue(sp.getReadFields(1).contains(0)); assertTrue(sp.getReadFields(1).contains(1)); sp.addReadFields(1, new FieldSet(3)); assertEquals(3, sp.getReadFields(1).size()); assertTrue(sp.getReadFields(1).contains(0)); assertTrue(sp.getReadFields(1).contains(1)); assertTrue(sp.getReadFields(1).contains(3)); }
@Test public void testForwardedReadDual() { String[] forwardedFieldsFirst = { "f1->f2; f2->f3" }; String[] forwardedFieldsSecond = { "f1->f1; f2->f0" }; String[] readFieldsFirst = {"0;2"}; String[] readFieldsSecond = {"1"}; DualInputSemanticProperties dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null, null, readFieldsFirst, readFieldsSecond, fourIntTupleType, fourIntTupleType, fourIntTupleType); assertTrue(dsp.getForwardingTargetFields(0, 1).contains(2)); assertTrue(dsp.getForwardingTargetFields(0, 2).contains(3)); assertTrue(dsp.getForwardingTargetFields(1, 1).contains(1)); assertTrue(dsp.getForwardingTargetFields(1, 2).contains(0)); assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0); assertTrue(dsp.getForwardingTargetFields(1, 3).size() == 0); assertTrue(dsp.getReadFields(0).size() == 2); assertTrue(dsp.getReadFields(0).contains(0)); assertTrue(dsp.getReadFields(0).contains(2)); assertTrue(dsp.getReadFields(1).size() == 1); assertTrue(dsp.getReadFields(1).contains(1)); }
String[] forwardedFieldsFirst = { "f1->f2; f2->f3" }; String[] forwardedFieldsSecond = { "f1->f1; f2->f0" }; DualInputSemanticProperties dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null, null, null, null, fourIntTupleType, fourIntTupleType, fourIntTupleType); dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null, null, null, null, fourIntTupleType, pojoType, pojoInTupleType); dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null, null, null, null, deepNestedTupleType, pojoType, nestedPojoType); dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst2, forwardedFieldsSecond2, null, null, null, null, deepNestedTupleType, pojoType, nestedPojoType);
@Test public void testAddSourceFieldOffsets() { DualInputSemanticProperties semProps = new DualInputSemanticProperties(); semProps.addForwardedField(0, 0, 1); semProps.addForwardedField(0, 3, 3); assertTrue(offsetProps.getReadFields(1).contains(6)); semProps = new DualInputSemanticProperties(); SemanticPropUtil.addSourceFieldOffsets(semProps, 4, 3, 2, 2);
@Test public void testDualGroupingPreserved() { DualInputSemanticProperties dprops = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[]{"1->0;3;2->4"}, new String[]{"0->7;1"}, null, null, null, null, tupleInfo, tupleInfo, tupleInfo); RequestedLocalProperties lprops1 = new RequestedLocalProperties(); lprops1.setGroupedFields(new FieldSet(0,3,4)); RequestedLocalProperties lprops2 = new RequestedLocalProperties(); lprops2.setGroupedFields(new FieldSet(7, 1)); RequestedLocalProperties filtered1 = lprops1.filterBySemanticProperties(dprops, 0); RequestedLocalProperties filtered2 = lprops2.filterBySemanticProperties(dprops, 1); assertNotNull(filtered1); assertNotNull(filtered1.getGroupedFields()); assertEquals(3, filtered1.getGroupedFields().size()); assertTrue(filtered1.getGroupedFields().contains(1)); assertTrue(filtered1.getGroupedFields().contains(2)); assertTrue(filtered1.getGroupedFields().contains(3)); assertNull(filtered1.getOrdering()); assertNotNull(filtered2); assertNotNull(filtered2.getGroupedFields()); assertEquals(2, filtered2.getGroupedFields().size()); assertTrue(filtered2.getGroupedFields().contains(0)); assertTrue(filtered2.getGroupedFields().contains(1)); assertNull(filtered2.getOrdering()); }
@Test public void testDualHashPartitioningPreserved() { DualInputSemanticProperties dprops = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[]{"0;2;4"}, new String[]{"1->3;4->6;3->7"}, null, null, null, null, tupleInfo, tupleInfo, tupleInfo);