@Test public void testRejectJoinOnHashAndRangePartitioning() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L, 0L)); Configuration cfg = new Configuration(); cfg.setString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH); cfg.setString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE); input.join(input).where(0).equalTo(0) .withParameters(cfg) .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>()); Plan p = env.createProgramPlan(); try { compileNoStats(p); fail("This should fail with an exception"); } catch (CompilerException e) { // expected } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } }
.join(input2, JoinHint.REPARTITION_HASH_FIRST) .where("b").equalTo("a").withPartitioner(partitioner) .output(new DiscardingOutputFormat<Tuple2<Pojo2, Pojo3>>());
.output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple3<Long, Long, Long>>>());
.equalTo(new Pojo3KeySelector()) .withPartitioner(partitioner) .output(new DiscardingOutputFormat<Tuple2<Pojo2, Pojo3>>());
.withBroadcastSet(input1, "bc2") .withBroadcastSet(result1, "bc3") .output(new DiscardingOutputFormat<String>());
@Test public void testBinaryForwardedInLine1() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L)); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L)); input1.join(input2).where(0).equalTo(0).with(new NoAnnotationJoin<Long>()) .withForwardedFieldsFirst("0->1; 1->2").withForwardedFieldsSecond("1->0") .output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); Plan plan = env.createProgramPlan(); GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next(); InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput(); DualInputSemanticProperties semantics = join.getSemanticProperties(); assertNotNull(semantics.getForwardingTargetFields(1, 0)); assertEquals(1, semantics.getForwardingTargetFields(0, 0).size()); assertEquals(1, semantics.getForwardingTargetFields(0, 1).size()); assertEquals(1, semantics.getForwardingTargetFields(1, 1).size()); assertTrue(semantics.getForwardingTargetFields(0, 0).contains(1)); assertTrue(semantics.getForwardingTargetFields(0, 1).contains(2)); assertTrue(semantics.getForwardingTargetFields(1, 1).contains(0)); assertEquals(0, semantics.getForwardingTargetFields(1, 0).size()); }
@Test public void testBinaryForwardedAnnotationInLineMixed1() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L)); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L)); input1.join(input2).where(0).equalTo(0).with(new ForwardedFirstAnnotationJoin<Long>()) .withForwardedFieldsSecond("1") .output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); Plan plan = env.createProgramPlan(); GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next(); InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput(); DualInputSemanticProperties semantics = join.getSemanticProperties(); assertNotNull(semantics.getForwardingTargetFields(0, 1)); assertNotNull(semantics.getForwardingTargetFields(1, 0)); assertNotNull(semantics.getForwardingTargetFields(0, 0)); assertNotNull(semantics.getForwardingTargetFields(1, 1)); assertEquals(1, semantics.getForwardingTargetFields(0, 0).size()); assertEquals(1, semantics.getForwardingTargetFields(1, 1).size()); assertTrue(semantics.getForwardingTargetFields(0, 0).contains(2)); assertTrue(semantics.getForwardingTargetFields(1, 1).contains(1)); assertEquals(0, semantics.getForwardingTargetFields(0, 1).size()); assertEquals(0, semantics.getForwardingTargetFields(1, 0).size()); }
@Test public void testBinaryForwardedAnnotationInLineMixed2() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L)); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L)); input1.join(input2).where(0).equalTo(0).with(new ForwardedSecondAnnotationJoin<Long>()) .withForwardedFieldsFirst("0->1") .output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); Plan plan = env.createProgramPlan(); GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next(); InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput(); DualInputSemanticProperties semantics = join.getSemanticProperties(); assertNotNull(semantics.getForwardingTargetFields(0, 1)); assertNotNull(semantics.getForwardingTargetFields(1, 0)); assertNotNull(semantics.getForwardingTargetFields(0, 0)); assertNotNull(semantics.getForwardingTargetFields(1, 1)); assertEquals(1, semantics.getForwardingTargetFields(0, 0).size()); assertEquals(1, semantics.getForwardingTargetFields(1, 1).size()); assertTrue(semantics.getForwardingTargetFields(0, 0).contains(1)); assertTrue(semantics.getForwardingTargetFields(1, 1).contains(2)); assertEquals(0, semantics.getForwardingTargetFields(0, 1).size()); assertEquals(0, semantics.getForwardingTargetFields(1, 0).size()); }
@Test public void testBinaryForwardedInLine2() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L)); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L)); input1.join(input2).where(0).equalTo(0).with(new ReadSetJoin<Long>()) .withForwardedFieldsFirst("0->1; 1->2").withForwardedFieldsSecond("1->0") .output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); Plan plan = env.createProgramPlan(); GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next(); InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput(); DualInputSemanticProperties semantics = join.getSemanticProperties(); assertNotNull(semantics.getForwardingTargetFields(1, 0)); assertEquals(1, semantics.getForwardingTargetFields(0, 0).size()); assertEquals(1, semantics.getForwardingTargetFields(0, 1).size()); assertEquals(1, semantics.getForwardingTargetFields(1, 1).size()); assertTrue(semantics.getForwardingTargetFields(0, 0).contains(1)); assertTrue(semantics.getForwardingTargetFields(0, 1).contains(2)); assertTrue(semantics.getForwardingTargetFields(1, 1).contains(0)); assertNotNull(semantics.getReadFields(0)); assertNotNull(semantics.getReadFields(1)); assertEquals(1, semantics.getReadFields(0).size()); assertEquals(1, semantics.getReadFields(1).size()); assertTrue(semantics.getReadFields(0).contains(1)); assertTrue(semantics.getReadFields(1).contains(0)); assertEquals(0, semantics.getForwardingTargetFields(1, 0).size()); }
.output(new DiscardingOutputFormat<Long>()).setParallelism(5);
union123.join(src4, JoinOperatorBase.JoinHint.BROADCAST_HASH_FIRST) .where(0).equalTo(0).name("join") .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>()).name("out");