@Override public Set<AccumulatorRequest> getAccumulatorRequests() { LOG.info("Kudu output requesting accumulators"); return Sets.newHashSet(new AccumulatorRequest(ACCUMULATOR_NUMBER_OF_SCANNERS, Long.class), new AccumulatorRequest(ACCUMULATOR_NUMBER_OF_FILTERS_SCANNED, Long.class), new AccumulatorRequest(ACCUMULATOR_SECONDS_SCANNING, Double.class)); }
@Override public boolean equals(Object other) { if (other == null) return false; if (!(other instanceof AccumulatorRequest)) return false; // Accumulator requests are unique only by their name. If multiple objects request accumulators // with the same name but different classes then it is not defined which one Envelope will request from Spark. if (!((AccumulatorRequest)other).getName().equals(this.getName())) return false; return true; }
@Test public void testNewAccumulatorRequest() { AccumulatorRequest requestInt = new AccumulatorRequest("hello", Long.class); assertEquals(requestInt.getName(), "hello"); assertEquals(requestInt.getClazz(), Long.class); AccumulatorRequest requestDouble = new AccumulatorRequest("world", Double.class); assertEquals(requestDouble.getName(), "world"); assertEquals(requestDouble.getClazz(), Double.class); }
@Test public void testEquality() { AccumulatorRequest request1; AccumulatorRequest request2; request1 = new AccumulatorRequest("hello", Long.class); request2 = new AccumulatorRequest("hello", Long.class); assertTrue(request1.equals(request2)); request1 = new AccumulatorRequest("hello", Long.class); request2 = new AccumulatorRequest("hello", Long.class); assertTrue(request1.equals(request2)); request1 = new AccumulatorRequest("hello", Long.class); request2 = new AccumulatorRequest("hello", Double.class); assertTrue(request1.equals(request2)); request1 = new AccumulatorRequest("hello", Long.class); request2 = new AccumulatorRequest("world", Long.class); assertFalse(request1.equals(request2)); }
public Accumulators(Set<AccumulatorRequest> requests) { for (AccumulatorRequest request : requests) { String name = request.getName(); Class<?> clazz = request.getClazz(); if (clazz == Long.class) { LongAccumulator acc = Contexts.getSparkSession().sparkContext().longAccumulator(name); longAccumulators.put(name, acc); } if (clazz == Double.class) { DoubleAccumulator acc = Contexts.getSparkSession().sparkContext().doubleAccumulator(name); doubleAccumulators.put(name, acc); } LOG.info("Processed accumulator request: " + name); } }
@Test (expected = IllegalArgumentException.class) public void testUnsupportedClass() { new AccumulatorRequest("hello", Float.class); }
@Override public Set<AccumulatorRequest> getAccumulatorRequests() { Set<AccumulatorRequest> requests = Sets.newHashSet(); if (hasInput() && getInput(false) instanceof UsesAccumulators) { requests.addAll(((UsesAccumulators)getInput(false)).getAccumulatorRequests()); } if (hasDeriver() && getDeriver(false) instanceof UsesAccumulators) { requests.addAll(((UsesAccumulators)getDeriver(false)).getAccumulatorRequests()); } if (hasPlanner() && getPlanner(false) instanceof UsesAccumulators) { requests.addAll(((UsesAccumulators)getPlanner(false)).getAccumulatorRequests()); } if (hasOutput() && getOutput(false) instanceof UsesAccumulators) { requests.addAll(((UsesAccumulators)getOutput(false)).getAccumulatorRequests()); } requests.add(new AccumulatorRequest(ACCUMULATOR_SECONDS_PLANNING, Double.class)); requests.add(new AccumulatorRequest(ACCUMULATOR_SECONDS_APPLYING, Double.class)); requests.add(new AccumulatorRequest(ACCUMULATOR_SECONDS_EXISTING, Double.class)); requests.add(new AccumulatorRequest(ACCUMULATOR_SECONDS_EXTRACTING_KEYS, Double.class)); return requests; }
@Test public void testRequestMany() { AccumulatorRequest request1 = new AccumulatorRequest("hello", Long.class); AccumulatorRequest request2 = new AccumulatorRequest("world", Double.class); Accumulators accumulators = new Accumulators(Sets.newHashSet(request1, request2)); LongAccumulator accumulator1 = accumulators.getLongAccumulators().get("hello"); assertEquals(accumulator1.name().get(), "hello"); DoubleAccumulator accumulator2 = accumulators.getDoubleAccumulators().get("world"); assertEquals(accumulator2.name().get(), "world"); }
@Test public void testRequestOne() { AccumulatorRequest request = new AccumulatorRequest("hello", Long.class); Accumulators accumulators = new Accumulators(Collections.singleton(request)); LongAccumulator accumulator = accumulators.getLongAccumulators().get("hello"); assertEquals(accumulator.name().get(), "hello"); }