@Override public LocatableInputSplitAssigner getInputSplitAssigner(FileInputSplit[] splits) { return new LocatableInputSplitAssigner(splits); }
@Override public void run() { LocatableInputSplit split; while ((split = ia.getNextInputSplit(null, 0)) != null) { splitsRetrieved.incrementAndGet(); sumOfIds.addAndGet(split.getSplitNumber()); } } };
LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits); InputSplit is = null; while ((is = ia.getNextInputSplit("testhost", 0)) != null) { assertTrue(splits.remove(is)); assertNull(ia.getNextInputSplit("anotherHost", 0)); assertEquals(NUM_SPLITS, ia.getNumberOfRemoteAssignments()); assertEquals(0, ia.getNumberOfLocalAssignments());
final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits); assertNull(ia.getNextInputSplit("testhost", 0)); assertTrue(ia.getNumberOfLocalAssignments() >= NUM_SPLITS / hosts.length);
final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits); LocatableInputSplit split = ia.getNextInputSplit(requestingHosts[rand.nextInt(requestingHosts.length)], 0); assertTrue(split != null); assertTrue(splits.remove(split)); assertNull(ia.getNextInputSplit("testHost", 0));
LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits); InputSplit is = null; while ((is = ia.getNextInputSplit("testhost", 0)) != null) { assertTrue(splits.remove(is)); assertNull(ia.getNextInputSplit("", 0)); assertEquals(0, ia.getNumberOfRemoteAssignments()); assertEquals(NUM_SPLITS, ia.getNumberOfLocalAssignments());
@Override public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) { return new LocatableInputSplitAssigner(inputSplits); }
LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits); InputSplit is = null; while ((is = ia.getNextInputSplit(null, 0)) != null) { assertTrue(splits.remove(is)); assertNull(ia.getNextInputSplit("", 0)); assertEquals(NUM_SPLITS, ia.getNumberOfRemoteAssignments()); assertEquals(0, ia.getNumberOfLocalAssignments());
@Override public void run() { LocatableInputSplit split; while ((split = ia.getNextInputSplit("testhost", 0)) != null) { splitsRetrieved.incrementAndGet(); sumOfIds.addAndGet(split.getSplitNumber()); } } };
@Override public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) { return new LocatableInputSplitAssigner(inputSplits); }
@Test public void testSerialSplitAssignmentMixedLocalHost() { try { final String[] hosts = { "host1", "host1", "host1", "host2", "host2", "host3" }; final int NUM_SPLITS = 10 * hosts.length; // load some splits Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>(); for (int i = 0; i < NUM_SPLITS; i++) { splits.add(new LocatableInputSplit(i, hosts[i % hosts.length])); } // get all available splits LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits); InputSplit is = null; int i = 0; while ((is = ia.getNextInputSplit(hosts[i++ % hosts.length], 0)) != null) { assertTrue(splits.remove(is)); } // check we had all assertTrue(splits.isEmpty()); assertNull(ia.getNextInputSplit("anotherHost", 0)); assertEquals(0, ia.getNumberOfRemoteAssignments()); assertEquals(NUM_SPLITS, ia.getNumberOfLocalAssignments()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
@Override public void run() { final String threadHost = hosts[(int) (Math.random() * hosts.length)]; LocatableInputSplit split; while ((split = ia.getNextInputSplit(threadHost, 0)) != null) { splitsRetrieved.incrementAndGet(); sumOfIds.addAndGet(split.getSplitNumber()); } } };
@Override public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) { return new LocatableInputSplitAssigner(inputSplits); }
LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits); InputSplit is = null; int i = 0; while ((is = ia.getNextInputSplit(hosts[i++ % hosts.length], 0)) != null) { assertTrue(splits.remove(is)); assertNull(ia.getNextInputSplit("anotherHost", 0)); assertEquals(NUM_REMOTE_SPLITS, ia.getNumberOfRemoteAssignments()); assertEquals(NUM_LOCAL_SPLITS, ia.getNumberOfLocalAssignments());
@Override public InputSplitAssigner getInputSplitAssigner(TableInputSplit[] inputSplits) { return new LocatableInputSplitAssigner(inputSplits); }
final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits); assertNull(ia.getNextInputSplit("", 0)); assertEquals(NUM_SPLITS, ia.getNumberOfRemoteAssignments()); assertEquals(0, ia.getNumberOfLocalAssignments());
@Override public LocatableInputSplitAssigner getInputSplitAssigner(FileInputSplit[] splits) { return new LocatableInputSplitAssigner(splits); }
final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits); assertNull(ia.getNextInputSplit("testhost", 0)); assertEquals(0, ia.getNumberOfRemoteAssignments()); assertEquals(NUM_SPLITS, ia.getNumberOfLocalAssignments());
@Override public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) { return new LocatableInputSplitAssigner(inputSplits); }
LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits); LocatableInputSplit is = null; for (int i = 0; i < NUM_SPLITS; i++) { String host = requestingHosts[i % requestingHosts.length]; is = ia.getNextInputSplit(host, 0); assertNull(ia.getNextInputSplit("anotherHost", 0)); assertEquals(NUM_REMOTE_SPLITS, ia.getNumberOfRemoteAssignments()); assertEquals(NUM_LOCAL_SPLITS, ia.getNumberOfLocalAssignments());