@ReadOnly @Override public CloseableIterator<KeyValue<byte[], T>> scan(byte[] startRow, byte[] stopRow) { final CloseableIterator<KeyValue<byte[], byte[]>> keyValueIterator = kvTable.scan(startRow, stopRow); return new AbstractCloseableIterator<KeyValue<byte[], T>>() { boolean closed = false; @Override protected KeyValue<byte[], T> computeNext() { Preconditions.checkState(!closed); if (keyValueIterator.hasNext()) { KeyValue<byte[], byte[]> row = keyValueIterator.next(); return new KeyValue<>(row.getKey(), decode(row.getValue())); } close(); return null; } @Override public void close() { keyValueIterator.close(); endOfData(); closed = true; } }; }
@ReadOnly @Override public CloseableIterator<KeyValue<byte[], T>> scan(byte[] startRow, byte[] stopRow) { final CloseableIterator<KeyValue<byte[], byte[]>> keyValueIterator = kvTable.scan(startRow, stopRow); return new AbstractCloseableIterator<KeyValue<byte[], T>>() { boolean closed = false; @Override protected KeyValue<byte[], T> computeNext() { Preconditions.checkState(!closed); if (keyValueIterator.hasNext()) { KeyValue<byte[], byte[]> row = keyValueIterator.next(); return new KeyValue<>(row.getKey(), decode(row.getValue())); } close(); return null; } @Override public void close() { keyValueIterator.close(); endOfData(); closed = true; } }; }
try (CloseableIterator<KeyValue<byte[], byte[]>> filesIterable = table.scan(null, null)) { while (filesIterable.hasNext()) { KeyValue<byte[], byte[]> file = filesIterable.next();
try (CloseableIterator<KeyValue<byte[], byte[]>> iterator = processedFileTrackingTable.scan(null, null)) { while (iterator.hasNext()) { KeyValue<byte[], byte[]> keyValue = iterator.next();
@Override public void apply() throws Exception { // scan with start row '0' and end row '1000' and make sure we have 1000 records Iterator<KeyValue<byte[], byte[]>> keyValueIterator = t.scan(Bytes.toBytes(0), Bytes.toBytes(1000)); int rowCount = 0; while (keyValueIterator.hasNext()) { rowCount++; keyValueIterator.next(); } Assert.assertEquals(1000, rowCount); } });
@Override public void run(DatasetContext context) throws Exception { int uniqueWordCount = 0; try (CloseableIterator<KeyValue<byte[], byte[]>> scanner = wordCount.scan(null, null)) { while (scanner.hasNext()) { scanner.next(); uniqueWordCount++; } } result.write("UniqueWordCount", String.valueOf(uniqueWordCount)); metrics.gauge("unique.words", uniqueWordCount); } });
@Override public void run(DatasetContext dsContext) throws Exception { KeyValueTable stateTable = dsContext.getDataset(ETLRealtimeApplication.STATE_TABLE); byte[] startKey = Bytes.toBytes(String.format("%s%s", appName, SEPARATOR)); // Scan the table for appname: prefixes and remove rows which doesn't match the unique id of this application. try (CloseableIterator<KeyValue<byte[], byte[]>> rows = stateTable.scan(startKey, Bytes.stopKeyForPrefix(startKey))) { while (rows.hasNext()) { KeyValue<byte[], byte[]> row = rows.next(); if (Bytes.compareTo(stateStoreKeyBytes, row.getKey()) != 0) { stateTable.delete(row.getKey()); } } } } }, Exception.class);
private void assertWorkerDatasetWrites(byte[] startRow, byte[] endRow, int expectedCount, int expectedTotalCount) throws Exception { DataSetManager<KeyValueTable> datasetManager = getDataset(testSpace.dataset(AppUsingGetServiceURL.WORKER_INSTANCES_DATASET)); KeyValueTable instancesTable = datasetManager.get(); try (CloseableIterator<KeyValue<byte[], byte[]>> instancesIterator = instancesTable.scan(startRow, endRow)) { List<KeyValue<byte[], byte[]>> workerInstances = Lists.newArrayList(instancesIterator); // Assert that the worker starts with expectedCount instances Assert.assertEquals(expectedCount, workerInstances.size()); // Assert that each instance of the worker knows the total number of instances for (KeyValue<byte[], byte[]> keyValue : workerInstances) { Assert.assertEquals(expectedTotalCount, Bytes.toInt(keyValue.getValue())); } } }
try (CloseableIterator<KeyValue<byte[], byte[]>> itor = resultTable.scan(null, null)) { return ImmutableSet.copyOf(Iterators.transform(itor, input -> { String word = Bytes.toString(input.getKey());
private void assertDatasetIsEmpty(NamespaceId namespaceId, String datasetName) throws Exception { DataSetManager<KeyValueTable> outTableManager = getDataset(namespaceId.dataset(datasetName)); KeyValueTable outputTable = outTableManager.get(); try (CloseableIterator<KeyValue<byte[], byte[]>> scanner = outputTable.scan(null, null)) { Assert.assertFalse(scanner.hasNext()); } }
); try (CloseableIterator<KeyValue<byte[], byte[]>> scan = logStatsTable.scan(null, null)) {
@Override public void apply() throws Exception { // scan with start row '0' and end row '1000' and make sure we have 1000 records CloseableIterator<KeyValue<byte[], byte[]>> keyValueIterator = t.scan(Bytes.toBytes(0), Bytes.toBytes(200)); int rowCount = 0; while (keyValueIterator.hasNext() && (rowCount < 100)) { rowCount++; keyValueIterator.next(); } keyValueIterator.close(); try { keyValueIterator.next(); Assert.fail("Reading after closing Scanner returned result."); } catch (NoSuchElementException e) { } } });
private void testSparkWithLocalFiles(Class<? extends Application> appClass, String sparkProgram, String prefix) throws Exception { ApplicationManager applicationManager = deploy(NamespaceId.DEFAULT, appClass); URI localFile = createLocalPropertiesFile(prefix); SparkManager sparkManager = applicationManager.getSparkManager(sparkProgram) .start(Collections.singletonMap(SparkAppUsingLocalFiles.LOCAL_FILE_RUNTIME_ARG, localFile.toString())); sparkManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS); sparkManager.waitForStopped(120, TimeUnit.SECONDS); DataSetManager<KeyValueTable> kvTableManager = getDataset(SparkAppUsingLocalFiles.OUTPUT_DATASET_NAME); KeyValueTable kvTable = kvTableManager.get(); Map<String, String> expected = ImmutableMap.of("a", "1", "b", "2", "c", "3"); List<byte[]> deleteKeys = new ArrayList<>(); try (CloseableIterator<KeyValue<byte[], byte[]>> scan = kvTable.scan(null, null)) { for (int i = 0; i < 3; i++) { KeyValue<byte[], byte[]> next = scan.next(); Assert.assertEquals(expected.get(Bytes.toString(next.getKey())), Bytes.toString(next.getValue())); deleteKeys.add(next.getKey()); } Assert.assertFalse(scan.hasNext()); } // Cleanup after run kvTableManager.flush(); for (byte[] key : deleteKeys) { kvTable.delete(key); } kvTableManager.flush(); }
CloseableIterator<KeyValue<byte[], byte[]>> iter = trackingTable.scan(null, null);
private void testSparkWithGetDataset(Class<? extends Application> appClass, String sparkProgram) throws Exception { ApplicationManager applicationManager = deploy(appClass); DataSetManager<FileSet> filesetManager = getDataset("logs"); FileSet fileset = filesetManager.get(); Location location = fileset.getLocation("nn"); prepareInputFileSetWithLogData(location); Map<String, String> inputArgs = new HashMap<>(); FileSetArguments.setInputPath(inputArgs, "nn"); Map<String, String> args = new HashMap<>(); args.putAll(RuntimeArguments.addScope(Scope.DATASET, "logs", inputArgs)); args.put("input", "logs"); args.put("output", "logStats"); SparkManager sparkManager = applicationManager.getSparkManager(sparkProgram); sparkManager.startAndWaitForRun(args, ProgramRunStatus.COMPLETED, 2, TimeUnit.MINUTES); DataSetManager<KeyValueTable> logStatsManager = getDataset("logStats"); KeyValueTable logStatsTable = logStatsManager.get(); validateGetDatasetOutput(logStatsTable); // Cleanup after run location.delete(true); logStatsManager.flush(); try (CloseableIterator<KeyValue<byte[], byte[]>> scan = logStatsTable.scan(null, null)) { while (scan.hasNext()) { logStatsTable.delete(scan.next().getKey()); } } logStatsManager.flush(); }