private KuduSplit toKuduSplit(KuduTableHandle tableHandle, KuduScanToken token, int primaryKeyColumnCount) { try { byte[] serializedScanToken = token.serialize(); return new KuduSplit(tableHandle, primaryKeyColumnCount, serializedScanToken); } catch (IOException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } } }
/** * Serializes this {@code KuduScanToken} into a byte array. * @return the serialized scan token * @throws IOException */ public byte[] serialize() throws IOException { return serialize(message); }
private KuduSplit toKuduSplit(KuduTableHandle tableHandle, KuduScanToken token, int primaryKeyColumnCount) { try { byte[] pb = token.serialize(); return new KuduSplit(tableHandle, primaryKeyColumnCount, pb); } catch (IOException e) { throw new RuntimeException(e); } }
private KuduSplit toKuduSplit(KuduTableHandle tableHandle, KuduScanToken token, int primaryKeyColumnCount) { try { byte[] serializedScanToken = token.serialize(); return new KuduSplit(tableHandle, primaryKeyColumnCount, serializedScanToken); } catch (IOException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } } }
List<Thread> threads = new ArrayList<>(); for (final KuduScanToken token : tokens) { final byte[] serializedToken = token.serialize(); Thread thread = new Thread(new Runnable() { @Override
@Override public KuduInputSplit[] createInputSplits(int minNumSplits) throws IOException { startTableContext(); Preconditions.checkNotNull(tableContext,"tableContext should not be null"); List<KuduScanToken> tokens = tableContext.scanTokens(tableFilters, tableProjections, rowsLimit); KuduInputSplit[] splits = new KuduInputSplit[tokens.size()]; for (int i = 0; i < tokens.size(); i++) { KuduScanToken token = tokens.get(i); List<String> locations = new ArrayList<>(token.getTablet().getReplicas().size()); for (LocatedTablet.Replica replica : token.getTablet().getReplicas()) { locations.add(getLocation(replica.getRpcHost(), replica.getRpcPort())); } KuduInputSplit split = new KuduInputSplit( token.serialize(), i, locations.toArray(new String[locations.size()]) ); splits[i] = split; } if (splits.length < minNumSplits) { LOG.warn(" The minimum desired number of splits with your configured parallelism level " + "is {}. Current kudu splits = {}. {} instances will remain idle.", minNumSplits, splits.length, (minNumSplits - splits.length) ); } return splits; }
/** * Collects the rows from a set of scan tokens. * * @param scanTokens the scan token builder * @return the rows */ private Set<Row> collectRows(KuduScanToken.KuduScanTokenBuilder scanTokens) throws Exception { Set<Row> rows = new HashSet<>(); for (KuduScanToken token : scanTokens.build()) { LOG.debug("Scanning token: {}", KuduScanToken.stringifySerializedToken(token.serialize(), client)); int existingCount = rows.size(); Set<Row> newRows = collectRows(token.intoScanner(client)); rows.addAll(newRows); assertEquals(existingCount + newRows.size(), rows.size()); } return rows; }
aMetaForThisQuery.setSerializedKuduScanToken(aTokenForThisOperator.serialize()); returnOfAssignments.add(aMetaForThisQuery); LOG.debug("Added query scan for this operator " + aMetaForThisQuery + " with scan tablet as " +
@Test(timeout = 100000) public void testScanTokenReadMode() throws Exception { ScanTokenPB.Builder pbBuilder = ScanTokenPB.newBuilder(); pbBuilder.setTableName(table.getName()); pbBuilder.setReadMode(Common.ReadMode.READ_YOUR_WRITES); Client.ScanTokenPB scanTokenPB = pbBuilder.build(); final byte[] serializedToken = KuduScanToken.serialize(scanTokenPB); // Deserialize scan tokens and make sure the read mode is updated accordingly. KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serializedToken, client); assertEquals(AsyncKuduScanner.ReadMode.READ_YOUR_WRITES, scanner.getReadMode()); }
/** Test that scanRequestTimeout makes it from the scan token to the underlying Scanner class. */ @Test public void testScanRequestTimeout() throws IOException { final int NUM_ROWS_DESIRED = 100; final int SCAN_REQUEST_TIMEOUT_MS = 20; KuduTable table = createDefaultTable(client, testTableName); loadDefaultTable(client, testTableName, NUM_ROWS_DESIRED); KuduScanToken.KuduScanTokenBuilder builder = new KuduScanToken.KuduScanTokenBuilder(asyncClient, table); builder.scanRequestTimeout(SCAN_REQUEST_TIMEOUT_MS); List<KuduScanToken> tokens = builder.build(); for (KuduScanToken token : tokens) { byte[] serialized = token.serialize(); KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serialized, client); assertEquals(SCAN_REQUEST_TIMEOUT_MS, scanner.getScanRequestTimeout()); } } }
@Test(timeout = 100000) public void testScanTokenReplicaSelections() throws Exception { ScanTokenPB.Builder pbBuilder = ScanTokenPB.newBuilder(); pbBuilder.setTableName(table.getName()); pbBuilder.setReplicaSelection(Common.ReplicaSelection.CLOSEST_REPLICA); Client.ScanTokenPB scanTokenPB = pbBuilder.build(); final byte[] serializedToken = KuduScanToken.serialize(scanTokenPB); // Deserialize the scan token into a scanner, and make sure it is using // 'CLOSEST_REPLICA' selection policy. KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serializedToken, client); assertEquals(ReplicaSelection.CLOSEST_REPLICA, scanner.getReplicaSelection()); assertEquals(9, countRowsInScan(scanner)); }
LOG.debug(KuduScanToken.stringifySerializedToken(token.serialize(), client));
LOG.debug(KuduScanToken.stringifySerializedToken(token.serialize(), client));
@Test(timeout = 100000) public void testScanTokenPropagatesTimestamp() throws Exception { // Initially, the client does not have the timestamp set. assertEquals(AsyncKuduClient.NO_TIMESTAMP, asyncClient.getLastPropagatedTimestamp()); assertEquals(KuduClient.NO_TIMESTAMP, client.getLastPropagatedTimestamp()); AsyncKuduScanner scanner = asyncClient.newScannerBuilder(table).build(); KuduScanner syncScanner = new KuduScanner(scanner); // Let the client receive the propagated timestamp in the scanner response. syncScanner.nextRows().getNumRows(); final long tsPrev = asyncClient.getLastPropagatedTimestamp(); final long tsPropagated = tsPrev + 1000000; ScanTokenPB.Builder pbBuilder = ScanTokenPB.newBuilder(); pbBuilder.setTableName(table.getName()); pbBuilder.setPropagatedTimestamp(tsPropagated); Client.ScanTokenPB scanTokenPB = pbBuilder.build(); final byte[] serializedToken = KuduScanToken.serialize(scanTokenPB); // Deserialize scan tokens and make sure the client's last propagated // timestamp is updated accordingly. assertEquals(tsPrev, asyncClient.getLastPropagatedTimestamp()); KuduScanToken.deserializeIntoScanner(serializedToken, client); assertEquals(tsPropagated, asyncClient.getLastPropagatedTimestamp()); }