EXCEPTION_HANDLER, serviceOptions.getClock()); String cursor = result.getPageToken(); return Tuple.of( new PageImpl<>( new TableDataPageFetcher(tableId, serviceOptions, cursor, optionsMap), cursor, transformTableData(result.getRows())), result.getTotalRows()); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e);
PROJECT, DATASET, TABLE, Collections.<BigQueryRpc.Option, Object>emptyMap())) .andReturn( new TableDataList() .setPageToken("") .setRows(ImmutableList.of(TABLE_ROW)) .setTotalRows(1L));
.setPageToken(pageToken) .execute(); List<TableRow> rows = queryResult.getRows(); System.out.print("\nQuery Results, Page #" + page + ":\n------------\n"); for (TableRow row : rows) { if (queryResult.getPageToken() != null) { pageToken = queryResult.getPageToken(); moreResults = true; page++;
@VisibleForTesting boolean isTableEmpty(TableReference tableRef, BackOff backoff, Sleeper sleeper) throws IOException, InterruptedException { TableDataList dataList = executeWithRetries( client .tabledata() .list(tableRef.getProjectId(), tableRef.getDatasetId(), tableRef.getTableId()), String.format( "Unable to list table data: %s, aborting after %d retries.", tableRef.getTableId(), MAX_RPC_RETRIES), sleeper, backoff, DONT_RETRY_NOT_FOUND); return dataList.getRows() == null || dataList.getRows().isEmpty(); }
@Test public void testIsTableEmptySucceeds() throws Exception { TableReference tableRef = new TableReference() .setProjectId("projectId") .setDatasetId("datasetId") .setTableId("tableId"); TableDataList testDataList = new TableDataList().setRows(ImmutableList.of(new TableRow())); // First response is 403 rate limited, second response has valid payload. when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(403).thenReturn(200); when(response.getContent()) .thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403))) .thenReturn(toStream(testDataList)); BigQueryServicesImpl.DatasetServiceImpl datasetService = new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create()); assertFalse(datasetService.isTableEmpty(tableRef, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT)); verify(response, times(2)).getStatusCode(); verify(response, times(2)).getContent(); verify(response, times(2)).getContentType(); }
@Test public void testTableGet() throws InterruptedException, IOException { onTableGet(basicTableSchema()); TableDataList dataList = new TableDataList().setTotalRows(0L); onTableList(dataList); BigQueryServicesImpl.DatasetServiceImpl services = new BigQueryServicesImpl.DatasetServiceImpl(mockClient, options); services.getTable( new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table")); verifyTableGet(); }
@Override public Tuple<String, Iterable<TableRow>> listTableData(String datasetId, String tableId, Map<Option, ?> options) { try { TableDataList tableDataList = bigquery.tabledata() .list(this.options.projectId(), datasetId, tableId) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .setStartIndex(START_INDEX.getLong(options) != null ? BigInteger.valueOf(START_INDEX.getLong(options)) : null) .execute(); return Tuple.<String, Iterable<TableRow>>of(tableDataList.getPageToken(), tableDataList.getRows()); } catch (IOException ex) { throw translate(ex); } }
private List<TableRow> getTableRows(Bigquery bq) { try { return bq.tabledata() .list( pipelineOptions.getProject(), pipelineOptions.getTargetDataset(), table.getTableReference().getTableId()) .execute() .getRows(); } catch (IOException e) { throw new RuntimeException(e); } }
ref.getTableId(), ref.getDatasetId())); pageToken = result.getPageToken(); iteratorOverCurrentBatch = result.getRows() != null ? result.getRows().iterator() : Collections.<TableRow>emptyIterator(); if (pageToken == null || result.getTotalRows() != null && result.getTotalRows() == 0) { lastPage = true;
@Override public Tuple<String, Iterable<TableRow>> listTableData(String datasetId, String tableId, Map<Option, ?> options) { try { TableDataList tableDataList = bigquery.tabledata() .list(this.options.projectId(), datasetId, tableId) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .setStartIndex(START_INDEX.getLong(options) != null ? BigInteger.valueOf(START_INDEX.getLong(options)) : null) .execute(); return Tuple.<String, Iterable<TableRow>>of(tableDataList.getPageToken(), tableDataList.getRows()); } catch (IOException ex) { throw translate(ex); } }
PROJECT, DATASET, TABLE, Collections.<BigQueryRpc.Option, Object>emptyMap())) .andReturn( new TableDataList() .setPageToken("") .setRows(ImmutableList.of(TABLE_ROW)) .setTotalRows(1L));
EXCEPTION_HANDLER, serviceOptions.getClock()); String cursor = result.getPageToken(); return Tuple.of( new PageImpl<>( new TableDataPageFetcher(tableId, serviceOptions, cursor, optionsMap), cursor, transformTableData(result.getRows())), result.getTotalRows()); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e);
EasyMock.expect(bigqueryRpcMock.listTableData(PROJECT, DATASET, TABLE, optionMap)) .andReturn( new TableDataList() .setPageToken("") .setRows(ImmutableList.of(TABLE_ROW)) .setTotalRows(1L));