/** * Waits up to 10 minutes for all of the outstanding mutations to be resolved. * * @see #close(Duration) for more infortmation. */ @Override public void close() throws InterruptedException, TimeoutException { close(Duration.ofMinutes(10)); }
@AfterClass public static void afterClass() throws IOException, InterruptedException, TimeoutException { helper.stop(Duration.ofMinutes(1)); }
/** Example of partitioning data to a list of Google Cloud Storage files. */ // [TARGET extract(String, List, JobOption...)] // [VARIABLE "CSV"] // [VARIABLE "gs://my_bucket/PartitionA_*.csv"] // [VARIABLE "gs://my_bucket/PartitionB_*.csv"] public Job extractList(String format, String gcsUrl1, String gcsUrl2) { // [START ] List<String> destinationUris = new ArrayList<>(); destinationUris.add(gcsUrl1); destinationUris.add(gcsUrl2); Job job = table.extract(format, destinationUris); // Wait for the job to complete try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END ] return job; }
/** Example loading data from a list of Google Cloud Storage files. */ // [TARGET load(FormatOptions, List, JobOption...)] // [VARIABLE "gs://my_bucket/filename1.csv"] // [VARIABLE "gs://my_bucket/filename2.csv"] public Job loadList(String gcsUrl1, String gcsUrl2) { // [START ] List<String> sourceUris = new ArrayList<>(); sourceUris.add(gcsUrl1); sourceUris.add(gcsUrl2); Job job = table.load(FormatOptions.csv(), sourceUris); // Wait for the job to complete try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END ] return job; }
@Test public void testEmulatorHelper() throws IOException, InterruptedException, TimeoutException { Process process = EasyMock.createStrictMock(Process.class); InputStream stream = new ByteArrayInputStream(BLOCK_UNTIL.getBytes(Charsets.UTF_8)); EmulatorRunner emulatorRunner = EasyMock.createStrictMock(EmulatorRunner.class); EasyMock.expect(process.getInputStream()).andReturn(stream); EasyMock.expect(emulatorRunner.isAvailable()).andReturn(true); emulatorRunner.start(); EasyMock.expectLastCall(); EasyMock.expect(emulatorRunner.getProcess()).andReturn(process); emulatorRunner.waitFor(Duration.ofMinutes(1)); EasyMock.expectLastCall().andReturn(0); EasyMock.replay(process, emulatorRunner); TestEmulatorHelper helper = new TestEmulatorHelper(ImmutableList.of(emulatorRunner), BLOCK_UNTIL); helper.start(); helper.stop(Duration.ofMinutes(1)); EasyMock.verify(); }
@Test public void testEmulatorHelperMultipleRunners() throws IOException, InterruptedException, TimeoutException { Process process = EasyMock.createStrictMock(Process.class); InputStream stream = new ByteArrayInputStream(BLOCK_UNTIL.getBytes(Charsets.UTF_8)); EmulatorRunner firstRunner = EasyMock.createStrictMock(EmulatorRunner.class); EmulatorRunner secondRunner = EasyMock.createStrictMock(EmulatorRunner.class); EasyMock.expect(process.getInputStream()).andReturn(stream); EasyMock.expect(firstRunner.isAvailable()).andReturn(false); EasyMock.expect(secondRunner.isAvailable()).andReturn(true); secondRunner.start(); EasyMock.expectLastCall(); EasyMock.expect(secondRunner.getProcess()).andReturn(process); secondRunner.waitFor(Duration.ofMinutes(1)); EasyMock.expectLastCall().andReturn(0); EasyMock.replay(process, secondRunner); TestEmulatorHelper helper = new TestEmulatorHelper(ImmutableList.of(firstRunner, secondRunner), BLOCK_UNTIL); helper.start(); helper.stop(Duration.ofMinutes(1)); EasyMock.verify(); }
/** Example extracting data to single Google Cloud Storage file. */ // [TARGET extract(String, String, JobOption...)] // [VARIABLE "CSV"] // [VARIABLE "gs://my_bucket/filename.csv"] public Job extractSingle(String format, String gcsUrl) { // [START bigquery_extract_table] Job job = table.extract(format, gcsUrl); // Wait for the job to complete try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END bigquery_extract_table] return job; }
/** Example of copying the table to a destination table. */ // [TARGET copy(String, String, JobOption...)] // [VARIABLE "my_dataset"] // [VARIABLE "my_destination_table"] public Job copy(String datasetName, String tableName) { // [START ] Job job = table.copy(datasetName, tableName); // Wait for the job to complete. try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END ] return job; }
/** Example usage of {@code waitFor()} with checking period and timeout. */ // [TARGET waitFor(RetryOption...)] public boolean waitForWithOptions() throws InterruptedException { try { // [START ] Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(1))); if (completedJob == null) { // job no longer exists } else if (completedJob.getStatus().getError() != null) { // job failed, handle error } else { // job completed successfully } // [END ] } catch (BigQueryException e) { if (e.getCause() instanceof PollException) { return false; } throw e; } return true; }
/** Example loading data from a single Google Cloud Storage file. */ // [TARGET load(FormatOptions, String, JobOption...)] // [VARIABLE "gs://my_bucket/filename.csv"] public Job loadSingle(String sourceUri) { // [START bigquery_load_table_gcs_csv] Job job = table.load(FormatOptions.csv(), sourceUri); // Wait for the job to complete try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END bigquery_load_table_gcs_csv] return job; } }
/** Example copying the table to a destination table. */ // [TARGET copy(TableId, JobOption...)] // [VARIABLE "my_dataset"] // [VARIABLE "my_destination_table"] public Job copyTableId(String dataset, String tableName) throws BigQueryException { // [START bigquery_copy_table] TableId destinationId = TableId.of(dataset, tableName); JobOption options = JobOption.fields(JobField.STATUS, JobField.USER_EMAIL); Job job = table.copy(destinationId, options); // Wait for the job to complete. try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully. } else { // Handle error case. } } catch (InterruptedException e) { // Handle interrupted wait } // [END bigquery_copy_table] return job; }
processor, Duration.ofSeconds(5), Duration.ofMinutes(60), new Distribution(Subscriber.MAX_ACK_DEADLINE_SECONDS + 1), flowController,
@Test public void testReadRowsSettings() { ServerStreamingCallSettings.Builder<ReadRowsRequest, ReadRowsResponse> builder = EnhancedBigQueryStorageStubSettings.newBuilder().readRowsSettings(); assertThat(builder.getRetryableCodes()).containsAllOf(Code.DEADLINE_EXCEEDED, Code.UNAVAILABLE); RetrySettings retrySettings = builder.getRetrySettings(); assertThat(retrySettings.getInitialRetryDelay()).isEqualTo(Duration.ofMillis(100L)); assertThat(retrySettings.getRetryDelayMultiplier()).isWithin(1e-6).of(1.3); assertThat(retrySettings.getMaxRetryDelay()).isEqualTo(Duration.ofMinutes(1L)); assertThat(retrySettings.getInitialRpcTimeout()).isEqualTo(Duration.ofDays(1L)); assertThat(retrySettings.getRpcTimeoutMultiplier()).isWithin(1e-6).of(1.0); assertThat(retrySettings.getMaxRpcTimeout()).isEqualTo(Duration.ofDays(1L)); assertThat(retrySettings.getTotalTimeout()).isEqualTo(Duration.ofDays(1L)); assertThat(builder.getIdleTimeout()).isEqualTo(Duration.ZERO); }
.setIdleTimeout(Duration.ofMinutes(5)) .setRetryableCodes(Code.ABORTED, Code.DEADLINE_EXCEEDED) .setRetrySettings(retrySettings) assertThat(builder.readRowsSettings().getIdleTimeout()).isEqualTo(Duration.ofMinutes(5)); assertThat(builder.readRowsSettings().getRetryableCodes()) .containsAllOf(Code.ABORTED, Code.DEADLINE_EXCEEDED); .isEqualTo(Duration.ofMinutes(5)); assertThat(builder.build().readRowsSettings().getRetryableCodes()) .containsAllOf(Code.ABORTED, Code.DEADLINE_EXCEEDED); .isEqualTo(Duration.ofMinutes(5)); assertThat(builder.build().toBuilder().readRowsSettings().getRetryableCodes()) .containsAllOf(Code.ABORTED, Code.DEADLINE_EXCEEDED);
@Test public void testStartStopReset() throws IOException, InterruptedException, TimeoutException { LocalDatastoreHelper helper = LocalDatastoreHelper.create(); helper.start(); Datastore datastore = helper.getOptions().getService(); Key key = datastore.newKeyFactory().setKind("kind").newKey("name"); datastore.put(Entity.newBuilder(key).build()); assertNotNull(datastore.get(key)); helper.reset(); assertNull(datastore.get(key)); helper.stop(Duration.ofMinutes(1)); thrown.expect(DatastoreException.class); datastore.get(key); } }
.setSimpleTimeoutNoRetries(Duration.ofMinutes(10)) .setRetryableCodes(settings.bulkMutateRowsSettings().getRetryableCodes()) .setIdleTimeout(Duration.ZERO);
assertTrue(createdTable.delete()); Job completedJob = remoteJob.waitFor(RetryOption.totalTimeout(Duration.ofMinutes(1)));
/** * Waits up to 10 minutes for all of the outstanding mutations to be resolved. * * @see #close(Duration) for more infortmation. */ @Override public void close() throws InterruptedException, TimeoutException { close(Duration.ofMinutes(10)); }
remoteJob.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(1))); assertNotNull(completedJob); assertNull(completedJob.getStatus().getError());
FakeRetryingFuture(ServerStreamingAttemptCallable<String, String> attemptCallable) { this.attemptCallable = attemptCallable; attemptSettings = TimedAttemptSettings.newBuilder() .setGlobalSettings( RetrySettings.newBuilder().setTotalTimeout(Duration.ofHours(1)).build()) .setFirstAttemptStartTimeNanos(0) .setAttemptCount(0) .setOverallAttemptCount(0) .setRandomizedRetryDelay(Duration.ofMillis(1)) .setRetryDelay(Duration.ofMillis(1)) .setRpcTimeout(Duration.ofMinutes(1)) .build(); }