public static CloudDebugger.Builder newClouddebuggerClient(DataflowPipelineOptions options) { return new CloudDebugger.Builder( getTransport(), getJsonFactory(), chainHttpRequestInitializer( options.getGcpCredential(), new RetryHttpRequestInitializer())) .setApplicationName(options.getAppName()) .setGoogleClientRequestInitializer(options.getGoogleApiTrace()); }
/** Returns a BigQuery client builder using the specified {@link BigQueryOptions}. */ private static Bigquery.Builder newBigQueryClient(BigQueryOptions options) { RetryHttpRequestInitializer httpRequestInitializer = new RetryHttpRequestInitializer(ImmutableList.of(404)); httpRequestInitializer.setWriteTimeout(options.getHTTPWriteTimeout()); return new Bigquery.Builder( Transport.getTransport(), Transport.getJsonFactory(), chainHttpRequestInitializer( options.getGcpCredential(), // Do not log 404. It clutters the output and is possibly even required by the caller. httpRequestInitializer)) .setApplicationName(options.getAppName()) .setGoogleClientRequestInitializer(options.getGoogleApiTrace()); }
private static Bigquery newBigQueryClient(BigQueryOptions options) { return new Bigquery.Builder( Transport.getTransport(), Transport.getJsonFactory(), chainHttpRequestInitializer( options.getGcpCredential(), // Do not log 404. It clutters the output and is possibly even required by the // caller. new RetryHttpRequestInitializer(ImmutableList.of(404)))) .setApplicationName(options.getAppName()) .setGoogleClientRequestInitializer(options.getGoogleApiTrace()) .build(); }
@Override public PubsubClient newClient( @Nullable String timestampAttribute, @Nullable String idAttribute, PubsubOptions options) throws IOException { Pubsub pubsub = new Builder( Transport.getTransport(), Transport.getJsonFactory(), chainHttpRequestInitializer( options.getGcpCredential(), // Do not log 404. It clutters the output and is possibly even required by the caller. new RetryHttpRequestInitializer(ImmutableList.of(404)))) .setRootUrl(options.getPubsubRootUrl()) .setApplicationName(options.getAppName()) .setGoogleClientRequestInitializer(options.getGoogleApiTrace()) .build(); return new PubsubJsonClient(timestampAttribute, idAttribute, pubsub); }
/** Returns a BigQuery client builder using the specified {@link BigQueryOptions}. */ private static Bigquery.Builder newBigQueryClient(BigQueryOptions options) { return new Bigquery.Builder( Transport.getTransport(), Transport.getJsonFactory(), chainHttpRequestInitializer( options.getGcpCredential(), // Do not log 404. It clutters the output and is possibly even required by the caller. new RetryHttpRequestInitializer(ImmutableList.of(404)))) .setApplicationName(options.getAppName()) .setGoogleClientRequestInitializer(options.getGoogleApiTrace()); }
/** * Builds a Cloud Datastore client for the given pipeline options, project and an optional * locahost. */ public Datastore getDatastore( PipelineOptions pipelineOptions, String projectId, @Nullable String localhost) { Credentials credential = pipelineOptions.as(GcpOptions.class).getGcpCredential(); HttpRequestInitializer initializer; if (credential != null) { initializer = new ChainingHttpRequestInitializer( new HttpCredentialsAdapter(credential), new RetryHttpRequestInitializer()); } else { initializer = new RetryHttpRequestInitializer(); } DatastoreOptions.Builder builder = new DatastoreOptions.Builder().projectId(projectId).initializer(initializer); if (localhost != null) { builder.localHost(localhost); } else { builder.host("batch-datastore.googleapis.com"); } return DatastoreFactory.get().create(builder.build()); }
/** * Returns a CloudResourceManager client builder using the specified {@link * CloudResourceManagerOptions}. */ @VisibleForTesting static CloudResourceManager.Builder newCloudResourceManagerClient( CloudResourceManagerOptions options) { Credentials credentials = options.getGcpCredential(); if (credentials == null) { NullCredentialInitializer.throwNullCredentialException(); } return new CloudResourceManager.Builder( Transport.getTransport(), Transport.getJsonFactory(), chainHttpRequestInitializer( credentials, // Do not log 404. It clutters the output and is possibly even required by the caller. new RetryHttpRequestInitializer(ImmutableList.of(404)))) .setApplicationName(options.getAppName()) .setGoogleClientRequestInitializer(options.getGoogleApiTrace()); }
public PubSubClient(PubSubDatastoreProperties datastore, boolean runOnDataflow) { Credentials credentials = null; if (runOnDataflow || datastore.serviceAccountFile.getValue() == null) { try { credentials = GoogleCredentials.getApplicationDefault().createScoped(PubsubScopes.all()); } catch (IOException e) { throw TalendRuntimeException.createUnexpectedException(e); } } else { credentials = createCredentials(datastore); } this.PROJECT_NAME = datastore.projectName.getValue(); this.client = new Pubsub.Builder(Transport.getTransport(), Transport.getJsonFactory(), chainHttpRequestInitializer(credentials, // Do not log 404. It clutters the output and is possibly even required by the caller. new RetryHttpRequestInitializer(ImmutableList.of(404)))).build(); }
/** Returns a Pubsub client builder using the specified {@link PubsubOptions}. */ private static Pubsub.Builder newPubsubClient(PubsubOptions options) { return new Pubsub.Builder( Transport.getTransport(), Transport.getJsonFactory(), chainHttpRequestInitializer( options.getGcpCredential(), // Do not log 404. It clutters the output and is possibly even required by the caller. new RetryHttpRequestInitializer(ImmutableList.of(404)))) .setRootUrl(options.getPubsubRootUrl()) .setApplicationName(options.getAppName()) .setGoogleClientRequestInitializer(options.getGoogleApiTrace()); }
/** Returns a Cloud Storage client builder using the specified {@link GcsOptions}. */ public static Storage.Builder newStorageClient(GcsOptions options) { String servicePath = options.getGcsEndpoint(); Storage.Builder storageBuilder = new Storage.Builder( getTransport(), getJsonFactory(), chainHttpRequestInitializer( options.getGcpCredential(), // Do not log the code 404. Code up the stack will deal with 404's if needed, and // logging it by default clutters the output during file staging. new RetryHttpRequestInitializer( ImmutableList.of(404), new UploadIdResponseInterceptor()))) .setApplicationName(options.getAppName()) .setGoogleClientRequestInitializer(options.getGoogleApiTrace()); if (servicePath != null) { ApiComponents components = apiComponentsFromUrl(servicePath); storageBuilder.setRootUrl(components.rootUrl); storageBuilder.setServicePath(components.servicePath); storageBuilder.setBatchPath(Paths.get("batch/", components.servicePath).toString()); } return storageBuilder; }
/** Returns a Google Cloud Dataflow client builder. */ public static Dataflow.Builder newDataflowClient(DataflowPipelineOptions options) { String servicePath = options.getDataflowEndpoint(); ApiComponents components; if (servicePath.contains("://")) { components = apiComponentsFromUrl(servicePath); } else { components = new ApiComponents(options.getApiRootUrl(), servicePath); } return new Dataflow.Builder( getTransport(), getJsonFactory(), chainHttpRequestInitializer( options.getGcpCredential(), // Do not log 404. It clutters the output and is possibly even required by the caller. new RetryHttpRequestInitializer(ImmutableList.of(404)))) .setApplicationName(options.getAppName()) .setRootUrl(components.rootUrl) .setServicePath(components.servicePath) .setGoogleClientRequestInitializer(options.getGoogleApiTrace()); }
@Before public void setUp() { MockitoAnnotations.initMocks(this); // Set up the MockHttpRequest for future inspection request = new MockLowLevelHttpRequest() { @Override public LowLevelHttpResponse execute() throws IOException { return response; } }; // A mock transport that lets us mock the API responses. MockHttpTransport transport = new MockHttpTransport.Builder().setLowLevelHttpRequest(request).build(); // A sample BigQuery API client that uses default JsonFactory and RetryHttpInitializer. bigquery = new Bigquery.Builder( transport, Transport.getJsonFactory(), new RetryHttpRequestInitializer()) .build(); }
/** Build a new datastore client. */ static Datastore getDatastore(PipelineOptions pipelineOptions, String projectId) { Credentials credential = pipelineOptions.as(GcpOptions.class).getGcpCredential(); HttpRequestInitializer initializer; if (credential != null) { initializer = new ChainingHttpRequestInitializer( new HttpCredentialsAdapter(credential), new RetryHttpRequestInitializer()); } else { initializer = new RetryHttpRequestInitializer(); } DatastoreOptions.Builder builder = new DatastoreOptions.Builder().projectId(projectId).initializer(initializer); return DatastoreFactory.get().create(builder.build()); }
@Before public void setUp() { MockitoAnnotations.initMocks(this); HttpTransport lowLevelTransport = new HttpTransport() { @Override protected LowLevelHttpRequest buildRequest(String method, String url) throws IOException { return mockLowLevelRequest; } }; // Retry initializer will pass through to credential, since we can have // only a single HttpRequestInitializer, and we use multiple Credential // types in the SDK, not all of which allow for retry configuration. RetryHttpRequestInitializer initializer = new RetryHttpRequestInitializer( new MockNanoClock(), millis -> {}, Arrays.asList(418 /* I'm a teapot */), mockHttpResponseInterceptor); storage = new Storage.Builder(lowLevelTransport, jsonFactory, initializer) .setApplicationName("test") .build(); }
new Storage(mockTransport, Transport.getJsonFactory(), new RetryHttpRequestInitializer())); gcsUtil.remove(Arrays.asList("gs://some-bucket/already-deleted"));
transport, Transport.getJsonFactory(), new RetryHttpRequestInitializer( fakeClockAndSleeper, fakeClockAndSleeper, Collections.emptyList(), null)) .build();
new Storage(mockTransport, Transport.getJsonFactory(), new RetryHttpRequestInitializer())); gcsUtil.fileSizes(ImmutableList.of(GcsPath.fromComponents("testbucket", "testobject")));