@Override public String create(PipelineOptions options) { return options.as(GcpOptions.class).getProject(); } }
@Override public String create(PipelineOptions options) { return "projects/" + options.as(GcpOptions.class).getProject() + "/topics/" + options.getJobName(); } }
@Override public String create(PipelineOptions options) { return "projects/" + options.as(GcpOptions.class).getProject() + "/subscriptions/" + options.getJobName(); } }
projectPath = project.get(); } else { String projectId = options.as(GcpOptions.class).getProject(); checkState( projectId != null, "Cannot create subscription to topic %s because pipeline option 'project' not specified", topicPath); projectPath = PubsubClient.projectPathFromId(options.as(GcpOptions.class).getProject());
@Before public void setup() { PipelineOptionsFactory.register(V1TestOptions.class); options = TestPipeline.testingPipelineOptions().as(V1TestOptions.class); project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); ancestor = UUID.randomUUID().toString(); }
@Before public void setup() throws Exception { PipelineOptionsFactory.register(V1TestOptions.class); options = TestPipeline.testingPipelineOptions().as(V1TestOptions.class); project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); ancestor = UUID.randomUUID().toString(); // Create entities and write them to datastore writeEntitiesToDatastore(options, project, ancestor, numEntities); }
@Before public void setupBqEnvironment() { Long timeSeed = System.currentTimeMillis(); Integer random = new Random(timeSeed).nextInt(900) + 100; this.bigQueryDatasetId = "bq_query_to_table_" + timeSeed.toString() + "_" + random.toString(); PipelineOptionsFactory.register(BigQueryToTableOptions.class); options = TestPipeline.testingPipelineOptions().as(BigQueryToTableOptions.class); options.setTempLocation(options.getTempRoot() + "/bq_it_temp"); project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); bqOption = options.as(BigQueryOptions.class); bqClient = new BigqueryClient(bqOption.getAppName()); bqClient.createNewDataset(project, this.bigQueryDatasetId); outputTable = project + ":" + this.bigQueryDatasetId + "." + BigQueryToTableIT.OUTPUT_TABLE_NAME; }
private void setupTestEnvironment(String recordSize, boolean enableCustomBigquery) { PipelineOptionsFactory.register(BigQueryIOReadOptions.class); options = TestPipeline.testingPipelineOptions().as(BigQueryIOReadOptions.class); options.setNumRecords(numOfRecords.get(recordSize)); options.setTempLocation(options.getTempRoot() + "/temp-it/"); project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); options.setInputTable(project + ":" + datasetId + "." + tablePrefix + recordSize); if (enableCustomBigquery) { options.setExperiments( ImmutableList.of("enable_custom_bigquery_sink", "enable_custom_bigquery_source")); } }
@Before public void setupTestEnvironment() { PipelineOptionsFactory.register(TrafficRoutesOptions.class); this.options = TestPipeline.testingPipelineOptions().as(TrafficRoutesOptions.class); this.projectId = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); this.bqClient = new BigqueryClient("TrafficRoutesIT"); this.bqClient.createNewDataset(this.projectId, this.outputDatasetId); }
@Before public void setupTestEnvironment() { PipelineOptionsFactory.register(TrafficMaxLaneFlowOptions.class); this.options = TestPipeline.testingPipelineOptions().as(TrafficMaxLaneFlowOptions.class); this.projectId = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); this.bqClient = new BigqueryClient("TrafficMaxLaneFlowIT"); this.bqClient.createNewDataset(this.projectId, this.outputDatasetId); }
pipeline.getOptions().as(GcpOptions.class).getProject(), Duration.standardMinutes(1)); eventsTopic.publish(messages); assertThat(queryResult.get(2, TimeUnit.MINUTES).size(), equalTo(3));
@Before public void setUp() throws Exception { PipelineOptionsFactory.register(SpannerTestPipelineOptions.class); options = TestPipeline.testingPipelineOptions().as(SpannerTestPipelineOptions.class); project = options.as(GcpOptions.class).getProject(); spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService(); databaseName = generateDatabaseName(); databaseAdminClient = spanner.getDatabaseAdminClient(); // Delete database if exists. databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName); Operation<Database, CreateDatabaseMetadata> op = databaseAdminClient.createDatabase( options.getInstanceId(), databaseName, Collections.singleton( "CREATE TABLE " + options.getTable() + " (" + " Key INT64," + " Value STRING(MAX) NOT NULL," + ") PRIMARY KEY (Key)")); op.waitFor(); }
"WriteTeamScoreSums", new WriteWindowedToBigQuery<>( options.as(GcpOptions.class).getProject(), options.getDataset(), options.getLeaderBoardTableName() + "_team", "WriteUserScoreSums", new WriteToBigQuery<>( options.as(GcpOptions.class).getProject(), options.getDataset(), options.getLeaderBoardTableName() + "_user",
@Test public void testE2EBigtableRead() throws Exception { PipelineOptionsFactory.register(BigtableTestOptions.class); BigtableTestOptions options = TestPipeline.testingPipelineOptions().as(BigtableTestOptions.class); String project = options.getBigtableProject(); if (project.equals("")) { project = options.as(GcpOptions.class).getProject(); } BigtableOptions.Builder bigtableOptionsBuilder = new BigtableOptions.Builder().setProjectId(project).setInstanceId(options.getInstanceId()); final String tableId = "BigtableReadTest"; final long numRows = 1000L; Pipeline p = Pipeline.create(options); PCollection<Long> count = p.apply(BigtableIO.read().withBigtableOptions(bigtableOptionsBuilder).withTableId(tableId)) .apply(Count.globally()); PAssert.thatSingleton(count).isEqualTo(numRows); p.run(); } }
@Before public void setUp() throws Exception { PipelineOptionsFactory.register(SpannerTestPipelineOptions.class); options = TestPipeline.testingPipelineOptions().as(SpannerTestPipelineOptions.class); project = options.as(GcpOptions.class).getProject(); spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService(); databaseName = generateDatabaseName(); databaseAdminClient = spanner.getDatabaseAdminClient(); // Delete database if exists. databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName); Operation<Database, CreateDatabaseMetadata> op = databaseAdminClient.createDatabase( options.getInstanceId(), databaseName, Collections.singleton( "CREATE TABLE " + options.getTable() + " (" + " Key INT64," + " Value STRING(MAX)," + ") PRIMARY KEY (Key)")); op.waitFor(); makeTestData(); }
@Before public void setup() throws Exception { PipelineOptionsFactory.register(BigtableTestOptions.class); options = TestPipeline.testingPipelineOptions().as(BigtableTestOptions.class); project = options.as(GcpOptions.class).getProject(); bigtableOptions = new Builder() .setProjectId(project) .setInstanceId(options.getInstanceId()) .setUserAgent("apache-beam-test") .build(); session = new BigtableSession( bigtableOptions .toBuilder() .setCredentialOptions( CredentialOptions.credential(options.as(GcpOptions.class).getGcpCredential())) .build()); tableAdminClient = session.getTableAdminClient(); }
"WriteTeamLeaders", new WriteWindowedToBigQuery<>( options.as(GcpOptions.class).getProject(), options.getDataset(), options.getLeaderBoardTableName() + "_team_leader",
"WriteTeamSums", new WriteWindowedToBigQuery<>( options.as(GcpOptions.class).getProject(), options.getDataset(), options.getGameStatsTablePrefix() + "_team", "WriteAvgSessionLength", new WriteWindowedToBigQuery<>( options.as(GcpOptions.class).getProject(), options.getDataset(), options.getGameStatsTablePrefix() + "_sessions",