@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString()); loadConfigurationPb.setWriteDisposition(writeDisposition.toString()); loadConfigurationPb.setNullMarker(nullMarker); .setFieldDelimiter(csvOptions.getFieldDelimiter()) .setAllowJaggedRows(csvOptions.allowJaggedRows()) .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) .setEncoding(csvOptions.getEncoding()) .setQuote(csvOptions.getQuote()); if (csvOptions.getSkipLeadingRows() != null) { loadConfigurationPb.setSkipLeadingRows(Ints.checkedCast(csvOptions.getSkipLeadingRows())); loadConfigurationPb.setSchema(schema.toPb()); loadConfigurationPb.setSourceFormat(formatOptions.getType()); loadConfigurationPb.setMaxBadRecords(maxBadRecords); loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); if (getDatastoreBackupOptions() != null) { DatastoreBackupOptions backOptions = getDatastoreBackupOptions(); loadConfigurationPb.setProjectionFields(backOptions.getProjectionFields());
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { this(); JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad(); this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable()); if (loadConfigurationPb.getCreateDisposition() != null) { this.createDisposition = JobInfo.CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition()); if (loadConfigurationPb.getWriteDisposition() != null) { this.writeDisposition = JobInfo.WriteDisposition.valueOf(loadConfigurationPb.getWriteDisposition()); if (loadConfigurationPb.getSourceFormat() != null) { this.formatOptions = FormatOptions.of(loadConfigurationPb.getSourceFormat()); if (loadConfigurationPb.getNullMarker() != null) { this.nullMarker = loadConfigurationPb.getNullMarker(); if (loadConfigurationPb.getAllowJaggedRows() != null || loadConfigurationPb.getAllowQuotedNewlines() != null || loadConfigurationPb.getEncoding() != null || loadConfigurationPb.getFieldDelimiter() != null || loadConfigurationPb.getQuote() != null || loadConfigurationPb.getSkipLeadingRows() != null) { CsvOptions.Builder builder = CsvOptions.newBuilder() .setEncoding(loadConfigurationPb.getEncoding()) .setFieldDelimiter(loadConfigurationPb.getFieldDelimiter()) .setQuote(loadConfigurationPb.getQuote()); if (loadConfigurationPb.getAllowJaggedRows() != null) {
@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString()); loadConfigurationPb.setWriteDisposition(writeDisposition.toString()); loadConfigurationPb.setFieldDelimiter(csvOptions.fieldDelimiter()) .setAllowJaggedRows(csvOptions.allowJaggedRows()) .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) .setEncoding(csvOptions.encoding()) .setQuote(csvOptions.quote()) .setSkipLeadingRows(csvOptions.skipLeadingRows()); loadConfigurationPb.setSchema(schema.toPb()); loadConfigurationPb.setSourceFormat(formatOptions.type()); loadConfigurationPb.setMaxBadRecords(maxBadRecords); loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); loadConfigurationPb.setProjectionFields(projectionFields); if (sourceUris != null) { loadConfigurationPb.setSourceUris(ImmutableList.copyOf(sourceUris));
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { this(); JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad(); this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable()); if (loadConfigurationPb.getCreateDisposition() != null) { this.createDisposition = JobInfo.CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition()); if (loadConfigurationPb.getWriteDisposition() != null) { this.writeDisposition = JobInfo.WriteDisposition.valueOf(loadConfigurationPb.getWriteDisposition()); if (loadConfigurationPb.getSourceFormat() != null) { this.formatOptions = FormatOptions.of(loadConfigurationPb.getSourceFormat()); if (loadConfigurationPb.getAllowJaggedRows() != null || loadConfigurationPb.getAllowQuotedNewlines() != null || loadConfigurationPb.getEncoding() != null || loadConfigurationPb.getFieldDelimiter() != null || loadConfigurationPb.getQuote() != null || loadConfigurationPb.getSkipLeadingRows() != null) { CsvOptions.Builder builder = CsvOptions.builder() .encoding(loadConfigurationPb.getEncoding()) .fieldDelimiter(loadConfigurationPb.getFieldDelimiter()) .quote(loadConfigurationPb.getQuote()); if (loadConfigurationPb.getAllowJaggedRows() != null) { builder.allowJaggedRows(loadConfigurationPb.getAllowJaggedRows()); if (loadConfigurationPb.getAllowQuotedNewlines() != null) { builder.allowQuotedNewLines(loadConfigurationPb.getAllowQuotedNewlines());
CreateDisposition createDisposition) { JobConfigurationLoad loadConfig = new JobConfigurationLoad() .setDestinationTable(ref) .setSchema(schema) .setSourceUris(gcsUris) .setWriteDisposition(writeDisposition.name()) .setCreateDisposition(createDisposition.name()) .setSourceFormat("NEWLINE_DELIMITED_JSON") .setIgnoreUnknownValues(ignoreUnknownValues); if (timePartitioning != null) { loadConfig.setTimePartitioning(timePartitioning);
/** * Defines names of counters we track for each operation. * * There are two types of counters: -- METHOD_NAME : Number of successful invocations of method * METHOD. -- METHOD_NAME_TIME : Total inclusive time spent in method METHOD. */ public enum Counter { BYTES_WRITTEN, CLOSE_CALLS, CLOSE_TOTAL_TIME, JOBS_INSERTED, WRITE_CALLS, WRITE_TOTAL_TIME, }
JobConfigurationLoad loadConfig = new JobConfigurationLoad(); loadConfig.setSchema(schema); loadConfig.setSourceFormat(sourceFormat.getFormatIdentifier()); loadConfig.setSourceUris(gcsPaths); loadConfig.setDestinationTable(tableRef); loadConfig.setWriteDisposition(writeDisposition); if (!Strings.isNullOrEmpty(kmsKeyName)) { loadConfig.setDestinationEncryptionConfiguration( new EncryptionConfiguration().setKmsKeyName(kmsKeyName)); loadConfig.setAutodetect(true); } else { logger.atInfo().log("Using provided import schema '%s'.", schema.toString());
TableSchema schema = new TableSchema(); schema.setFields(new ArrayList<TableFieldSchema>()); JacksonFactory JACKSON = new JacksonFactory(); JACKSON.createJsonParser(new FileInputStream("schema.json")) .parseArrayAndClose(schema.getFields(), TableFieldSchema.class, null); schema.setFactory(JACKSON); TableReference destTable = new TableReference(); destTable.setProjectId(projectId); destTable.setDatasetId(datasetId); destTable.setTableId(tableId); FileContent content = new FileContent("application/octet-stream", new File(csv)); Job job = new Job(); JobConfiguration config = new JobConfiguration(); JobConfigurationLoad configLoad = new JobConfigurationLoad(); configLoad.setSchema(schema); configLoad.setDestinationTable(destTable); configLoad.setEncoding("UTF-8"); configLoad.setCreateDisposition("CREATE_IF_NEEDED"); config.setLoad(configLoad); job.setConfiguration(config); Insert insert = bigquery.jobs().insert(projectId, job, content); insert.setProjectId(projectId); JobReference jobRef = insert.execute().getJobReference(); String jobId = jobRef.getJobId();
@Override protected JobConfiguration jobConfiguration(String projectId) { JobConfigurationLoad cfg = new JobConfigurationLoad() .setSourceUris(sourceUris(params)); if (params.has("schema")) { cfg.setSchema(tableSchema(params)); } Optional<DatasetReference> defaultDataset = params.getOptional("dataset", String.class) .transform(Bq::datasetReference); String destinationTable = params.get("destination_table", String.class); cfg.setDestinationTable(tableReference(projectId, defaultDataset, destinationTable)); params.getOptional("create_disposition", String.class).transform(cfg::setCreateDisposition); params.getOptional("write_disposition", String.class).transform(cfg::setWriteDisposition); params.getOptional("source_format", String.class).transform(cfg::setSourceFormat); params.getOptional("field_delimiter", String.class).transform(cfg::setFieldDelimiter); params.getOptional("skip_leading_rows", int.class).transform(cfg::setSkipLeadingRows); params.getOptional("encoding", String.class).transform(cfg::setEncoding); params.getOptional("quote", String.class).transform(cfg::setQuote); params.getOptional("max_bad_records", int.class).transform(cfg::setMaxBadRecords); params.getOptional("allow_quoted_newlines", boolean.class).transform(cfg::setAllowQuotedNewlines); params.getOptional("allow_jagged_rows", boolean.class).transform(cfg::setAllowJaggedRows); params.getOptional("ignore_unknown_values", boolean.class).transform(cfg::setIgnoreUnknownValues); params.getOptional("projection_fields", new TypeReference<List<String>>() {}).transform(cfg::setProjectionFields); params.getOptional("autodetect", boolean.class).transform(cfg::setAutodetect); params.getOptional("schema_update_options", new TypeReference<List<String>>() {}).transform(cfg::setSchemaUpdateOptions); return new JobConfiguration() .setLoad(cfg); }
JobConfigurationLoad loadConfig = new JobConfigurationLoad(); loadConfig.setSourceUris(Arrays.asList(gsUrl)); loadConfig.set("sourceFormat", "DATASTORE_BACKUP"); loadConfig.set("allowQuotedNewlines", true); table.setDatasetId(exporterConfig.getBigqueryDatasetId()); table.setTableId(kind + datatableSuffix); loadConfig.setDestinationTable(table);
private JobStatus runLoadJob(JobReference jobRef, JobConfigurationLoad load) throws InterruptedException, IOException { TableReference destination = load.getDestinationTable(); TableSchema schema = load.getSchema(); checkArgument(schema != null, "No schema specified"); List<ResourceId> sourceFiles = filesForLoadJobs.get(jobRef.getProjectId(), jobRef.getJobId()); WriteDisposition writeDisposition = WriteDisposition.valueOf(load.getWriteDisposition()); CreateDisposition createDisposition = CreateDisposition.valueOf(load.getCreateDisposition()); checkArgument("NEWLINE_DELIMITED_JSON".equals(load.getSourceFormat())); Table existingTable = datasetService.getTable(destination); if (!validateDispositions(existingTable, createDisposition, writeDisposition)) { .setTableId(BigQueryHelpers.stripPartitionDecorator(destination.getTableId())); existingTable = new Table().setTableReference(strippedDestination).setSchema(schema); if (load.getTimePartitioning() != null) { existingTable = existingTable.setTimePartitioning(load.getTimePartitioning());
static String jobToPrettyString(@Nullable Job job) throws IOException { if (job != null && job.getConfiguration().getLoad() != null) { // Removing schema and sourceUris from error messages for load jobs since these fields can be // quite long and error message might not be displayed properly in runner specific logs. job = job.clone(); job.getConfiguration().getLoad().setSchema(null); job.getConfiguration().getLoad().setSourceUris(null); } return job == null ? "null" : job.toPrettyString(); }
@Test public void testIncomplete() { // https://github.com/googleapis/google-cloud-java/issues/2357 com.google.api.services.bigquery.model.Job job = new com.google.api.services.bigquery.model.Job() .setStatistics( new com.google.api.services.bigquery.model.JobStatistics() .setCreationTime(1234L) .setStartTime(5678L)); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setCopy(new com.google.api.services.bigquery.model.JobConfigurationTableCopy())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(CopyStatistics.class); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setLoad(new com.google.api.services.bigquery.model.JobConfigurationLoad())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(LoadStatistics.class); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setExtract(new com.google.api.services.bigquery.model.JobConfigurationExtract())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(ExtractStatistics.class); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setQuery(new com.google.api.services.bigquery.model.JobConfigurationQuery())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(QueryStatistics.class); }
@Test public void testToPbAndFromPb() { assertNull(LOAD_CONFIGURATION_CSV.toPb().getLoad().getSourceUris()); compareLoadConfiguration( LOAD_CONFIGURATION_CSV, WriteChannelConfiguration.fromPb(LOAD_CONFIGURATION_CSV.toPb())); WriteChannelConfiguration configuration = WriteChannelConfiguration.of(TABLE_ID); compareLoadConfiguration(configuration, WriteChannelConfiguration.fromPb(configuration.toPb())); }
com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString()); } if (writeDisposition != null) { loadConfigurationPb.setWriteDisposition(writeDisposition.toString()); } if (csvOptions() != null) { CsvOptions csvOptions = csvOptions(); loadConfigurationPb.setFieldDelimiter(csvOptions.fieldDelimiter()) .setAllowJaggedRows(csvOptions.allowJaggedRows()) .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) .setEncoding(csvOptions.encoding()) .setQuote(csvOptions.quote()) .setSkipLeadingRows(csvOptions.skipLeadingRows()); } if (schema != null) { loadConfigurationPb.setSchema(schema.toPb()); } if (formatOptions != null) { loadConfigurationPb.setSourceFormat(formatOptions.type()); } loadConfigurationPb.setMaxBadRecords(maxBadRecords); loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); loadConfigurationPb.setProjectionFields(projectionFields); return new com.google.api.services.bigquery.model.JobConfiguration() .setLoad(loadConfigurationPb); }
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad(); this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable()); if (loadConfigurationPb.getCreateDisposition() != null) { this.createDisposition = CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition()); if (loadConfigurationPb.getWriteDisposition() != null) { this.writeDisposition = WriteDisposition.valueOf(loadConfigurationPb.getWriteDisposition()); if (loadConfigurationPb.getSourceFormat() != null) { this.formatOptions = FormatOptions.of(loadConfigurationPb.getSourceFormat()); if (loadConfigurationPb.getAllowJaggedRows() != null || loadConfigurationPb.getAllowQuotedNewlines() != null || loadConfigurationPb.getEncoding() != null || loadConfigurationPb.getFieldDelimiter() != null || loadConfigurationPb.getQuote() != null || loadConfigurationPb.getSkipLeadingRows() != null) { CsvOptions.Builder builder = CsvOptions.builder() .encoding(loadConfigurationPb.getEncoding()) .fieldDelimiter(loadConfigurationPb.getFieldDelimiter()) .quote(loadConfigurationPb.getQuote()); if (loadConfigurationPb.getAllowJaggedRows() != null) { builder.allowJaggedRows(loadConfigurationPb.getAllowJaggedRows()); if (loadConfigurationPb.getAllowQuotedNewlines() != null) { builder.allowQuotedNewLines(loadConfigurationPb.getAllowQuotedNewlines()); if (loadConfigurationPb.getSkipLeadingRows() != null) {
JobConfigurationLoad loadConfig = new JobConfigurationLoad() .setDestinationTable(tableReference) .setSchema(tableSchema)
@Override public void startLoadJob(JobReference jobRef, JobConfigurationLoad loadConfig) throws IOException { synchronized (allJobs) { verifyUniqueJobId(jobRef.getJobId()); Job job = new Job(); job.setJobReference(jobRef); job.setConfiguration(new JobConfiguration().setLoad(loadConfig)); job.setKind(" bigquery#job"); job.setStatus(new JobStatus().setState("PENDING")); // Copy the files to a new location for import, as the temporary files will be deleted by // the caller. if (loadConfig.getSourceUris().size() > 0) { ImmutableList.Builder<ResourceId> sourceFiles = ImmutableList.builder(); ImmutableList.Builder<ResourceId> loadFiles = ImmutableList.builder(); for (String filename : loadConfig.getSourceUris()) { sourceFiles.add(FileSystems.matchNewResource(filename, false /* isDirectory */)); loadFiles.add( FileSystems.matchNewResource( filename + ThreadLocalRandom.current().nextInt(), false /* isDirectory */)); } FileSystems.copy(sourceFiles.build(), loadFiles.build()); filesForLoadJobs.put(jobRef.getProjectId(), jobRef.getJobId(), loadFiles.build()); } allJobs.put(jobRef.getProjectId(), jobRef.getJobId(), new JobInfo(job)); } }
com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString()); loadConfigurationPb.setWriteDisposition(writeDisposition.toString()); loadConfigurationPb.setNullMarker(nullMarker); .setFieldDelimiter(csvOptions.getFieldDelimiter()) .setAllowJaggedRows(csvOptions.allowJaggedRows()) .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) .setEncoding(csvOptions.getEncoding()) .setQuote(csvOptions.getQuote()); if (csvOptions.getSkipLeadingRows() != null) { loadConfigurationPb.setSkipLeadingRows(Ints.checkedCast(csvOptions.getSkipLeadingRows())); loadConfigurationPb.setSchema(schema.toPb()); loadConfigurationPb.setSourceFormat(formatOptions.getType()); loadConfigurationPb.setMaxBadRecords(maxBadRecords); loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); if (getDatastoreBackupOptions() != null) { DatastoreBackupOptions backupOptions = getDatastoreBackupOptions(); loadConfigurationPb.setProjectionFields(backupOptions.getProjectionFields()); loadConfigurationPb.setSchemaUpdateOptions(schemaUpdateOptionsBuilder.build());
com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString()); loadConfigurationPb.setWriteDisposition(writeDisposition.toString()); loadConfigurationPb.setFieldDelimiter(csvOptions.fieldDelimiter()) .setAllowJaggedRows(csvOptions.allowJaggedRows()) .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) .setEncoding(csvOptions.encoding()) .setQuote(csvOptions.quote()); if (csvOptions.skipLeadingRows() != null) { loadConfigurationPb.setSkipLeadingRows(Ints.checkedCast(csvOptions.skipLeadingRows())); loadConfigurationPb.setSchema(schema.toPb()); loadConfigurationPb.setSourceFormat(formatOptions.type()); loadConfigurationPb.setMaxBadRecords(maxBadRecords); loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); loadConfigurationPb.setProjectionFields(projectionFields); return new com.google.api.services.bigquery.model.JobConfiguration() .setLoad(loadConfigurationPb);