@SuppressWarnings("unchecked") static ExternalTableDefinition fromPb(Table tablePb) { Builder builder = newBuilder().table(tablePb); com.google.api.services.bigquery.model.ExternalDataConfiguration externalDataConfiguration = tablePb.getExternalDataConfiguration(); if (externalDataConfiguration != null) { if (externalDataConfiguration.getSourceUris() != null) { builder.setSourceUris(ImmutableList.copyOf(externalDataConfiguration.getSourceUris())); } if (externalDataConfiguration.getSourceFormat() != null) { builder.setFormatOptions(FormatOptions.of(externalDataConfiguration.getSourceFormat())); } builder.setCompression(externalDataConfiguration.getCompression()); builder.setIgnoreUnknownValues(externalDataConfiguration.getIgnoreUnknownValues()); if (externalDataConfiguration.getCsvOptions() != null) { builder.setFormatOptions(CsvOptions.fromPb(externalDataConfiguration.getCsvOptions())); } if (externalDataConfiguration.getGoogleSheetsOptions() != null) { builder.setFormatOptions( GoogleSheetsOptions.fromPb(externalDataConfiguration.getGoogleSheetsOptions())); } if (externalDataConfiguration.getBigtableOptions() != null) { builder.setFormatOptions( BigtableOptions.fromPb(externalDataConfiguration.getBigtableOptions())); } builder.setMaxBadRecords(externalDataConfiguration.getMaxBadRecords()); builder.setAutodetect(externalDataConfiguration.getAutodetect()); } return builder.build(); }
@VisibleForTesting static String getCommaSeparatedGcsPathList(final Table table) { Preconditions.checkNotNull(table.getExternalDataConfiguration()); for (String uri : table.getExternalDataConfiguration().getSourceUris()) { Preconditions.checkArgument(uri.startsWith("gs://"), "Invalid GCS resource: '%s'", uri); } // FileInputFormat accepts a comma separated list of potentially globbed paths. return Joiner.on(",").join(table.getExternalDataConfiguration().getSourceUris()); }
public NoopFederatedExportToCloudStorage( Configuration configuration, ExportFileFormat fileFormat, BigQueryHelper bigQueryHelper, String projectId, Table table, @Nullable InputFormat<LongWritable, Text> delegateInputFormat) { super( configuration, getCommaSeparatedGcsPathList(table), fileFormat, bigQueryHelper, projectId, table, delegateInputFormat); Preconditions.checkNotNull(table.getExternalDataConfiguration()); String inputType = fileFormat.getFormatIdentifier(); String tableType = table.getExternalDataConfiguration().getSourceFormat(); Preconditions.checkArgument( inputType.equals(tableType), "MapReduce fileFormat '%s' does not match BigQuery sourceFormat '%s'. Use the " + "appropriate InputFormat.", inputType, tableType); gcsPaths = table.getExternalDataConfiguration().getSourceUris(); }
private static TableDefinition tableDefinition(Table model) { ExternalDataConfiguration external = model.getExternalDataConfiguration(); Schema schema = schema(model.getSchema()); if (external != null) { return ExternalTableDefinition.of(external.getSourceUris(), schema, formatOptions(external)); } return StandardTableDefinition.of(schema); }
private Builder(Table tablePb) { super(tablePb); com.google.api.services.bigquery.model.ExternalDataConfiguration externalDataConfiguration = tablePb.getExternalDataConfiguration(); if (externalDataConfiguration != null) { if (externalDataConfiguration.getSourceUris() != null) { this.sourceUris = ImmutableList.copyOf(externalDataConfiguration.getSourceUris()); } if (externalDataConfiguration.getSourceFormat() != null) { this.formatOptions = FormatOptions.of(externalDataConfiguration.getSourceFormat()); } this.compression = externalDataConfiguration.getCompression(); this.ignoreUnknownValues = externalDataConfiguration.getIgnoreUnknownValues(); if (externalDataConfiguration.getCsvOptions() != null) { this.formatOptions = CsvOptions.fromPb(externalDataConfiguration.getCsvOptions()); } this.maxBadRecords = externalDataConfiguration.getMaxBadRecords(); } }
private Builder(Table tablePb) { super(tablePb); com.google.api.services.bigquery.model.ExternalDataConfiguration externalDataConfiguration = tablePb.getExternalDataConfiguration(); if (externalDataConfiguration != null) { if (externalDataConfiguration.getSourceUris() != null) { this.sourceUris = ImmutableList.copyOf(externalDataConfiguration.getSourceUris()); } if (externalDataConfiguration.getSourceFormat() != null) { this.formatOptions = FormatOptions.of(externalDataConfiguration.getSourceFormat()); } this.compression = externalDataConfiguration.getCompression(); this.ignoreUnknownValues = externalDataConfiguration.getIgnoreUnknownValues(); if (externalDataConfiguration.getCsvOptions() != null) { this.formatOptions = CsvOptions.fromPb(externalDataConfiguration.getCsvOptions()); } this.maxBadRecords = externalDataConfiguration.getMaxBadRecords(); } }
table.setView(currentStateViewGenerator.createViewDefinition(shape, dataSource)); if (table.getExternalDataConfiguration() != null) { table.setType("EXTERNAL"); tableVisitor.visit(dataSource, table);
@SuppressWarnings("unchecked") static ExternalTableDefinition fromPb(Table tablePb) { Builder builder = newBuilder().table(tablePb); com.google.api.services.bigquery.model.ExternalDataConfiguration externalDataConfiguration = tablePb.getExternalDataConfiguration(); if (externalDataConfiguration != null) { if (externalDataConfiguration.getSourceUris() != null) { builder.setSourceUris(ImmutableList.copyOf(externalDataConfiguration.getSourceUris())); } if (externalDataConfiguration.getSourceFormat() != null) { builder.setFormatOptions(FormatOptions.of(externalDataConfiguration.getSourceFormat())); } builder.setCompression(externalDataConfiguration.getCompression()); builder.setIgnoreUnknownValues(externalDataConfiguration.getIgnoreUnknownValues()); if (externalDataConfiguration.getCsvOptions() != null) { builder.setFormatOptions(CsvOptions.fromPb(externalDataConfiguration.getCsvOptions())); } if (externalDataConfiguration.getGoogleSheetsOptions() != null) { builder.setFormatOptions( GoogleSheetsOptions.fromPb(externalDataConfiguration.getGoogleSheetsOptions())); } if (externalDataConfiguration.getBigtableOptions() != null) { builder.setFormatOptions( BigtableOptions.fromPb(externalDataConfiguration.getBigtableOptions())); } builder.setMaxBadRecords(externalDataConfiguration.getMaxBadRecords()); builder.setAutodetect(externalDataConfiguration.getAutodetect()); } return builder.build(); }