@SuppressWarnings("unchecked") static ModelTableDefinition fromPb(Table tablePb) { Builder builder = newBuilder().table(tablePb); return builder.setNumBytes(tablePb.getNumBytes()).setLocation(tablePb.getLocation()).build(); } }
BuilderImpl(Table tablePb) { this.tableId = TableId.fromPb(tablePb.getTableReference()); if (tablePb.getLastModifiedTime() != null) { this.setLastModifiedTime(tablePb.getLastModifiedTime().longValue()); } this.description = tablePb.getDescription(); this.expirationTime = tablePb.getExpirationTime(); this.friendlyName = tablePb.getFriendlyName(); this.creationTime = tablePb.getCreationTime(); this.etag = tablePb.getEtag(); this.generatedId = tablePb.getId(); this.selfLink = tablePb.getSelfLink(); this.numBytes = tablePb.getNumBytes(); this.numRows = tablePb.getNumRows(); this.definition = TableDefinition.fromPb(tablePb); if (tablePb.getEncryptionConfiguration() != null) { this.encryptionConfiguration = new EncryptionConfiguration.Builder(tablePb.getEncryptionConfiguration()).build(); } this.labels = Labels.fromPb(tablePb.getLabels()); }
@SuppressWarnings("unchecked") static StandardTableDefinition fromPb(Table tablePb) { Builder builder = newBuilder().table(tablePb); if (tablePb.getNumRows() != null) { builder.setNumRows(tablePb.getNumRows().longValue()); } if (tablePb.getStreamingBuffer() != null) { builder.setStreamingBuffer(StreamingBuffer.fromPb(tablePb.getStreamingBuffer())); } if (tablePb.getTimePartitioning() != null) { builder.setTimePartitioning(TimePartitioning.fromPb(tablePb.getTimePartitioning())); } if (tablePb.getClustering() != null) { builder.setClustering(Clustering.fromPb(tablePb.getClustering())); } return builder.setNumBytes(tablePb.getNumBytes()).setLocation(tablePb.getLocation()).build(); } }
@Override public boolean isTableEmpty(TableReference tableRef) throws IOException, InterruptedException { Long numBytes = getTable(tableRef).getNumBytes(); return numBytes == null || numBytes == 0L; }
@Override public List<String> getExportPaths() throws IOException { List<String> paths = new ArrayList<>(); long numTableRows = tableToExport.getNumRows().longValue(); long numTableBytes = tableToExport.getNumBytes(); int numShards = computeNumShards(numTableBytes); logger.atInfo().log("Computed '%s' shards for sharded BigQuery export.", numShards); for (int i = 0; i < numShards; ++i) { String exportPattern = String.format( "%s/shard-%d/%s", gcsPath, i, fileFormat.getFilePattern()); paths.add(exportPattern); } logger.atInfo().log( "Table '%s' to be exported has %s rows and %s bytes", BigQueryStrings.toString(tableToExport.getTableReference()), numTableRows, numTableBytes); return paths; }
long addRow(TableRow row, String id) { rows.add(row); ids.add(id); long rowSize = row.toString().length(); Long tableSize = table.getNumBytes(); if (tableSize == null) { table.setNumBytes(rowSize); } else { table.setNumBytes(tableSize + rowSize); } return rowSize; }
@Override public synchronized long getEstimatedSizeBytes(PipelineOptions options) throws Exception { if (tableSizeBytes.get() == null) { TableReference table = setDefaultProjectIfAbsent( options.as(BigQueryOptions.class), BigQueryIO.JSON_FACTORY.fromString(jsonTable.get(), TableReference.class)); Table tableRef = bqServices.getDatasetService(options.as(BigQueryOptions.class)).getTable(table); Long numBytes = tableRef.getNumBytes(); if (tableRef.getStreamingBuffer() != null) { numBytes += tableRef.getStreamingBuffer().getEstimatedBytes().longValue(); } tableSizeBytes.compareAndSet(null, numBytes); } return tableSizeBytes.get(); }
private Builder(Table tablePb) { super(tablePb); if (tablePb.getNumRows() != null) { this.numRows(tablePb.getNumRows().longValue()); } this.numBytes = tablePb.getNumBytes(); this.location = tablePb.getLocation(); if (tablePb.getStreamingBuffer() != null) { this.streamingBuffer = StreamingBuffer.fromPb(tablePb.getStreamingBuffer()); } }
@SuppressWarnings("unchecked") static ModelTableDefinition fromPb(Table tablePb) { Builder builder = newBuilder().table(tablePb); return builder.setNumBytes(tablePb.getNumBytes()).setLocation(tablePb.getLocation()).build(); } }
private Builder(Table tablePb) { super(tablePb); if (tablePb.getNumRows() != null) { this.numRows(tablePb.getNumRows().longValue()); } this.numBytes = tablePb.getNumBytes(); this.location = tablePb.getLocation(); if (tablePb.getStreamingBuffer() != null) { this.streamingBuffer = StreamingBuffer.fromPb(tablePb.getStreamingBuffer()); } if (tablePb.getTimePartitioning() != null) { this.timePartitioning = TimePartitioning.fromPb(tablePb.getTimePartitioning()); } }
BuilderImpl(Table tablePb) { this.tableId = TableId.fromPb(tablePb.getTableReference()); if (tablePb.getLastModifiedTime() != null) { this.setLastModifiedTime(tablePb.getLastModifiedTime().longValue()); } this.description = tablePb.getDescription(); this.expirationTime = tablePb.getExpirationTime(); this.friendlyName = tablePb.getFriendlyName(); this.creationTime = tablePb.getCreationTime(); this.etag = tablePb.getEtag(); this.generatedId = tablePb.getId(); this.selfLink = tablePb.getSelfLink(); this.numBytes = tablePb.getNumBytes(); this.numRows = tablePb.getNumRows(); this.definition = TableDefinition.fromPb(tablePb); if (tablePb.getEncryptionConfiguration() != null) { this.encryptionConfiguration = new EncryptionConfiguration.Builder(tablePb.getEncryptionConfiguration()).build(); } this.labels = Labels.fromPb(tablePb.getLabels()); }
@SuppressWarnings("unchecked") static StandardTableDefinition fromPb(Table tablePb) { Builder builder = newBuilder().table(tablePb); if (tablePb.getNumRows() != null) { builder.setNumRows(tablePb.getNumRows().longValue()); } if (tablePb.getStreamingBuffer() != null) { builder.setStreamingBuffer(StreamingBuffer.fromPb(tablePb.getStreamingBuffer())); } if (tablePb.getTimePartitioning() != null) { builder.setTimePartitioning(TimePartitioning.fromPb(tablePb.getTimePartitioning())); } if (tablePb.getClustering() != null) { builder.setClustering(Clustering.fromPb(tablePb.getClustering())); } return builder.setNumBytes(tablePb.getNumBytes()).setLocation(tablePb.getLocation()).build(); } }