TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (getSubFields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(getSubFields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
/** @see TableFieldSchema#setMode(String) */ public BigQueryTableFieldSchema setMode(String mode) { fieldSchema.setMode(mode); return this; }
private TableFieldSchema tryArrayFieldSchema(org.apache.avro.Schema.Field field) { String fieldName = field.name(); TableFieldSchema tableFieldSchema = new TableFieldSchema().setName(fieldName); boolean nullable = AvroUtils.isNullable(field.schema()); if (!nullable) { tableFieldSchema = tableFieldSchema.setMode(REQUIRED_MODE); } org.apache.avro.Schema fieldSchema = AvroUtils.unwrapIfNullable(field.schema()); if (fieldSchema.getType() == org.apache.avro.Schema.Type.ARRAY) { return tryFieldSchema(tableFieldSchema.setMode(REPEATED_MODE), fieldSchema.getElementType()); } return tryFieldSchema(tableFieldSchema, fieldSchema); }
private TableFieldSchema tryArrayFieldSchema(org.apache.avro.Schema.Field field) { String fieldName = field.name(); TableFieldSchema tableFieldSchema = new TableFieldSchema().setName(fieldName); boolean nullable = AvroUtils.isNullable(field.schema()); if (!nullable) { tableFieldSchema = tableFieldSchema.setMode(REQUIRED_MODE); } org.apache.avro.Schema fieldSchema = AvroUtils.unwrapIfNullable(field.schema()); if (fieldSchema.getType() == org.apache.avro.Schema.Type.ARRAY) { return tryFieldSchema(tableFieldSchema.setMode(REPEATED_MODE), fieldSchema.getElementType()); } return tryFieldSchema(tableFieldSchema, fieldSchema); }
fieldDef.setMode(jsonObject.get("mode").getAsString());
@Override public TableSchema apply(String jsonPath) { TableSchema tableSchema = new TableSchema(); List<TableFieldSchema> fields = new ArrayList<>(); SchemaParser schemaParser = new SchemaParser(); JSONObject jsonSchema; try { jsonSchema = schemaParser.parseSchema(jsonPath); JSONArray bqSchemaJsonArray = jsonSchema.getJSONArray(BIGQUERY_SCHEMA); for (int i = 0; i < bqSchemaJsonArray.length(); i++) { JSONObject inputField = bqSchemaJsonArray.getJSONObject(i); TableFieldSchema field = new TableFieldSchema() .setName(inputField.getString(NAME)) .setType(inputField.getString(TYPE)); if (inputField.has(MODE)) { field.setMode(inputField.getString(MODE)); } fields.add(field); } tableSchema.setFields(fields); } catch (Exception e) { throw new RuntimeException(e); } return tableSchema; } }))
private List<TableFieldSchema> langStringFields(PropertyConstraint p) { List<TableFieldSchema> list = new ArrayList<>(); TableFieldSchema stringValue = new TableFieldSchema(); list.add(stringValue); stringValue.setName("stringValue"); stringValue.setMode(FieldMode.REQUIRED.name()); stringValue.setType(BigQueryDatatype.STRING.name()); TableFieldSchema languageCode = new TableFieldSchema(); list.add(languageCode); languageCode.setName("languageCode"); languageCode.setMode(FieldMode.REQUIRED.name()); languageCode.setType(BigQueryDatatype.STRING.name()); return list; }
TableFieldSchema elementSchema = fieldSchema.clone().setMode("REQUIRED"); @SuppressWarnings("unchecked") List<Map<String, Object>> rawCells = (List<Map<String, Object>>) v;
/** Defines the BigQuery schema used for the output. */ static TableSchema getSchema() { List<TableFieldSchema> tagFields = new ArrayList<>(); tagFields.add(new TableFieldSchema().setName("count").setType("INTEGER")); tagFields.add(new TableFieldSchema().setName("tag").setType("STRING")); List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("prefix").setType("STRING")); fields.add( new TableFieldSchema() .setName("tags") .setType("RECORD") .setMode("REPEATED") .setFields(tagFields)); return new TableSchema().setFields(fields); } }
@Override public TableSchema getSchema(Long destination) { return new TableSchema() .setFields( ImmutableList.of( new TableFieldSchema() .setName("year") .setType("INTEGER") .setMode("REQUIRED"), new TableFieldSchema() .setName("month") .setType("INTEGER") .setMode("REQUIRED"), new TableFieldSchema() .setName("day") .setType("INTEGER") .setMode("REQUIRED"), new TableFieldSchema() .setName("maxTemp") .setType("FLOAT") .setMode("NULLABLE"))); } })
tfs.setName(fieldName); tfs.setType(fieldType); tfs.setMode(fieldMode);
TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (getSubFields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(getSubFields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
private List<TableFieldSchema> listFields(Shape shape, Traversal traversal) { List<TableFieldSchema> list = new ArrayList<>(); List<PropertyConstraint> plist = shape.getProperty(); if (shape.getNodeKind() == NodeKind.IRI) { TableFieldSchema idField = new TableFieldSchema(); idField.setName("id"); idField.setType(BigQueryDatatype.STRING.name()); idField.setMode(FieldMode.REQUIRED.name()); list.add(idField); } else if (shape.getNodeKind() == NodeKind.BlankNodeOrIRI) { TableFieldSchema idField = new TableFieldSchema(); idField.setName("id"); idField.setType(BigQueryDatatype.STRING.name()); idField.setMode(FieldMode.NULLABLE.name()); list.add(idField); } for (PropertyConstraint p : plist) { if(!SqlTableGeneratorUtil.isValidRdbmsShape(shape) && SqlKeyType.SYNTHETIC_KEY.equals(SqlTableGeneratorUtil.getKeyType(p))){ logger.error("konig:synthicKey is applicable only for shapes with datasource GoogleCloudSqlTable or AwsAurora"); p.setStereotype(null); } TableFieldSchema field = toField(p, traversal); list.add(field); } return list; }
List<TableFieldSchema> fields = new ArrayList<>(); TableFieldSchema column = new TableFieldSchema().setName("rawFields"); column.setType("RECORD"); List<TableFieldSchema> list = new ArrayList<>(); list.add(new TableFieldSchema().setName("key").setType("STRING")); list.add(new TableFieldSchema().setName("value").setType("STRING")); column.setFields(list); column.setMode("REPEATED"); fields.add(column); TableSchema schema = new TableSchema().setFields(fields); TableRow row = new TableRow(); List<TableRow> rawFields = new ArrayList<>(); rawFields.add(new TableRow().set("key","foo").set("value", "bar")); row.set("rawFields", rawFields); Pipeline p = Pipeline.create(options); PCollection<TableRow> c = p.apply(Create.of(row, row).withCoder(TableRowJsonCoder.of())); c.apply(BigQueryIO.Write.named("BigQuery-Write") .to(options.getOutput()) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE) .withSchema(schema)); p.run();
TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.value().name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (fields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(fields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.value().name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (fields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(fields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
private static List<TableFieldSchema> toTableFieldSchema(Schema schema) { List<TableFieldSchema> fields = new ArrayList<>(schema.getFieldCount()); for (Field schemaField : schema.getFields()) { FieldType type = schemaField.getType(); TableFieldSchema field = new TableFieldSchema().setName(schemaField.getName()); if (schemaField.getDescription() != null && !"".equals(schemaField.getDescription())) { field.setDescription(schemaField.getDescription()); } if (!schemaField.getNullable()) { field.setMode(Mode.REQUIRED.toString()); } if (TypeName.ARRAY == type.getTypeName()) { type = type.getCollectionElementType(); field.setMode(Mode.REPEATED.toString()); } if (TypeName.ROW == type.getTypeName()) { Schema subType = type.getRowSchema(); field.setFields(toTableFieldSchema(subType)); } field.setType(toStandardSQLTypeName(type).toString()); fields.add(field); } return fields; }
static TableFieldSchema getAlignmentFieldSchema(String fieldName) { TableFieldSchema fieldSchema = new TableFieldSchema().setName(fieldName).setType(RECORD); fieldSchema.setFields(new ArrayList<TableFieldSchema>() { private static final long serialVersionUID = 0; { // Position record. add(getPositionFieldSchema(POSITION)); add(new TableFieldSchema().setName(MAPPING_QUALITY).setType(INT64)); // CIGAR record. TableFieldSchema cigarSchema = new TableFieldSchema().setName(CIGAR).setType(RECORD).setMode(REPEATED); cigarSchema.setFields(new ArrayList<TableFieldSchema>() { private static final long serialVersionUID = 0; { // Operation is of type enum, but BQ doesn't accept enum. Hence, converting to INT64. add(new TableFieldSchema().setName(OPERATION).setType(INT64)); add(new TableFieldSchema().setName(OPERATION_LENGTH).setType(INT64)); add(new TableFieldSchema().setName(REFERENCE_SEQUENCE).setType(STRING)); } }); add(cigarSchema); } }); return fieldSchema; }
result.setMode(fieldMode.name());
static TableSchema getSchema() { return new TableSchema().setFields(new ArrayList<TableFieldSchema>() { private static final long serialVersionUID = 0; // Compose the list of TableFieldSchema from tableSchema. { add(new TableFieldSchema().setName(ID).setType(STRING)); add(new TableFieldSchema().setName(READ_GROUP_ID).setType(STRING)); add(new TableFieldSchema().setName(READ_GROUP_SET_ID).setType(STRING)); add(new TableFieldSchema().setName(FRAGMENT_NAME).setType(STRING)); add(new TableFieldSchema().setName(PROPER_PLACEMENT).setType(BOOL)); add(new TableFieldSchema().setName(DUPLICATE_FRAGMENT).setType(BOOL)); add(new TableFieldSchema().setName(FRAGMENT_LENGTH).setType(INT64)); add(new TableFieldSchema().setName(READ_NUMBER).setType(INT64)); add(new TableFieldSchema().setName(NUMBER_READS).setType(INT64)); add(new TableFieldSchema().setName(FAILED_VENDOR_QUALITY_CHECKS).setType(BOOL)); add(getAlignmentFieldSchema(ALIGNMENT)); add(new TableFieldSchema().setName(SECONDARY_ALIGNMENT).setType(BOOL)); add(new TableFieldSchema().setName(SUPPLEMENTARY_ALIGNMENT).setType(BOOL)); add(new TableFieldSchema().setName(ALIGNED_SEQUENCE).setType(STRING)); add(new TableFieldSchema().setName(ALIGNED_QUALITY).setType(INT64).setMode(REPEATED)); add(getPositionFieldSchema(NEXT_MATE_POSITION)); // TODO(#222): store info field into bigquery. } }); }