private Schema getOutputSchema(List<FieldAggregation> fieldAggregations) { Schema.Builder outputSchema = Schema.builder(); for (FieldAggregation aggregation : fieldAggregations) { outputSchema.addField(aggregation.outputField); } return outputSchema.build(); }
/** Returns true if two schemas are equal ignoring field names and descriptions. */ public boolean typesEqual(Schema other) { if (uuid != null && other.uuid != null && Objects.equals(uuid, other.uuid)) { return true; } if (getFieldCount() != other.getFieldCount()) { return false; } if (!Objects.equals(fieldIndices.values(), other.fieldIndices.values())) { return false; } for (int i = 0; i < getFieldCount(); ++i) { if (!getField(i).typesEqual(other.getField(i))) { return false; } } return true; }
/** Return the count of fields. */ public int getFieldCount() { return getFields().size(); } }
private boolean equivalent(Schema other, EquivalenceNullablePolicy nullablePolicy) { if (other.getFieldCount() != getFieldCount()) { return false; } List<Field> otherFields = other .getFields() .stream() .sorted(Comparator.comparing(Field::getName)) .collect(Collectors.toList()); List<Field> actualFields = getFields() .stream() .sorted(Comparator.comparing(Field::getName)) .collect(Collectors.toList()); for (int i = 0; i < otherFields.size(); ++i) { Field otherField = otherFields.get(i); Field actualField = actualFields.get(i); if (!otherField.equivalent(actualField, nullablePolicy)) { return false; } } return true; }
@Test public void testMapNotEquivalent() { Schema schema1 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.BOOLEAN).build(); Schema schema2 = Schema.builder().addMapField("foo", FieldType.DATETIME, FieldType.BOOLEAN).build(); assertNotEquals(schema1, schema2); assertFalse(schema1.equivalent(schema2)); schema1 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.BOOLEAN).build(); schema2 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.STRING).build(); assertNotEquals(schema1, schema2); assertFalse(schema1.equivalent(schema2)); }
/** * Tries to parse the JSON {@link TableRow} from BigQuery. * * <p>Only supports basic types and arrays. Doesn't support date types. */ public static Row toBeamRow(Schema rowSchema, TableSchema bqSchema, TableRow jsonBqRow) { List<TableFieldSchema> bqFields = bqSchema.getFields(); Map<String, Integer> bqFieldIndices = IntStream.range(0, bqFields.size()) .boxed() .collect(toMap(i -> bqFields.get(i).getName(), i -> i)); List<Object> rawJsonValues = rowSchema .getFields() .stream() .map(field -> bqFieldIndices.get(field.getName())) .map(index -> jsonBqRow.getF().get(index).getV()) .collect(toList()); return IntStream.range(0, rowSchema.getFieldCount()) .boxed() .map(index -> toBeamValue(rowSchema.getField(index).getType(), rawJsonValues.get(index))) .collect(toRow(rowSchema)); }
@Test public void testCollector() { Schema schema = Stream.of( Schema.Field.of("f_int", FieldType.INT32), Schema.Field.of("f_string", FieldType.STRING)) .collect(toSchema()); assertEquals(2, schema.getFieldCount()); assertEquals("f_int", schema.getField(0).getName()); assertEquals(FieldType.INT32, schema.getField(0).getType()); assertEquals("f_string", schema.getField(1).getName()); assertEquals(FieldType.STRING, schema.getField(1).getType()); }
left.getFields().stream().map(Schema.Field::getName), right.getFields().stream().map(Schema.Field::getName)) .distinct(); left.getFields().stream().map(Schema.Field::getName).filter(right::hasField); zipFold, context.withPathPart(name).withParent(TypeName.ROW), left.getField(name).getType(), right.getField(name).getType())) .reduce(node, zipFold::accumulate); Optional<Field> field1 = Optional.empty(); if (left.hasField(name)) { field0 = Optional.of(left.getField(name)); if (right.hasField(name)) { field1 = Optional.of(right.getField(name));
public static Row castRow(Row input, Schema inputSchema, Schema outputSchema) { if (input == null) { return null; } Row.Builder output = Row.withSchema(outputSchema); for (int i = 0; i < outputSchema.getFieldCount(); i++) { Schema.Field outputField = outputSchema.getField(i); int fromFieldIdx = inputSchema.indexOf(outputField.getName()); Schema.Field inputField = inputSchema.getField(fromFieldIdx); Object inputValue = input.getValue(fromFieldIdx); Object outputValue = castValue(inputValue, inputField.getType(), outputField.getType()); output.addValue(outputValue); } return output.build(); }
private void validatePubsubMessageSchema(Table tableDefinition) { Schema schema = tableDefinition.getSchema(); if (schema.getFieldCount() != 3 || !fieldPresent(schema, TIMESTAMP_FIELD, TIMESTAMP) || !fieldPresent(schema, ATTRIBUTES_FIELD, Schema.FieldType.map(VARCHAR, VARCHAR)) || !(schema.hasField(PAYLOAD_FIELD) && ROW.equals(schema.getField(PAYLOAD_FIELD).getType().getTypeName()))) { throw new IllegalArgumentException( "Unsupported schema specified for Pubsub source in CREATE TABLE. " + "CREATE TABLE for Pubsub topic should define exactly the following fields: " + "'event_timestamp' field of type 'TIMESTAMP', 'attributes' field of type " + "MAP<VARCHAR, VARCHAR>, and 'payload' field of type 'ROW<...>' which matches the " + "payload JSON format."); } }
/** Generate {@link Schema} from {@code RelDataType} which is used to create table. */ public static Schema toSchema(RelDataType tableInfo) { return tableInfo.getFieldList().stream().map(CalciteUtils::toField).collect(Schema.toSchema()); }
private Schema.Field toField(Schema schema, Integer fieldIndex) { Schema.Field original = schema.getField(fieldIndex); return original.withName("c" + fieldIndex); } }
private List<Object> verify(Schema schema, List<Object> values) { List<Object> verifiedValues = Lists.newArrayListWithCapacity(values.size()); if (schema.getFieldCount() != values.size()) { throw new IllegalArgumentException( String.format( "Field count in Schema (%s) and values (%s) must match", schema.getFieldNames(), values)); } for (int i = 0; i < values.size(); ++i) { Object value = values.get(i); Schema.Field field = schema.getField(i); if (value == null) { if (!field.getNullable()) { throw new IllegalArgumentException( String.format("Field %s is not nullable", field.getName())); } verifiedValues.add(null); } else { verifiedValues.add(verify(value, field.getType(), field.getName())); } } return verifiedValues; }
private static int validateFieldId(Schema schema, int fieldId) { if (fieldId < 0 || fieldId >= schema.getFieldCount()) { throw new IllegalArgumentException("Invalid field id " + fieldId + " for schema " + schema); } return fieldId; } }
/** Get value by field name, {@link ClassCastException} is thrown if type doesn't match. */ @Nullable @SuppressWarnings("TypeParameterUnusedInFormals") public <T> T getValue(String fieldName) { return getValue(getSchema().indexOf(fieldName)); }
/** Returns true if two Schemas have the same fields, but possibly in different orders. */ public boolean equivalent(Schema other) { return equivalent(other, EquivalenceNullablePolicy.SAME); }