/** * Find merge strategy. * * @param root the root * @param arrayFieldName the array field name * @return the string */ private String findMergeStrategy(Schema root, String arrayFieldName) { if (root.getType() != Type.RECORD) { return null; } List<Field> fields = root.getFields(); for (Field field : fields) { if (arrayFieldName.equals(field.name())) { return field.getProp(FIELD_OVERRIDE_STRATEGY); } } return null; } }
/** * Initialize converters per each schema field * * @param schema design schema * @review */ protected void initConverters(Schema schema) { schemaFields = schema.getFields(); avroConverters = new AvroConverter[schemaFields.size()]; for (int i = 0; i < schemaFields.size(); i++) { Schema.Field field = schemaFields.get(i); Schema fieldSchema = AvroUtils.unwrapIfNullable(field.schema()); if (LogicalTypeUtils.isLogicalTimestampMillis(fieldSchema)) { String datePattern = field.getProp( SchemaConstants.TALEND_COLUMN_PATTERN); avroConverters[i] = new StringTimestampConverter(datePattern); } else { Schema.Type type = fieldSchema.getType(); avroConverters[i] = _converterRegistry.get(type); } } }
hiveColumns.get().put(field.name(), type); String flattenSource = field.getProp("flatten_source"); if (StringUtils.isBlank(flattenSource)) { flattenSource = field.name();
public static List<String> getAllSchemaFieldDBNames(Schema schema) { List<String> values = new ArrayList<>(); if (schema == null) { return values; } for (Schema.Field field : schema.getFields()) { values.add(field.getProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME)); } return values; }
StringToDateConverter(Schema.Field field) { super(field); String pattern = field.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); // TODO: null handling format = new SimpleDateFormat(pattern); }
@Override public void onVisit(Field field, Collection<String> breadcrumb) { if (field.getProp(LogicalType.LOGICAL_TYPE_PROP) != null) { String path = breadcrumb.stream().collect(joining("/", "/", "")); throw new IllegalArgumentException("Invalid logical type declared on field at " + path); } }
@Override @Nullable public String getProp(final String name) { return wrapped.getProp(name); }
StringToDateConverter(Schema.Field field) { super(field); String pattern = field.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); // TODO: null handling format = new SimpleDateFormat(pattern); }
@Override public void onVisit(Field field, Collection<String> breadcrumb) { if (PII.equalsIgnoreCase(field.getProp(SENSITIVITY))) { Schema schema = field.schema(); if (!isStringOrBytes(schema) && !isNullableStringOrBytes(schema)) { throw new InvalidPiiAnnotationException(breadcrumb); } onPiiField(field, breadcrumb); } }
DateConverter(Schema.Field field, FileDelimitedProperties properties) { super(field, properties); pattern = field.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); if (properties instanceof TFileInputDelimitedProperties) { isLenient = ((TFileInputDelimitedProperties) properties).checkDate.getValue(); } }
/** * Return internal (NetSuite specific) name for a given <code>schema field</code>. * * @param field schema field * @return name */ public static String getNsFieldName(Schema.Field field) { String name = field.getProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME); return name != null ? toInitialLower(name) : toInitialLower(field.name()); }
public static List<String> getAllSchemaFieldDBNames(Schema schema) { List<String> values = new ArrayList<>(); if (schema == null) { return values; } for (Schema.Field field : schema.getFields()) { values.add(field.getProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME)); } return values; }
StringToDateConverter(Schema.Field field) { super(field); String pattern = field.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); // TODO: null handling format = new SimpleDateFormat(pattern); }
List<Schema.Field> fieldList = outputOrcSchema.getFields(); for (Schema.Field field : fieldList) { String flattenSource = field.getProp("flatten_source"); String colName; if (StringUtils.isNotBlank(flattenSource)) { for (Schema.Field evolvedField : evolvedFieldList) { if (evolvedField.name().equalsIgnoreCase(field.getName())) { String flattenSource = evolvedField.getProp("flatten_source"); if (StringUtils.isNotBlank(flattenSource)) { colName = flattenSource;
String flattenSource = evolvedSchema.getField(evolvedColumn.getKey()).getProp("flatten_source"); if (StringUtils.isBlank(flattenSource)) { flattenSource = evolvedSchema.getField(evolvedColumn.getKey()).name();
public static Schema extractSchemaFromAvroWithoutTime(File avroFile) throws IOException { DataFileStream<GenericRecord> dataStream = new DataFileStream<GenericRecord>(new FileInputStream(avroFile), new GenericDatumReader<GenericRecord>()); Schema schema = new Schema(); for (final Field field : dataStream.getSchema().getFields()) { try { getColumnType(field); } catch (Exception e) { LOGGER.warn("Caught exception while converting Avro field {} of type {}, field will not be in schema.", field.name(), field.schema().getType()); continue; } final String columnName = field.name(); final String pinotType = field.getProp("pinotType"); final FieldSpec fieldSpec; if (pinotType != null && "METRIC".equals(pinotType)) { fieldSpec = new MetricFieldSpec(); } else { fieldSpec = new DimensionFieldSpec(); } fieldSpec.setName(columnName); fieldSpec.setDataType(getColumnType(dataStream.getSchema().getField(columnName))); fieldSpec.setSingleValueField(isSingleValueField(dataStream.getSchema().getField(columnName))); schema.addField(fieldSpec); } dataStream.close(); return schema; }
public static Schema mergeRuntimeSchemaWithDesignSchemaDynamic( Schema designSchema, Schema runtimeSchema) { List<Schema.Field> designFields = designSchema.getFields(); Set<String> designFieldSet = new HashSet<>(); for (Schema.Field designField : designFields) { String name = designField.getProp( SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME); designFieldSet.add(name); } List<Schema.Field> dynamicFields = new ArrayList<>(); for (Schema.Field runtimeField : runtimeSchema.getFields()) { String name = runtimeField.getProp( SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME); if (!designFieldSet.contains(name)) { dynamicFields.add(runtimeField); } } dynamicFields = _cloneFieldsAndResetPosition(dynamicFields); int dynamicPosition = Integer.valueOf( designSchema.getProp("di.dynamic.column.position")); return newSchema( designSchema, designSchema.getName(), dynamicFields, dynamicPosition); }
@Test public void testMessageCustomProperties() throws IOException{ Protocol protocol = getSimpleProtocol(); final Message msg = protocol.getMessages().get("hello"); assertNotNull(msg); final Schema.Field field = msg.getRequest().getField("greeting"); assertNotNull(field); assertEquals("customValue", field.getProp("customProp")); } }