/** * Checks {@link TalendType#convertFromAvro(Schema)} converts boolean avro type to "id_Boolean" di type */ @Test public void testConvertFromAvroBoolean() { TalendType expectedType = TalendType.BOOLEAN; Schema fieldSchema = AvroUtils._boolean(); assertEquals(expectedType, TalendType.convertFromAvro(fieldSchema)); }
/** * Checks {@link AvroTypeConverter#convertToAvro(String, String)} returns Boolean avro schema in case TalendType.BOOLEAN * Talend type * is passed */ @Test public void testConvertToAvroBoolean() { Schema expectedSchema = AvroUtils._boolean(); assertEquals(expectedSchema, AvroTypeConverter.convertToAvro(TalendType.BOOLEAN, null)); }
/** * Infers an Avro schema for the given DataPrep Field. This can be an expensive operation so the schema should be * cached where possible. The return type will be the Avro Schema that can contain the field data without loss of * precision. * * @param field the Field to analyse. * @return the schema for data that the field describes. */ private Schema inferSchemaField(DataPrepField field) { Schema base; switch (field.getType()) { case "boolean": base = AvroUtils._boolean(); break; case "double": base = AvroUtils._double(); break; case "integer": base = AvroUtils._int(); break; case "float": base = AvroUtils._float(); break; default: base = AvroUtils._string(); break; } // TODO add handling for numeric, any and date. return base; }
/** * A helper method to convert the String representation of a datum in the DataPrep system to the Avro type that * matches the Schema generated for it. * * @param f is field in Avro Schema. * @return converter for a given type. */ public AvroConverter<String, ?> getConverterFromString(org.apache.avro.Schema.Field f) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); // FIXME use avro type to decide the converter is not correct if the user change the avro type, Date to String // for instance if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new StringToBooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) { return new StringToDecimalConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return new StringToDoubleConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._int())) { return new StringToIntegerConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new StringToDateConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) { return super.getConverter(String.class); } throw new UnsupportedOperationException("The type " + fieldSchema.getType() + " is not supported."); //$NON-NLS-1$ //$NON-NLS-2$ }
private Object convertField(FieldValue fieldValue, org.apache.avro.Schema fieldSchema) { boolean nullable = AvroUtils.isNullable(fieldSchema); if (nullable && fieldValue.isNull()) { return null; } fieldSchema = AvroUtils.unwrapIfNullable(fieldSchema); switch (fieldValue.getAttribute()) { case PRIMITIVE: if (BigQueryType.TIMESTAMP.toString().equals(fieldSchema.getProp(TALEND_COLUMN_DB_TYPE))) { Double doubleValue = ((Long) fieldValue.getTimestampValue()) / 1000000.0; return formatTimestamp(doubleValue.toString()); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return fieldValue.getDoubleValue(); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return fieldValue.getBooleanValue(); } else { return fieldValue.getValue(); } case REPEATED: List<Object> listValue = new ArrayList<>(); List<FieldValue> repeatedChildValue = fieldValue.getRepeatedValue(); for (FieldValue childValue : repeatedChildValue) { listValue.add(convertField(childValue, fieldSchema.getElementType())); } return listValue; case RECORD: return convertFileds(fieldValue.getRecordValue(), fieldSchema); } throw TalendRuntimeException.build(CommonErrorCodes.UNEXPECTED_ARGUMENT).create(); }
private Object convertField(FieldValue fieldValue, org.apache.avro.Schema fieldSchema) { boolean nullable = AvroUtils.isNullable(fieldSchema); if (nullable && fieldValue.isNull()) { return null; } fieldSchema = AvroUtils.unwrapIfNullable(fieldSchema); switch (fieldValue.getAttribute()) { case PRIMITIVE: if (BigQueryType.TIMESTAMP.toString().equals(fieldSchema.getProp(TALEND_COLUMN_DB_TYPE))) { Double doubleValue = ((Long) fieldValue.getTimestampValue()) / 1000000.0; return formatTimestamp(doubleValue.toString()); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return fieldValue.getDoubleValue(); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return fieldValue.getBooleanValue(); } else { return fieldValue.getValue(); } case REPEATED: List<Object> listValue = new ArrayList<>(); List<FieldValue> repeatedChildValue = fieldValue.getRepeatedValue(); for (FieldValue childValue : repeatedChildValue) { listValue.add(convertField(childValue, fieldSchema.getElementType())); } return listValue; case RECORD: return convertFileds(fieldValue.getRecordValue(), fieldSchema); } throw TalendRuntimeException.build(CommonErrorCodes.UNEXPECTED_ARGUMENT).create(); }
/** * A helper method to convert the String representation of a datum in the Salesforce system to the Avro type that * matches the Schema generated for it. * * @param f * @return */ public AvroConverter<String, ?> getConverterFromString(org.apache.avro.Schema.Field f) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); // FIXME use avro type to decide the converter is not correct if the user change the avro type, Date to String // for instance if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new StringToBooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) { return new StringToDecimalConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return new StringToDoubleConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._int())) { return new StringToIntegerConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new StringToDateConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._bytes())) { return new StringToBytesConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) { return super.getConverter(String.class); } throw new UnsupportedOperationException("The type " + fieldSchema.getType() + " is not supported."); //$NON-NLS-1$ //$NON-NLS-2$ }
return AvroUtils._string(); case BOOLEAN: return AvroUtils._boolean(); case INTEGER: return AvroUtils._int();
return AvroUtils._string(); case BOOLEAN: return AvroUtils._boolean(); case INTEGER: return AvroUtils._int();
/** * Get an Avro schema using {@link AvroUtils#wrapAsNullable(Schema)} by node type. * * @param node Json node. * @return an Avro schema using {@link AvroUtils#wrapAsNullable(Schema)} by node type. */ @VisibleForTesting Schema getAvroSchema(JsonNode node) { if (node instanceof TextNode) { return AvroUtils.wrapAsNullable(AvroUtils._string()); } else if (node instanceof IntNode) { return AvroUtils.wrapAsNullable(AvroUtils._int()); } else if (node instanceof LongNode) { return AvroUtils.wrapAsNullable(AvroUtils._long()); } else if (node instanceof DoubleNode) { return AvroUtils.wrapAsNullable(AvroUtils._double()); } else if (node instanceof BooleanNode) { return AvroUtils.wrapAsNullable(AvroUtils._boolean()); } else if (node instanceof NullNode) { return AvroUtils.wrapAsNullable(AvroUtils._string()); } else { return createSubRecord(node); } }
switch (field.getType()) { case _boolean: base = AvroUtils._boolean(); break; case _double:
if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new StringToBooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) {
if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new StringToBooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) {
} else if (AvroUtils.isSameType(basicSchema, AvroUtils._float())) { return Types.FLOAT; } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { return Types.BOOLEAN; } else if (AvroUtils.isSameType(basicSchema, AvroUtils._short())) {
public AvroConverter<String, ?> getConverter(Schema.Field f, FileDelimitedProperties properties) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new BooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) { return new DecimalConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return new DoubleConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._float())) { return new FloatConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._int())) { return new IntegerConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new DateConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._long())) { return new LongConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._bytes())) { return new BytesConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._byte())) { return new ByteConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._short())) { return new ShortConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._character())) { return new CharacterConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) { return super.getConverter(String.class); } throw new UnsupportedOperationException("The type " + fieldSchema.getType() + " is not supported."); //$NON-NLS-1$ //$NON-NLS-2$ }
} else if (AvroUtils.isSameType(basicSchema, AvroUtils._float())) { return Types.FLOAT; } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { return Types.BOOLEAN; } else if (AvroUtils.isSameType(basicSchema, AvroUtils._short())) {
} else if (AvroUtils.isSameType(basicSchema, AvroUtils._float())) { writer = new FloatTypeWriter(statement, statementIndex, inputValueLocation); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { writer = new BooleanTypeWriter(statement, statementIndex, inputValueLocation); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._short())) {
} else if (AvroUtils.isSameType(basicSchema, AvroUtils._float())) { writer = new FloatTypeWriter(statement, statementIndex, inputValueLocation); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { writer = new BooleanTypeWriter(statement, statementIndex, inputValueLocation); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._short())) {
public static DTEConverter createConverter(final Field f, final String mappedName) { Schema basicSchema = AvroUtils.unwrapIfNullable(f.schema()); AzureStorageDTEConverters converters = new AzureStorageDTEConverters(); DTEConverter dteConverter; if (AvroUtils.isSameType(basicSchema, AvroUtils._string()) || AvroUtils.isSameType(basicSchema, AvroUtils._character())) { dteConverter = converters.new StringDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._int()) || AvroUtils.isSameType(basicSchema, AvroUtils._short()) || AvroUtils.isSameType(basicSchema, AvroUtils._byte())) { dteConverter = converters.new IntegerDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._date())) { dteConverter = converters.new DateDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._decimal()) || AvroUtils.isSameType(basicSchema, AvroUtils._double()) || AvroUtils.isSameType(basicSchema, AvroUtils._float())) { dteConverter = converters.new DoubleDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) { dteConverter = converters.new LongDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { dteConverter = converters.new BooleanDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) { dteConverter = converters.new ByteArrayDTEConverter(mappedName); } else { dteConverter = converters.new StringDTEConverter(f, mappedName); } return dteConverter; }
public static DTEConverter createConverter(final Field f, final String mappedName) { Schema basicSchema = AvroUtils.unwrapIfNullable(f.schema()); AzureStorageDTEConverters converters = new AzureStorageDTEConverters(); DTEConverter dteConverter; if (AvroUtils.isSameType(basicSchema, AvroUtils._string()) || AvroUtils.isSameType(basicSchema, AvroUtils._character())) { dteConverter = converters.new StringDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._int()) || AvroUtils.isSameType(basicSchema, AvroUtils._short()) || AvroUtils.isSameType(basicSchema, AvroUtils._byte())) { dteConverter = converters.new IntegerDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._date())) { dteConverter = converters.new DateDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._decimal()) || AvroUtils.isSameType(basicSchema, AvroUtils._double()) || AvroUtils.isSameType(basicSchema, AvroUtils._float())) { dteConverter = converters.new DoubleDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) { dteConverter = converters.new LongDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { dteConverter = converters.new BooleanDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) { dteConverter = converters.new ByteArrayDTEConverter(mappedName); } else { dteConverter = converters.new StringDTEConverter(f, mappedName); } return dteConverter; }