@Override public org.apache.avro.Schema getSchema() { return AvroUtils._double(); }
@Override public org.apache.avro.Schema getSchema() { return AvroUtils._double(); }
/** * Checks {@link AvroTypeConverter#convertToAvro(String, String)} returns Double avro schema in case TalendType.DOUBLE Talend * type is * passed */ @Test public void testConvertToAvroDouble() { Schema expectedSchema = AvroUtils._double(); assertEquals(expectedSchema, AvroTypeConverter.convertToAvro(TalendType.DOUBLE, null)); }
/** * Checks {@link TalendType#convertFromAvro(Schema)} converts double avro type to "id_Double" di type */ @Test public void testConvertFromAvroDouble() { TalendType expectedType = TalendType.DOUBLE; Schema fieldSchema = AvroUtils._double(); assertEquals(expectedType, TalendType.convertFromAvro(fieldSchema)); }
/** * Infers an Avro schema for the given DataPrep Field. This can be an expensive operation so the schema should be * cached where possible. The return type will be the Avro Schema that can contain the field data without loss of * precision. * * @param field the Field to analyse. * @return the schema for data that the field describes. */ private Schema inferSchemaField(DataPrepField field) { Schema base; switch (field.getType()) { case "boolean": base = AvroUtils._boolean(); break; case "double": base = AvroUtils._double(); break; case "integer": base = AvroUtils._int(); break; case "float": base = AvroUtils._float(); break; default: base = AvroUtils._string(); break; } // TODO add handling for numeric, any and date. return base; }
/** * A helper method to convert the String representation of a datum in the DataPrep system to the Avro type that * matches the Schema generated for it. * * @param f is field in Avro Schema. * @return converter for a given type. */ public AvroConverter<String, ?> getConverterFromString(org.apache.avro.Schema.Field f) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); // FIXME use avro type to decide the converter is not correct if the user change the avro type, Date to String // for instance if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new StringToBooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) { return new StringToDecimalConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return new StringToDoubleConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._int())) { return new StringToIntegerConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new StringToDateConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) { return super.getConverter(String.class); } throw new UnsupportedOperationException("The type " + fieldSchema.getType() + " is not supported."); //$NON-NLS-1$ //$NON-NLS-2$ }
private Object convertField(FieldValue fieldValue, org.apache.avro.Schema fieldSchema) { boolean nullable = AvroUtils.isNullable(fieldSchema); if (nullable && fieldValue.isNull()) { return null; } fieldSchema = AvroUtils.unwrapIfNullable(fieldSchema); switch (fieldValue.getAttribute()) { case PRIMITIVE: if (BigQueryType.TIMESTAMP.toString().equals(fieldSchema.getProp(TALEND_COLUMN_DB_TYPE))) { Double doubleValue = ((Long) fieldValue.getTimestampValue()) / 1000000.0; return formatTimestamp(doubleValue.toString()); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return fieldValue.getDoubleValue(); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return fieldValue.getBooleanValue(); } else { return fieldValue.getValue(); } case REPEATED: List<Object> listValue = new ArrayList<>(); List<FieldValue> repeatedChildValue = fieldValue.getRepeatedValue(); for (FieldValue childValue : repeatedChildValue) { listValue.add(convertField(childValue, fieldSchema.getElementType())); } return listValue; case RECORD: return convertFileds(fieldValue.getRecordValue(), fieldSchema); } throw TalendRuntimeException.build(CommonErrorCodes.UNEXPECTED_ARGUMENT).create(); }
private Object convertField(FieldValue fieldValue, org.apache.avro.Schema fieldSchema) { boolean nullable = AvroUtils.isNullable(fieldSchema); if (nullable && fieldValue.isNull()) { return null; } fieldSchema = AvroUtils.unwrapIfNullable(fieldSchema); switch (fieldValue.getAttribute()) { case PRIMITIVE: if (BigQueryType.TIMESTAMP.toString().equals(fieldSchema.getProp(TALEND_COLUMN_DB_TYPE))) { Double doubleValue = ((Long) fieldValue.getTimestampValue()) / 1000000.0; return formatTimestamp(doubleValue.toString()); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return fieldValue.getDoubleValue(); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return fieldValue.getBooleanValue(); } else { return fieldValue.getValue(); } case REPEATED: List<Object> listValue = new ArrayList<>(); List<FieldValue> repeatedChildValue = fieldValue.getRepeatedValue(); for (FieldValue childValue : repeatedChildValue) { listValue.add(convertField(childValue, fieldSchema.getElementType())); } return listValue; case RECORD: return convertFileds(fieldValue.getRecordValue(), fieldSchema); } throw TalendRuntimeException.build(CommonErrorCodes.UNEXPECTED_ARGUMENT).create(); }
/** * A helper method to convert the String representation of a datum in the Salesforce system to the Avro type that * matches the Schema generated for it. * * @param f * @return */ public AvroConverter<String, ?> getConverterFromString(org.apache.avro.Schema.Field f) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); // FIXME use avro type to decide the converter is not correct if the user change the avro type, Date to String // for instance if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new StringToBooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) { return new StringToDecimalConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return new StringToDoubleConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._int())) { return new StringToIntegerConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new StringToDateConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._bytes())) { return new StringToBytesConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) { return super.getConverter(String.class); } throw new UnsupportedOperationException("The type " + fieldSchema.getType() + " is not supported."); //$NON-NLS-1$ //$NON-NLS-2$ }
return AvroUtils._long(); case DOUBLE: return AvroUtils._double(); case FLOAT: return AvroUtils._float();
return AvroUtils._long(); case DOUBLE: return AvroUtils._double(); case FLOAT: return AvroUtils._float();
/** * Get an Avro schema using {@link AvroUtils#wrapAsNullable(Schema)} by node type. * * @param node Json node. * @return an Avro schema using {@link AvroUtils#wrapAsNullable(Schema)} by node type. */ @VisibleForTesting Schema getAvroSchema(JsonNode node) { if (node instanceof TextNode) { return AvroUtils.wrapAsNullable(AvroUtils._string()); } else if (node instanceof IntNode) { return AvroUtils.wrapAsNullable(AvroUtils._int()); } else if (node instanceof LongNode) { return AvroUtils.wrapAsNullable(AvroUtils._long()); } else if (node instanceof DoubleNode) { return AvroUtils.wrapAsNullable(AvroUtils._double()); } else if (node instanceof BooleanNode) { return AvroUtils.wrapAsNullable(AvroUtils._boolean()); } else if (node instanceof NullNode) { return AvroUtils.wrapAsNullable(AvroUtils._string()); } else { return createSubRecord(node); } }
public AvroConverter<String, ?> getConverter(Schema.Field f, FileDelimitedProperties properties) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new BooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) { return new DecimalConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return new DoubleConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._float())) { return new FloatConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._int())) { return new IntegerConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new DateConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._long())) { return new LongConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._bytes())) { return new BytesConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._byte())) { return new ByteConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._short())) { return new ShortConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._character())) { return new CharacterConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) { return super.getConverter(String.class); } throw new UnsupportedOperationException("The type " + fieldSchema.getType() + " is not supported."); //$NON-NLS-1$ //$NON-NLS-2$ }
} else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) { return Types.BIGINT; } else if (AvroUtils.isSameType(basicSchema, AvroUtils._double())) { return Types.DOUBLE; } else if (AvroUtils.isSameType(basicSchema, AvroUtils._float())) {
} else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) { return Types.BIGINT; } else if (AvroUtils.isSameType(basicSchema, AvroUtils._double())) { return Types.DOUBLE; } else if (AvroUtils.isSameType(basicSchema, AvroUtils._float())) {
public static DTEConverter createConverter(final Field f, final String mappedName) { Schema basicSchema = AvroUtils.unwrapIfNullable(f.schema()); AzureStorageDTEConverters converters = new AzureStorageDTEConverters(); DTEConverter dteConverter; if (AvroUtils.isSameType(basicSchema, AvroUtils._string()) || AvroUtils.isSameType(basicSchema, AvroUtils._character())) { dteConverter = converters.new StringDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._int()) || AvroUtils.isSameType(basicSchema, AvroUtils._short()) || AvroUtils.isSameType(basicSchema, AvroUtils._byte())) { dteConverter = converters.new IntegerDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._date())) { dteConverter = converters.new DateDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._decimal()) || AvroUtils.isSameType(basicSchema, AvroUtils._double()) || AvroUtils.isSameType(basicSchema, AvroUtils._float())) { dteConverter = converters.new DoubleDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) { dteConverter = converters.new LongDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { dteConverter = converters.new BooleanDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) { dteConverter = converters.new ByteArrayDTEConverter(mappedName); } else { dteConverter = converters.new StringDTEConverter(f, mappedName); } return dteConverter; }
public static DTEConverter createConverter(final Field f, final String mappedName) { Schema basicSchema = AvroUtils.unwrapIfNullable(f.schema()); AzureStorageDTEConverters converters = new AzureStorageDTEConverters(); DTEConverter dteConverter; if (AvroUtils.isSameType(basicSchema, AvroUtils._string()) || AvroUtils.isSameType(basicSchema, AvroUtils._character())) { dteConverter = converters.new StringDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._int()) || AvroUtils.isSameType(basicSchema, AvroUtils._short()) || AvroUtils.isSameType(basicSchema, AvroUtils._byte())) { dteConverter = converters.new IntegerDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._date())) { dteConverter = converters.new DateDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._decimal()) || AvroUtils.isSameType(basicSchema, AvroUtils._double()) || AvroUtils.isSameType(basicSchema, AvroUtils._float())) { dteConverter = converters.new DoubleDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) { dteConverter = converters.new LongDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { dteConverter = converters.new BooleanDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) { dteConverter = converters.new ByteArrayDTEConverter(mappedName); } else { dteConverter = converters.new StringDTEConverter(f, mappedName); } return dteConverter; }
@Before public void createSchema() { schema = SchemaBuilder.builder() .record("main") .fields() .name("id") .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .type(AvroUtils._int()) .withDefault(1) .name("name") .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255") .prop(SchemaConstants.TALEND_COLUMN_DEFAULT, "\"ok\"") .type(AvroUtils._string()) .noDefault() .name("date") .type(AvroUtils._logicalDate()) .noDefault() .name("salary") .prop(SchemaConstants.TALEND_COLUMN_DB_TYPE, "MY_DOUBLE") .prop(SchemaConstants.TALEND_COLUMN_PRECISION, "38") .prop(SchemaConstants.TALEND_COLUMN_SCALE, "4") .type(AvroUtils._double()) .withDefault("0") .name("updated") .type(AvroUtils._logicalTimestamp()) .noDefault() .endRecord(); }
@Before public void createSchema(){ schema = SchemaBuilder.builder() .record("main") .fields() .name("id") .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .type(AvroUtils._int()) .withDefault(1) .name("name") .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255") .prop(SchemaConstants.TALEND_COLUMN_DEFAULT, "\"ok\"") .type(AvroUtils._string()) .noDefault() .name("date") .type(AvroUtils._logicalDate()) .noDefault() .name("salary") .prop(SchemaConstants.TALEND_COLUMN_DB_TYPE, "MY_DOUBLE") .prop(SchemaConstants.TALEND_COLUMN_PRECISION, "38") .prop(SchemaConstants.TALEND_COLUMN_SCALE, "4") .type(AvroUtils._double()) .withDefault("0") .name("updated") .type(AvroUtils._logicalTimestamp()) .noDefault() .endRecord(); }
@Before public void createSchema() { schema = SchemaBuilder.builder() .record("main") .fields() .name("id") .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .type(Schema.createUnion(AvroUtils._int(), Schema.create(Schema.Type.NULL))) .withDefault(1) .name("name") .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255") .prop(SchemaConstants.TALEND_COLUMN_DEFAULT, "\"ok\"") .type(Schema.createUnion(AvroUtils._string(), Schema.create(Schema.Type.NULL))) .noDefault() .name("date") .type(Schema.createUnion(AvroUtils._logicalDate(), Schema.create(Schema.Type.NULL))) .noDefault() .name("salary") .prop(SchemaConstants.TALEND_COLUMN_DB_TYPE, "MY_DOUBLE") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "38") .prop(SchemaConstants.TALEND_COLUMN_PRECISION, "4") .type(Schema.createUnion(AvroUtils._double(), Schema.create(Schema.Type.NULL))) .withDefault("0") .name("updated") .type(Schema.createUnion(AvroUtils._logicalTimestamp(), Schema.create(Schema.Type.NULL))) .noDefault() .endRecord(); }