/** * Checks {@link AvroTypeConverter#convertToAvro(String, String)} returns Integer avro schema in case TalendType.INTEGER * Talend type * is passed */ @Test public void testConvertToAvroInteger() { Schema expectedSchema = AvroUtils._int(); assertEquals(expectedSchema, AvroTypeConverter.convertToAvro(TalendType.INTEGER, null)); }
/** * Checks {@link TalendType#convertFromAvro(Schema)} converts int avro type to "id_Integer" di type */ @Test public void testConvertFromAvroInteger() { TalendType expectedType = TalendType.INTEGER; Schema fieldSchema = AvroUtils._int(); assertEquals(expectedType, TalendType.convertFromAvro(fieldSchema)); }
public static Schema getEventSchema() { Schema.Field[] fields = new Schema.Field[10]; Schema.Field field; fields[EventSchemaField.EVENT_IDX] = new Schema.Field("event", AvroUtils._string(), "Type of event", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.PARTITION_IDX] = new Schema.Field("partition", AvroUtils._short(), "Partition number", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.KEY_IDX] = new Schema.Field("key", AvroUtils._string(), "Key", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.CAS_IDX] = new Schema.Field("cas", AvroUtils._long(), "CAS", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.SEQNO_IDX] = new Schema.Field("bySeqno", AvroUtils._long(), "Sequence number", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.REV_SEQNO_IDX] = new Schema.Field("revSeqno", AvroUtils._long(), "Revision sequence number", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.EXPIRATION_IDX] = new Schema.Field("expiration", AvroUtils.wrapAsNullable(AvroUtils._int()), "Expiration", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.FLAGS_IDX] = new Schema.Field("flags", AvroUtils.wrapAsNullable(AvroUtils._int()), "Flags", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.LOCK_TIME_IDX] = new Schema.Field("lockTime", AvroUtils.wrapAsNullable(AvroUtils._int()), "Lock time", (Object) null, Schema.Field.Order.ASCENDING); field = new Schema.Field("content", AvroUtils.wrapAsNullable(AvroUtils._bytes()), "Content", (Object) null, Schema.Field.Order.ASCENDING); field.addProp(TALEND_IS_LOCKED, "false"); fields[EventSchemaField.CONTENT_IDX] = field; Schema schema = Schema.createRecord("DcpMessage", "Couchbase DCP message", "com.couchbase", false, Arrays.asList(fields)); schema.addProp(TALEND_IS_LOCKED, "true"); return schema; }
/** * A helper method to convert the String representation of a datum in the DataPrep system to the Avro type that * matches the Schema generated for it. * * @param f is field in Avro Schema. * @return converter for a given type. */ public AvroConverter<String, ?> getConverterFromString(org.apache.avro.Schema.Field f) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); // FIXME use avro type to decide the converter is not correct if the user change the avro type, Date to String // for instance if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new StringToBooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) { return new StringToDecimalConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return new StringToDoubleConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._int())) { return new StringToIntegerConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new StringToDateConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) { return super.getConverter(String.class); } throw new UnsupportedOperationException("The type " + fieldSchema.getType() + " is not supported."); //$NON-NLS-1$ //$NON-NLS-2$ }
/** * A helper method to convert the String representation of a datum in the Salesforce system to the Avro type that * matches the Schema generated for it. * * @param f * @return */ public AvroConverter<String, ?> getConverterFromString(org.apache.avro.Schema.Field f) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); // FIXME use avro type to decide the converter is not correct if the user change the avro type, Date to String // for instance if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new StringToBooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) { return new StringToDecimalConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return new StringToDoubleConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._int())) { return new StringToIntegerConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new StringToDateConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._bytes())) { return new StringToBytesConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) { return super.getConverter(String.class); } throw new UnsupportedOperationException("The type " + fieldSchema.getType() + " is not supported."); //$NON-NLS-1$ //$NON-NLS-2$ }
return AvroUtils._boolean(); case INTEGER: return AvroUtils._int(); case LONG: return AvroUtils._long();
/** * Get an Avro schema using {@link AvroUtils#wrapAsNullable(Schema)} by node type. * * @param node Json node. * @return an Avro schema using {@link AvroUtils#wrapAsNullable(Schema)} by node type. */ @VisibleForTesting Schema getAvroSchema(JsonNode node) { if (node instanceof TextNode) { return AvroUtils.wrapAsNullable(AvroUtils._string()); } else if (node instanceof IntNode) { return AvroUtils.wrapAsNullable(AvroUtils._int()); } else if (node instanceof LongNode) { return AvroUtils.wrapAsNullable(AvroUtils._long()); } else if (node instanceof DoubleNode) { return AvroUtils.wrapAsNullable(AvroUtils._double()); } else if (node instanceof BooleanNode) { return AvroUtils.wrapAsNullable(AvroUtils._boolean()); } else if (node instanceof NullNode) { return AvroUtils.wrapAsNullable(AvroUtils._string()); } else { return createSubRecord(node); } }
public static Schema createSPSchema1(String tablename) { FieldAssembler<Schema> builder = SchemaBuilder.builder().record(tablename).fields(); Schema schema = AvroUtils._int(); schema = wrap(schema); builder = builder.name("PARAMETER").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "PARAMETER").type(schema) .noDefault(); return builder.endRecord(); }
public static DTEConverter createConverter(final Field f, final String mappedName) { Schema basicSchema = AvroUtils.unwrapIfNullable(f.schema()); AzureStorageDTEConverters converters = new AzureStorageDTEConverters(); DTEConverter dteConverter; if (AvroUtils.isSameType(basicSchema, AvroUtils._string()) || AvroUtils.isSameType(basicSchema, AvroUtils._character())) { dteConverter = converters.new StringDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._int()) || AvroUtils.isSameType(basicSchema, AvroUtils._short()) || AvroUtils.isSameType(basicSchema, AvroUtils._byte())) { dteConverter = converters.new IntegerDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._date())) { dteConverter = converters.new DateDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._decimal()) || AvroUtils.isSameType(basicSchema, AvroUtils._double()) || AvroUtils.isSameType(basicSchema, AvroUtils._float())) { dteConverter = converters.new DoubleDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) { dteConverter = converters.new LongDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { dteConverter = converters.new BooleanDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) { dteConverter = converters.new ByteArrayDTEConverter(mappedName); } else { dteConverter = converters.new StringDTEConverter(f, mappedName); } return dteConverter; }
public static Schema createTestSchema(String tablename) { FieldAssembler<Schema> builder = SchemaBuilder.builder().record(tablename).fields(); Schema schema = AvroUtils._int(); schema = wrap(schema); builder = builder.name("ID").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "ID").type(schema).noDefault(); schema = AvroUtils._string(); schema = wrap(schema); builder = builder.name("NAME").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "NAME").type(schema).noDefault(); return builder.endRecord(); }
public static Schema createSPSchema3(String tablename) { FieldAssembler<Schema> builder = SchemaBuilder.builder().record(tablename).fields(); Schema schema = AvroUtils._int(); schema = wrap(schema); builder = builder.name("PARAMETER1").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "PARAMETER1").type(schema) .noDefault(); schema = AvroUtils._string(); schema = wrap(schema); builder = builder.name("PARAMETER2").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "PARAMETER2").type(schema) .noDefault(); return builder.endRecord(); }
public static DTEConverter createConverter(final Field f, final String mappedName) { Schema basicSchema = AvroUtils.unwrapIfNullable(f.schema()); AzureStorageDTEConverters converters = new AzureStorageDTEConverters(); DTEConverter dteConverter; if (AvroUtils.isSameType(basicSchema, AvroUtils._string()) || AvroUtils.isSameType(basicSchema, AvroUtils._character())) { dteConverter = converters.new StringDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._int()) || AvroUtils.isSameType(basicSchema, AvroUtils._short()) || AvroUtils.isSameType(basicSchema, AvroUtils._byte())) { dteConverter = converters.new IntegerDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._date())) { dteConverter = converters.new DateDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._decimal()) || AvroUtils.isSameType(basicSchema, AvroUtils._double()) || AvroUtils.isSameType(basicSchema, AvroUtils._float())) { dteConverter = converters.new DoubleDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) { dteConverter = converters.new LongDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { dteConverter = converters.new BooleanDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) { dteConverter = converters.new ByteArrayDTEConverter(mappedName); } else { dteConverter = converters.new StringDTEConverter(f, mappedName); } return dteConverter; }
private Schema createTestSchema() { FieldAssembler<Schema> builder = SchemaBuilder.builder().record("TEST").fields(); Schema schema = AvroUtils._int(); schema = wrap(schema); builder = builder.name("ID").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "ID1") .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true").type(schema).noDefault(); schema = AvroUtils._string(); schema = wrap(schema); builder = builder.name("NAME").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "NAME1").type(schema).noDefault(); return builder.endRecord(); }
public static Schema createTestSchema2(String tablename) { FieldAssembler<Schema> builder = SchemaBuilder.builder().record(tablename).fields(); Schema schema = AvroUtils._int(); schema = wrap(schema); builder = builder.name("ID").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "ID") .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true").type(schema).noDefault(); schema = AvroUtils._string(); schema = wrap(schema); builder = builder.name("NAME").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "NAME").type(schema).noDefault(); return builder.endRecord(); }
public Schema getQueueSchema() { return SchemaBuilder.builder().record("Main").fields()// .name(AzureStorageQueueProperties.FIELD_MESSAGE_ID).prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "100").type(AvroUtils._string()).noDefault()// .name(AzureStorageQueueProperties.FIELD_MESSAGE_CONTENT).type(AvroUtils._string()).noDefault() // .name(AzureStorageQueueProperties.FIELD_INSERTION_TIME) .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss").type(AvroUtils._date()).noDefault() // .name(AzureStorageQueueProperties.FIELD_EXPIRATION_TIME) .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss").type(AvroUtils._date()).noDefault() // .name(AzureStorageQueueProperties.FIELD_NEXT_VISIBLE_TIME) .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss").type(AvroUtils._date()).noDefault() // .name(AzureStorageQueueProperties.FIELD_DEQUEUE_COUNT).type(AvroUtils._int()).noDefault() // .name(AzureStorageQueueProperties.FIELD_POP_RECEIPT).type(AvroUtils._string()).noDefault() // .endRecord(); }
public Schema getQueueSchema() { return SchemaBuilder.builder().record("Main").fields()// .name(AzureStorageQueueProperties.FIELD_MESSAGE_ID).prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "100").type(AvroUtils._string()).noDefault()// .name(AzureStorageQueueProperties.FIELD_MESSAGE_CONTENT).type(AvroUtils._string()).noDefault() // .name(AzureStorageQueueProperties.FIELD_INSERTION_TIME) .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss").type(AvroUtils._date()).noDefault() // .name(AzureStorageQueueProperties.FIELD_EXPIRATION_TIME) .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss").type(AvroUtils._date()).noDefault() // .name(AzureStorageQueueProperties.FIELD_NEXT_VISIBLE_TIME) .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss").type(AvroUtils._date()).noDefault() // .name(AzureStorageQueueProperties.FIELD_DEQUEUE_COUNT).type(AvroUtils._int()).noDefault() // .name(AzureStorageQueueProperties.FIELD_POP_RECEIPT).type(AvroUtils._string()).noDefault() // .endRecord(); }
public static Schema createTestSchema5(String tablename) { FieldAssembler<Schema> builder = SchemaBuilder.builder().record(tablename).fields(); Schema schema = AvroUtils._int(); schema = wrap(schema); builder = builder.name("ID").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "ID").type(schema).noDefault(); schema = AvroUtils._string(); schema = wrap(schema); builder = builder.name("NAME").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "NAME").type(schema).noDefault(); schema = AvroUtils._string();// TODO : fix it as should be object type schema = wrap(schema); builder = builder.name("RESULTSET").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "RESULTSET").type(schema) .noDefault(); return builder.endRecord(); }
@Before public void createSchema() { schema = SchemaBuilder.builder() .record("main") .fields() .name("integer_fld") .type(Schema.createUnion(AvroUtils._int(), Schema.create(Schema.Type.NULL))) .withDefault(1) .name("string_fld") .type(Schema.createUnion(AvroUtils._string(), Schema.create(Schema.Type.NULL))) .noDefault() .name("date_fld") .type(Schema.createUnion(AvroUtils._logicalDate(), Schema.create(Schema.Type.NULL))) .noDefault() .endRecord(); }
@Override public void setupProperties() { super.setupProperties(); dieOnError.setValue(true); Schema s = SchemaBuilder.builder().record("Main").fields()// .name(FIELD_MESSAGE_ID).prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "100").type(AvroUtils._string()).noDefault()// .name(FIELD_MESSAGE_CONTENT).type(AvroUtils._string()).noDefault() // .name(FIELD_INSERTION_TIME).prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss") .type(AvroUtils._date()).noDefault() // .name(FIELD_EXPIRATION_TIME).prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss") .type(AvroUtils._date()).noDefault() // .name(FIELD_NEXT_VISIBLE_TIME).prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss") .type(AvroUtils._date()).noDefault() // .name(FIELD_DEQUEUE_COUNT).type(AvroUtils._int()).noDefault() // .name(FIELD_POP_RECEIPT).type(AvroUtils._string()).noDefault() // .endRecord(); schema.schema.setValue(s); numberOfMessages.setValue(32); peekMessages.setValue(true); deleteMessages.setValue(false); visibilityTimeoutInSeconds.setValue(30); }
@Override public void setupProperties() { super.setupProperties(); dieOnError.setValue(true); Schema s = SchemaBuilder.builder().record("Main").fields()// .name(FIELD_MESSAGE_ID).prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "100").type(AvroUtils._string()).noDefault()// .name(FIELD_MESSAGE_CONTENT).type(AvroUtils._string()).noDefault() // .name(FIELD_INSERTION_TIME).prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss") .type(AvroUtils._date()).noDefault() // .name(FIELD_EXPIRATION_TIME).prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss") .type(AvroUtils._date()).noDefault() // .name(FIELD_NEXT_VISIBLE_TIME).prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss") .type(AvroUtils._date()).noDefault() // .name(FIELD_DEQUEUE_COUNT).type(AvroUtils._int()).noDefault() // .name(FIELD_POP_RECEIPT).type(AvroUtils._string()).noDefault() // .endRecord(); schema.schema.setValue(s); numberOfMessages.setValue(32); peekMessages.setValue(true); deleteMessages.setValue(false); visibilityTimeoutInSeconds.setValue(30); }