private void setupMainAndFlowSchemas() { Schema stringSchema = AvroUtils._string(); Schema objectSchema = AvroUtils._bytes(); // Talend type should be Object (but avro type is bytes[]) objectSchema.addProp(SchemaConstants.JAVA_CLASS_FLAG, "java.lang.Object"); Schema.Field docIdField = new Schema.Field("docId", stringSchema, null, (Object) null, Schema.Field.Order.ASCENDING); docIdField.addProp(SchemaConstants.TALEND_COLUMN_IS_KEY, "true"); Schema.Field docContentField = new Schema.Field("docContent", objectSchema, null, (Object) null, Schema.Field.Order.IGNORE); List<Schema.Field> fields = new ArrayList<>(); fields.add(docIdField); fields.add(docContentField); Schema initialSchema = Schema.createRecord("marklogic", null, null, false, fields); initialSchema.addProp(TALEND_IS_LOCKED, "true"); fields.clear(); // Do we need this? datasetProperties.main.schema.setValue(initialSchema); schemaFlow.schema.setValue(initialSchema); }
/** * Checks {@link TalendType#convertFromAvro(Schema)} converts bytes avro type to "id_byte[]" di type */ @Test public void testConvertFromAvroBytes() { TalendType expectedType = TalendType.BYTES; Schema fieldSchema = AvroUtils._bytes(); assertEquals(expectedType, TalendType.convertFromAvro(fieldSchema)); }
/** * A helper method to convert the String representation of a datum in the Salesforce system to the Avro type that * matches the Schema generated for it. * * @param f * @return */ public AvroConverter<String, ?> getConverterFromString(org.apache.avro.Schema.Field f) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); // FIXME use avro type to decide the converter is not correct if the user change the avro type, Date to String // for instance if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new StringToBooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) { return new StringToDecimalConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return new StringToDoubleConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._int())) { return new StringToIntegerConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new StringToDateConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._bytes())) { return new StringToBytesConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) { return super.getConverter(String.class); } throw new UnsupportedOperationException("The type " + fieldSchema.getType() + " is not supported."); //$NON-NLS-1$ //$NON-NLS-2$ }
break; case base64: base = AvroUtils._bytes(); break; default:
break; case base64: base = AvroUtils._bytes(); break; default:
break; case base64: base = AvroUtils._bytes(); break; default:
} else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new StringToDateConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._bytes())) { return new StringToBytesConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) {
} else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new StringToDateConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._bytes())) { return new StringToBytesConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) {
public AvroConverter<String, ?> getConverter(Schema.Field f, FileDelimitedProperties properties) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new BooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) { return new DecimalConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return new DoubleConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._float())) { return new FloatConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._int())) { return new IntegerConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new DateConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._long())) { return new LongConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._bytes())) { return new BytesConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._byte())) { return new ByteConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._short())) { return new ShortConverter(f, properties); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._character())) { return new CharacterConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) { return super.getConverter(String.class); } throw new UnsupportedOperationException("The type " + fieldSchema.getType() + " is not supported."); //$NON-NLS-1$ //$NON-NLS-2$ }
public static Schema getEventSchema() { Schema.Field[] fields = new Schema.Field[10]; Schema.Field field; fields[EventSchemaField.EVENT_IDX] = new Schema.Field("event", AvroUtils._string(), "Type of event", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.PARTITION_IDX] = new Schema.Field("partition", AvroUtils._short(), "Partition number", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.KEY_IDX] = new Schema.Field("key", AvroUtils._string(), "Key", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.CAS_IDX] = new Schema.Field("cas", AvroUtils._long(), "CAS", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.SEQNO_IDX] = new Schema.Field("bySeqno", AvroUtils._long(), "Sequence number", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.REV_SEQNO_IDX] = new Schema.Field("revSeqno", AvroUtils._long(), "Revision sequence number", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.EXPIRATION_IDX] = new Schema.Field("expiration", AvroUtils.wrapAsNullable(AvroUtils._int()), "Expiration", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.FLAGS_IDX] = new Schema.Field("flags", AvroUtils.wrapAsNullable(AvroUtils._int()), "Flags", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.LOCK_TIME_IDX] = new Schema.Field("lockTime", AvroUtils.wrapAsNullable(AvroUtils._int()), "Lock time", (Object) null, Schema.Field.Order.ASCENDING); field = new Schema.Field("content", AvroUtils.wrapAsNullable(AvroUtils._bytes()), "Content", (Object) null, Schema.Field.Order.ASCENDING); field.addProp(TALEND_IS_LOCKED, "false"); fields[EventSchemaField.CONTENT_IDX] = field; Schema schema = Schema.createRecord("DcpMessage", "Couchbase DCP message", "com.couchbase", false, Arrays.asList(fields)); schema.addProp(TALEND_IS_LOCKED, "true"); return schema; }
} else if (AvroUtils.isSameType(basicSchema, AvroUtils._byte())) { writer = new ByteTypeWriter(statement, statementIndex, inputValueLocation); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) { writer = new BytesTypeWriter(statement, statementIndex, inputValueLocation); } else {
} else if (AvroUtils.isSameType(basicSchema, AvroUtils._byte())) { writer = new ByteTypeWriter(statement, statementIndex, inputValueLocation); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) { writer = new BytesTypeWriter(statement, statementIndex, inputValueLocation); } else {
return BigQueryType.BOOLEAN.toString(); if (AvroUtils.isSameType(schema, AvroUtils._bytes())) { return BigQueryType.BYTES.toString();
public static DTEConverter createConverter(final Field f, final String mappedName) { Schema basicSchema = AvroUtils.unwrapIfNullable(f.schema()); AzureStorageDTEConverters converters = new AzureStorageDTEConverters(); DTEConverter dteConverter; if (AvroUtils.isSameType(basicSchema, AvroUtils._string()) || AvroUtils.isSameType(basicSchema, AvroUtils._character())) { dteConverter = converters.new StringDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._int()) || AvroUtils.isSameType(basicSchema, AvroUtils._short()) || AvroUtils.isSameType(basicSchema, AvroUtils._byte())) { dteConverter = converters.new IntegerDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._date())) { dteConverter = converters.new DateDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._decimal()) || AvroUtils.isSameType(basicSchema, AvroUtils._double()) || AvroUtils.isSameType(basicSchema, AvroUtils._float())) { dteConverter = converters.new DoubleDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) { dteConverter = converters.new LongDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { dteConverter = converters.new BooleanDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) { dteConverter = converters.new ByteArrayDTEConverter(mappedName); } else { dteConverter = converters.new StringDTEConverter(f, mappedName); } return dteConverter; }
public static DTEConverter createConverter(final Field f, final String mappedName) { Schema basicSchema = AvroUtils.unwrapIfNullable(f.schema()); AzureStorageDTEConverters converters = new AzureStorageDTEConverters(); DTEConverter dteConverter; if (AvroUtils.isSameType(basicSchema, AvroUtils._string()) || AvroUtils.isSameType(basicSchema, AvroUtils._character())) { dteConverter = converters.new StringDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._int()) || AvroUtils.isSameType(basicSchema, AvroUtils._short()) || AvroUtils.isSameType(basicSchema, AvroUtils._byte())) { dteConverter = converters.new IntegerDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._date())) { dteConverter = converters.new DateDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._decimal()) || AvroUtils.isSameType(basicSchema, AvroUtils._double()) || AvroUtils.isSameType(basicSchema, AvroUtils._float())) { dteConverter = converters.new DoubleDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) { dteConverter = converters.new LongDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { dteConverter = converters.new BooleanDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) { dteConverter = converters.new ByteArrayDTEConverter(mappedName); } else { dteConverter = converters.new StringDTEConverter(f, mappedName); } return dteConverter; }
public void setupProperties() { super.setupProperties(); Schema schema = SchemaBuilder.builder().record(GoogleDriveGetDefinition.COMPONENT_NAME).fields() // .name(GoogleDriveGetDefinition.RETURN_CONTENT).type(AvroUtils._bytes()).noDefault() // .endRecord(); schema.addProp(SchemaConstants.TALEND_IS_LOCKED, "true"); schemaMain.schema.setValue(schema); fileAccessMethod.setPossibleValues(AccessMethod.values()); fileAccessMethod.setValue(AccessMethod.Name); file.setValue(""); storeToLocal.setValue(false); outputFileName.setValue(""); // exportDocument.setPossibleValues(HTML, HTML_ZIPPED, TEXT, RTF, OO_DOCUMENT, PDF, WORD, EPUB); exportDocument.setValue(WORD); exportDrawing.setPossibleValues(JPG, PNG, SVG, PDF); exportDrawing.setValue(PNG); exportPresentation.setPossibleValues(POWERPOINT, OO_PRESENTATION, PDF, TEXT); exportPresentation.setValue(PDF); exportSpreadsheet.setPossibleValues(EXCEL, OO_SPREADSHEET, PDF, CSV, CSV_TAB, HTML_ZIPPED); exportSpreadsheet.setValue(EXCEL); setOutputExt.setValue(false); }
public void setupProperties() { super.setupProperties(); Schema schema = SchemaBuilder.builder().record(GoogleDriveGetDefinition.COMPONENT_NAME).fields() // .name(GoogleDriveGetDefinition.RETURN_CONTENT).type(AvroUtils._bytes()).noDefault() // .endRecord(); schema.addProp(SchemaConstants.TALEND_IS_LOCKED, "true"); schemaMain.schema.setValue(schema); fileAccessMethod.setPossibleValues(AccessMethod.values()); fileAccessMethod.setValue(AccessMethod.Name); file.setValue(""); storeToLocal.setValue(false); outputFileName.setValue(""); // exportDocument.setPossibleValues(HTML, HTML_ZIPPED, TEXT, RTF, OO_DOCUMENT, PDF, WORD, EPUB); exportDocument.setValue(WORD); exportDrawing.setPossibleValues(JPG, PNG, SVG, PDF); exportDrawing.setValue(PNG); exportPresentation.setPossibleValues(POWERPOINT, OO_PRESENTATION, PDF, TEXT); exportPresentation.setValue(PDF); exportSpreadsheet.setPossibleValues(EXCEL, OO_SPREADSHEET, PDF, CSV, CSV_TAB, HTML_ZIPPED); exportSpreadsheet.setValue(EXCEL); setOutputExt.setValue(false); }
public void setupProperties() { super.setupProperties(); Schema schema = SchemaBuilder.builder().record(GoogleDrivePutDefinition.COMPONENT_NAME).fields() // .name(GoogleDrivePutDefinition.RETURN_CONTENT).type(AvroUtils._bytes()).noDefault() // .name(GoogleDrivePutDefinition.RETURN_PARENT_FOLDER_ID).type().nullable().stringType().noDefault()// .name(GoogleDrivePutDefinition.RETURN_FILE_ID).type().nullable().stringType().noDefault()// .endRecord(); schema.addProp(SchemaConstants.TALEND_IS_LOCKED, "true"); schemaMain.schema.setValue(schema); fileName.setValue(""); destinationFolderAccessMethod.setPossibleValues(AccessMethod.values()); destinationFolderAccessMethod.setValue(AccessMethod.Name); destinationFolder.setValue(""); uploadMode.setPossibleValues(UploadMode.values()); uploadMode.setValue(UploadMode.READ_CONTENT_FROM_INPUT); localFilePath.setValue(""); overwrite.setValue(false); }
public void setupProperties() { super.setupProperties(); Schema schema = SchemaBuilder.builder().record(GoogleDrivePutDefinition.COMPONENT_NAME).fields() // .name(GoogleDrivePutDefinition.RETURN_CONTENT).type(AvroUtils._bytes()).noDefault() // .name(GoogleDrivePutDefinition.RETURN_PARENT_FOLDER_ID).type().nullable().stringType().noDefault()// .name(GoogleDrivePutDefinition.RETURN_FILE_ID).type().nullable().stringType().noDefault()// .endRecord(); schema.addProp(SchemaConstants.TALEND_IS_LOCKED, "true"); schemaMain.schema.setValue(schema); fileName.setValue(""); destinationFolderAccessMethod.setPossibleValues(AccessMethod.values()); destinationFolderAccessMethod.setValue(AccessMethod.Name); destinationFolder.setValue(""); uploadMode.setPossibleValues(UploadMode.values()); uploadMode.setValue(UploadMode.READ_CONTENT_FROM_INPUT); localFilePath.setValue(""); overwrite.setValue(false); }
public static Schema getBasicSchema(String pattern) { return SchemaBuilder.builder().record("Schema").fields() // .name("TestBoolean").type().booleanType().noDefault() // .name("TestByte").type(AvroUtils._byte()).noDefault() // .name("TestBytes").type(AvroUtils._bytes()).noDefault() // .name("TestChar").type(AvroUtils._character()).noDefault() // .name("TestDate").prop(SchemaConstants.TALEND_COLUMN_PATTERN, pattern)// .type(AvroUtils._date()).noDefault() // .name("TestDouble").type().doubleType().noDefault() // .name("TestFloat").type().floatType().noDefault() // .name("TestBigDecimal").type(AvroUtils._decimal()).noDefault()// .name("TestInteger").type().intType().noDefault() // .name("TestLong").type().longType().noDefault() // .name("TestObject").type(AvroUtils._bytes()).noDefault().endRecord(); }