return toBigInt(value, fieldName); case BOOLEAN: return toBoolean(value, fieldName); case BYTE: return toByte(value, fieldName); case CHAR: return toCharacter(value, fieldName); case DATE: return toDate(value, dateFormat, fieldName); case DOUBLE: return toDouble(value, fieldName); case FLOAT: return toFloat(value, fieldName); case INT: return toInteger(value, fieldName); case LONG: return toLong(value, fieldName); case SHORT: return toShort(value, fieldName); case STRING: return toString(value, () -> getDateFormat(dataType.getFieldType(), dateFormat, timeFormat, timestampFormat), charset); case TIME: return toTime(value, timeFormat, fieldName); case TIMESTAMP: return toTimestamp(value, timestampFormat, fieldName); case ARRAY: return toArray(value, fieldName, ((ArrayDataType)dataType).getElementType(), charset); case MAP: return toMap(value, fieldName);
public static Map<String, Object> convertRecordMapToJavaMap(final Map<String, Object> map, DataType valueDataType) { if (map == null) { return null; } Map<String, Object> resultMap = new LinkedHashMap<>(); for (Map.Entry<String, Object> entry : map.entrySet()) { resultMap.put(entry.getKey(), convertRecordFieldtoObject(entry.getValue(), valueDataType)); } return resultMap; }
public static boolean isCompatibleDataType(final Object value, final DataType dataType) { switch (dataType.getFieldType()) { case ARRAY: return isArrayTypeCompatible(value, ((ArrayDataType) dataType).getElementType()); case BIGINT: return isBigIntTypeCompatible(value); case BOOLEAN: return isBooleanTypeCompatible(value); case BYTE: return isByteTypeCompatible(value); case CHAR: return isCharacterTypeCompatible(value); case DATE: return isDateTypeCompatible(value, dataType.getFormat()); case DOUBLE: return isDoubleTypeCompatible(value); case FLOAT: return isFloatTypeCompatible(value); case INT: return isIntegerTypeCompatible(value); case LONG: return isLongTypeCompatible(value); case RECORD: { if (value == null) { if (!isCompatibleDataType(childValue, childField.getDataType())) { return false; return isShortTypeCompatible(value); case TIME:
public static Object[] convertRecordArrayToJavaArray(final Object[] array, DataType elementDataType) { if (array == null || array.length == 0 || isScalarValue(elementDataType, array[0])) { return array; } else { // Must be an array of complex types, build an array of converted values Object[] resultArray = new Object[array.length]; for (int i = 0; i < array.length; i++) { resultArray[i] = convertRecordFieldtoObject(array[i], elementDataType); } return resultArray; } }
public static boolean isDateTypeCompatible(final Object value, final String format) { if (value == null) { return false; } if (value instanceof java.util.Date || value instanceof Number) { return true; } if (value instanceof String) { if (format == null) { return isInteger((String) value); } try { getDateFormat(format).parse((String) value); return true; } catch (final ParseException e) { return false; } } return false; }
@Override public Date getAsDate(final String fieldName, final String format) { return DataTypeUtils.toDate(getValue(fieldName), () -> DataTypeUtils.getDateFormat(format), fieldName); }
final DataType chosenDataType = dataType.getFieldType() == RecordFieldType.CHOICE ? DataTypeUtils.chooseDataType(value, (ChoiceDataType) dataType) : dataType; final Object coercedValue = DataTypeUtils.convertType(value, chosenDataType, fieldName); if (coercedValue == null) { generator.writeNull(); final String stringValue = DataTypeUtils.toString(coercedValue, () -> DataTypeUtils.getDateFormat(this.dateFormat)); if (DataTypeUtils.isLongTypeCompatible(stringValue)) { generator.writeNumber(DataTypeUtils.toLong(coercedValue, fieldName)); } else { generator.writeString(stringValue); final String stringValue = DataTypeUtils.toString(coercedValue, () -> DataTypeUtils.getDateFormat(this.timeFormat)); if (DataTypeUtils.isLongTypeCompatible(stringValue)) { generator.writeNumber(DataTypeUtils.toLong(coercedValue, fieldName)); } else { generator.writeString(stringValue); final String stringValue = DataTypeUtils.toString(coercedValue, () -> DataTypeUtils.getDateFormat(this.timestampFormat)); if (DataTypeUtils.isLongTypeCompatible(stringValue)) { generator.writeNumber(DataTypeUtils.toLong(coercedValue, fieldName)); } else { generator.writeString(stringValue); generator.writeNumber(DataTypeUtils.toDouble(coercedValue, fieldName)); break; case FLOAT: generator.writeNumber(DataTypeUtils.toFloat(coercedValue, fieldName)); break; case LONG:
final LogicalType logicalType = fieldSchema.getLogicalType(); if (logicalType == null) { return DataTypeUtils.toInteger(rawValue, fieldName); final Date date = DataTypeUtils.toDate(rawValue, () -> DataTypeUtils.getDateFormat(format), fieldName); final Duration duration = Duration.between(new Date(0L).toInstant(), new Date(date.getTime()).toInstant()); final long days = duration.toDays(); } else if (LOGICAL_TYPE_TIME_MILLIS.equals(logicalType.getName())) { final String format = AvroTypeUtil.determineDataType(fieldSchema).getFormat(); final Time time = DataTypeUtils.toTime(rawValue, () -> DataTypeUtils.getDateFormat(format), fieldName); final Date date = new Date(time.getTime()); final Duration duration = Duration.between(date.toInstant().truncatedTo(ChronoUnit.DAYS), date.toInstant()); return DataTypeUtils.toInteger(rawValue, fieldName); return DataTypeUtils.toLong(rawValue, fieldName); } else if (LOGICAL_TYPE_TIMESTAMP_MILLIS.equals(logicalType.getName())) { final String format = AvroTypeUtil.determineDataType(fieldSchema).getFormat(); Timestamp t = DataTypeUtils.toTimestamp(rawValue, () -> DataTypeUtils.getDateFormat(format), fieldName); return getLongFromTimestamp(rawValue, fieldSchema, fieldName); } else if (LOGICAL_TYPE_TIMESTAMP_MICROS.equals(logicalType.getName())) { return DataTypeUtils.toLong(rawValue, fieldName); return DataTypeUtils.toBoolean(rawValue, fieldName); case DOUBLE: return DataTypeUtils.toDouble(rawValue, fieldName);
private String convertToString(final Object value, final String format) { if (value == null) { return null; } return DataTypeUtils.toString(value, format); }
final boolean lhsLongCompatible = DataTypeUtils.isLongTypeCompatible(value); final boolean lhsDoubleCompatible; if (lhsLongCompatible) { lhsNumeric = true; } else { lhsDoubleCompatible = DataTypeUtils.isDoubleTypeCompatible(value); lhsNumeric = lhsDoubleCompatible; final boolean rhsLongCompatible = DataTypeUtils.isLongTypeCompatible(rhsValue); final boolean rhsDoubleCompatible; if (rhsLongCompatible) { rhsNumeric = true; } else { rhsDoubleCompatible = DataTypeUtils.isDoubleTypeCompatible(rhsValue); rhsNumeric = rhsDoubleCompatible; final Number lhsNumber = lhsLongCompatible ? DataTypeUtils.toLong(value, fieldName) : DataTypeUtils.toDouble(value, fieldName); final Number rhsNumber = rhsLongCompatible ? DataTypeUtils.toLong(rhsValue, fieldName) : DataTypeUtils.toDouble(rhsValue, fieldName); return compare(lhsNumber, rhsNumber);
public JsonTreeRowRecordReader(final InputStream in, final ComponentLog logger, final RecordSchema schema, final String dateFormat, final String timeFormat, final String timestampFormat) throws IOException, MalformedRecordException { super(in, logger); this.schema = schema; final DateFormat df = dateFormat == null ? null : DataTypeUtils.getDateFormat(dateFormat); final DateFormat tf = timeFormat == null ? null : DataTypeUtils.getDateFormat(timeFormat); final DateFormat tsf = timestampFormat == null ? null : DataTypeUtils.getDateFormat(timestampFormat); LAZY_DATE_FORMAT = () -> df; LAZY_TIME_FORMAT = () -> tf; LAZY_TIMESTAMP_FORMAT = () -> tsf; }
public static Object convertType(final Object value, final DataType dataType, final Supplier<DateFormat> dateFormat, final Supplier<DateFormat> timeFormat, final Supplier<DateFormat> timestampFormat, final String fieldName) { return convertType(value, dataType, dateFormat, timeFormat, timestampFormat, fieldName, StandardCharsets.UTF_8); }
case CHAR: case SHORT: if (DataTypeUtils.isCompatibleDataType(trimmed, dataType)) { return DataTypeUtils.convertType(trimmed, dataType, LAZY_DATE_FORMAT, LAZY_TIME_FORMAT, LAZY_TIMESTAMP_FORMAT, fieldName); if (DataTypeUtils.isDateTypeCompatible(trimmed, dateFormat)) { return DataTypeUtils.convertType(trimmed, dataType, LAZY_DATE_FORMAT, LAZY_TIME_FORMAT, LAZY_TIMESTAMP_FORMAT, fieldName); if (DataTypeUtils.isTimeTypeCompatible(trimmed, timeFormat)) { return DataTypeUtils.convertType(trimmed, dataType, LAZY_DATE_FORMAT, LAZY_TIME_FORMAT, LAZY_TIMESTAMP_FORMAT, fieldName); if (DataTypeUtils.isTimestampTypeCompatible(trimmed, timestampFormat)) { return DataTypeUtils.convertType(trimmed, dataType, LAZY_DATE_FORMAT, LAZY_TIME_FORMAT, LAZY_TIMESTAMP_FORMAT, fieldName);
final String stringValue = DataTypeUtils.toString(coercedValue, LAZY_DATE_FORMAT); writer.writeCharacters(stringValue); writer.writeEndElement(); final String stringValue = DataTypeUtils.toString(coercedValue, LAZY_TIME_FORMAT); writer.writeCharacters(stringValue); writer.writeEndElement(); final String stringValue = DataTypeUtils.toString(coercedValue, LAZY_TIMESTAMP_FORMAT); writer.writeCharacters(stringValue); writer.writeEndElement(); for (Object element : arrayValues) { final DataType chosenDataType = elementType.getFieldType() == RecordFieldType.CHOICE ? DataTypeUtils.chooseDataType(element, (ChoiceDataType) elementType) : elementType; final Object coercedElement = DataTypeUtils.convertType(element, chosenDataType, LAZY_DATE_FORMAT, LAZY_TIME_FORMAT, LAZY_TIMESTAMP_FORMAT, elementName); final DataType chosenDataType = valueDataType.getFieldType() == RecordFieldType.CHOICE ? DataTypeUtils.chooseDataType(entry.getValue(), (ChoiceDataType) valueDataType) : valueDataType; final Object coercedElement = DataTypeUtils.convertType(entry.getValue(), chosenDataType, LAZY_DATE_FORMAT, LAZY_TIME_FORMAT, LAZY_TIMESTAMP_FORMAT, key);
private static Long getLongFromTimestamp(final Object rawValue, final Schema fieldSchema, final String fieldName) { final String format = AvroTypeUtil.determineDataType(fieldSchema).getFormat(); Timestamp t = DataTypeUtils.toTimestamp(rawValue, () -> DataTypeUtils.getDateFormat(format), fieldName); return t.getTime(); }
private static LocalDate getLocalDateFromEpochTime(String fieldName, Object coercedValue) { Long date = DataTypeUtils.toLong(coercedValue, fieldName); return Instant.ofEpochMilli(date).atZone(ZoneId.systemDefault()).toLocalDate(); }
public static DataType chooseDataType(final Object value, final ChoiceDataType choiceType) { for (final DataType subType : choiceType.getPossibleSubTypes()) { if (isCompatibleDataType(value, subType)) { if (subType.getFieldType() == RecordFieldType.CHOICE) { return chooseDataType(value, (ChoiceDataType) subType); } return subType; } } return null; }
private boolean iterateThroughRecordUsingSchema(Deque<String> tagsToOpen, Record record, RecordSchema schema) throws XMLStreamException { boolean loopHasWritten = false; for (RecordField field : schema.getFields()) { String fieldName = field.getFieldName(); DataType dataType = field.getDataType(); Object value = record.getValue(field); final DataType chosenDataType = dataType.getFieldType() == RecordFieldType.CHOICE ? DataTypeUtils.chooseDataType(value, (ChoiceDataType) dataType) : dataType; final Object coercedValue = DataTypeUtils.convertType(value, chosenDataType, LAZY_DATE_FORMAT, LAZY_TIME_FORMAT, LAZY_TIMESTAMP_FORMAT, fieldName); if (coercedValue != null) { boolean hasWritten = writeFieldForType(tagsToOpen, coercedValue, chosenDataType, fieldName); if (hasWritten) { loopHasWritten = true; } } else { if (nullSuppression.equals(NullSuppression.NEVER_SUPPRESS) || nullSuppression.equals(NullSuppression.SUPPRESS_MISSING) && recordHasField(field, record)) { writeAllTags(tagsToOpen, fieldName); writer.writeEndElement(); loopHasWritten = true; } } } return loopHasWritten; }
private OptionalInt getIndex(final RecordPathSegment indexSegment, final RecordPathEvaluationContext context) { final Optional<FieldValue> firstFieldValueOption = indexSegment.evaluate(context).findFirst(); if (!firstFieldValueOption.isPresent()) { return OptionalInt.empty(); } final FieldValue fieldValue = firstFieldValueOption.get(); final Object indexObject = fieldValue.getValue(); if (!DataTypeUtils.isIntegerTypeCompatible(indexObject)) { return OptionalInt.empty(); } final String fieldName; final RecordField field = fieldValue.getField(); fieldName = field == null ? "<Unknown Field>" : field.getFieldName(); return OptionalInt.of(DataTypeUtils.toInteger(indexObject, fieldName)); } }
if (fieldValue == null) { recordMap.put(fieldName, null); } else if (isScalarValue(fieldDataType, fieldValue)) { recordMap.put(fieldName, fieldValue); recordMap.put(fieldName, convertRecordFieldtoObject(nestedRecord, fieldDataType)); recordMap.put(fieldName, convertRecordMapToJavaMap((Map) fieldValue, ((MapDataType)fieldDataType).getValueType())); recordMap.put(fieldName, convertRecordArrayToJavaArray((Object[])fieldValue, ((ArrayDataType) fieldDataType).getElementType())); } else { throw new IllegalTypeConversionException("Cannot convert value [" + fieldValue + "] of type " + fieldDataType.toString() return convertRecordMapToJavaMap((Map)value, ((MapDataType)dataType).getValueType()); } else if (dataType != null && isScalarValue(dataType, value)) { return value;