public void addSchemaField(final String fieldName, final RecordFieldType type, boolean isNullable) { fields.add(new RecordField(fieldName, type.getDataType(), isNullable)); }
static RecordSchema createRecordSchema(final Grok grok) { final List<RecordField> fields = new ArrayList<>(); String grokExpression = grok.getOriginalGrokPattern(); populateSchemaFieldNames(grok, grokExpression, fields); fields.add(new RecordField(GrokRecordReader.STACK_TRACE_COLUMN_NAME, RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(GrokRecordReader.RAW_MESSAGE_NAME, RecordFieldType.STRING.getDataType(), true)); final RecordSchema schema = new SimpleRecordSchema(fields); return schema; }
private List<RecordField> getRecordFields() { if (this.recordFields != null) { return this.recordFields; } // Use a SortedMap keyed by index of the field so that we can get a List of field names in the correct order final SortedMap<Integer, String> sortedMap = new TreeMap<>(); for (final Map.Entry<String, Integer> entry : csvParser.getHeaderMap().entrySet()) { sortedMap.put(entry.getValue(), entry.getKey()); } final List<RecordField> fields = new ArrayList<>(); final List<String> rawFieldNames = new ArrayList<>(sortedMap.values()); for (final String rawFieldName : rawFieldNames) { final Optional<RecordField> option = schema.getField(rawFieldName); if (option.isPresent()) { fields.add(option.get()); } else { fields.add(new RecordField(rawFieldName, RecordFieldType.STRING.getDataType())); } } this.recordFields = fields; return fields; }
@Override public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) { Stream<FieldValue> concatenated = Stream.empty(); for (final RecordPathSegment valuePath : valuePaths) { final Stream<FieldValue> stream = valuePath.evaluate(context); concatenated = Stream.concat(concatenated, stream); } final StringBuilder sb = new StringBuilder(); concatenated.forEach(fv -> sb.append(DataTypeUtils.toString(fv.getValue(), (String) null))); final RecordField field = new RecordField("concat", RecordFieldType.STRING.getDataType()); final FieldValue responseValue = new StandardFieldValue(sb.toString(), field, null); return Stream.of(responseValue); }
static RecordSchema createRecordSchema() { final List<RecordField> fields = new ArrayList<>(); fields.add(new RecordField(SyslogAttributes.PRIORITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.SEVERITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.FACILITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.VERSION.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.TIMESTAMP.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.HOSTNAME.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.BODY.key(), RecordFieldType.STRING.getDataType(), true)); SchemaIdentifier schemaIdentifier = new StandardSchemaIdentifier.Builder().name(GENERIC_SYSLOG_SCHEMA_NAME).build(); final RecordSchema schema = new SimpleRecordSchema(fields,schemaIdentifier); return schema; }
private static RecordSchema createSchema(final ResultSet rs, final RecordSchema readerSchema) throws SQLException { final ResultSetMetaData metadata = rs.getMetaData(); final int numCols = metadata.getColumnCount(); final List<RecordField> fields = new ArrayList<>(numCols); for (int i = 0; i < numCols; i++) { final int column = i + 1; final int sqlType = metadata.getColumnType(column); final DataType dataType = getDataType(sqlType, rs, column, readerSchema); final String fieldName = metadata.getColumnLabel(column); final int nullableFlag = metadata.isNullable(column); final boolean nullable; if (nullableFlag == ResultSetMetaData.columnNoNulls) { nullable = false; } else { nullable = true; } final RecordField field = new RecordField(fieldName, dataType, nullable); fields.add(field); } return new SimpleRecordSchema(fields); }
static RecordSchema createRecordSchema() { final List<RecordField> fields = new ArrayList<>(); fields.add(new RecordField(SyslogAttributes.PRIORITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.SEVERITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.FACILITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.VERSION.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.TIMESTAMP.key(), RecordFieldType.TIMESTAMP.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.HOSTNAME.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.BODY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(Syslog5424Attributes.APP_NAME.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(Syslog5424Attributes.PROCID.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(Syslog5424Attributes.MESSAGEID.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(Syslog5424Attributes.STRUCTURED_BASE.key(), RecordFieldType.MAP.getMapDataType(RecordFieldType.MAP.getMapDataType(RecordFieldType.STRING.getDataType())))); SchemaIdentifier schemaIdentifier = new StandardSchemaIdentifier.Builder().name(RFC_5424_SCHEMA_NAME).build(); final RecordSchema schema = new SimpleRecordSchema(fields,schemaIdentifier); return schema; }
/** * Adds fields to <tt>recordFields</tt> list. * @param recordFields - record fields are added to this list. * @param field - the field * @param fieldName - field name * @param fieldSchema - field schema * @param dataType - data type * @param nullable - is nullable? */ private static void addFieldToList(final List<RecordField> recordFields, final Field field, final String fieldName, final Schema fieldSchema, final DataType dataType, final boolean nullable) { if (field.defaultVal() == JsonProperties.NULL_VALUE) { recordFields.add(new RecordField(fieldName, dataType, field.aliases(), nullable)); } else { Object defaultValue = field.defaultVal(); if (fieldSchema.getType() == Schema.Type.ARRAY && !DataTypeUtils.isArrayTypeCompatible(defaultValue, ((ArrayDataType) dataType).getElementType())) { defaultValue = defaultValue instanceof List ? ((List<?>) defaultValue).toArray() : new Object[0]; } recordFields.add(new RecordField(fieldName, dataType, defaultValue, field.aliases(), nullable)); } }
private List<RecordField> getDefaultFields() { final List<RecordField> fields = new ArrayList<>(); fields.add(new RecordField("id", RecordFieldType.INT.getDataType())); fields.add(new RecordField("name", RecordFieldType.STRING.getDataType())); fields.add(new RecordField("address", RecordFieldType.STRING.getDataType())); fields.add(new RecordField("city", RecordFieldType.STRING.getDataType())); fields.add(new RecordField("state", RecordFieldType.STRING.getDataType())); fields.add(new RecordField("zipCode", RecordFieldType.STRING.getDataType())); fields.add(new RecordField("country", RecordFieldType.STRING.getDataType())); return fields; }
private RecordSchema getTransactionSchema() { final List<RecordField> transactionFields = new ArrayList<>(); transactionFields.add(new RecordField("id", RecordFieldType.INT.getDataType())); transactionFields.add(new RecordField("amount", RecordFieldType.DOUBLE.getDataType())); return new SimpleRecordSchema(transactionFields); }
private RecordSchema getAccountSchema() { final List<RecordField> accountFields = new ArrayList<>(); accountFields.add(new RecordField("id", RecordFieldType.INT.getDataType())); accountFields.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType())); return new SimpleRecordSchema(accountFields); }
@Override public RecordSchema getSchema(Map<String, String> variables, RecordSchema readSchema) throws SchemaNotFoundException, IOException { final List<RecordField> recordFields = columnNames.stream() .map(name -> new RecordField(name, RecordFieldType.STRING.getDataType())) .collect(Collectors.toList()); return new SimpleRecordSchema(recordFields); }
@Override public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) { final RecordField field = new RecordField("root", RecordFieldType.RECORD.getRecordDataType(context.getRecord().getSchema())); final FieldValue fieldValue = new StandardFieldValue(context.getRecord(), field, null); return Stream.of(fieldValue); } }
private RecordSchema getAccountWithTransactionSchema() { final List<RecordField> accountFields = new ArrayList<>(); accountFields.add(new RecordField("id", RecordFieldType.INT.getDataType())); accountFields.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType())); final DataType transactionRecordType = RecordFieldType.RECORD.getRecordDataType(getTransactionSchema()); final DataType transactionsType = RecordFieldType.ARRAY.getArrayDataType(transactionRecordType); accountFields.add(new RecordField("transactions", transactionsType)); return new SimpleRecordSchema(accountFields); }
private FieldValue missingChild(final FieldValue parent) { final RecordField field = new RecordField(childName, RecordFieldType.CHOICE.getChoiceDataType(RecordFieldType.STRING.getDataType(), RecordFieldType.RECORD.getDataType())); return new StandardFieldValue(null, field, parent); }
@Override @SuppressWarnings("unchecked") public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) { final Stream<FieldValue> parentResult = getParentPath().evaluate(context); return parentResult .filter(Filters.fieldTypeFilter(RecordFieldType.MAP)) .flatMap(fieldValue -> { final Map<String, ?> map = (Map<String, ?>) fieldValue.getValue(); return mapKeys.stream().map(key -> { final DataType valueType = ((MapDataType) fieldValue.getField().getDataType()).getValueType(); final RecordField elementField = new RecordField(fieldValue.getField().getFieldName(), valueType); return new MapEntryFieldValue(map.get(key), elementField, fieldValue, key); }); }); } }
@Override public RecordSchema getSchema(Map<String, String> variables, final InputStream contentStream, final RecordSchema readSchema) throws SchemaNotFoundException { if (this.context == null) { throw new SchemaNotFoundException("Schema Access Strategy intended only for validation purposes and cannot obtain schema"); } try { final CSVFormat csvFormat = CSVUtils.createCSVFormat(context).withFirstRecordAsHeader(); try (final Reader reader = new InputStreamReader(new BOMInputStream(contentStream)); final CSVParser csvParser = new CSVParser(reader, csvFormat)) { final List<RecordField> fields = new ArrayList<>(); for (final String columnName : csvParser.getHeaderMap().keySet()) { fields.add(new RecordField(columnName, RecordFieldType.STRING.getDataType(), true)); } return new SimpleRecordSchema(fields); } } catch (final Exception e) { throw new SchemaNotFoundException("Failed to read Header line from CSV", e); } }
@Override public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) { final Stream<FieldValue> parentResult = getParentPath().evaluate(context); return parentResult .filter(Filters.fieldTypeFilter(RecordFieldType.MAP)) .map(fieldValue -> { final DataType valueType = ((MapDataType) fieldValue.getField().getDataType()).getValueType(); final RecordField elementField = new RecordField(fieldValue.getField().getFieldName(), valueType); return new MapEntryFieldValue(getMapValue(fieldValue), elementField, fieldValue, mapKey); }); }
@Override public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) { final Stream<FieldValue> parentResult = getParentPath().evaluate(context); return parentResult .filter(Filters.fieldTypeFilter(RecordFieldType.ARRAY)) .filter(fieldValue -> fieldValue.getValue() != null && ((Object[]) fieldValue.getValue()).length > getArrayIndex(((Object[]) fieldValue.getValue()).length)) .map(fieldValue -> { final ArrayDataType arrayDataType = (ArrayDataType) fieldValue.getField().getDataType(); final DataType elementDataType = arrayDataType.getElementType(); final RecordField arrayField = new RecordField(fieldValue.getField().getFieldName(), elementDataType); final Object[] values = (Object[]) fieldValue.getValue(); final int arrayIndex = getArrayIndex(values.length); final RecordField elementField = new RecordField(arrayField.getFieldName(), elementDataType); final FieldValue result = new ArrayIndexFieldValue(values[arrayIndex], elementField, fieldValue, arrayIndex); return result; }); }
public static RecordField merge(final RecordField thisField, final RecordField otherField) { final String fieldName = thisField.getFieldName(); final Set<String> aliases = new HashSet<>(); aliases.addAll(thisField.getAliases()); aliases.addAll(otherField.getAliases()); final Object defaultValue; if (thisField.getDefaultValue() == null && otherField.getDefaultValue() != null) { defaultValue = otherField.getDefaultValue(); } else { defaultValue = thisField.getDefaultValue(); } final DataType dataType; if (thisField.getDataType().equals(otherField.getDataType())) { dataType = thisField.getDataType(); } else { dataType = RecordFieldType.CHOICE.getChoiceDataType(thisField.getDataType(), otherField.getDataType()); } return new RecordField(fieldName, dataType, defaultValue, aliases, thisField.isNullable() || otherField.isNullable()); }