private Record createContainerRecord(final Record geoRecord, final Record ispRecord, final String domainName, final String connectionType, final Record anonymousIpRecord) { final Map<String, Object> values = new HashMap<>(4); values.put("geo", geoRecord); values.put("isp", ispRecord); values.put("domainName", domainName); values.put("connectionType", connectionType); values.put("anonymousIp", anonymousIpRecord); final Record containerRecord = new MapRecord(ContainerSchema.CONTAINER_SCHEMA, values); return containerRecord; } }
private Record applyMappings(Record record, Map<String, Object> source) { Record _rec = new MapRecord(record.getSchema(), new HashMap<>()); mappings.entrySet().forEach(entry -> { try { Object o = JsonPath.read(source, entry.getKey()); RecordPath path = entry.getValue(); Optional<FieldValue> first = path.evaluate(_rec).getSelectedFields().findFirst(); if (first.isPresent()) { first.get().updateValue(o); } } catch (Exception ex) { throw new RuntimeException(ex); } }); return _rec; }
private Record createRecord(final Subdivision subdivision) { if (subdivision == null) { return null; } final Map<String, Object> values = new HashMap<>(2); values.put(CitySchema.SUBDIVISION_NAME.getFieldName(), subdivision.getName()); values.put(CitySchema.SUBDIVISION_ISO.getFieldName(), subdivision.getIsoCode()); return new MapRecord(CitySchema.SUBDIVISION_SCHEMA, values); }
private Record createRecord(final Country country) { if (country == null) { return null; } final Map<String, Object> values = new HashMap<>(2); values.put(CitySchema.COUNTRY_NAME.getFieldName(), country.getName()); values.put(CitySchema.COUNTRY_ISO.getFieldName(), country.getIsoCode()); return new MapRecord(CitySchema.COUNTRY_SCHEMA, values); }
protected Record createRecord(final ResultSet rs) throws SQLException { final Map<String, Object> values = new HashMap<>(schema.getFieldCount()); for (final RecordField field : schema.getFields()) { final String fieldName = field.getFieldName(); final Object value; if (rsColumnNames.contains(fieldName)) { value = normalizeValue(rs.getObject(fieldName)); } else { value = null; } values.put(fieldName, value); } return new MapRecord(schema, values); }
private Object getReplacementObject(final List<FieldValue> selectedFields) { if (selectedFields.size() > 1) { final List<RecordField> fields = selectedFields.stream().map(FieldValue::getField).collect(Collectors.toList()); final RecordSchema schema = new SimpleRecordSchema(fields); final Record record = new MapRecord(schema, new HashMap<>()); for (final FieldValue fieldVal : selectedFields) { record.setValue(fieldVal.getField().getFieldName(), fieldVal.getValue()); } return record; } if (selectedFields.isEmpty()) { return null; } else { return selectedFields.get(0).getValue(); } } }
@Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknown) throws IOException, MalformedRecordException { if (failAfterN > -1 && recordCount >= failAfterN) { throw new MalformedRecordException("Intentional Unit Test Exception because " + recordCount + " records have been read"); } final String nextLine = reader.readLine(); if (nextLine == null) { return null; } recordCount++; final String[] values = nextLine.split(","); final Map<String, Object> valueMap = new HashMap<>(); int i = 0; for (final RecordField field : fields) { final String fieldName = field.getFieldName(); valueMap.put(fieldName, values[i++].trim()); } return new MapRecord(new SimpleRecordSchema(fields), valueMap); }
@Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknown) throws IOException, MalformedRecordException { if (failAfterN >= 0 && recordCount >= failAfterN) { throw new MalformedRecordException("Intentional Unit Test Exception because " + recordCount + " records have been read"); } recordCount++; if (!itr.hasNext()) { return null; } final Object[] values = itr.next(); final Map<String, Object> valueMap = new HashMap<>(); int i = 0; for (final RecordField field : fields) { final String fieldName = field.getFieldName(); valueMap.put(fieldName, values[i++]); } return new MapRecord(new SimpleRecordSchema(fields), valueMap); }
@Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException { GenericRecord record = nextAvroRecord(); if (record == null) { return null; } final RecordSchema schema = getSchema(); final Map<String, Object> values = AvroTypeUtil.convertAvroRecordToMap(record, schema); return new MapRecord(schema, values); }
private Record createRecord(final IspResponse isp) { if (isp == null) { return null; } final Map<String, Object> values = new HashMap<>(4); values.put(IspSchema.ASN.getFieldName(), isp.getAutonomousSystemNumber()); values.put(IspSchema.ASN_ORG.getFieldName(), isp.getAutonomousSystemOrganization()); values.put(IspSchema.NAME.getFieldName(), isp.getIsp()); values.put(IspSchema.ORG.getFieldName(), isp.getOrganization()); return new MapRecord(IspSchema.ISP_SCHEMA, values); }
private Record createRecord(final AnonymousIpResponse anonymousIp) { if (anonymousIp == null) { return null; } final Map<String, Object> values = new HashMap<>(5); values.put(AnonymousIpSchema.ANONYMOUS.getFieldName(), anonymousIp.isAnonymous()); values.put(AnonymousIpSchema.ANONYMOUS_VPN.getFieldName(), anonymousIp.isAnonymousVpn()); values.put(AnonymousIpSchema.HOSTING_PROVIDER.getFieldName(), anonymousIp.isHostingProvider()); values.put(AnonymousIpSchema.PUBLIC_PROXY.getFieldName(), anonymousIp.isPublicProxy()); values.put(AnonymousIpSchema.TOR_EXIT_NODE.getFieldName(), anonymousIp.isTorExitNode()); return new MapRecord(AnonymousIpSchema.ANONYMOUS_IP_SCHEMA, values); }
@Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException { if (currentRecordStartTag == null) { return null; } try { final Record record = parseRecord(currentRecordStartTag, this.schema, coerceTypes, dropUnknownFields); setNextRecordStartTag(); if (record != null) { return record; } else { return new MapRecord(this.schema, Collections.EMPTY_MAP); } } catch (XMLStreamException e) { throw new MalformedRecordException("Could not parse XML", e); } }
private Record updateRecord(final List<FieldValue> destinationFields, final List<FieldValue> selectedFields, final Record record) { if (destinationFields.size() == 1 && !destinationFields.get(0).getParentRecord().isPresent()) { final Object replacement = getReplacementObject(selectedFields); if (replacement == null) { return record; } if (replacement instanceof Record) { return (Record) replacement; } final List<RecordField> fields = selectedFields.stream().map(FieldValue::getField).collect(Collectors.toList()); final RecordSchema schema = new SimpleRecordSchema(fields); final Record mapRecord = new MapRecord(schema, new HashMap<>()); for (final FieldValue selectedField : selectedFields) { mapRecord.setValue(selectedField.getField().getFieldName(), selectedField.getValue()); } return mapRecord; } else { for (final FieldValue fieldVal : destinationFields) { fieldVal.updateValue(getReplacementObject(selectedFields)); } return record; } }
private Record getByQuery(final Map<String, Object> query, Map<String, String> context) throws LookupFailureException { try { final String json = mapper.writeValueAsString(buildQuery(query)); SearchResponse response = clientService.search(json, index, type); if (response.getNumberOfHits() == 0) { return null; } else { final Map<String, Object> source = (Map)response.getHits().get(0).get("_source"); RecordSchema toUse = getSchema(context, source, null); Record record = new MapRecord(toUse, source); if (mappings.size() > 0) { record = applyMappings(record, source); } return record; } } catch (Exception e) { throw new LookupFailureException(e); } }
private Record getById(final String _id, Map<String, String> context) throws IOException, LookupFailureException, SchemaNotFoundException { Map<String, Object> query = new HashMap<String, Object>(){{ put("query", new HashMap<String, Object>() {{ put("match", new HashMap<String, String>(){{ put("_id", _id); }}); }}); }}; String json = mapper.writeValueAsString(query); SearchResponse response = clientService.search(json, index, type); if (response.getNumberOfHits() > 1) { throw new LookupFailureException(String.format("Expected 1 response, got %d for query %s", response.getNumberOfHits(), json)); } else if (response.getNumberOfHits() == 0) { return null; } final Map<String, Object> source = (Map)response.getHits().get(0).get("_source"); RecordSchema toUse = getSchema(context, source, null); Record record = new MapRecord(toUse, source); if (mappings.size() > 0) { record = applyMappings(record, source); } return record; }
/** * Writes each SolrDocument to a record. */ public static RecordSet solrDocumentsToRecordSet(final List<SolrDocument> docs, final RecordSchema schema) { final List<Record> lr = new ArrayList<Record>(); for (SolrDocument doc : docs) { final Map<String, Object> recordValues = new LinkedHashMap<>(); for (RecordField field : schema.getFields()){ final Object fieldValue = doc.getFieldValue(field.getFieldName()); if (fieldValue != null) { if (field.getDataType().getFieldType().equals(RecordFieldType.ARRAY)){ recordValues.put(field.getFieldName(), ((List<Object>) fieldValue).toArray()); } else { recordValues.put(field.getFieldName(), fieldValue); } } } lr.add(new MapRecord(schema, recordValues)); } return new ListRecordSet(schema, lr); }
@Override public Record nextRecord() throws IOException { if (initialized && lastRecord == null) { return null; } lastRecord = parquetReader.read(); initialized = true; if (lastRecord == null) { return null; } if (recordSchema == null) { recordSchema = AvroTypeUtil.createSchema(lastRecord.getSchema()); } final Map<String, Object> values = AvroTypeUtil.convertAvroRecordToMap(lastRecord, recordSchema); return new MapRecord(recordSchema, values); }
@Override public Record nextRecord(boolean coerceTypes, boolean dropUnknownFields) throws IOException, MalformedRecordException { String line = reader.readLine(); if ( line == null ) { // a null return from readLine() signals the end of the stream return null; } if (StringUtils.isBlank(line)) { // while an empty string is an error throw new MalformedRecordException("Encountered a blank message!"); } final MalformedRecordException malformedRecordException; Syslog5424Event event = parser.parseEvent(ByteBuffer.wrap(line.getBytes(parser.getCharsetName()))); if (!event.isValid()) { if (event.getException() != null) { malformedRecordException = new MalformedRecordException( String.format("Failed to parse %s as a Syslog message: it does not conform to any of the RFC "+ "formats supported", line), event.getException()); } else { malformedRecordException = new MalformedRecordException( String.format("Failed to parse %s as a Syslog message: it does not conform to any of the RFC" + " formats supported", line)); } throw malformedRecordException; } Map<String,Object> modifiedMap = new HashMap<>(event.getFieldMap()); modifiedMap.put(SyslogAttributes.TIMESTAMP.key(),convertTimeStamp((String)event.getFieldMap().get(SyslogAttributes.TIMESTAMP.key()))); return new MapRecord(schema,modifiedMap); }
@Override public Record nextRecord(boolean coerceTypes, boolean dropUnknownFields) throws IOException, MalformedRecordException { String line = reader.readLine(); if (line == null) { // a null return from readLine() signals the end of the stream return null; } if (StringUtils.isBlank(line)) { // while an empty string is an error throw new MalformedRecordException("Encountered a blank message!"); } final MalformedRecordException malformedRecordException; SyslogEvent event = parser.parseEvent(ByteBuffer.wrap(line.getBytes(parser.getCharsetName()))); if (!event.isValid()) { malformedRecordException = new MalformedRecordException( String.format("Failed to parse %s as a Syslog message: it does not conform to any of the RFC" + " formats supported", line)); throw malformedRecordException; } final Map<String, Object> syslogMap = new HashMap<>(8); syslogMap.put(SyslogAttributes.PRIORITY.key(), event.getPriority()); syslogMap.put(SyslogAttributes.SEVERITY.key(), event.getSeverity()); syslogMap.put(SyslogAttributes.FACILITY.key(), event.getFacility()); syslogMap.put(SyslogAttributes.VERSION.key(), event.getVersion()); syslogMap.put(SyslogAttributes.TIMESTAMP.key(), event.getTimeStamp()); syslogMap.put(SyslogAttributes.HOSTNAME.key(), event.getHostName()); syslogMap.put(SyslogAttributes.BODY.key(), event.getMsgBody()); return new MapRecord(schema, syslogMap); }
private Record createRecord(final CityResponse city) { if (city == null) { return null; } final Map<String, Object> values = new HashMap<>(); values.put(CitySchema.CITY.getFieldName(), city.getCity().getName()); final Location location = city.getLocation(); values.put(CitySchema.ACCURACY.getFieldName(), location.getAccuracyRadius()); values.put(CitySchema.METRO_CODE.getFieldName(), location.getMetroCode()); values.put(CitySchema.TIMEZONE.getFieldName(), location.getTimeZone()); values.put(CitySchema.LATITUDE.getFieldName(), location.getLatitude()); values.put(CitySchema.LONGITUDE.getFieldName(), location.getLongitude()); values.put(CitySchema.CONTINENT.getFieldName(), city.getContinent().getName()); values.put(CitySchema.POSTALCODE.getFieldName(), city.getPostal().getCode()); values.put(CitySchema.COUNTRY.getFieldName(), createRecord(city.getCountry())); final Object[] subdivisions = new Object[city.getSubdivisions().size()]; int i = 0; for (final Subdivision subdivision : city.getSubdivisions()) { subdivisions[i++] = createRecord(subdivision); } values.put(CitySchema.SUBDIVISIONS.getFieldName(), subdivisions); return new MapRecord(CitySchema.GEO_SCHEMA, values); }