protected void update(List<datawave.data.type.Type<?>> types, RawRecordContainer event, Collection<NormalizedContentInterface> norms, String tokenDesignator, long countDelta, String loadDate, MetadataWithMostRecentDate mostRecentDatesByIdentifier, MetadataCounterGroup counts) { Type dataType = event.getDataType(); long eventDate = event.getDate(); for (NormalizedContentInterface norm : norms) { for (datawave.data.type.Type<?> type : types) { mostRecentDatesByIdentifier.createOrUpdate(norm.getIndexedFieldName() + tokenDesignator, dataType.outputName(), type.getClass().getName(), eventDate); if (null != loadDate) { updateLoadDateCounters(counts, event, norm.getIndexedFieldName() + tokenDesignator, countDelta, loadDate); } } } }
@Test public void testNoReplacement() { MetadataWithMostRecentDate counters = new MetadataWithMostRecentDate(COLUMN_FAMILY); counters.createOrUpdate(fieldName, dataTypeName, normalizerClassName, date); counters.createOrUpdate(fieldName, dataTypeName, normalizerClassName, 123L); assertOneEntryWithExpectedDate(counters, date); }
@Test public void testReplacement() { MetadataWithMostRecentDate counters = new MetadataWithMostRecentDate(COLUMN_FAMILY); counters.createOrUpdate(fieldName, dataTypeName, normalizerClassName, 123L); counters.createOrUpdate(fieldName, dataTypeName, normalizerClassName, date); assertOneEntryWithExpectedDate(counters, date); }
protected void addEventField(IngestHelperInterface helper, RawRecordContainer event, String fieldName, long countDelta, boolean frequency) { // if only indexing this field, then do not add to event and frequency maps if (helper.isIndexOnlyField(fieldName)) { log.debug(fieldName + " is indexonly, not adding to event"); return; } if (helper.isCompositeField(fieldName) && !helper.isOverloadedCompositeField(fieldName)) { log.debug(fieldName + " is a composite, not adding to event"); return; } log.debug("createOrUpdate for " + fieldName); eventFieldsInfo.createOrUpdate(fieldName, event.getDataType().outputName(), MetadataWithMostRecentDate.IGNORED_NORMALIZER_CLASS, event.getDate()); if (frequency) { addToFrequencyCounts(event, fieldName, countDelta); } }
protected void update(RawRecordContainer event, Collection<NormalizedContentInterface> norms, String tokenDesignator, long countDelta, String loadDate, MetadataWithMostRecentDate mostRecentDatesByIdentifier, MetadataCounterGroup counts) { Type dataType = event.getDataType(); long eventDate = event.getDate(); for (NormalizedContentInterface norm : norms) { mostRecentDatesByIdentifier.createOrUpdate(norm.getIndexedFieldName() + tokenDesignator, dataType.outputName(), null, eventDate); if (null != loadDate) { updateLoadDateCounters(counts, event, norm.getIndexedFieldName() + tokenDesignator, countDelta, loadDate); } } }
protected void update(String[] names, RawRecordContainer event, Collection<NormalizedContentInterface> norms, @SuppressWarnings("UnusedParameters") String tokenDesignator, long countDelta, String loadDate, MetadataWithMostRecentDate mostRecentDatesByIdentifier, MetadataCounterGroup counts) { Type dataType = event.getDataType(); long eventDate = event.getDate(); for (NormalizedContentInterface norm : norms) { int counter = 0; for (String name : names) { // mostRecentDatesByIdentifier.createOrUpdate(norm.getIndexedFieldName() + tokenDesignator, dataType.outputName(), name, eventDate); mostRecentDatesByIdentifier.createOrUpdate(name, dataType.outputName(), norm.getIndexedFieldName() + "," + counter++, eventDate); if (null != loadDate) { // updateLoadDateCounters(counts, event, norm.getIndexedFieldName() + tokenDesignator, countDelta, loadDate); updateLoadDateCounters(counts, event, name, countDelta, loadDate); } } } }
this.compositeFieldsInfo.createOrUpdate(fieldName, event.getDataType().outputName(), String.join(",", componentFields), event.getDate()); this.compositeSeparators.createOrUpdate(fieldName, event.getDataType().outputName(), helper.getCompositeFieldSeparators().get(fieldName), event.getDate());
for (String field : fields.keySet()) { if (h.isTermFrequencyField(field)) { termFrequencyFieldsInfo.createOrUpdate(field, event.getDataType().outputName(), MetadataWithMostRecentDate.IGNORED_NORMALIZER_CLASS, event.getDate()); if (h.isContentIndexField(field)) { updateForIndexedField(helper, event, fields, countDelta, loadDate, tokenDesignator, field); termFrequencyFieldsInfo.createOrUpdate(field, event.getDataType().outputName(), MetadataWithMostRecentDate.IGNORED_NORMALIZER_CLASS, event.getDate());
@Test public void testCanTrackSeparately() { MetadataWithMostRecentDate metadata = new MetadataWithMostRecentDate(COLUMN_FAMILY); metadata.createOrUpdate(fieldName, dataTypeName, normalizerClassName, 345); metadata.createOrUpdate(fieldName, dataTypeName + "2", normalizerClassName, 123); metadata.createOrUpdate(fieldName, dataTypeName + "2", normalizerClassName, 124); Assert.assertEquals(2, metadata.entries().size()); // Need to use a junk class here because we can't make instances of MostRecentEventDateAndKeyComponents as it is // not an enclosing class Collection<Tuple> expected = Lists.newArrayList(); expected.add(new Tuple("sham", "wow", "sham.wow.pick.up.Messes", 345L)); expected.add(new Tuple("sham", "wow2", "sham.wow.pick.up.Messes", 124L)); Collection<Tuple> actual = Lists.newArrayList(); for (MetadataWithMostRecentDate.MostRecentEventDateAndKeyComponents m : metadata.entries()) { actual.add(new Tuple(m.getFieldName(), m.getDataType(), m.getNormalizerClassName(), m.getMostRecentDate())); } Assert.assertTrue(CollectionUtils.isEqualCollection(expected, actual)); }
@Test public void testAssignments() { MetadataWithMostRecentDate counters = new MetadataWithMostRecentDate(COLUMN_FAMILY); counters.createOrUpdate(fieldName, dataTypeName, normalizerClassName, date); MetadataWithMostRecentDate.MostRecentEventDateAndKeyComponents entry = getOnlyEntry(counters); Assert.assertEquals(fieldName, entry.getFieldName()); Assert.assertEquals(dataTypeName, entry.getDataType()); Assert.assertEquals(normalizerClassName, entry.getNormalizerClassName()); Assert.assertEquals(date, entry.getMostRecentDate()); }