public HollowWriteStateEngine patch() { Map<String, BitSet> baseMatches = findMatches(base); TransitiveSetTraverser.addTransitiveMatches(base, baseMatches); TransitiveSetTraverser.removeReferencedOutsideClosure(base, baseMatches); Map<String, BitSet> patchFromMatches = findMatches(patchFrom); HollowCombinerCopyDirector combineDirector = new HollowPatcherCombinerCopyDirector(base, baseMatches, patchFrom, patchFromMatches); HollowCombiner combiner = new HollowCombiner(combineDirector, base, patchFrom); combiner.addIgnoredTypes(ignoredTypes); combiner.combine(); return combiner.getCombinedStateEngine(); }
@Override public int getMappedOrdinal(String type, int originalOrdinal) { int typeMapping[] = typeMappings.get(type); if(typeMapping == null) return originalOrdinal; if(typeMapping[originalOrdinal] == -1) typeMapping[originalOrdinal] = combiner.copyOrdinal(type, originalOrdinal); return typeMapping[originalOrdinal]; }
/** * @param copyDirector a {@link HollowCombinerCopyDirector} which will specify which specific records to copy from the input(s). * @param output the {@link HollowWriteStateEngine} to use as the destination. * @param inputs the set of {@link HollowReadStateEngine} to combine data from. */ public HollowCombiner(HollowCombinerCopyDirector copyDirector, HollowWriteStateEngine output, HollowReadStateEngine... inputs) { this.inputs = inputs; this.output = output; this.typeNamesWithDefinedHashCodes = getAllTypesWithDefinedHashCodes(); this.ordinalRemappers = new OrdinalRemapper[inputs.length]; this.copiersPerType = new ThreadLocal<Map<String, HollowCombinerCopier>>(); this.hashOrderIndependentOrdinalMaps = new HashMap<String, ByteArrayOrdinalMap>(); this.ignoredTypes = new HashSet<String>(); this.copyDirector = copyDirector; initializePrimaryKeys(); }
@Test public void testExplicitlyExcludedButOtherwiseReferencedKeys() throws IOException { HollowPrimaryKeyIndex cIdx3 = new HollowPrimaryKeyIndex(input3, "TypeC", "key"); HollowCombinerExcludePrimaryKeysCopyDirector director = new HollowCombinerExcludePrimaryKeysCopyDirector(); director.excludeKey(cIdx3, 8); HollowCombiner combiner = new HollowCombiner(director, input1, input2, input3); combiner.setPrimaryKeys(new PrimaryKey("TypeB", "key", "c.key"), new PrimaryKey("TypeC", "key")); combiner.combine(); HollowReadStateEngine output = StateEngineRoundTripper.roundTripSnapshot(combiner.getCombinedStateEngine()); assertObject(output, 9, 3, 8, 3, 8, 3); }
@Test public void combinesEvenIfReferencedTypesAreFiltered() throws IOException { HollowWriteStateEngine writeEngine = new HollowWriteStateEngine(); HollowObjectMapper mapper = new HollowObjectMapper(writeEngine); mapper.add(new TypeA(1, 1)); mapper.add(new TypeA(2, 2)); mapper.add(new TypeA(3, 3)); HollowReadStateEngine readEngine = new HollowReadStateEngine(); HollowFilterConfig filter = new HollowFilterConfig(true); filter.addType("TypeB"); StateEngineRoundTripper.roundTripSnapshot(writeEngine, readEngine, filter); HollowCombiner combiner = new HollowCombiner(readEngine); combiner.combine(); HollowWriteStateEngine combined = combiner.getCombinedStateEngine(); HollowReadStateEngine combinedReadStateEngine = StateEngineRoundTripper.roundTripSnapshot(combined); HollowTypeReadState typeState = combinedReadStateEngine.getTypeState("TypeA"); Assert.assertEquals(3, typeState.getPopulatedOrdinals().cardinality()); Assert.assertNull(combinedReadStateEngine.getTypeState("TypeB")); }
HollowCombiner combiner = new HollowCombiner(roundTrip(sEngine), roundTrip(shard1)); HollowReadStateEngine combinedResult = roundTrip(combiner.getCombinedStateEngine()); Assert.assertEquals(1, extractPrimaryKeys(combinedResult).size()); Assert.assertEquals(pKeys, extractPrimaryKeys(combinedResult)); Assert.assertEquals(oldPK, newPK); HollowCombiner combiner = new HollowCombiner(roundTrip(sEngine), roundTrip(shard1)); combiner.setPrimaryKeys(newPK); Assert.assertEquals(1, combiner.getPrimaryKeys().size()); Assert.assertEquals(oldPK, combiner.getPrimaryKeys().get(0)); Assert.assertEquals(newPK, combiner.getPrimaryKeys().get(0)); Assert.assertNotEquals(oldPK, newPK); HollowCombiner combiner = new HollowCombiner(roundTrip(sEngine), roundTrip(shard1)); combiner.setPrimaryKeys(newPK); Assert.assertEquals(1, combiner.getPrimaryKeys().size()); Assert.assertNotEquals(oldPK, combiner.getPrimaryKeys().get(0)); Assert.assertEquals(newPK, combiner.getPrimaryKeys().get(0)); Assert.assertNotEquals(oldPK, newPK); HollowCombiner combiner = new HollowCombiner(roundTrip(sEngine), roundTrip(shard1)); combiner.setPrimaryKeys(newPK); Assert.assertEquals(2, combiner.getPrimaryKeys().size()); Assert.assertTrue(combiner.getPrimaryKeys().contains(oldPK)); Assert.assertTrue(combiner.getPrimaryKeys().contains(newPK));
final int numThreads = executor.getCorePoolSize(); createOrdinalRemappers(); createHashOrderIndependentOrdinalMaps(); if(!isAnySelectedPrimaryKeyADependencyOf(key.getType(), selectedPrimaryKeys)) { selectedPrimaryKeys.add(key); if(selectedPrimaryKeys.isEmpty() || isAnySelectedPrimaryKeyDependentOn(schema.getName(), selectedPrimaryKeys)) { for(PrimaryKey pk : selectedPrimaryKeys) { if(pk.getType().equals(schema.getName())) {
@Test public void multipleTypes() throws IOException { HollowWriteStateEngine combineInto = new HollowWriteStateEngine(); HollowObjectMapper mapper = new HollowObjectMapper(combineInto); mapper.initializeTypeState(TypeA.class); mapper.initializeTypeState(TypeB.class); HollowWriteStateEngine combineFromWriteEngine1 = new HollowWriteStateEngine(); mapper = new HollowObjectMapper(combineFromWriteEngine1); mapper.add(new TypeB(1)); HollowReadStateEngine combineFrom1 = StateEngineRoundTripper.roundTripSnapshot(combineFromWriteEngine1); HollowWriteStateEngine combineFromWriteEngine2 = new HollowWriteStateEngine(); mapper = new HollowObjectMapper(combineFromWriteEngine2); mapper.add(new TypeB(2)); mapper.add(new TypeA(2)); HollowReadStateEngine combineFrom2 = StateEngineRoundTripper.roundTripSnapshot(combineFromWriteEngine2); HollowCombiner combiner = new HollowCombiner(combineInto, combineFrom1, combineFrom2); combiner.combine(); HollowReadStateEngine combined = StateEngineRoundTripper.roundTripSnapshot(combineInto); assertEquals(2, combined.getTypeState("TypeB").getPopulatedOrdinals().cardinality()); assertEquals(1, combined.getTypeState("TypeA").getPopulatedOrdinals().cardinality()); }
@Test public void testCompoundKeys() throws IOException { HollowCombiner combiner = new HollowCombiner(input1, input2, input3); combiner.setPrimaryKeys(new PrimaryKey("TypeB", "key", "c.key")); combiner.combine(); HollowReadStateEngine output = StateEngineRoundTripper.roundTripSnapshot(combiner.getCombinedStateEngine()); assertObject(output, 1, 1, 1, 1, 1, 1); assertObject(output, 2, 1, 2, 1, 2, 1); assertObject(output, 3, 1, 3, 1, 3, 1); assertObject(output, 4, 2, 2, 2, 3, 2); assertObject(output, 5, 2, 4, 2, 4, 2); assertObject(output, 6, 2, 6, 2, 6, 2); assertObject(output, 7, 3, 2, 2, 3, 2); assertObject(output, 8, 3, 7, 3, 6, 3); assertObject(output, 9, 3, 8, 3, 8, 3); assertObject(output, 10,3, 4, 3, 10,3); }
@Test public void testCombiner() throws IOException { addRecord(shard1, 1, "C1", "C2", "C3"); addRecord(shard1, 2, "C2", "C3", "C4"); addRecord(shard1, 3, "C1", "C2", "C3"); addRecord(shard2, 4, "C2", "C3", "C4"); addRecord(shard2, 5, "C1", "C4", "C5"); addRecord(shard2, 6, "C1", "C2", "C3"); addRecord(shard3, 7, "C4", "C5", "C6"); HollowCombiner combiner = new HollowCombiner(roundTrip(shard1), roundTrip(shard2), roundTrip(shard3)); combiner.combine(); HollowReadStateEngine combinedResult = roundTrip(combiner.getCombinedStateEngine()); Assert.assertEquals(6, combinedResult.getTypeState("A").maxOrdinal()); Assert.assertEquals(3, combinedResult.getTypeState("B").maxOrdinal()); Assert.assertEquals(5, combinedResult.getTypeState("C").maxOrdinal()); HollowSetTypeReadState bTypeState = (HollowSetTypeReadState)combinedResult.getTypeState("B"); Assert.assertTrue(setOrderingExists(bTypeState, "C1", "C2", "C3")); Assert.assertTrue(setOrderingExists(bTypeState, "C2", "C3", "C4")); Assert.assertTrue(setOrderingExists(bTypeState, "C1", "C4", "C5")); Assert.assertTrue(setOrderingExists(bTypeState, "C4", "C5", "C6")); }
@Test public void testCompoundCascadingKeys() throws IOException { HollowCombiner combiner = new HollowCombiner(input1, input2, input3); combiner.setPrimaryKeys(new PrimaryKey("TypeB", "key", "c.key"), new PrimaryKey("TypeC", "key")); combiner.combine(); HollowReadStateEngine output = StateEngineRoundTripper.roundTripSnapshot(combiner.getCombinedStateEngine()); assertObject(output, 1, 1, 1, 1, 1, 1); assertObject(output, 2, 1, 2, 1, 2, 1); assertObject(output, 3, 1, 3, 1, 3, 1); assertObject(output, 4, 2, 2, 2, 3, 1); assertObject(output, 5, 2, 4, 2, 4, 2); assertObject(output, 6, 2, 6, 2, 6, 2); assertObject(output, 7, 3, 2, 2, 3, 1); assertObject(output, 8, 3, 7, 3, 6, 2); assertObject(output, 9, 3, 8, 3, 8, 3); assertObject(output, 10,3, 4, 3, 10,3); }
@Test public void testCascadingKeys() throws IOException { HollowCombiner combiner = new HollowCombiner(input1, input2, input3); combiner.setPrimaryKeys(new PrimaryKey("TypeB", "key"), new PrimaryKey("TypeC", "key")); combiner.combine(); HollowReadStateEngine output = StateEngineRoundTripper.roundTripSnapshot(combiner.getCombinedStateEngine()); assertObject(output, 1, 1, 1, 1, 1, 1); assertObject(output, 2, 1, 2, 1, 2, 1); assertObject(output, 3, 1, 3, 1, 3, 1); assertObject(output, 4, 2, 2, 1, 2, 1); assertObject(output, 5, 2, 4, 2, 4, 2); assertObject(output, 6, 2, 6, 2, 6, 2); assertObject(output, 7, 3, 2, 1, 2, 1); assertObject(output, 8, 3, 7, 3, 6, 2); assertObject(output, 9, 3, 8, 3, 8, 3); assertObject(output, 10,3, 4, 2, 4, 2); }
@Test public void testExcludeReferencedObjectsFromPrimaryKeyCopyDirector() throws IOException { HollowPrimaryKeyIndex aIdx3 = new HollowPrimaryKeyIndex(input3, "TypeA", "key"); HollowCombinerExcludePrimaryKeysCopyDirector director = new HollowCombinerExcludePrimaryKeysCopyDirector(); director.excludeKey(aIdx3, 9); HollowCombiner combiner = new HollowCombiner(director, input1, input2, input3); combiner.setPrimaryKeys(new PrimaryKey("TypeB", "key", "c.key"), new PrimaryKey("TypeC", "key")); combiner.combine(); HollowReadStateEngine output = StateEngineRoundTripper.roundTripSnapshot(combiner.getCombinedStateEngine()); Assert.assertEquals(-1, new HollowPrimaryKeyIndex(output, "TypeA", "key").getMatchingOrdinal(9)); Assert.assertNotEquals(-1, new HollowPrimaryKeyIndex(output, "TypeB", "key").getMatchingOrdinal(8)); Assert.assertNotEquals(-1, new HollowPrimaryKeyIndex(output, "TypeC", "key").getMatchingOrdinal(8)); director.excludeReferencedObjects(); combiner = new HollowCombiner(director, input1, input2, input3); combiner.setPrimaryKeys(new PrimaryKey("TypeB", "key", "c.key"), new PrimaryKey("TypeC", "key")); combiner.combine(); output = StateEngineRoundTripper.roundTripSnapshot(combiner.getCombinedStateEngine()); Assert.assertEquals(-1, new HollowPrimaryKeyIndex(output, "TypeA", "key").getMatchingOrdinal(9)); Assert.assertEquals(-1, new HollowPrimaryKeyIndex(output, "TypeB", "key").getMatchingOrdinal(8)); Assert.assertEquals(-1, new HollowPrimaryKeyIndex(output, "TypeC", "key").getMatchingOrdinal(8)); }
@Test public void testExplicitlyExcludedAndRemappedKeys() throws IOException { HollowPrimaryKeyIndex cIdx1 = new HollowPrimaryKeyIndex(input1, "TypeC", "key"); HollowCombinerExcludePrimaryKeysCopyDirector director = new HollowCombinerExcludePrimaryKeysCopyDirector(); director.excludeKey(cIdx1, 3); HollowCombiner combiner = new HollowCombiner(director, input1, input2, input3); combiner.setPrimaryKeys(new PrimaryKey("TypeB", "key", "c.key"), new PrimaryKey("TypeC", "key")); combiner.combine(); HollowReadStateEngine output = StateEngineRoundTripper.roundTripSnapshot(combiner.getCombinedStateEngine()); assertObject(output, 1, 1, 1, 1, 1, 1); assertObject(output, 2, 1, 2, 1, 2, 1); assertObject(output, 3, 1, 3, 1, 3, 2); assertObject(output, 4, 2, 2, 2, 3, 2); assertObject(output, 5, 2, 4, 2, 4, 2); assertObject(output, 6, 2, 6, 2, 6, 2); assertObject(output, 7, 3, 2, 2, 3, 2); assertObject(output, 8, 3, 7, 3, 6, 2); assertObject(output, 9, 3, 8, 3, 8, 3); assertObject(output, 10,3, 4, 3, 10,3); }