@Test public void testNormalizedKeyReadWriter() { try { T[] data = getSortedData(); T reuse = getSortedData()[0]; TypeComparator<T> comp1 = getComparator(true); if(!comp1.supportsSerializationWithKeyNormalization()){ return; } TypeComparator<T> comp2 = comp1.duplicate(); comp2.setReference(reuse); TestOutputView out = new TestOutputView(); TestInputView in; for (T value : data) { comp1.setReference(value); comp1.writeWithKeyNormalization(value, out); in = out.getInputView(); comp1.readWithKeyDenormalization(reuse, in); assertTrue(comp1.compareToReference(comp2) == 0); } } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); fail("Exception in test: " + e.getMessage()); } }
DefaultInMemorySorterFactory( @Nonnull TypeSerializerFactory<T> typeSerializerFactory, @Nonnull TypeComparator<T> typeComparator, int thresholdForInPlaceSorting) { this.typeSerializerFactory = typeSerializerFactory; this.typeComparator = typeComparator; TypeSerializer<T> typeSerializer = typeSerializerFactory.getSerializer(); this.useFixedLengthRecordSorter = typeComparator.supportsSerializationWithKeyNormalization() && typeSerializer.getLength() > 0 && typeSerializer.getLength() <= thresholdForInPlaceSorting; }
DefaultInMemorySorterFactory( @Nonnull TypeSerializerFactory<T> typeSerializerFactory, @Nonnull TypeComparator<T> typeComparator, int thresholdForInPlaceSorting) { this.typeSerializerFactory = typeSerializerFactory; this.typeComparator = typeComparator; TypeSerializer<T> typeSerializer = typeSerializerFactory.getSerializer(); this.useFixedLengthRecordSorter = typeComparator.supportsSerializationWithKeyNormalization() && typeSerializer.getLength() > 0 && typeSerializer.getLength() <= thresholdForInPlaceSorting; }
if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = this.config.getStubParameters(); BatchTask.openUserCode(this.combiner, stubConfig); // ----------------- Set up the sorter ------------------------- // instantiate the serializer / comparator final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader); this.serializer = serializerFactory.getSerializer(); TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator(); this.groupingComparator = groupingComparatorFactory.createComparator(); MemoryManager memManager = this.parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver()); this.memory = memManager.allocatePages(this.parent, numMemoryPages); // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory); } else { this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory); } if (LOG.isDebugEnabled()) { LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = this.config.getStubParameters(); BatchTask.openUserCode(this.combiner, stubConfig); // ----------------- Set up the sorter ------------------------- // instantiate the serializer / comparator final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader); this.serializer = serializerFactory.getSerializer(); TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator(); this.groupingComparator = groupingComparatorFactory.createComparator(); MemoryManager memManager = this.parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver()); this.memory = memManager.allocatePages(this.parent, numMemoryPages); // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory); } else { this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory); } if (LOG.isDebugEnabled()) { LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = this.config.getStubParameters(); BatchTask.openUserCode(this.reducer, stubConfig); // ----------------- Set up the sorter ------------------------- // instantiate the serializer / comparator final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader); this.serializer = serializerFactory.getSerializer(); TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator(); this.groupingComparator = groupingComparatorFactory.createComparator(); MemoryManager memManager = this.parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver()); this.memory = memManager.allocatePages(this.parent, numMemoryPages); // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory); } else { this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory); } if (LOG.isDebugEnabled()) { LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = this.config.getStubParameters(); BatchTask.openUserCode(this.reducer, stubConfig); // ----------------- Set up the sorter ------------------------- // instantiate the serializer / comparator final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader); this.serializer = serializerFactory.getSerializer(); TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator(); this.groupingComparator = groupingComparatorFactory.createComparator(); MemoryManager memManager = this.parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver()); this.memory = memManager.allocatePages(this.parent, numMemoryPages); // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory); } else { this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory); } if (LOG.isDebugEnabled()) { LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = this.config.getStubParameters(); BatchTask.openUserCode(this.combiner, stubConfig); // ----------------- Set up the sorter ------------------------- // instantiate the serializer / comparator final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader); this.serializer = serializerFactory.getSerializer(); TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator(); this.groupingComparator = groupingComparatorFactory.createComparator(); MemoryManager memManager = this.parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver()); this.memory = memManager.allocatePages(this.parent, numMemoryPages); // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory); } else { this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory); } if (LOG.isDebugEnabled()) { LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = this.config.getStubParameters(); BatchTask.openUserCode(this.reducer, stubConfig); // ----------------- Set up the sorter ------------------------- // instantiate the serializer / comparator final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader); this.serializer = serializerFactory.getSerializer(); TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator(); this.groupingComparator = groupingComparatorFactory.createComparator(); MemoryManager memManager = this.parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver()); this.memory = memManager.allocatePages(this.parent, numMemoryPages); // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory); } else { this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory); } if (LOG.isDebugEnabled()) { LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = this.config.getStubParameters(); BatchTask.openUserCode(this.reducer, stubConfig); // ----------------- Set up the sorter ------------------------- // instantiate the serializer / comparator final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader); this.serializer = serializerFactory.getSerializer(); TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator(); this.groupingComparator = groupingComparatorFactory.createComparator(); MemoryManager memManager = this.parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver()); this.memory = memManager.allocatePages(this.parent, numMemoryPages); // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory); } else { this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory); } if (LOG.isDebugEnabled()) { LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = this.config.getStubParameters(); BatchTask.openUserCode(this.combiner, stubConfig); // ----------------- Set up the sorter ------------------------- // instantiate the serializer / comparator final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader); this.serializer = serializerFactory.getSerializer(); TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator(); this.groupingComparator = groupingComparatorFactory.createComparator(); MemoryManager memManager = this.parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver()); this.memory = memManager.allocatePages(this.parent, numMemoryPages); // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory); } else { this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory); } if (LOG.isDebugEnabled()) { LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); } }
case SORTED_PARTIAL_REDUCE: if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory);
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = config.getStubParameters(); BatchTask.openUserCode(reducer, stubConfig); // instantiate the serializer / comparator serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer(); comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator(); MemoryManager memManager = parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver()); memory = memManager.allocatePages(parent, numMemoryPages); LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); switch (strategy) { case SORTED_PARTIAL_REDUCE: // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory); } else { sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory); } break; case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); table.open(); reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled); break; } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = config.getStubParameters(); BatchTask.openUserCode(reducer, stubConfig); // instantiate the serializer / comparator serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer(); comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator(); MemoryManager memManager = parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver()); memory = memManager.allocatePages(parent, numMemoryPages); LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); switch (strategy) { case SORTED_PARTIAL_REDUCE: // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory); } else { sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory); } break; case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); table.open(); reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled); break; } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = config.getStubParameters(); BatchTask.openUserCode(reducer, stubConfig); // instantiate the serializer / comparator serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer(); comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator(); MemoryManager memManager = parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver()); memory = memManager.allocatePages(parent, numMemoryPages); LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); switch (strategy) { case SORTED_PARTIAL_REDUCE: // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory); } else { sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory); } break; case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); table.open(); reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled); break; } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = config.getStubParameters(); BatchTask.openUserCode(reducer, stubConfig); // instantiate the serializer / comparator serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer(); comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator(); MemoryManager memManager = parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver()); memory = memManager.allocatePages(parent, numMemoryPages); LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); switch (strategy) { case SORTED_PARTIAL_REDUCE: // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory); } else { sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory); } break; case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); table.open(); reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled); break; } }