/** * Initialize the hash table */ @Override public void open() { open(calcInitialNumBucketSegments()); }
/** * Initialize the hash table */ @Override public void open() { open(calcInitialNumBucketSegments()); }
/** * Initialize the hash table */ @Override public void open() { open(calcInitialNumBucketSegments()); }
/** * Initialize the hash table */ @Override public void open() { open(calcInitialNumBucketSegments()); }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = config.getStubParameters(); BatchTask.openUserCode(reducer, stubConfig); // instantiate the serializer / comparator serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer(); comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator(); MemoryManager memManager = parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver()); memory = memManager.allocatePages(parent, numMemoryPages); LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); switch (strategy) { case SORTED_PARTIAL_REDUCE: // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory); } else { sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory); } break; case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); table.open(); reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled); break; } }
table.open(); if (objectReuseEnabled) { T value = serializer.createInstance();
table.open(); if (objectReuseEnabled) { T value = serializer.createInstance();
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = config.getStubParameters(); BatchTask.openUserCode(reducer, stubConfig); // instantiate the serializer / comparator serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer(); comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator(); MemoryManager memManager = parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver()); memory = memManager.allocatePages(parent, numMemoryPages); LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); switch (strategy) { case SORTED_PARTIAL_REDUCE: // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory); } else { sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory); } break; case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); table.open(); reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled); break; } }
table.open(); if (objectReuseEnabled) { T value = serializer.createInstance();
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = config.getStubParameters(); BatchTask.openUserCode(reducer, stubConfig); // instantiate the serializer / comparator serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer(); comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator(); MemoryManager memManager = parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver()); memory = memManager.allocatePages(parent, numMemoryPages); LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); switch (strategy) { case SORTED_PARTIAL_REDUCE: // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory); } else { sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory); } break; case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); table.open(); reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled); break; } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = config.getStubParameters(); BatchTask.openUserCode(reducer, stubConfig); // instantiate the serializer / comparator serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer(); comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator(); MemoryManager memManager = parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver()); memory = memManager.allocatePages(parent, numMemoryPages); LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); switch (strategy) { case SORTED_PARTIAL_REDUCE: // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory); } else { sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory); } break; case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); table.open(); reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled); break; } }
table.open(); if (objectReuseEnabled) { T value = serializer.createInstance();