protected void complete(final MapReduce<K, V, ?, ?, ?> mapReduce) { if (!this.doReduce && mapReduce.getMapKeySort().isPresent()) { final Comparator<K> comparator = mapReduce.getMapKeySort().get(); final List<KeyValue<K, V>> list = new ArrayList<>(this.mapQueue); Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator)); this.mapQueue.clear(); this.mapQueue.addAll(list); } else if (mapReduce.getMapKeySort().isPresent()) { final Comparator<K> comparator = mapReduce.getMapKeySort().get(); final List<Map.Entry<K, Queue<V>>> list = new ArrayList<>(); list.addAll(this.reduceMap.entrySet()); Collections.sort(list, Comparator.comparing(Map.Entry::getKey, comparator)); this.reduceMap = new LinkedHashMap<>(); list.forEach(entry -> this.reduceMap.put(entry.getKey(), entry.getValue())); } } }
protected void complete(final MapReduce<K, V, ?, ?, ?> mapReduce) { if (!this.doReduce && mapReduce.getMapKeySort().isPresent()) { final Comparator<K> comparator = mapReduce.getMapKeySort().get(); final List<KeyValue<K, V>> list = new ArrayList<>(this.mapQueue); list.sort(Comparator.comparing(KeyValue::getKey, comparator)); this.mapQueue.clear(); this.mapQueue.addAll(list); } else if (mapReduce.getMapKeySort().isPresent()) { final Comparator<K> comparator = mapReduce.getMapKeySort().get(); final List<Map.Entry<K, Queue<V>>> list = new ArrayList<>(this.reduceMap.entrySet()); list.sort(Comparator.comparing(Map.Entry::getKey, comparator)); this.reduceMap = new LinkedHashMap<>(); list.forEach(entry -> this.reduceMap.put(entry.getKey(), entry.getValue())); } } }
protected void complete(final MapReduce<K, V, ?, ?, ?> mapReduce) { if (!this.doReduce && mapReduce.getMapKeySort().isPresent()) { final Comparator<K> comparator = mapReduce.getMapKeySort().get(); final List<KeyValue<K, V>> list = new ArrayList<>(this.mapQueue); Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator)); this.mapQueue.clear(); this.mapQueue.addAll(list); } else if (mapReduce.getMapKeySort().isPresent()) { final Comparator<K> comparator = mapReduce.getMapKeySort().get(); final List<Map.Entry<K, Queue<V>>> list = new ArrayList<>(); list.addAll(this.reduceMap.entrySet()); Collections.sort(list, Comparator.comparing(Map.Entry::getKey, comparator)); this.reduceMap = new LinkedHashMap<>(); list.forEach(entry -> this.reduceMap.put(entry.getKey(), entry.getValue())); } } }
public static <K, V> JavaPairRDD<K, V> executeMap( final JavaPairRDD<Object, VertexWritable> graphRDD, final MapReduce<K, V, ?, ?, ?> mapReduce, final Configuration graphComputerConfiguration) { JavaPairRDD<K, V> mapRDD = graphRDD.mapPartitionsToPair(partitionIterator -> { KryoShimServiceLoader.applyConfiguration(graphComputerConfiguration); return new MapIterator<>(MapReduce.<MapReduce<K, V, ?, ?, ?>>createMapReduce(HadoopGraph.open(graphComputerConfiguration), graphComputerConfiguration), partitionIterator); }); if (mapReduce.getMapKeySort().isPresent()) mapRDD = mapRDD.sortByKey(mapReduce.getMapKeySort().get(), true, 1); return mapRDD; }
@Override public void setConf(final Configuration configuration) { super.setConf(configuration); final org.apache.commons.configuration.Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(configuration); this.comparator = MapReduce.<MapReduce<?,?,?,?,?>>createMapReduce(HadoopGraph.open(apacheConfiguration),apacheConfiguration).getMapKeySort().get(); } }
ConfUtil.mergeApacheIntoHadoopConfiguration(apacheConfiguration, newConfiguration); final Optional<Comparator<?>> mapSort = mapReduce.getMapKeySort(); final Optional<Comparator<?>> reduceSort = mapReduce.getReduceKeySort(); newConfiguration.setClass(Constants.GREMLIN_HADOOP_MAP_REDUCE_CLASS, mapReduce.getClass(), MapReduce.class);
protected void complete(final MapReduce<K, V, ?, ?, ?> mapReduce) { if (!this.doReduce && mapReduce.getMapKeySort().isPresent()) { final Comparator<K> comparator = mapReduce.getMapKeySort().get(); final List<KeyValue<K, V>> list = new ArrayList<>(this.mapQueue); Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator)); this.mapQueue.clear(); this.mapQueue.addAll(list); } else if (mapReduce.getMapKeySort().isPresent()) { final Comparator<K> comparator = mapReduce.getMapKeySort().get(); final List<Map.Entry<K, Queue<V>>> list = new ArrayList<>(); list.addAll(this.reduceMap.entrySet()); Collections.sort(list, Comparator.comparing(Map.Entry::getKey, comparator)); this.reduceMap = new LinkedHashMap<>(); list.forEach(entry -> this.reduceMap.put(entry.getKey(), entry.getValue())); } } }
protected void complete(MapReduce<K, V, ?, ?, ?> mapReduce) { if (!doReduce && mapReduce.getMapKeySort().isPresent()) { Comparator<K> comparator = mapReduce.getMapKeySort().get(); List<KeyValue<K, V>> list = new ArrayList<>(mapQueue); Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator)); mapQueue.clear(); mapQueue.addAll(list); } else if (mapReduce.getMapKeySort().isPresent()) { Comparator<K> comparator = mapReduce.getMapKeySort().get(); List<Map.Entry<K, Queue<V>>> list = new ArrayList<>(); list.addAll(reduceMap.entrySet()); Collections.sort(list, Comparator.comparing(Map.Entry::getKey, comparator)); reduceMap.clear(); list.forEach(entry -> reduceMap.put(entry.getKey(), entry.getValue())); } }
protected void complete(final MapReduce<K, V, ?, ?, ?> mapReduce) { if (!this.doReduce && mapReduce.getMapKeySort().isPresent()) { final Comparator<K> comparator = mapReduce.getMapKeySort().get(); final List<KeyValue<K, V>> list = new ArrayList<>(this.mapQueue); Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator)); this.mapQueue.clear(); this.mapQueue.addAll(list); } else if (mapReduce.getMapKeySort().isPresent()) { final Comparator<K> comparator = mapReduce.getMapKeySort().get(); final List<Map.Entry<K, Queue<V>>> list = new ArrayList<>(); list.addAll(this.reduceMap.entrySet()); Collections.sort(list, Comparator.comparing(Map.Entry::getKey, comparator)); this.reduceMap = new LinkedHashMap<>(); list.forEach(entry -> this.reduceMap.put(entry.getKey(), entry.getValue())); } } }
protected void complete(final MapReduce<K, V, ?, ?, ?> mapReduce) { if (!this.doReduce && mapReduce.getMapKeySort().isPresent()) { final Comparator<K> comparator = mapReduce.getMapKeySort().get(); final List<KeyValue<K, V>> list = new ArrayList<>(this.mapQueue); Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator)); this.mapQueue.clear(); this.mapQueue.addAll(list); } else if (mapReduce.getMapKeySort().isPresent()) { final Comparator<K> comparator = mapReduce.getMapKeySort().get(); final List<Map.Entry<K, Queue<V>>> list = new ArrayList<>(); list.addAll(this.reduceMap.entrySet()); Collections.sort(list, Comparator.comparing(Map.Entry::getKey, comparator)); this.reduceMap = new LinkedHashMap<>(); list.forEach(entry -> this.reduceMap.put(entry.getKey(), entry.getValue())); } } }
public static <K, V> JavaPairRDD<K, V> executeMap( final JavaPairRDD<Object, VertexWritable> graphRDD, final MapReduce<K, V, ?, ?, ?> mapReduce, final Configuration graphComputerConfiguration) { JavaPairRDD<K, V> mapRDD = graphRDD.mapPartitionsToPair(partitionIterator -> { KryoShimServiceLoader.applyConfiguration(graphComputerConfiguration); return new MapIterator<>(MapReduce.<MapReduce<K, V, ?, ?, ?>>createMapReduce(HadoopGraph.open(graphComputerConfiguration), graphComputerConfiguration), partitionIterator); }); if (mapReduce.getMapKeySort().isPresent()){ mapRDD = mapRDD.sortByKey(mapReduce.getMapKeySort().get(), true, 1);} return mapRDD; }
public static <K, V> JavaPairRDD<K, V> executeMap( final JavaPairRDD<Object, VertexWritable> graphRDD, final MapReduce<K, V, ?, ?, ?> mapReduce, final Configuration graphComputerConfiguration) { JavaPairRDD<K, V> mapRDD = graphRDD.mapPartitionsToPair(partitionIterator -> { KryoShimServiceLoader.applyConfiguration(graphComputerConfiguration); return new MapIterator<>(MapReduce.<MapReduce<K, V, ?, ?, ?>>createMapReduce(HadoopGraph.open(graphComputerConfiguration), graphComputerConfiguration), partitionIterator); }); if (mapReduce.getMapKeySort().isPresent()) mapRDD = mapRDD.sortByKey(mapReduce.getMapKeySort().get(), true, 1); return mapRDD; }
@Override public void setConf(final Configuration configuration) { super.setConf(configuration); final org.apache.commons.configuration.Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(configuration); this.comparator = MapReduce.<MapReduce<?,?,?,?,?>>createMapReduce(HadoopGraph.open(apacheConfiguration),apacheConfiguration).getMapKeySort().get(); } }
ConfUtil.mergeApacheIntoHadoopConfiguration(apacheConfiguration, newConfiguration); final Optional<Comparator<?>> mapSort = mapReduce.getMapKeySort(); final Optional<Comparator<?>> reduceSort = mapReduce.getReduceKeySort(); newConfiguration.setClass(Constants.GREMLIN_HADOOP_MAP_REDUCE_CLASS, mapReduce.getClass(), MapReduce.class);