@Override public int compareTo(final ObjectWritable objectWritable) { if (null == this.t) return objectWritable.isEmpty() ? 0 : -1; else if (this.t instanceof Comparable && !objectWritable.isEmpty()) return ((Comparable) this.t).compareTo(objectWritable.get()); else if (this.t.equals(objectWritable.get())) return 0; else return -1; }
@Override public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final ObjectWritable<T> starGraph) { kryo.writeClassAndObject(output, starGraph.get()); }
@Override public int compare(byte[] bytes, int i, int i1, byte[] bytes1, int i2, int i3) { try { this.objectWritable1.readFields(new DataInputStream(new ByteArrayInputStream(bytes, i, i1))); this.objectWritable2.readFields(new DataInputStream(new ByteArrayInputStream(bytes1, i2, i3))); //System.out.println(objectWritable1 + "<=>" + objectWritable2 + ":::" + this.comparator.compare(objectWritable1.get(), objectWritable2.get())); return this.comparator.compare(this.objectWritable1.get(), this.objectWritable2.get()); } catch (final Exception e) { LOGGER.error(e.getMessage()); throw new IllegalStateException(e.getMessage()); } }
@Override public KeyValue next() { try { if (this.available) { this.available = false; return new KeyValue<>(this.key.get(), this.value.get()); } else { while (true) { if (this.readers.isEmpty()) throw new NoSuchElementException(); if (this.readers.peek().next(this.key, this.value)) { return new KeyValue<>(this.key.get(), this.value.get()); } else this.readers.remove().close(); } } } catch (final IOException e) { throw new IllegalStateException(e.getMessage(), e); } } }
@Override public boolean equals(final Object other) { if (!(other instanceof ObjectWritable)) return false; else if (this.isEmpty()) return ((ObjectWritable) other).isEmpty(); else return this.t.equals(((ObjectWritable) other).get()); }
@Override public <K, V> JavaPairRDD<K, V> readMemoryRDD(final Configuration configuration, final String memoryKey, final JavaSparkContext sparkContext) { return sparkContext.newAPIHadoopRDD(ConfUtil.makeHadoopConfiguration(configuration), SequenceFileInputFormat.class, ObjectWritable.class, ObjectWritable.class) .mapToPair(tuple -> new Tuple2<>((K) ((Tuple2<ObjectWritable, ObjectWritable>) tuple)._1().get(), (V) ((Tuple2<ObjectWritable, ObjectWritable>) tuple)._2().get())); } }
@Override public void reduce(final ObjectWritable key, final Iterable<ObjectWritable> values, final Reducer<ObjectWritable, ObjectWritable, ObjectWritable, ObjectWritable>.Context context) throws IOException, InterruptedException { this.reduceEmitter.setContext(context); this.mapReduce.reduce(key.get(), IteratorUtils.map(values.iterator(), ObjectWritable::get), this.reduceEmitter); }
@Override public void reduce(final ObjectWritable key, final Iterable<ObjectWritable> values, final Reducer<ObjectWritable, ObjectWritable, ObjectWritable, ObjectWritable>.Context context) throws IOException, InterruptedException { this.combineEmitter.setContext(context); this.mapReduce.combine(key.get(), IteratorUtils.map(values.iterator(), ObjectWritable::get), this.combineEmitter); }
@Override public <R> R get(final String key) throws IllegalArgumentException { if (!this.memoryComputeKeys.containsKey(key)) throw Memory.Exceptions.memoryDoesNotExist(key); if (this.inExecute && !this.memoryComputeKeys.get(key).isBroadcast()) throw Memory.Exceptions.memoryDoesNotExist(key); final ObjectWritable<R> r = (ObjectWritable<R>) (this.inExecute ? this.broadcast.value().get(key) : this.sparkMemory.get(key).value()); if (null == r || r.isEmpty()) throw Memory.Exceptions.memoryDoesNotExist(key); else return r.get(); }
@Override public int compareTo(final ObjectWritable objectWritable) { if (null == this.t) return objectWritable.isEmpty() ? 0 : -1; else if (this.t instanceof Comparable && !objectWritable.isEmpty()) return ((Comparable) this.t).compareTo(objectWritable.get()); else if (this.t.equals(objectWritable.get())) return 0; else return -1; }
@Override public int compare(byte[] bytes, int i, int i1, byte[] bytes1, int i2, int i3) { try { this.objectWritable1.readFields(new DataInputStream(new ByteArrayInputStream(bytes, i, i1))); this.objectWritable2.readFields(new DataInputStream(new ByteArrayInputStream(bytes1, i2, i3))); //System.out.println(objectWritable1 + "<=>" + objectWritable2 + ":::" + this.comparator.compare(objectWritable1.get(), objectWritable2.get())); return this.comparator.compare(this.objectWritable1.get(), this.objectWritable2.get()); } catch (final Exception e) { LOGGER.error(e.getMessage()); throw new IllegalStateException(e.getMessage()); } }
@Override public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final ObjectWritable<T> starGraph) { kryo.writeClassAndObject(output, starGraph.get()); }
@Override public boolean equals(final Object other) { if (!(other instanceof ObjectWritable)) return false; else if (this.isEmpty()) return ((ObjectWritable) other).isEmpty(); else return this.t.equals(((ObjectWritable) other).get()); }
@Override public <K, V> JavaPairRDD<K, V> readMemoryRDD(final Configuration configuration, final String memoryKey, final JavaSparkContext sparkContext) { return sparkContext.newAPIHadoopRDD(ConfUtil.makeHadoopConfiguration(configuration), SequenceFileInputFormat.class, ObjectWritable.class, ObjectWritable.class) .mapToPair(tuple -> new Tuple2<>((K) ((Tuple2<ObjectWritable, ObjectWritable>) tuple)._1().get(), (V) ((Tuple2<ObjectWritable, ObjectWritable>) tuple)._2().get())); } }
@Override public void reduce(final ObjectWritable key, final Iterable<ObjectWritable> values, final Reducer<ObjectWritable, ObjectWritable, ObjectWritable, ObjectWritable>.Context context) throws IOException, InterruptedException { this.reduceEmitter.setContext(context); this.mapReduce.reduce(key.get(), IteratorUtils.map(values.iterator(), ObjectWritable::get), this.reduceEmitter); }
@Override public void reduce(final ObjectWritable key, final Iterable<ObjectWritable> values, final Reducer<ObjectWritable, ObjectWritable, ObjectWritable, ObjectWritable>.Context context) throws IOException, InterruptedException { this.combineEmitter.setContext(context); this.mapReduce.combine(key.get(), IteratorUtils.map(values.iterator(), ObjectWritable::get), this.combineEmitter); }
@Override public <R> R get(final String key) throws IllegalArgumentException { if (!this.memoryComputeKeys.containsKey(key)) throw Memory.Exceptions.memoryDoesNotExist(key); if (this.inExecute && !this.memoryComputeKeys.get(key).isBroadcast()) throw Memory.Exceptions.memoryDoesNotExist(key); final ObjectWritable<R> r = (ObjectWritable<R>) (this.inExecute ? this.broadcast.value().get(key) : this.sparkMemory.get(key).value()); if (null == r || r.isEmpty()) throw Memory.Exceptions.memoryDoesNotExist(key); else return r.get(); }