/** * Make a copy of a writable object using serialization to a buffer. * @param orig The object to copy * @return The copied object */ public static <T extends Writable> T clone(T orig, Configuration conf) { try { @SuppressWarnings("unchecked") // Unchecked cast from Class to Class<T> T newInst = ReflectionUtils.newInstance((Class<T>) orig.getClass(), conf); ReflectionUtils.copy(conf, orig, newInst); return newInst; } catch (IOException e) { throw new RuntimeException("Error writing/reading clone buffer", e); } }
/** * Enqueue most recent record read, and dequeue earliest result in the queue. * * @param job * Current job configuration. * * @param recordreader * Record reader. * * @param key * Key of current reading record. * * @param value * Value of current reading record. * * @return Return false if reaches the end of file, otherwise return true. */ public boolean updateBuffer(JobConf job, RecordReader recordreader, WritableComparable key, Writable value) throws IOException { key = ReflectionUtils.copy(job, (WritableComparable)buffer.get(cur).getFirst(), key); value = ReflectionUtils.copy(job, (Writable)buffer.get(cur).getSecond(), value); boolean notEOF = recordreader.next(buffer.get(cur).getFirst(), buffer.get(cur).getSecond()); if (notEOF) { cur = (++cur) % buffer.size(); } return notEOF; }
/** * Enqueue most recent record read, and dequeue earliest result in the queue. * * @param job * Current job configuration. * * @param recordreader * Record reader. * * @param key * Key of current reading record. * * @param value * Value of current reading record. * * @return Return false if reaches the end of file, otherwise return true. */ public boolean updateBuffer(JobConf job, RecordReader recordreader, WritableComparable key, Writable value) throws IOException { key = ReflectionUtils.copy(job, (WritableComparable)buffer.get(cur).getFirst(), key); value = ReflectionUtils.copy(job, (Writable)buffer.get(cur).getSecond(), value); boolean notEOF = recordreader.next(buffer.get(cur).getFirst(), buffer.get(cur).getSecond()); if (notEOF) { cur = (++cur) % buffer.size(); } return notEOF; }
/** * Length of region need to be properly serialized. * */ @Test public void testLengthIsSerialized() throws Exception { TableSplit split1 = new TableSplit(TableName.valueOf(name.getMethodName()), "row-start".getBytes(), "row-end".getBytes(), "location", 666); TableSplit deserialized = new TableSplit(TableName.valueOf(name.getMethodName()), "row-start2".getBytes(), "row-end2".getBytes(), "location1"); ReflectionUtils.copy(new Configuration(), split1, deserialized); Assert.assertEquals(666, deserialized.getLength()); }
@SuppressWarnings("unchecked") @Override public void reduce(KEY key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException { KEY keyCopy = (KEY) ReflectionUtils.newInstance(key.getClass(), conf); ReflectionUtils.copy(conf, key, keyCopy); allSplits.add(keyCopy); }
@SuppressWarnings("unchecked") @Override public void map(KEY key, VALUE value, Context context) throws IOException, InterruptedException { KEY keyCopy = (KEY) ReflectionUtils.newInstance(key.getClass(), conf); ReflectionUtils.copy(conf, key, keyCopy); allKeys.add(keyCopy); }
LOG.info(String.format("Fill: Collected %d samples from %d splits", counter, i)); counter++; samples.add(ReflectionUtils.copy(job.getConfiguration(), reader.getCurrentKey(), null)); } else { samples.set(ind, ReflectionUtils.copy(job.getConfiguration(), reader.getCurrentKey(), null)); if (counter % 1000 == 0)
/** * Clone the key at the head of this RR into the object supplied. */ public void key(K qkey) throws IOException { ReflectionUtils.copy(conf, key, qkey); }
/** * Clone the key at the head of this RR into the object supplied. */ public void key(K qkey) throws IOException { ReflectionUtils.copy(conf, key, qkey); }
/** * Clone the key at the head of this RR into the object supplied. */ public void key(K qkey) throws IOException { ReflectionUtils.copy(conf, key, qkey); }
/** * Clone the key at the head of this RR into the object supplied. */ public void key(K qkey) throws IOException { ReflectionUtils.copy(conf, key, qkey); }
@SuppressWarnings("unchecked") @Override public void map(KEY key, VALUE value, Context context) throws IOException, InterruptedException { KEY keyCopy = (KEY) ReflectionUtils.newInstance(key.getClass(), conf); ReflectionUtils.copy(conf, key, keyCopy); allKeys.add(keyCopy); }
@SuppressWarnings("unchecked") @Override public void reduce(KEY key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException { KEY keyCopy = (KEY) ReflectionUtils.newInstance(key.getClass(), conf); ReflectionUtils.copy(conf, key, keyCopy); allSplits.add(keyCopy); }
/** * Clone the key at the top of this RR into the given object. */ public void key(K key) throws IOException { ReflectionUtils.copy(conf, key(), key); }
/** * Clone the key at the top of this RR into the given object. */ public void key(K key) throws IOException { ReflectionUtils.copy(conf, key(), key); }
@SuppressWarnings("unchecked") private void writeToQueue(KEYOUT key, VALUEOUT value) throws IOException, InterruptedException { this.keyout = (KEYOUT) ReflectionUtils.newInstance(keyClass, conf); this.valueout = (VALUEOUT) ReflectionUtils.newInstance(valueClass, conf); ReflectionUtils.copy(conf, key, this.keyout); ReflectionUtils.copy(conf, value, this.valueout); // wait to write output to queuue outputQueue.enqueue(new KeyValuePair<KEYOUT, VALUEOUT>(keyout, valueout)); }
/** * Clone the key at the top of this RR into the given object. */ public void key(K key) throws IOException { ReflectionUtils.copy(conf, key(), key); }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { synchronized(outer) { if(!outer.nextKeyValue()) { return false; } key = ReflectionUtils.copy(outer.getConfiguration(), outer.getCurrentKey(), key); value = ReflectionUtils.copy(conf, outer.getCurrentValue(), value); return true; } }