public static VectorWritable merge(Iterator<VectorWritable> vectors) { return new VectorWritable(mergeToVector(vectors)); }
public static Vector readVector(DataInput in, byte vectorFlags, int size) throws IOException { VectorWritable v = new VectorWritable(); v.readFields(in, vectorFlags, size); return v.get(); }
public static Vector readVector(DataInput in) throws IOException { VectorWritable v = new VectorWritable(); v.readFields(in); return v.get(); }
private static void doTestVectorWritableEquals(Vector v) throws IOException { Writable vectorWritable = new VectorWritable(v); VectorWritable vectorWritable2 = new VectorWritable(); writeAndRead(vectorWritable, vectorWritable2); Vector v2 = vectorWritable2.get(); if (v instanceof NamedVector) { assertTrue(v2 instanceof NamedVector); NamedVector nv = (NamedVector) v; NamedVector nv2 = (NamedVector) v2; assertEquals(nv.getName(), nv2.getName()); assertEquals("Victor", nv.getName()); } assertEquals(v, v2); }
samplesInCurrFile = 0; out.append(new LongWritable(samplesInCurrFile++), new VectorWritable(p)); sampleNum++;
Vector p = new RandomAccessSparseVector(dimension); p.assign(vec); output.collect(new LongWritable(count), new VectorWritable(p)); reporter.setStatus(Long.toString(count + 1) + " samples generated"); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED,
@Override protected void cleanup(Context ctx) throws IOException, InterruptedException { ctx.write(new IntWritable(NORM_VECTOR_MARKER), new VectorWritable(norms)); ctx.write(new IntWritable(NUM_NON_ZERO_ENTRIES_VECTOR_MARKER), new VectorWritable(nonZeroEntries)); ctx.write(new IntWritable(MAXVALUE_VECTOR_MARKER), new VectorWritable(maxValues)); } }
@Override protected void reduce(IntWritable row, Iterable<VectorWritable> partialVectors, Context ctx) throws IOException, InterruptedException { ctx.write(row, new VectorWritable(Vectors.merge(partialVectors))); } }
@Override public void close() throws IOException { if (out != null) { out.collect(NullWritable.get(), new VectorWritable(outputVector)); } }
@Override protected void reduce(WritableComparable<?> key, Iterable<VectorWritable> values, Context ctx) throws IOException, InterruptedException { ctx.write(key, new VectorWritable(Vectors.sum(values.iterator()))); } }
@Override protected void map(Text key, VectorWritable value, Context context) throws IOException, InterruptedException { Vector result = classifier.classifyFull(value.get()); //the key is the expected value context.write(new Text(SLASH.split(key.toString())[1]), new VectorWritable(result)); } }
@Override protected void cleanup(Context ctx) throws IOException, InterruptedException { ctx.write(new Text(TrainNaiveBayesJob.LABEL_THETA_NORMALIZER), new VectorWritable(trainer.retrievePerLabelThetaNormalizer())); super.cleanup(ctx); } }
@Override protected void reduce(IntWritable row, Iterable<VectorWritable> partials, Context ctx) throws IOException, InterruptedException { Vector allSimilarities = Vectors.merge(partials); Vector topKSimilarities = Vectors.topKElements(maxSimilaritiesPerRow, allSimilarities); ctx.write(row, new VectorWritable(topKSimilarities)); } }
@Override protected void map(IntWritable row, VectorWritable vector, Context context) throws IOException, InterruptedException { context.write(row, new VectorWritable(vector.get().normalize(2))); }
@Override public void write(DataOutput out) throws IOException { VectorWritable vw = new VectorWritable(vector); vw.setWritesLaxPrecision(true); vw.write(out); Varint.writeUnsignedVarInt(userIDs.size(), out); for (int i = 0; i < userIDs.size(); i++) { Varint.writeSignedVarLong(userIDs.get(i), out); out.writeFloat(values.get(i)); } }
private void saveOutputVector(Path path, Vector v) throws IOException { FileSystem fs = path.getFileSystem(conf); SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class); try { writer.append(new IntWritable(0), new VectorWritable(v)); } finally { writer.close(); } }
public static Vector readVector(DataInput in) throws IOException { VectorWritable v = new VectorWritable(); v.readFields(in); return v.get(); }
@Override protected void reduce(IntWritable key, Iterable<WeightedVectorWritable> values, Context context) throws IOException, InterruptedException { // find the most distant point WeightedVectorWritable mdp = null; for (WeightedVectorWritable dpw : values) { if (mdp == null || mdp.getWeight() < dpw.getWeight()) { mdp = new WeightedVectorWritable(dpw.getWeight(), dpw.getVector()); } } context.write(new IntWritable(key.get()), new VectorWritable(mdp.getVector())); }
@Override public void map(IntWritable r, VectorWritable v, OutputCollector<IntWritable, VectorWritable> out, Reporter reporter) throws IOException { int row = r.get(); for (Vector.Element e : v.get().nonZeroes()) { RandomAccessSparseVector tmp = new RandomAccessSparseVector(newNumCols, 1); tmp.setQuick(row, e.get()); r.set(e.index()); out.collect(r, new VectorWritable(tmp)); } } }
@Override @Before public void setUp() throws Exception { super.setUp(); ctx = EasyMock.createMock(Mapper.Context.class); instance = new VectorWritable(new DenseVector(new double[] { 1, 0, 1, 1, 0 })); labelIndex = new OpenObjectIntHashMap<String>(); labelIndex.put("bird", 0); labelIndex.put("cat", 1); }