@Override public long getMemorySize() { return realTuple.getMemorySize(); }
@Override public long getMemorySize() { return t.getMemorySize(); }
@Override public long getMemorySize() { return t.getMemorySize(); }
/** * Update the average tuple size base on newly sampled tuple t * and recalculate skipInterval * @param t - tuple */ private void updateSkipInterval(Tuple t) { avgTupleMemSz = ((avgTupleMemSz*numRowsSampled) + t.getMemorySize())/(numRowsSampled + 1); skipInterval = memToSkipPerSample/avgTupleMemSz; // skipping fewer number of rows the first few times, to reduce the // probability of first tuples size (if much smaller than rest) // resulting in very few samples being sampled. Sampling a little extra // is OK if(numRowsSampled < 5) skipInterval = skipInterval/(10-numRowsSampled); ++numRowsSampled; }
/** * Update the average tuple size base on newly sampled tuple t * and recalculate skipInterval * @param t - tuple */ protected void updateSkipInterval(Tuple t) { avgTupleMemSz = ((avgTupleMemSz*numRowsSampled) + t.getMemorySize())/(numRowsSampled + 1); skipInterval = memToSkipPerSample/avgTupleMemSz; // skipping fewer number of rows the first few times, to reduce the // probability of first tuples size (if much smaller than rest) // resulting in very few samples being sampled. Sampling a little extra // is OK if(numRowsSampled < 5) { skipInterval = skipInterval/(10-numRowsSampled); } ++numRowsSampled; }
@Override public long getMemorySize() { // The protobuf estimate is obviously inaccurate. return msg_.getSerializedSize() + realTuple.getMemorySize(); } }
/** * Sample every SPILL_SAMPLE_FREQUENCYth tuple * until we reach a max of SPILL_SAMPLE_SIZE * to get an estimate of the tuple sizes. */ protected void sampleContents() { synchronized (mContents) { Iterator<Tuple> iter = mContents.iterator(); for (int i = 0; i < sampled * SPILL_SAMPLE_FREQUENCY && iter.hasNext(); i++) { iter.next(); } for (int i = sampled; iter.hasNext() && sampled < SPILL_SAMPLE_SIZE; i++) { Tuple t = iter.next(); if (t != null && i % SPILL_SAMPLE_FREQUENCY == 0) { aggSampleTupleSize += t.getMemorySize(); sampled += 1; } } } }
/** * This adds the additional overhead of the append Tuple */ @Override public long getMemorySize() { return SizeUtil.roundToEight(appendedFields.getMemorySize()) + super.getMemorySize(); }
@Override public void add(Tuple t) { synchronized(mContents) { if(mReadStarted) { throw new IllegalStateException("InternalDistinctBag is closed for adding new tuples"); } if (mContents.size() > memLimit.getCacheLimit()) { proactive_spill(null); } if (mContents.add(t)) { mSize ++; // check how many tuples memory can hold by getting average // size of first 100 tuples if(mSize < 100 && (mSpillFiles == null || mSpillFiles.isEmpty())) { memLimit.addNewObjSize(t.getMemorySize()); } } markSpillableIfNecessary(); } }
@Override public void add(Tuple t) { synchronized(mContents) { if(mReadStarted) { throw new IllegalStateException("InternalSortedBag is closed for adding new tuples"); } if (mContents.size() > memLimit.getCacheLimit()) { proactive_spill(mComp); } mContents.add(t); // check how many tuples memory can hold by getting average // size of first 100 tuples if(mSize < 100 && (mSpillFiles == null || mSpillFiles.isEmpty())&&t!=null) { memLimit.addNewObjSize(t.getMemorySize()); } mSize++; markSpillableIfNecessary(); } }
if(mContents.size() < 100) memLimit.addNewObjSize(t.getMemorySize());
for (Tuple t : entry.getValue()) { estTuples += 1; int mem = (int) t.getMemorySize(); estTotalMem += mem; memLimits.addNewObjSize(mem);
while ((tuple = outputHandler.getNext()) != null) { processOutput(tuple); outputBytes += tuple.getMemorySize();
return t.getMemorySize();
inputBytes += t.getMemorySize(); inputRecords++;
/** * @param in - input tuple * @return - tuple having size in memory of this tuple and numRows if this * is specially marked tuple having number of rows field */ public Tuple exec(Tuple in) throws IOException { if (in == null) { return null; } long memSize = in.getMemorySize(); long numRows = 0; // if this is specially marked tuple, get the number of rows int tSize = in.size(); if(tSize >=2 && PoissonSampleLoader.NUMROWS_TUPLE_MARKER.equals(in.get(tSize-2)) ){ memSize -= SizeUtil.getPigObjMemSize(PoissonSampleLoader.NUMROWS_TUPLE_MARKER); numRows = (Long)in.get(tSize-1); } //create tuple to be returned Tuple t = factory.newTuple(2); t.set(0, memSize); t.set(1, numRows); return t; }