try { FileService fileService = new FileService(); FileList files = fileService.getPublicFiles(null); if(files != null && ! files.isEmpty()) { for (File file : files) { out.println("<a href=\"" + file.getDownloadLink() + "\"> " + file.getLabel() + "</a><br/>" ); } } else { out.println("No Results"); } } catch (Throwable e) {}
/** * Clear out the contents of the bag, both on disk and in memory. * Any attempts to read after this is called will produce undefined * results. */ @Override public void clear() { synchronized (mContents) { mContents.clear(); if (mSpillFiles != null) { for (int i = 0; i < mSpillFiles.size(); i++) { boolean res = mSpillFiles.get(i).delete(); if (!res) warn ("DefaultAbstractBag.clear: failed to delete " + mSpillFiles.get(i), PigWarning.DELETE_FAILED, null); } mSpillFiles.clear(); } mSize = 0; aggSampleTupleSize = 0; sampled = 0; // not changing spillableRegistered -- clear doesn't change that. } }
public CachedBagIterator() { iter = mContents.iterator(); if(mSpillFiles != null && mSpillFiles.size() > 0) { File file = mSpillFiles.get(0); try { in = new DataInputStream(new BufferedInputStream(new FileInputStream(file))); } catch(FileNotFoundException fnfe) { String msg = "Unable to find our spill file."; throw new RuntimeException(msg, fnfe); } } }
@Override public long size() { if (mSpillFiles != null && mSpillFiles.size() > 0){ //We need to racalculate size to guarantee a count of unique //entries including those on disk Iterator<Tuple> iter = iterator(); int newSize = 0; while (iter.hasNext()) { newSize++; iter.next(); } mSize = newSize; } return mSize; }
new FileInputStream(mSpillFiles.get(mFilePtr++)))); } catch (FileNotFoundException fnfe) {
private long totalSizeFromAvgTupleSize(long avgTupleSize, int numInMem) { long used = avgTupleSize * numInMem; long mFields_size = roundToEight(4 + numInMem*4); /* mContents fixed + per entry */ // in java hotspot 32bit vm, there seems to be a minimum bag size of 188 bytes // some of the extra bytes is probably from a minimum size of this array list mFields_size = Math.max(40, mFields_size); // the fixed overhead for this object and other object variables = 84 bytes // 8 - object header // 4 + 8 + 8 - sampled + aggSampleTupleSize + mSize // 8 + 8 - mContents ref + mSpillFiles ref // 4 - spillableRegistered +4 instead of 1 to round it to eight // 36 - mContents fixed used += 84 + mFields_size; // add up overhead for mSpillFiles ArrayList, Object[] inside ArrayList, // object variable inside ArrayList and references to spill files if (mSpillFiles != null) { used += roundToEight(36 /* mSpillFiles fixed overhead*/ + mSpillFiles.size()*4); if(mSpillFiles.size() > 0){ //a rough estimate of memory used by each file entry // the auto generated files are likely to have same length long approx_per_entry_size = roundToEight(mSpillFiles.get(0).toString().length() * 2 + 38); used += mSpillFiles.size() * approx_per_entry_size; } } return used; }
@Override public long size() { if (mSpillFiles != null && mSpillFiles.size() > 0){ //We need to racalculate size to guarantee a count of unique //entries including those on disk Iterator<Tuple> iter = iterator(); int newSize = 0; while (iter.hasNext()) { newSize++; iter.next(); } synchronized(mContents) { //we don't want adds to change our numbers //the lock may need to cover more of the method mSize = newSize; } } return mSize; }
@Override public void add(Tuple t) { synchronized(mContents) { if(mReadStarted) { throw new IllegalStateException("InternalDistinctBag is closed for adding new tuples"); } if (mContents.size() > memLimit.getCacheLimit()) { proactive_spill(null); } if (mContents.add(t)) { mSize ++; // check how many tuples memory can hold by getting average // size of first 100 tuples if(mSize < 100 && (mSpillFiles == null || mSpillFiles.isEmpty())) { memLimit.addNewObjSize(t.getMemorySize()); } } markSpillableIfNecessary(); } }
@Override public Tuple next() { // This will report progress every 1024 times through next. // This should be much faster than using mod. if ((mCntr++ & 0x3ff) == 0) reportProgress(); // If there's one in the buffer, use that one. if (mBuf != null) { Tuple t = mBuf; mBuf = null; return t; } // Check to see if we just need to read from memory. if (mSpillFiles == null || mSpillFiles.size() == 0) { return readFromMemory(); } // We have spill files, so we need to read the next tuple from // one of those files or from memory. return readFromTree(); }