protected ProcessResult write(CrawlURI curi, long recordLength, InputStream in, String ip) throws IOException { WriterPoolMember writer = getPool().borrowFile(); long position = writer.getPosition(); getPool().invalidateFile(writer); setTotalBytesWritten(getTotalBytesWritten() + (writer.getPosition() - position)); getPool().returnFile(writer);
protected AtomicInteger getSerialNo() { return ((WriterPool)getPool()).getSerialNo(); }
/** * @return State of the pool string */ protected String getPoolState() { StringBuffer buffer = new StringBuffer("Active "); buffer.append(getNumActive()); buffer.append(" of max "); buffer.append(maxActive); buffer.append(", idle "); buffer.append(getNumIdle()); return buffer.toString(); }
/** * Close all {@link WriterPoolMember}s in pool. */ public void close() { Collection<WriterPoolMember> writers = drainAllWriters(); for (WriterPoolMember writer: writers) { try { destroyWriter(writer); } catch (IOException e) { logger.log(Level.WARNING,"problem closing writer",e); } } }
public JSONArray jsonStatus() throws JSONException { Collection<WriterPoolMember> writers = drainAllWriters(); JSONArray ja = new JSONArray(); for (WriterPoolMember w: writers) { JSONObject jo = new JSONObject(); jo.put("file", w.getFile()); jo.put("position", w.getPosition()); ja.put(jo); } availableWriters.addAll(writers); return ja; } }
/** * Return a writer, for likely reuse unless (1) writer's current file has * reached its target size; and (2) there's been no demand for additional * writers since the last time a new writer-file was rolled-over. In that * case, the possibly-superfluous writer instance is discarded. * @param writer Writer to return to the pool. * @throws IOException Problem returning File to pool. */ public void returnFile(WriterPoolMember writer) throws IOException { synchronized(this) { if(writer.isOversize()) { // maybe retire writer rather than recycle if(lastWriterNeededTime<=lastWriterRolloverTime) { // no timeouts waiting for recycled writer since last writer rollover destroyWriter(writer); return; } else { // reuse writer instance, causing new file to be created lastWriterRolloverTime = System.currentTimeMillis(); } } } if(!availableWriters.offer(writer)) { logger.log(Level.WARNING, "writer unreturnable to available pool; closing early"); destroyWriter(writer); } }
/** * @return Number of {@link WriterPoolMember}s checked out of pool. * @throws java.lang.UnsupportedOperationException */ public synchronized int getNumActive() throws UnsupportedOperationException { return currentActive - getNumIdle(); }
/** * Create a new writer instance, if still below maxActive count. * Remember times to help make later decision when writer should * be discarded. * * @return WriterPoolMember or null if already at max */ protected synchronized WriterPoolMember makeNewWriterIfAppropriate() { long now = System.currentTimeMillis(); lastWriterNeededTime = now; if(currentActive < maxActive) { currentActive++; lastWriterRolloverTime = now; return makeWriter(); } return null; }
/** * Check out a {@link WriterPoolMember}. * * This method should be followed by a call to * {@link #returnFile(WriterPoolMember)} or * {@link #invalidateFile(WriterPoolMember)} else pool starts leaking. * * @return Writer checked out of a pool of files or created * @throws IOException Problem getting Writer from pool (Converted * from Exception to IOException so this pool can live as a good citizen * down in depths of ARCSocketFactory). */ public WriterPoolMember borrowFile() throws IOException { WriterPoolMember writer = null; while(writer == null) { try { writer = availableWriters.poll(maxWait,TimeUnit.MILLISECONDS); } catch (InterruptedException e) { // nothing to do but proceed } if(writer==null) { writer = makeNewWriterIfAppropriate(); } } return writer; }
public void stop() { if (!isRunning()) { return; } super.stop(); // XXX happens at finish; move to teardown? this.pool.close(); }
/** * @return State of the pool string */ protected String getPoolState() { StringBuffer buffer = new StringBuffer("Active "); buffer.append(getNumActive()); buffer.append(" of max "); buffer.append(maxActive); buffer.append(", idle "); buffer.append(getNumIdle()); return buffer.toString(); }
/** * Close all {@link WriterPoolMember}s in pool. */ public void close() { Collection<WriterPoolMember> writers = drainAllWriters(); for (WriterPoolMember writer: writers) { try { destroyWriter(writer); } catch (IOException e) { logger.log(Level.WARNING,"problem closing writer",e); } } }
public void flush() { Collection<WriterPoolMember> writers = drainAllWriters(); for (WriterPoolMember writer: writers) { try { writer.flush(); } catch (IOException e) { logger.log(Level.WARNING, "problem flushing writer " + writer, e); } } availableWriters.addAll(writers); }
/** * Return a writer, for likely reuse unless (1) writer's current file has * reached its target size; and (2) there's been no demand for additional * writers since the last time a new writer-file was rolled-over. In that * case, the possibly-superfluous writer instance is discarded. * @param writer Writer to return to the pool. * @throws IOException Problem returning File to pool. */ public void returnFile(WriterPoolMember writer) throws IOException { synchronized(this) { if(writer.isOversize()) { // maybe retire writer rather than recycle if(lastWriterNeededTime<=lastWriterRolloverTime) { // no timeouts waiting for recycled writer since last writer rollover destroyWriter(writer); return; } else { // reuse writer instance, causing new file to be created lastWriterRolloverTime = System.currentTimeMillis(); } } } if(!availableWriters.offer(writer)) { logger.log(Level.WARNING, "writer unreturnable to available pool; closing early"); destroyWriter(writer); } }
/** * @return Number of {@link WriterPoolMember}s checked out of pool. * @throws java.lang.UnsupportedOperationException */ public synchronized int getNumActive() throws UnsupportedOperationException { return currentActive - getNumIdle(); }
/** * Create a new writer instance, if still below maxActive count. * Remember times to help make later decision when writer should * be discarded. * * @return WriterPoolMember or null if already at max */ protected synchronized WriterPoolMember makeNewWriterIfAppropriate() { long now = System.currentTimeMillis(); lastWriterNeededTime = now; if(currentActive < maxActive) { currentActive++; lastWriterRolloverTime = now; return makeWriter(); } return null; }
/** * Check out a {@link WriterPoolMember}. * * This method should be followed by a call to * {@link #returnFile(WriterPoolMember)} or * {@link #invalidateFile(WriterPoolMember)} else pool starts leaking. * * @return Writer checked out of a pool of files or created * @throws IOException Problem getting Writer from pool (Converted * from Exception to IOException so this pool can live as a good citizen * down in depths of ARCSocketFactory). */ public WriterPoolMember borrowFile() throws IOException { WriterPoolMember writer = null; while(writer == null) { try { writer = availableWriters.poll(maxWait,TimeUnit.MILLISECONDS); } catch (InterruptedException e) { // nothing to do but proceed } if(writer==null) { writer = makeNewWriterIfAppropriate(); } } return writer; }
public void stop() { if (!isRunning()) { return; } super.stop(); // XXX happens at finish; move to teardown? this.pool.close(); }
/** * @return State of the pool string */ protected String getPoolState() { StringBuffer buffer = new StringBuffer("Active "); buffer.append(getNumActive()); buffer.append(" of max "); buffer.append(maxActive); buffer.append(", idle "); buffer.append(getNumIdle()); return buffer.toString(); }