public static WatchFileReadStage newInstance(GraphManager graphManager, Pipe<?>[] inputs, Pipe<RawDataSchema>[] outputs, Pipe<?>[] ackRelease ) { return new WatchFileReadStage(graphManager,inputs, outputs, ackRelease ); }
public static FileBlobWriteStage newInstance(GraphManager graphManager, Pipe<RawDataSchema> input, String outputPathString) { return new FileBlobWriteStage(graphManager, input, false, outputPathString); }
public static WatchServiceStage newInstance(GraphManager graphManager, String pathText, String[] filePatterns, Pipe<FolderWatchSchema>[] output, int kindFlags ) { return new WatchServiceStage(graphManager, pathText, filePatterns, output, kindFlags); }
private static <S extends MessageSchema<S>> void processAvailData(TapeWriteStage<S> ss) { //only zero when we need to find the next block and have finished the previous if (0==ss.totalPrimaryCopy) { findStableCutPoint(ss); //we have established the point that we can read up to, this value is changed by the writer on the other side //get the start and stop locations for the copy //now find the point to start reading from, this is moved forward with each new read. if ((ss.totalPrimaryCopy = (ss.headPos - ss.cachedTail)) <= 0) { assert(ss.totalPrimaryCopy==0); return; //nothing to copy so come back later } setupBuffersToWriteFrom(ss); } copyToFile(ss); }
public static BlockStorageStage newInstance(GraphManager graphManager, String filePath, //single file accessed by multiple pipes Pipe<BlockStorageXmitSchema> input, Pipe<BlockStorageReceiveSchema> output) { return new BlockStorageStage(graphManager, filePath, new Pipe[]{input}, new Pipe[]{output}); }
public static FileBlobReadStage newInstance(GraphManager graphManager, //add input pipe to select file to read Pipe<RawDataSchema> output, String inputPathString, boolean shutDownAtEndOfFile) { return new FileBlobReadStage(graphManager, output, inputPathString, shutDownAtEndOfFile); }
protected WatchFileReadStage(GraphManager graphManager, Pipe<?>[] inputs, //requests for new files to read, must contain path in blob data unstructured. Pipe<RawDataSchema>[] outputs, //may be encrypted. Pipe<?>[] ackRelease //downstream is responsible for timeout triggers. //ack release will cause a -1 to be sent to the associated output to complete the round trip ) { super(graphManager, join(inputs,ackRelease), outputs); assert(inputs.length == outputs.length); assert(outputs.length == ackRelease.length); this.inputs = inputs; this.outputs = outputs; this.ackRelease = ackRelease; int i = inputs.length; this.readState = new ReadState[i]; while (--i>=0) { this.readState[i] = new ReadState(); } }
@Override public void run() { if (shutdownInProgress) { if (!Pipe.hasRoomForWrite(target, Pipe.EOF_SIZE)) { return; } requestShutdown(); return; } while (processAvailData(this)) { //keeps going while there is data to read and room to write it. } }
private static <S extends MessageSchema> int readPackedIntB(int a, ByteBuffer src, byte v) { assert(a!=0 || v!=0) : "malformed data"; return (v >= 0) ? readPackedInt((a | v) << 7, src) : a | (v & 0x7F); }
@Test public void writeReleaseAndReadTest() { writeReleaseAndReadImpl(false, false); }
@Override public void shutdown() { //if we are in the middle of a partial copy push the data out, this is blocking while (0!=totalPrimaryCopy) { //if all the copies are done then record it as complete, does as much work as possible each time its called. copyToFile(this); } }
@Override public void run() { int i = inputs.length; while (--i>=0) { process(inputs[i],outputs[i],ackRelease[i],readState[i]); } }
protected WatchServiceStage(GraphManager graphManager, String pathText, String[] filePatterns, Pipe<FolderWatchSchema>[] output, Pipe<FolderWatchSchema> defaultOutput, int kindFlags) { super(graphManager, NONE, null==defaultOutput? output : join(output,defaultOutput)); this.pathText = pathText; this.output = output; this.defaultOutput = defaultOutput; this.filePatterns = filePatterns; this.kindFlags = 0!=kindFlags? kindFlags : 7; }
@Override public void run() { writeProcessing(); }
@Override public void run() { processAvailData(this);//spin here makes little difference, its the ammount of work done which is the problem. }
@Test public void emptyReplayTest() { emptyReplayImpl(false, false); }
@Test public void encryptedWriteWithAckTest() { writeWithAckImpl(true, false); }
@Test public void writeAndReadTest() { writeAndReadImpl(false, false); }
public static <S extends MessageSchema> int readPackedInt(ByteBuffer src) { byte v = src.get(); int accumulator = (~((int)(((v>>6)&1)-1)))&0xFFFFFF80; return (v >= 0) ? readPackedInt((accumulator | v) << 7,src) : accumulator |(v & 0x7F); }
@Test public void encyptedWriteReleaseAndReadTest() { writeReleaseAndReadImpl(true, false); }