public PipeConfig<T> grow2x(){ PipeConfig<T> result = new PipeConfig<T>((byte)(1+slabBits), (byte)(0==blobBits ? 0 : 1+blobBits), byteConst, schema); result.showLabels = showLabels; return result; }
public static boolean isTooSmall(int queueLength, int maxMessageSize, PipeConfig<?> config) { return queueLength>config.minimumFragmentsOnPipe() || maxMessageSize>config.maxVarLenSize(); }
public PrivateTopic(String topic, int messageCount, int messageSize, boolean hideLabels, BuilderImpl builder) { this.topic = topic; this.config = new PipeConfig<MessagePrivate>(MessagePrivate.instance, messageCount, messageSize); if (hideLabels) { this.config.hideLabels(); //private topics can clutter if they show all the details. } this.builder = builder; assert(null!=config); }
this.schema = messageSchema; validate(messageSchema, minimumFragmentsOnRing, maximumLenghOfVariableLengthFields); } catch (UnsupportedOperationException t) { logger.info("unable to define pipe with size {},{} for type {} ",minimumFragmentsOnRing,maximumLenghOfVariableLengthFields,messageSchema); if ((showConfigsCreatedLargerThan>0) && (totalBytesAllocated() >= showConfigsCreatedLargerThan) ) { if (totalBytesAllocated() < (1<<11)) { new Exception(schema.getClass().getSimpleName()+" large config "+(totalBytesAllocated())+" B slab:"+slabBits+" blob:"+blobBits).printStackTrace(); } else { if (totalBytesAllocated() < (1<<21)) { new Exception(schema.getClass().getSimpleName()+" large config "+(totalBytesAllocated()>>10)+" KB slab:"+slabBits+" blob:"+blobBits).printStackTrace(); } else { new Exception(schema.getClass().getSimpleName()+" large config "+(totalBytesAllocated()>>20)+" MB slab:"+slabBits+" blob:"+blobBits).printStackTrace();
linesRingConfig = new PipeConfig(RawDataSchema.instance, 128, 512); fieldsRingConfig = new PipeConfig( new MessageSchemaDynamic(MetaMessageDefs.FROM), 512, 512); fieldsRingConfig = fieldsRingConfig; fieldsRingConfig2 = fieldsRingConfig.grow2x(); flatFileRingConfig = new PipeConfig(RawDataSchema.instance, 1024, 512);
public static <M extends MatrixSchema<M>> Pipe<RowSchema<M>> buildProductGraphRR(GraphManager gm, int targetThreadCount, Pipe<RowSchema<M>> left, Pipe<RowSchema<M>> right) { Pipe<ColumnSchema<M>>[] colResults = buildProductGraphRC(gm, left, right, targetThreadCount); ////////////////// ////////////////// MatrixSchema resultSchema2 = colResults[0].config().schema().rootSchema(); PipeConfig<RowSchema<M>> config = new PipeConfig<RowSchema<M>>(new RowSchema<M>(resultSchema2), resultSchema2.getRows()); config.hideLabels(); Pipe<RowSchema<M>> rowResults = new Pipe<RowSchema<M>>(config); ColumnsToRowsStage<M> ctr = new ColumnsToRowsStage( gm, colResults, rowResults); return rowResults; }
assertTrue(null!=from); PipeConfig busConfig = new PipeConfig(new MessageSchemaDynamic(from), 10, 64); PronghornStage generator = new TestGenerator(gm, seed, iterations, pipe(busConfig)); ReplicatorStage splitter = new ReplicatorStage(gm, getOutputPipe(gm, generator), pipe(busConfig.grow2x()), pipe(busConfig.grow2x())); PronghornStage validateResults = new TestValidator(gm, getOutputPipe(gm, splitter, 2), getOutputPipe(gm, splitter, 1));
assertTrue(null!=from); PipeConfig busConfig = new PipeConfig(new MessageSchemaDynamic(from), 10, 64); PronghornStage generator1 = new TestGenerator(gm, seed, iterations, pipe(busConfig)); PronghornStage generator2 = new TestGenerator(gm, seed, iterations, pipe(busConfig));
/** * * @param builder BuilderImpl used to initialize pipe * @param parallelInstance int used to initialize new Pipe array * @return all the rest requests */ public static Pipe<HTTPRequestSchema>[] newHTTPRequestPipes(BuilderImpl builder, final int parallelInstance) { Pipe<HTTPRequestSchema>[] restRequests = new Pipe[parallelInstance]; PipeConfig<HTTPRequestSchema> pipeConfig = builder.pcm.getConfig(HTTPRequestSchema.class).grow2x(); int idx = restRequests.length; while (--idx>=0) { restRequests[idx] = builder.newHTTPRequestPipe(pipeConfig); } return restRequests; }
Pipe<RowSchema<M>> left = new Pipe<RowSchema<M>>(new PipeConfig<RowSchema<M>>(leftRowSchema, leftRows)); Pipe<RowSchema<M>> right = new Pipe<RowSchema<M>>(new PipeConfig<RowSchema<M>>(rightRowSchema, rightRows)); MatrixSchema<M> leftSchema1 = left.config().schema().rootSchema(); MatrixSchema<M> rightSchema1 = right.config().schema().rootSchema(); MatrixSchema resultSchema = BuildMatrixCompute.buildResultSchema(leftSchema1, rightSchema1); RowSchema<M> rowResultSchema = new RowSchema<M>(resultSchema); Pipe<RowSchema<M>> result = new Pipe<RowSchema<M>>(new PipeConfig<RowSchema<M>>(rowResultSchema, resultSchema.getRows())); //NOTE: reqires 2 or JSON will not write out !! Pipe<DecimalSchema<M>> result2 = new Pipe<DecimalSchema<M>>(new PipeConfig<DecimalSchema<M>>(result2Schema, resultSchema.getRows())); //NOTE: reqires 2 or JSON will not write out !!
public static void growCommandCountRoom(MsgCommandChannel<?> cmd, int count) { if (cmd.isInit) { throw new UnsupportedOperationException("Too late, growCommandCountRoom method must be called in define behavior."); } PipeConfig<TrafficOrderSchema> goConfig = cmd.pcm.getConfig(TrafficOrderSchema.class); cmd.pcm.addConfig(count + goConfig.minimumFragmentsOnPipe(), 0, TrafficOrderSchema.class); }
public PipeConfig(T messageSchema, byte[] byteConst, int minimumFragmentsOnRing, int maximumLenghOfVariableLengthFields) { FieldReferenceOffsetManager from = MessageSchema.from(messageSchema); int biggestFragment = FieldReferenceOffsetManager.maxFragmentSize(from); int primaryMinSize = minimumFragmentsOnRing*biggestFragment; this.slabBits = (byte)(32 - Integer.numberOfLeadingZeros(primaryMinSize - 1)); try { int maxVarFieldsInRingAtOnce = FieldReferenceOffsetManager.maxVarLenFieldsPerPrimaryRingSize(from, 1<<slabBits); int totalBlobSize = maxVarFieldsInRingAtOnce * maximumLenghOfVariableLengthFields; this.blobBits = ((0==maximumLenghOfVariableLengthFields) | (0==maxVarFieldsInRingAtOnce))? (byte)0 : (byte)(32 - Integer.numberOfLeadingZeros(totalBlobSize - 1)); this.byteConst = byteConst; this.schema = messageSchema; validate(messageSchema, minimumFragmentsOnRing, maximumLenghOfVariableLengthFields); } catch (UnsupportedOperationException t) { logger.info("unable to define pipe with size {},{} for type {} ",minimumFragmentsOnRing,maximumLenghOfVariableLengthFields,messageSchema); throw(t); } }
private static <M extends MatrixSchema<M>> MatrixSchema resultSchema(Pipe<RowSchema<M>> left, Pipe<RowSchema<M>> right) { MatrixSchema<M> leftSchema1 = left.config().schema().rootSchema(); MatrixSchema<M> rightSchema1 = right.config().schema().rootSchema(); MatrixSchema resultSchema = BuildMatrixCompute.buildResultSchema(leftSchema1, rightSchema1); return resultSchema; }
while (--c>=0) { serverConfig.ensureServerCanWrite((spaceForEchos+temp[c].config().maxVarLenSize())); serverConfig.ensureServerCanWrite((spaceForEchos+errConfig.maxVarLenSize()));
public Pipe<MessagePrivate> getPipe(int activeIndex) { if (null==p) { p = new Pipe[builder.parallelTracks()]; } if (activeIndex>maxIndex) { throw new UnsupportedOperationException("can not span between primary and tracks with private topic"); } if (activeIndex<0) { //confirm that this topic was never used for tracks since we have a single request for(int i = 1; i<p.length; i++) { if (null!=p[i]) { throw new UnsupportedOperationException("can not span between primary and tracks with private topic"); } } maxIndex = 0; activeIndex = 0; } Pipe<MessagePrivate> result = p[activeIndex]; if (null == result) { result = p[activeIndex] = PipeConfig.pipe(config); } return result; }
try { while (i>limit) { PipeConfig<RawDataSchema> config = new PipeConfig<RawDataSchema>( RawDataSchema.instance, pipeLength, maxVarLength); Pipe<RawDataSchema> loadedDataPipe = new Pipe<RawDataSchema>(config.grow2x()); new TapeReadStage(gm, new RandomAccessFile(tapeFile,"rw"), loadedDataPipe);
MatrixSchema<M> rightSchema = rightInput.config().schema().rootSchema(); PipeConfig<ColumnSchema<M>> rightColumnConfig = new PipeConfig<ColumnSchema<M>>(columnsInputSchema,4); //just big enough to support batching of one consumer and one producer rightColumnConfig.hideLabels(); PipeConfig<ColumnSchema<M>> resultColumnConfig = new PipeConfig<ColumnSchema<M>>(columnsOutoutSchema,4); resultColumnConfig.hideLabels();
assertTrue(null!=from); PipeConfig busConfig = new PipeConfig(new MessageSchemaDynamic(from), 10, 64); Pipe inputRing1 = pipe(busConfig); inputRing1.initBuffers(); Pipe inputRing2 = pipe(busConfig); inputRing2.initBuffers();
private Pipe[] createPipes(BuilderImpl builder, int i, int matches, Object listener, Grouper g) { if (i<interfaces.size()) { final PipeConfig config = g.config(builder.schemaMapper(schemas.get(i))); final boolean isInUse = null!=config; final boolean doesMatch = interfaces.get(i).isInstance(listener) && isInUse; Pipe[] result = createPipes(builder, i+1, doesMatch ? 1+matches : matches, listener, g); if (doesMatch) { // logger.info("Does Match! {}", listener); result[matches] = new Pipe(config.grow2x()); } return result; } else { return new Pipe[matches]; } }