public OutputPort<IMonitoringRecord> getOutputPort() { return this.reader.getOutputPort(); }
public OutputPort<IMonitoringRecord> getOutputPort() { return this.directoryReaderStage.getOutputPort(); }
/** * Creates a composite stage to scan and read a set of Kieker log directories. * * @param configuration configuration for the enclosed filters */ public LogsReaderCompositeStage(final Configuration configuration) { final String[] directoryNames = configuration.getStringArrayProperty(LOG_DIRECTORIES, ":"); final File[] directories = new File[directoryNames.length]; int i = 0; for (final String name : directoryNames) { directories[i++] = new File(name); } this.directoryScannerStage = new DirectoryScannerStage(directories); this.directoryReaderStage = new DirectoryReaderStage(configuration); this.connectPorts(this.directoryScannerStage.getOutputPort(), this.directoryReaderStage.getInputPort()); }
public Dir2RecordsFilter(final ClassNameRegistryRepository classNameRegistryRepository) { this.classNameRegistryRepository = classNameRegistryRepository; // does not yet work with more than one thread due to classNameRegistryRepository: classNameRegistryRepository is set after the ctor // create stages final ClassNameRegistryCreationFilter tempClassNameRegistryCreationFilter = new ClassNameRegistryCreationFilter(this.classNameRegistryRepository); final Directory2FilesFilter directory2FilesFilter = new Directory2FilesFilter(); final FileExtensionSwitch fileExtensionSwitch = new FileExtensionSwitch(); final DatFile2RecordFilter datFile2RecordFilter = new DatFile2RecordFilter(this.classNameRegistryRepository); final BinaryFile2RecordFilter binaryFile2RecordFilter = new BinaryFile2RecordFilter(this.classNameRegistryRepository); this.recordMerger = new Merger<>(); // store ports due to readability reasons final OutputPort<File> datFileOutputPort = fileExtensionSwitch.addFileExtension(FSUtil.DAT_FILE_EXTENSION); final OutputPort<File> binFileOutputPort = fileExtensionSwitch.addFileExtension(BinaryCompressionMethod.NONE.getFileExtension()); // connect ports by pipes this.connectPorts(tempClassNameRegistryCreationFilter.getOutputPort(), directory2FilesFilter.getInputPort()); this.connectPorts(directory2FilesFilter.getOutputPort(), fileExtensionSwitch.getInputPort()); this.connectPorts(datFileOutputPort, datFile2RecordFilter.getInputPort()); this.connectPorts(binFileOutputPort, binaryFile2RecordFilter.getInputPort()); this.connectPorts(datFile2RecordFilter.getOutputPort(), this.recordMerger.getNewInputPort()); this.connectPorts(binaryFile2RecordFilter.getOutputPort(), this.recordMerger.getNewInputPort()); // prepare pipeline this.classNameRegistryCreationFilter = tempClassNameRegistryCreationFilter; }
public ReadingComposite(final File importDirectory) { final InitialElementProducer<File> producer = new InitialElementProducer<>(importDirectory); this.reader = new Dir2RecordsFilter(new ClassNameRegistryRepository()); super.connectPorts(producer.getOutputPort(), this.reader.getInputPort()); }
/** * Check whether the scanner produces nothing and terminates. */ @Test public void testEmptyArrayOfDirectories() { final DirectoryScannerStage producer = new DirectoryScannerStage(null); StageTester.test(producer).start(); Assert.assertThat(producer.getOutputPort(), StageTester.producesNothing()); }
private void scanDirectory(final File directory) { if (this.isKiekerDirectory(directory)) { this.logger.debug("Reading log data from {}", directory.getAbsolutePath()); this.numOfDirectories++; this.getOutputPort().send(directory); } else { for (final File subDirectory : directory.listFiles()) { // NOFB is guaranteed to be a directory if (subDirectory.isDirectory()) { this.scanDirectory(subDirectory); } } } }
/** * Execute the directory scanner. */ @Override protected void execute() { if (this.directories != null) { for (final File directory : this.directories) { this.logger.debug("Scanning directories in {} for Kieker logs", directory); if (directory.isDirectory()) { this.scanDirectory(directory); } } } else { this.logger.error("Cannot process an empty array of directories."); } this.logger.debug("Processed {} directories.", this.numOfDirectories); this.workCompleted(); }
@Override public void processDataStream(final InputStream inputStream, final OutputPort<IMonitoringRecord> outputPort) throws IOException { final byte[] buffer = new byte[this.charBuffer.length() * 2]; // NOPMD boolean endOfFile = false; int offset = 0; this.lineNumber = 0; while (!endOfFile) { final int numOfReadBytes = inputStream.read(buffer, offset, buffer.length - offset); if (numOfReadBytes == -1) { /** end of line. */ endOfFile = true; } else { final int numOfBufferedBytes = numOfReadBytes + offset; final int mark = this.processBuffer(buffer, offset, numOfBufferedBytes, outputPort); /** move remaining left. */ for (int j = mark + 1; j < numOfBufferedBytes; j++) { buffer[j - mark - 1] = buffer[j]; } offset = numOfBufferedBytes - mark; } } if (this.charBuffer.position() > 0) { this.createRecord(outputPort); } }
final String mapFileName = mapFile.getName(); try { this.readMapFile(new FileInputStream(mapFile), mapFileName, registry); } catch (final FileNotFoundException e) { this.logger.error("Cannot find map file {}.", mapFileName); final String logFileName = logFile.getName(); try { this.readLogFile(new FileInputStream(logFile), logFileName, registry); } catch (final FileNotFoundException e) { this.logger.error("Cannot find log file {}.", logFileName);
if (ch == '\n') { this.lineNumber++; this.createRecord(outputPort); mark = i; } else if (ch == '\r') { this.createRecord(outputPort); mark = i; } else {
private void processBuffer(final IValueDeserializer deserializer, final OutputPort<IMonitoringRecord> outputPort) throws IOException { this.buffer.flip(); try { /** Needs at least an record id. */ while ((this.buffer.position() + 4) <= this.buffer.limit()) { this.buffer.mark(); final IMonitoringRecord record = this.deserializeRecord(deserializer); if (record == null) { return; } else { outputPort.send(record); } } this.buffer.mark(); this.buffer.compact(); } catch (final BufferUnderflowException ex) { LOGGER.warn("Unexpected buffer underflow. Resetting and compacting buffer.", ex); this.buffer.reset(); this.buffer.compact(); throw ex; } }
@Override public void processDataStream(final InputStream chainInputStream, final OutputPort<IMonitoringRecord> outputPort) throws IOException { final BinaryValueDeserializer deserializer = BinaryValueDeserializer.create(this.buffer , this.registry); boolean endOfStreamReached = false; while (!endOfStreamReached) { byte[] bytes = this.buffer.array(); int bytesRead = chainInputStream.read(bytes, this.buffer.position(), this.buffer.remaining()); this.buffer.position(this.buffer.position() + bytesRead); while (bytesRead > 0) { bytes = this.buffer.array(); bytesRead = chainInputStream.read(bytes, this.buffer.position(), this.buffer.remaining()); if (bytesRead >= 0) { this.buffer.position(this.buffer.position() + bytesRead); } } if (bytesRead == -1) { endOfStreamReached = true; } this.processBuffer(deserializer, outputPort); if (endOfStreamReached) { chainInputStream.close(); } } }
continue; // continue on errors } else { this.registerEntry(registry, line.substring(0, split), line.substring(split + 1));
/** * Read a map file stream and initialize the registry. * * @param inputStream the input stream * @param logFileName the name of the log file used for user feedback * @param registry string registry */ private void readMapFile(final InputStream inputStream, final String mapFileName, final ReaderRegistry<String> registry) { final Class<? extends AbstractDecompressionFilter> decompressionClass = FSReaderUtil.findDecompressionFilterByExtension(mapFileName); this.logger.debug("Reading map file {}", mapFileName); /** detecting correct map file deserializer. */ final Class<? extends AbstractMapDeserializer> deserializerClass; if (decompressionClass.equals(NoneDecompressionFilter.class)) { deserializerClass = FSReaderUtil.findMapDeserializer(mapFileName); } else { final String baseName = mapFileName.substring(0, mapFileName.lastIndexOf('.') - 1); deserializerClass = FSReaderUtil.findMapDeserializer(baseName); } try { final AbstractDecompressionFilter decompressionFilter = decompressionClass.getConstructor(Configuration.class).newInstance(this.configuration); final AbstractMapDeserializer deserializer = deserializerClass.getConstructor(Configuration.class).newInstance(this.configuration); deserializer.processDataStream(decompressionFilter.chainInputStream(inputStream), registry, mapFileName); } catch (final IOException ex) { this.logger.error("Reading map file {} failed.", mapFileName); } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { this.logger.error("Cannot instantiate filter {} for decompression.", decompressionClass.getName()); } }
final AbstractEventDeserializer deserializer = deserializerClass.getConstructor(Configuration.class, ReaderRegistry.class).newInstance(this.configuration, registry); deserializer.processDataStream(decompressionFilter.chainInputStream(inputStream), this.outputPort); } catch (final IOException e) { this.logger.error("Reading log file {} failed.", logFileName);
public ReadingComposite(final File importDirectory) { this.producer = new InitialElementProducer<>(importDirectory); this.reader = new Dir2RecordsFilter(new ClassNameRegistryRepository()); this.connectPorts(this.producer.getOutputPort(), this.reader.getInputPort()); }
/** * Check whether the scanner produces a File handle for each directory. There should be three * directories. However, the sequence can vary depending on the operating system and file system * properties. therefore, we use our own matcher. */ @Test public void testPopulatedArrayOfDirectories() { final DirectoryScannerStage producer = new DirectoryScannerStage(this.directories); StageTester.test(producer).start(); Assert.assertThat(producer.getOutputPort(), new RandomContentMatcher(this.results)); }
public OutputPort<IMonitoringRecord> getOutputPort() { return this.reader.getOutputPort(); }