public void writeProcessorMsg(ProcessorMsg msg) throws IOException { serializer.writeProcessorMsg(msg); // Log any info sent on the error stream logErrorStream(); }
@Override public Long connect(Map<String, Object> conf, ShellContext context, List<String> outputStreams) throws IOException, NoOutputException { ConnectMsg connectMsg = new ConnectMsg(); connectMsg.setPidDir(context.getPidDir()); connectMsg.setConf(conf); connectMsg.setOutputStreams(outputStreams); connectMsg.setContext(context); writeConnectMsg(connectMsg); JsonNode node = readMessage(); JsonNode pidNode = node.get("pid"); Long pid = pidNode.asLong(); return pid; }
public ShellMsg readShellMsg() throws IOException { try { return serializer.readShellMsg(); } catch (NoOutputException e) { throw new RuntimeException(e + " ,Serializer Exception: " +getErrorsString() + "\n"); } }
public Long launch(Map<String, Object> conf, ShellContext context, List<String> outputStreams) { ProcessBuilder builder = new ProcessBuilder(command); if (!env.isEmpty()) { Map<String, String> buildEnv = builder.environment(); modifyEnvironment(buildEnv); } builder.directory(new File(context.getCodeDir())); shellLogger = LoggerFactory.getLogger(context.getComponentId()); this.componentName = context.getComponentId(); serializer = getSerializer(); try { LOG.info("Process Environment :" + builder.environment()); subprocess = builder.start(); processErrorStream = subprocess.getErrorStream(); serializer.initialize(subprocess.getOutputStream(), subprocess.getInputStream()); this.pid = serializer.connect(conf, context, outputStreams); } catch (IOException e) { throw new RuntimeException( "Error when launching multilang subprocess\n" + getErrorsString(), e); } catch (NoOutputException e) { throw new RuntimeException(e + " , Serializer Exception: " + getErrorsString() + "\n"); } return this.pid; }
private ProcessorMsg createProcessorMessage(StreamlineEvent event) { ProcessorMsg processorMsg = new ProcessorMsg(); processorMsg.setId(event.getId()); processorMsg.setSourceId(event.getDataSourceId()); processorMsg.setSourceStream(event.getSourceStream()); processorMsg.setFieldsAndValues(event); return processorMsg; }
private ShellContext getShellContext(TopologyContext context) { ShellContext shellContext = new ShellContext(); shellContext.setCodeDir(context.getCodeDir()); shellContext.setPidDir(context.getPIDDir()); shellContext.setComponentId(context.getThisComponentId()); return shellContext; }
@Override public void initialize(Map<String, Object> config) { command = (String[]) config.get(COMMAND); processTimeoutMills = (int) config.get(PROCESS_TIMEOUT_MILLS); Map<String, Object> processorConfig = (Map<String, Object>) config.get(PROCESS_CONFIG); ShellContext shellContext = (ShellContext) config.get(SHELL_CONTEXT); List<String> outputStreams = (List<String>) config.get(OUTPUT_STREAMS); Map<String, String> envMap = (Map<String, String>) config.get(SHELL_ENVIRONMENT); String className = (String) config.get(MULTILANG_SERIALIZER); shellProcess = new ShellProcess(command); if(className != null) shellProcess.setSerializerClassName(className); shellProcess.setEnv(envMap); //subprocesses must send their pid first thing Long subpid = shellProcess.launch(processorConfig, shellContext, outputStreams); LOG.info("Launched subprocess with pid " + subpid); LOG.info("Start checking heartbeat..."); setHeartbeat(); heartBeatExecutorService = MoreExecutors.getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1)); heartBeatExecutorService.scheduleAtFixedRate(new HeartbeatTimerTask(this), 1, 1, TimeUnit.SECONDS); }
public String getProcessTerminationInfoString() { return String.format(" exitCode:%s, errorString:%s ", getExitCode(), getErrorsString()); } }
private void die(Throwable exception) { String processInfo = shellProcess.getProcessInfoString() + shellProcess.getProcessTerminationInfoString(); this.exception = new RuntimeException(processInfo, exception); String message = String.format("Halting process: Processor died. Command: %s, ProcessInfo %s", Arrays.toString(command), processInfo); LOG.error(message, exception); if (running || (exception instanceof Error)) { //don't exit if not running, unless it is an Error System.exit(11); } }
protected String getJarPathFor(Long jarId) throws IOException { checkProperty(config, Constants.LOCAL_FILES_PATH); checkProperty(config, Constants.CATALOG_ROOT_URL); File filesDir = new File(config.get(Constants.LOCAL_FILES_PATH).toString()); ensureDirExists(filesDir); File localFile; do { localFile = new File(filesDir, jarId + "-" + UUID.randomUUID()); } while(localFile.exists()); localFile.deleteOnExit(); final CatalogRestClient catalogRestClient = new CatalogRestClient(config.get(Constants.CATALOG_ROOT_URL).toString()); try(final FileOutputStream output = new FileOutputStream(localFile); final InputStream inputStream = catalogRestClient.getFile(jarId)) { IOUtils.copy(inputStream, output); } return localFile.getAbsolutePath(); }
@Override public void writeProcessorMsg(ProcessorMsg processorMsg) throws IOException { String jsonString = objectMapper.writeValueAsString(processorMsg); writeString(jsonString); }
private JsonNode readMessage() throws IOException, NoOutputException { String jsonString = readString(); JsonNode msg = objectMapper.readValue(jsonString, JsonNode.class); if (msg != null) { return msg; } else { throw new IOException("unable to parse: " + jsonString); } }
@Override public void cleanup() { heartBeatExecutorService.shutdownNow(); shellProcess.destroy(); running = false; }
public InputStream getFile(Long jarId) { return getInputStream(jarId.toString(), FILE_DOWNLOAD_URL); }
public void logErrorStream() { String error = getErrorsString(); if (!error.isEmpty()) shellLogger.info(error); }
private String readString() throws IOException, NoOutputException { StringBuilder line = new StringBuilder(); while (true) { String subline = processOut.readLine(); if (subline == null) { StringBuilder errorMessage = new StringBuilder(); errorMessage.append("Pipe to subprocess seems to be broken!"); if (line.length() == 0) { errorMessage.append(" No output read.\n"); } else { errorMessage.append(" Currently read output: " + line.toString() + "\n"); } throw new NoOutputException(errorMessage.toString()); } if (subline.equals("end")) { break; } if (line.length() != 0) { line.append("\n"); } line.append(subline); } return line.toString(); }
private Result convertShellMsg(String stream, List<ShellMsg> shellMsgList, StreamlineEvent inputEvent) { List<StreamlineEvent> streamlineEvents = new LinkedList<>(); for (ShellMsg shellMsg: shellMsgList) { streamlineEvents.add(convertShellEvent(shellMsg.getStreamlineEvent(), inputEvent)); } return new Result(stream, streamlineEvents); }
private StreamlineEvent convertShellEvent(ShellMsg.ShellEvent shellEvent, StreamlineEvent inputEvent) { return StreamlineEventImpl.builder() .from(inputEvent) .fieldsAndValues(shellEvent.getFieldsAndValues()) .build(); }
private void writeConnectMsg(ConnectMsg connectMsg) throws IOException { String jsonString = objectMapper.writeValueAsString(connectMsg); writeString(jsonString); }
@Override public ShellMsg readShellMsg() throws IOException, NoOutputException { String jsonString = readString(); try { ShellMsg shellMsg = objectMapper.readValue(jsonString, ShellMsg.class); return shellMsg; } catch (IOException e) { LOG.error("Error during deserialization of output schema JSON string: {}", jsonString, e); throw new RuntimeException(e); } }