private void getActiveFlowsWithExecutorHelper( final List<Pair<ExecutableFlow, Optional<Executor>>> flows, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { flows.add(new Pair<>(ref.getSecond(), ref .getFirst().getExecutor())); } }
private void parseFlowId() { this.jobPath = new ArrayList<>(); // parsing pattern: flowRootName[,embeddedFlowName:embeddedFlowPath]* final String[] flowPairs = this.flowId.split(","); for (final String flowPair : flowPairs) { // splitting each embeddedFlowName:embeddedFlowPath pair by the first occurrence of ':' // only because embeddedFlowPath also uses ':' as delimiter. // Ex: "embeddedFlow3:rootFlow:embeddedFlow1:embeddedFlow2:embeddedFlow3" will result in // ["embeddedFlow3", "rootFlow:embeddedFlow1:embeddedFlow2:embeddedFlow3"] final String[] pairSplit = flowPair.split(":", 2); final Pair<String, String> pair; if (pairSplit.length == 1) { pair = new Pair<>(pairSplit[0], pairSplit[0]); } else { pair = new Pair<>(pairSplit[0], pairSplit[1]); } this.jobPath.add(pair); } this.immediateFlowId = this.jobPath.get(this.jobPath.size() - 1).getSecond(); }
/** * This is used to get the min/max memory size requirement by processes. SystemMemoryInfo can use * the info to determine if the memory request can be fulfilled. For Java process, this should be * Xms/Xmx setting. * * @return pair of min/max memory size */ protected Pair<Long, Long> getProcMemoryRequirement() throws Exception { return new Pair<>(0L, 0L); }
@Override public LogData getExecutionJobLog(final ExecutableFlow exFlow, final String jobId, final int offset, final int length, final int attempt) throws ExecutorManagerException { final Pair<ExecutionReference, ExecutableFlow> pair = this.executorLoader .fetchActiveFlowByExecId(exFlow.getExecutionId()); if (pair != null) { final Pair<String, String> typeParam = new Pair<>("type", "job"); final Pair<String, String> jobIdParam = new Pair<>("jobId", jobId); final Pair<String, String> offsetParam = new Pair<>("offset", String.valueOf(offset)); final Pair<String, String> lengthParam = new Pair<>("length", String.valueOf(length)); final Pair<String, String> attemptParam = new Pair<>("attempt", String.valueOf(attempt)); @SuppressWarnings("unchecked") final Map<String, Object> result = this.apiGateway.callWithReference(pair.getFirst(), ConnectorParams.LOG_ACTION, typeParam, jobIdParam, offsetParam, lengthParam, attemptParam); return LogData.createLogDataFromObject(result); } else { final LogData value = this.executorLoader.fetchLogs(exFlow.getExecutionId(), jobId, attempt, offset, length); return value; } }
@Test public void testAsciiUTF() throws IOException { final String foreignText = "abcdefghijklmnopqrstuvwxyz"; final byte[] utf8ByteArray = createUTF8ByteArray(foreignText); final int length = utf8ByteArray.length; System.out.println("char length:" + foreignText.length() + " utf8BytesLength:" + utf8ByteArray.length + " for:" + foreignText); final Pair<Integer, Integer> pair = FileIOUtils.getUtf8Range(utf8ByteArray, 1, length - 6); System.out.println("Pair :" + pair.toString()); final String recreatedString = new String(utf8ByteArray, 1, length - 6, "UTF-8"); System.out.println("recreatedString:" + recreatedString); final String correctString = new String(utf8ByteArray, pair.getFirst(), pair.getSecond(), "UTF-8"); System.out.println("correctString:" + correctString); assertEquals(pair, new Pair<>(1, 20)); // Two characters stripped from this. assertEquals(correctString.length(), foreignText.length() - 6); }
private void getActiveFlowHelper(final ArrayList<ExecutableFlow> flows, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { flows.add(ref.getSecond()); } }
public String getJobIdPath() { // Skip the first one because it's always just the root. String path = ""; for (int i = 1; i < this.jobPath.size(); ++i) { final Pair<String, String> pair = this.jobPath.get(i); path += pair.getFirst() + ":"; } path += this.jobId; return path; }
/** * <pre> * Enqueues all the elements of a collection * * @param collection * * @throws ExecutorManagerException * case 1: if blocking queue put method fails due to * InterruptedException * case 2: if there already an element with * same execution Id * </pre> */ public void enqueueAll( final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) throws ExecutorManagerException { for (final Pair<ExecutionReference, ExecutableFlow> pair : collection) { enqueue(pair.getSecond(), pair.getFirst()); } }
public List<Pair<String, Permission>> getUserPermissions() { final ArrayList<Pair<String, Permission>> permissions = new ArrayList<>(); for (final Map.Entry<String, Permission> entry : this.userPermissionMap.entrySet()) { permissions.add(new Pair<>(entry.getKey(), entry .getValue())); } return permissions; }
@Override public LogData getExecutionJobLog(final ExecutableFlow exFlow, final String jobId, final int offset, final int length, final int attempt) throws ExecutorManagerException { final Pair<ExecutionReference, ExecutableFlow> pair = this.runningExecutions.get().get(exFlow.getExecutionId()); if (pair != null) { final Pair<String, String> typeParam = new Pair<>("type", "job"); final Pair<String, String> jobIdParam = new Pair<>("jobId", jobId); final Pair<String, String> offsetParam = new Pair<>("offset", String.valueOf(offset)); final Pair<String, String> lengthParam = new Pair<>("length", String.valueOf(length)); final Pair<String, String> attemptParam = new Pair<>("attempt", String.valueOf(attempt)); @SuppressWarnings("unchecked") final Map<String, Object> result = this.apiGateway.callWithReference(pair.getFirst(), ConnectorParams.LOG_ACTION, typeParam, jobIdParam, offsetParam, lengthParam, attemptParam); return LogData.createLogDataFromObject(result); } else { final LogData value = this.executorLoader.fetchLogs(exFlow.getExecutionId(), jobId, attempt, offset, length); return value; } }
@Test public void testForeignUTF() throws IOException { final String foreignText = "안녕하세요, 제 이름은 박병호입니다"; final byte[] utf8ByteArray = createUTF8ByteArray(foreignText); final int length = utf8ByteArray.length; System.out.println("char length:" + foreignText.length() + " utf8BytesLength:" + utf8ByteArray.length + " for:" + foreignText); final Pair<Integer, Integer> pair = FileIOUtils.getUtf8Range(utf8ByteArray, 1, length - 6); System.out.println("Pair :" + pair.toString()); final String recreatedString = new String(utf8ByteArray, 1, length - 6, "UTF-8"); System.out.println("recreatedString:" + recreatedString); String correctString = new String(utf8ByteArray, pair.getFirst(), pair.getSecond(), "UTF-8"); System.out.println("correctString:" + correctString); assertEquals(pair, new Pair<>(3, 40)); // Two characters stripped from this. assertEquals(correctString.length(), foreignText.length() - 3); // Testing mixed bytes final String mixedText = "abc안녕하세요, 제 이름은 박병호입니다"; final byte[] mixedBytes = createUTF8ByteArray(mixedText); final Pair<Integer, Integer> pair2 = FileIOUtils.getUtf8Range(mixedBytes, 1, length - 4); correctString = new String(mixedBytes, pair2.getFirst(), pair2.getSecond(), "UTF-8"); System.out.println("correctString:" + correctString); assertEquals(pair2, new Pair<>(1, 45)); // Two characters stripped from this. assertEquals(correctString.length(), mixedText.length() - 3); }
@Test public void testExplicitlySpecifiedPriorities() throws IOException, InterruptedException { final ExecutableFlow flow1 = createExecutableFlow("exec1", 5, 3, 1); final ExecutableFlow flow2 = createExecutableFlow("exec2", 6, 3, 2); final ExecutableFlow flow3 = createExecutableFlow("exec3", 2, 3, 3); final ExecutionReference dummyRef = new ExecutionReference(0); final BlockingQueue<Pair<ExecutionReference, ExecutableFlow>> queue = new PriorityBlockingQueue<>(10, new ExecutableFlowPriorityComparator()); queue.put(new Pair<>(dummyRef, flow1)); queue.put(new Pair<>(dummyRef, flow2)); queue.put(new Pair<>(dummyRef, flow3)); Assert.assertEquals(flow2, queue.take().getSecond()); Assert.assertEquals(flow1, queue.take().getSecond()); Assert.assertEquals(flow3, queue.take().getSecond()); }
/** * Helper method to get all flows from collection. */ private void getFlowsHelper(final ArrayList<ExecutableFlow> flows, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { collection.stream().forEach(ref -> flows.add(ref.getSecond())); }
/** * Wraps BoundedQueue Take method to have a corresponding update in queuedFlowMap lookup table */ public Pair<ExecutionReference, ExecutableFlow> fetchHead() throws InterruptedException { final Pair<ExecutionReference, ExecutableFlow> pair = this.queuedFlowList.take(); if (pair != null && pair.getFirst() != null) { this.queuedFlowMap.remove(pair.getFirst().getExecId()); } return pair; }
/** * helper function to fill the request with header entries and posting body . */ private static HttpEntityEnclosingRequestBase completeRequest( final HttpEntityEnclosingRequestBase request, final List<Pair<String, String>> params) throws UnsupportedEncodingException { if (request != null) { if (null != params && !params.isEmpty()) { final List<NameValuePair> formParams = params.stream() .map(pair -> new BasicNameValuePair(pair.getFirst(), pair.getSecond())) .collect(Collectors.toList()); final HttpEntity entity = new UrlEncodedFormEntity(formParams, "UTF-8"); request.setEntity(entity); } } return request; }
private void getActiveFlowsWithExecutorHelper( final List<Pair<ExecutableFlow, Optional<Executor>>> flows, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { flows.add(new Pair<>(ref.getSecond(), ref .getFirst().getExecutor())); } }
public List<Pair<String, Permission>> getGroupPermissions() { final ArrayList<Pair<String, Permission>> permissions = new ArrayList<>(); for (final Map.Entry<String, Permission> entry : this.groupPermissionMap.entrySet()) { permissions.add(new Pair<>(entry.getKey(), entry .getValue())); } return permissions; }
@Override public List<Object> getExecutionJobStats(final ExecutableFlow exFlow, final String jobId, final int attempt) throws ExecutorManagerException { final Pair<ExecutionReference, ExecutableFlow> pair = this.executorLoader.fetchActiveFlowByExecId(exFlow.getExecutionId()); if (pair == null) { return this.executorLoader.fetchAttachments(exFlow.getExecutionId(), jobId, attempt); } final Pair<String, String> jobIdParam = new Pair<>("jobId", jobId); final Pair<String, String> attemptParam = new Pair<>("attempt", String.valueOf(attempt)); @SuppressWarnings("unchecked") final Map<String, Object> result = this.apiGateway.callWithReference(pair.getFirst(), ConnectorParams.ATTACHMENTS_ACTION, jobIdParam, attemptParam); @SuppressWarnings("unchecked") final List<Object> jobStats = (List<Object>) result .get("attachments"); return jobStats; }
@Test public void testMixedSpecifiedPriorities() throws IOException, InterruptedException { final ExecutableFlow flow1 = createExecutableFlow("exec1", 3, 3, 1); final ExecutableFlow flow2 = createExecutableFlow("exec2", 2, 3, 2); final ExecutableFlow flow3 = createExecutableFlow("exec3", -2, 3, 3); final ExecutionReference dummyRef = new ExecutionReference(0); final BlockingQueue<Pair<ExecutionReference, ExecutableFlow>> queue = new PriorityBlockingQueue<>(10, new ExecutableFlowPriorityComparator()); queue.put(new Pair<>(dummyRef, flow1)); queue.put(new Pair<>(dummyRef, flow2)); queue.put(new Pair<>(dummyRef, flow3)); Assert.assertEquals(flow3, queue.take().getSecond()); Assert.assertEquals(flow1, queue.take().getSecond()); Assert.assertEquals(flow2, queue.take().getSecond()); }