public String toKey() { // Used as ID in storage etc. Do not change. return componentId() + "#" + id(); }
@Override public List<Backfill> backfillsForWorkflow(boolean showAll, String workflow) throws IOException { Stream<Backfill> backfillStream = backfillStore.values().stream() .filter(backfill -> backfill.workflowId().id().equals(workflow)); if (!showAll) { backfillStream = backfillStream .filter(backfill -> backfill.halted() && backfill.allTriggered()); } return ImmutableList.copyOf(backfillStream.collect(Collectors.toList())); }
static Workflow parseWorkflowJson(Entity entity, WorkflowId workflowId) throws IOException { try { return OBJECT_MAPPER .readValue(entity.getString(PROPERTY_WORKFLOW_JSON), Workflow.class); } catch (IOException e) { LOG.error("Failed to read workflow for {}, {}", workflowId.componentId(), workflowId.id(), e); throw e; } }
private static List<EnvVar> buildEnv(WorkflowInstance workflowInstance, RunSpec runSpec, String styxEnvironment) { // store user provided env first to prevent accidentally/intentionally overwriting system ones final Map<String, String> env = new HashMap<>(runSpec.env()); env.put(COMPONENT_ID, workflowInstance.workflowId().componentId()); env.put(WORKFLOW_ID, workflowInstance.workflowId().id()); env.put(PARAMETER, workflowInstance.parameter()); env.put(COMMIT_SHA, runSpec.commitSha().orElse("")); env.put(SERVICE_ACCOUNT, runSpec.serviceAccount().orElse("")); env.put(DOCKER_ARGS, String.join(" ", runSpec.args())); env.put(DOCKER_IMAGE, runSpec.imageName()); env.put(EXECUTION_ID, runSpec.executionId()); env.put(TERMINATION_LOG, "/dev/termination-log"); env.put(TRIGGER_ID, runSpec.trigger().map(TriggerUtil::triggerId).orElse(null)); env.put(TRIGGER_TYPE, runSpec.trigger().map(TriggerUtil::triggerType).orElse(null)); env.put(ENVIRONMENT, styxEnvironment); env.put(LOGGING, "structured"); return env.entrySet().stream() .map(entry -> envVar(entry.getKey(), entry.getValue())) .collect(Collectors.toList()); }
@Override public void printWorkflow(Workflow wf, WorkflowState state) { System.out.println("Component: " + wf.componentId()); System.out.println(" Workflow: " + wf.id().id()); System.out.println(" Schedule: " + wf.configuration().schedule()); System.out.println(" Offset: " + wf.configuration().offset().orElse("")); System.out.println(" Image: " + wf.configuration().dockerImage().orElse("")); System.out.println(" Args: " + wf.configuration().dockerArgs().orElse(Collections.emptyList())); System.out.println(" TermLog: " + wf.configuration().dockerTerminationLogging()); System.out.println(" Secret: " + wf.configuration().secret().map(s -> s.name() + ':' + s.mountPath()).orElse("")); System.out.println(" Svc Acct: " + wf.configuration().serviceAccount().orElse("")); System.out.println("Resources: " + wf.configuration().resources()); System.out.println(" Env: " + Joiner.on(' ').withKeyValueSeparator('=').join(wf.configuration().env())); System.out.println(" Timeout: " + wf.configuration().runningTimeout().map(Duration::toString).orElse("")); System.out.println(" Commit: " + wf.configuration().commitSha().orElse("")); System.out.println(" Enabled: " + state.enabled().map(Object::toString).orElse("")); System.out.println(" Trig: " + state.nextNaturalTrigger().map(Object::toString).orElse("")); System.out.println("Ofst Trig: " + state.nextNaturalOffsetTrigger().map(Object::toString).orElse("")); }
@Override public void printBackfills(List<BackfillPayload> backfills, boolean noTruncate) { final int cidLength = backfills.stream() .map(x -> effectiveLongFieldLength( Optional.of(x.backfill().workflowId().componentId()), COMPONENT_HEADER, true)) .max(Comparator.naturalOrder()) .orElse(1); final int widLength = backfills.stream() .map(x -> effectiveLongFieldLength( Optional.of(x.backfill().workflowId().id()), WORKFLOW_HEADER, true)) .max(Comparator.naturalOrder()) .orElse(1); final int descriptionLength = backfills.stream() .map(x -> effectiveLongFieldLength( x.backfill().description(), DESCRIPTION_HEADER, noTruncate)) .max(Comparator.naturalOrder()) .orElse(1); printBackfillHeader(cidLength, widLength, descriptionLength); for (BackfillPayload backfillPayload : backfills) { printBackfill(backfillPayload.backfill(), cidLength, widLength, descriptionLength, noTruncate); if (backfillPayload.statuses().isPresent()) { System.out.println(); System.out.println(); printStates(backfillPayload.statuses().get()); } } }
@Override public void printBackfill(Backfill backfill, boolean noTruncate) { printBackfill(backfill, effectiveLongFieldLength( Optional.ofNullable(backfill.workflowId().componentId()), COMPONENT_HEADER, true), effectiveLongFieldLength( Optional.ofNullable(backfill.workflowId().id()), WORKFLOW_HEADER, true), effectiveLongFieldLength( backfill.description(), DESCRIPTION_HEADER, noTruncate), noTruncate); }
@Override public void printWorkflow(Workflow wf, WorkflowState state) { System.out.println(Joiner.on(' ').join( wf.componentId(), wf.id().id(), wf.configuration().schedule(), wf.configuration().offset().orElse(""), wf.configuration().dockerImage().orElse(""), wf.configuration().dockerArgs().orElse(Collections.emptyList()), wf.configuration().dockerTerminationLogging(), wf.configuration().secret().map(s -> s.name() + ':' + s.mountPath()).orElse(""), wf.configuration().serviceAccount().map(Object::toString).orElse(""), wf.configuration().resources(), wf.configuration().env(), wf.configuration().runningTimeout().map(Duration::toString).orElse(""), wf.configuration().commitSha().orElse(""), state.enabled().map(Object::toString).orElse(""), state.nextNaturalTrigger().map(Object::toString).orElse(""), state.nextNaturalOffsetTrigger().map(Object::toString).orElse(""))); }
@Override public void printStates(RunStateDataPayload runStateDataPayload) { System.out.println(String.format(" %-20s %-12s %-47s %-7s %s", "WORKFLOW INSTANCE", "STATE", "EXECUTION ID", "TRIES", "PREVIOUS EXECUTION MESSAGE")); CliUtil.groupStates(runStateDataPayload.activeStates()).entrySet().forEach(entry -> { System.out.println(); System.out.println(String.format("%s %s", colored(CYAN, entry.getKey().componentId()), colored(BLUE, entry.getKey().id()))); entry.getValue().forEach(runStateData -> { final StateData stateData = runStateData.stateData(); final Ansi ansiState = getAnsiForState(runStateData); final Message lastMessage = stateData.message().orElse(Message.create(Message.MessageLevel.UNKNOWN, "No info")); final Ansi ansiMessage = colored(messageColor(lastMessage.level()), lastMessage.line()); System.out.println(String.format(" %-20s %-20s %-47s %-7d %s", runStateData.workflowInstance().parameter(), ansiState, stateData.executionId().orElse("<no-execution-id>"), stateData.tries(), ansiMessage)); }); }); }
static Key workflowKey(KeyFactory keyFactory, WorkflowId workflowId) { return keyFactory.addAncestor(PathElement.of(KIND_COMPONENT, workflowId.componentId())) .setKind(KIND_WORKFLOW) .newKey(workflowId.id()); }
@Override public void printStates(RunStateDataPayload runStateDataPayload) { SortedMap<WorkflowId, SortedSet<RunStateDataPayload.RunStateData>> groupedStates = CliUtil.groupStates(runStateDataPayload.activeStates()); groupedStates.forEach((workflowId, value) -> value.forEach(RunStateData -> { final StateData stateData = RunStateData.stateData(); System.out.println(String.format( "%s %s %s %s %s %d %s", workflowId.componentId(), workflowId.id(), RunStateData.workflowInstance().parameter(), RunStateData.state(), stateData.executionId().orElse("<no-execution-id>"), stateData.tries(), stateData.message().map(Message::line).orElse("No info") )); })); }
List<Backfill> getBackfillsForWorkflowId(boolean showAll, WorkflowId workflowId) throws IOException { final EntityQuery query = backfillQueryBuilder( showAll, PropertyFilter.eq(PROPERTY_COMPONENT, workflowId.componentId()), PropertyFilter.eq(PROPERTY_WORKFLOW, workflowId.id())) .build(); return backfillsForQuery(query); }
@Override public void printBackfillPayload(BackfillPayload backfillPayload, boolean noTruncate) { printBackfillHeader( effectiveLongFieldLength( Optional.ofNullable(backfillPayload.backfill().workflowId().componentId()), COMPONENT_HEADER, true), effectiveLongFieldLength(Optional.ofNullable(backfillPayload.backfill().workflowId().id()), WORKFLOW_HEADER, true), effectiveLongFieldLength(backfillPayload.backfill().description(), DESCRIPTION_HEADER, noTruncate)); printBackfill(backfillPayload.backfill(), noTruncate); if (backfillPayload.statuses().isPresent()) { System.out.println(); System.out.println(); printStates(backfillPayload.statuses().get()); } }
@Override public void printBackfill(Backfill backfill, boolean ignored) { System.out.println(String.format("%s %s %s %s %s %s %s %s %s %s %s %s", backfill.id(), backfill.workflowId().componentId(), backfill.workflowId().id(), backfill.halted(), backfill.allTriggered(), backfill.concurrency(), backfill.start(), backfill.end(), backfill.reverse(), backfill.nextTrigger(), backfill.description().orElse(""), backfill.triggerParameters().map(triggerParameters -> formatMap( triggerParameters.env())).orElse(""))); }
private void printBackfill(Backfill backfill, int cidLength, int widLength, int descriptionLength, boolean noTruncate) { final Schedule schedule = backfill.schedule(); final WorkflowId workflowId = backfill.workflowId(); final String format = BACKFILL_FORMAT .replaceAll("<cid-length>", String.valueOf(cidLength)) .replaceAll("<wid-length>", String.valueOf(widLength)) .replaceAll("<description-length>", String.valueOf(descriptionLength)); System.out.println(String.format(format, backfill.id(), backfill.halted(), backfill.allTriggered(), backfill.concurrency(), toParameter(schedule, backfill.start()), toParameter(schedule, backfill.end()), backfill.reverse(), toParameter(schedule, backfill.nextTrigger()), workflowId.componentId(), workflowId.id(), formatLongField(backfill.description(), noTruncate), formatLongField(backfill.triggerParameters() .map(triggerParameters -> formatMap(triggerParameters.env())), noTruncate))); }
@Override public WorkflowId patchState(WorkflowId workflowId, WorkflowState state) throws IOException { final Key workflowKey = DatastoreStorage .workflowKey(tx.getDatastore().newKeyFactory(), workflowId); final Optional<Entity> workflowOpt = DatastoreStorage.getOpt(tx, workflowKey); if (!workflowOpt.isPresent()) { throw new ResourceNotFoundException( String.format("%s:%s doesn't exist.", workflowId.componentId(), workflowId.id())); } final Entity.Builder builder = Entity.newBuilder(workflowOpt.get()); state.enabled().ifPresent(x -> builder.set(PROPERTY_WORKFLOW_ENABLED, x)); state.nextNaturalTrigger() .ifPresent(x -> builder.set(PROPERTY_NEXT_NATURAL_TRIGGER, instantToTimestamp(x))); state.nextNaturalOffsetTrigger() .ifPresent(x -> builder.set(PROPERTY_NEXT_NATURAL_OFFSET_TRIGGER, instantToTimestamp(x))); tx.put(builder.build()); return workflowId; }
@Override public WorkflowId updateNextNaturalTrigger(WorkflowId workflowId, TriggerInstantSpec triggerSpec) throws IOException { final Key workflowKey = DatastoreStorage .workflowKey(tx.getDatastore().newKeyFactory(), workflowId); final Optional<Entity> workflowOpt = DatastoreStorage.getOpt(tx, workflowKey); if (!workflowOpt.isPresent()) { throw new ResourceNotFoundException( String.format("%s:%s doesn't exist.", workflowId.componentId(), workflowId.id())); } final Entity.Builder builder = Entity .newBuilder(workflowOpt.get()) .set(PROPERTY_NEXT_NATURAL_TRIGGER, instantToTimestamp(triggerSpec.instant())) .set(PROPERTY_NEXT_NATURAL_OFFSET_TRIGGER, instantToTimestamp(triggerSpec.offsetInstant())); tx.put(builder.build()); return workflowId; }
final Entity.Builder entity = Entity.newBuilder(key) .set(PROPERTY_COMPONENT, wfi.workflowId().componentId()) .set(PROPERTY_WORKFLOW, wfi.workflowId().id()) .set(PROPERTY_PARAMETER, wfi.parameter()) .set(PROPERTY_COUNTER, state.counter());
@Override public Backfill store(Backfill backfill) throws IOException { final Key key = DatastoreStorage.backfillKey(tx.getDatastore().newKeyFactory(), backfill.id()); Entity.Builder builder = Entity.newBuilder(key) .set(PROPERTY_CONCURRENCY, backfill.concurrency()) .set(PROPERTY_START, instantToTimestamp(backfill.start())) .set(PROPERTY_END, instantToTimestamp(backfill.end())) .set(PROPERTY_COMPONENT, backfill.workflowId().componentId()) .set(PROPERTY_WORKFLOW, backfill.workflowId().id()) .set(PROPERTY_SCHEDULE, backfill.schedule().toString()) .set(PROPERTY_NEXT_TRIGGER, instantToTimestamp(backfill.nextTrigger())) .set(PROPERTY_ALL_TRIGGERED, backfill.allTriggered()) .set(PROPERTY_HALTED, backfill.halted()) .set(PROPERTY_REVERSE, backfill.reverse()); backfill.description().ifPresent(x -> builder.set(PROPERTY_DESCRIPTION, StringValue .newBuilder(x).setExcludeFromIndexes(true).build())); if (backfill.triggerParameters().isPresent()) { final String json = OBJECT_MAPPER.writeValueAsString(backfill.triggerParameters().get()); builder.set(PROPERTY_TRIGGER_PARAMETERS, StringValue.newBuilder(json).setExcludeFromIndexes(true).build()); } tx.put(builder.build()); return backfill; }