programDescriptor = appMetaStore.loadProgram(schedule.getProgramId()); } catch (Exception e) { LOG.error("Exception occurs when looking up program descriptor for program {} in schedule {}", schedule.getProgramId(), schedule, e); throw new RuntimeException(String.format("Exception occurs when looking up program descriptor for" + " program %s in schedule %s", schedule.getProgramId(), schedule), e); userId = impersonator.getUGI(schedule.getProgramId()).getUserName(); } catch (IOException e) { LOG.error("Exception occurs when looking up user group information for program {} in schedule {}", schedule.getProgramId(), schedule, e); throw new RuntimeException(String.format("Exception occurs when looking up user group information for" + " program %s in schedule %s", schedule.getProgramId(), schedule), e); Map<String, String> newProperties = new HashMap<>(schedule.getProperties()); newProperties.putAll(additionalProperties); return new ProgramSchedule(schedule.getName(), schedule.getDescription(), schedule.getProgramId(), newProperties, schedule.getTrigger(), schedule.getConstraints(), schedule.getTimeoutMillis());
public ScheduleDetail toScheduleDetail() { ScheduleProgramInfo programInfo = new ScheduleProgramInfo(schedule.getProgramId().getType().getSchedulableType(), schedule.getProgramId().getProgram()); ScheduleId scheduleId = schedule.getScheduleId(); return new ScheduleDetail(scheduleId.getNamespace(), scheduleId.getApplication(), scheduleId.getVersion(), scheduleId.getSchedule(), schedule.getDescription(), programInfo, schedule.getProperties(), schedule.getTrigger(), schedule.getConstraints(), schedule.getTimeoutMillis(), meta.getStatus().name()); } }
@Override public ConstraintResult check(ProgramSchedule schedule, ConstraintContext context) { int numRunning = context.getProgramRuns(schedule.getProgramId(), ProgramRunStatus.RUNNING, maxConcurrency).size(); if (numRunning >= maxConcurrency) { LOG.debug("Skipping run of program {} from schedule {} because there are at least {} running runs.", schedule.getProgramId(), schedule.getName(), maxConcurrency); return notSatisfied(context); } int numSuspended = context.getProgramRuns(schedule.getProgramId(), ProgramRunStatus.SUSPENDED, maxConcurrency).size(); if (numRunning + numSuspended >= maxConcurrency) { LOG.debug("Skipping run of program {} from schedule {} because there are at least" + "{} running runs and at least {} suspended runs.", schedule.getProgramId(), schedule.getName(), numRunning, numSuspended); return notSatisfied(context); } return ConstraintResult.SATISFIED; }
private TriggeringScheduleInfo getTriggeringScheduleInfo(Job job) { TriggerInfoContext triggerInfoContext = new TriggerInfoContext(job, store); SatisfiableTrigger trigger = ((SatisfiableTrigger) job.getSchedule().getTrigger()); List<TriggerInfo> triggerInfo = trigger.getTriggerInfos(triggerInfoContext); ProgramSchedule schedule = job.getSchedule(); return new DefaultTriggeringScheduleInfo(schedule.getName(), schedule.getDescription(), triggerInfo, schedule.getProperties()); }
public void launch(Job job) throws Exception { ProgramSchedule schedule = job.getSchedule(); ProgramId programId = schedule.getProgramId(); Map<String, String> userArgs = Maps.newHashMap(); userArgs.putAll(schedule.getProperties()); userArgs.putAll(propertiesResolver.getUserProperties(Id.Program.fromEntityId(programId))); Map<String, String> systemArgs = Maps.newHashMap(); systemArgs.putAll(propertiesResolver.getSystemProperties(Id.Program.fromEntityId(programId))); // Let the triggers update the arguments first before setting the triggering schedule info ((SatisfiableTrigger) job.getSchedule().getTrigger()).updateLaunchArguments(job.getSchedule(), job.getNotifications(), userArgs, systemArgs); TriggeringScheduleInfo triggeringScheduleInfo = getTriggeringScheduleInfo(job); systemArgs.put(ProgramOptionConstants.TRIGGERING_SCHEDULE_INFO, GSON.toJson(triggeringScheduleInfo)); execute(programId, systemArgs, userArgs); LOG.info("Successfully started program {} in schedule {}.", schedule.getProgramId(), schedule.getName()); }
private void collectScheduleProfileMetadata(ProgramSchedule schedule, ProfileId programProfile, Map<MetadataEntity, Map<String, String>> updates) { ScheduleId scheduleId = schedule.getScheduleId(); LOG.trace("Updating profile metadata for {}", scheduleId); // if we are able to get profile from preferences or schedule properties, use it // otherwise default profile will be used Optional<ProfileId> scheduleProfileId = SystemArguments.getProfileIdFromArgs(scheduleId.getNamespaceId(), schedule.getProperties()); programProfile = scheduleProfileId.orElse(programProfile); addProfileMetadataUpdate(scheduleId, programProfile, updates); }
ProgramId program = schedule.getProgramId(); SchedulableProgramType programType = program.getType().getSchedulableType(); co.cask.cdap.api.schedule.Trigger trigger = schedule.getTrigger(); Map<String, TriggerKey> cronTriggerKeyMap = new HashMap<>(); String cron = ((TimeTrigger) timeTrigger).getCronExpression(); String triggerName = AbstractTimeSchedulerService.getTriggerName(program, programType, schedule.getName(), cron); cronTriggerKeyMap.put(cron, triggerKeyForName(triggerName)); String triggerName = AbstractTimeSchedulerService.scheduleIdFor(program, programType, schedule.getName()); cronTriggerKeyMap.put(((TimeTrigger) schedule.getTrigger()).getCronExpression(), triggerKeyForName(triggerName)); return cronTriggerKeyMap;
/** * @param scheduleLastUpdatedTime the last modification time of the schedule, at the time this job is created. * This serves as a way to detect whether the schedule was changed later-on, and * hence, the job would be obsolete in that case. */ public SimpleJob(ProgramSchedule schedule, long creationTime, List<Notification> notifications, State state, long scheduleLastUpdatedTime) { this.schedule = schedule; this.jobKey = new JobKey(schedule.getScheduleId(), creationTime); this.notifications = ImmutableList.copyOf(notifications); this.state = state; this.scheduleLastUpdatedTime = scheduleLastUpdatedTime; }
/** * Get the given schedule * * @return the schedule * @throws NotFoundException if the schedule could not be found * @throws UnauthorizedException if the principal is not authorized to access the schedule program * @throws Exception if any other errors occurred while performing the authorization enforcement check */ public ProgramSchedule get(ScheduleId scheduleId) throws Exception { ProgramSchedule schedule = scheduler.getSchedule(scheduleId); AuthorizationUtil.ensureAccess(schedule.getProgramId(), authorizationEnforcer, authenticationContext.getPrincipal()); return schedule; }
if (schedule.getTrigger() instanceof AbstractSatisfiableCompositeTrigger) { scheduleIds.add(getRowKeyPrefix(schedule.getScheduleId())); try (CloseableIterator<Job> jobs = getJobsForSchedule(schedule.getScheduleId())) { while (jobs.hasNext()) { Job job = jobs.next(); } else if (System.currentTimeMillis() - job.getCreationTime() > job.getSchedule().getTimeoutMillis()) {
/** * Adds and enables time based schedules for the given workflow at the given frequency. * * @param scheduleMins the number of minutes to wait before launching the given workflow each time * @param workflowId the ID of the scheduled workflow */ private ProgramSchedule initializeSchedules(int scheduleMins, WorkflowId workflowId) throws ConflictException, BadRequestException, NotFoundException { ProgramSchedule schedule = new ProgramSchedule(String.format("%dMinSchedule", scheduleMins), "time schedule", workflowId, Collections.emptyMap(), new TimeTrigger(String.format("*/%d * * * *", scheduleMins)), Collections.emptyList()); scheduler.addSchedule(schedule); scheduler.enableSchedule(schedule.getScheduleId()); return schedule; }
private void testContainingTrigger(ProtoTrigger proto, Trigger trigger) { ProgramSchedule proto1 = new ProgramSchedule("sched1", "one partition schedule", new NamespaceId("test").app("a").worker("ww"), ImmutableMap.of("prop3", "abc"), proto, ImmutableList.of()); ProgramSchedule sched1 = new ProgramSchedule("sched1", "one partition schedule", new NamespaceId("test").app("a").worker("ww"), ImmutableMap.of("prop3", "abc"), trigger, ImmutableList.of()); Assert.assertEquals(sched1, GSON.fromJson(GSON.toJson(proto1), ProgramSchedule.class)); } }
private boolean containsTimeTrigger(ProgramSchedule schedule) { // A composite trigger may contain a TimeTrigger return schedule.getTrigger() instanceof TimeTrigger || schedule.getTrigger() instanceof AbstractSatisfiableCompositeTrigger; }
long startTimeSecs, long endTimeSecs) throws Exception { ProgramId programId = schedule.getProgramId(); String userId = schedule.getProperties().get(ProgramOptionConstants.USER_ID); String artifactId = schedule.getProperties().get(ProgramOptionConstants.ARTIFACT_ID); ArtifactSummary artifactSummary = artifactId == null ? null : ArtifactSummary.from(GSON.fromJson(artifactId, ArtifactId.class));
ProgramSchedule schedule = new ProgramSchedule("SCHED1", "one partition schedule", WORKFLOW_ID, ImmutableMap.of("prop3", "abc"), new PartitionTrigger(DATASET_ID, 1), Map<String, String> systemArgs = ImmutableMap.of(ProgramOptionConstants.SCHEDULE_NAME, schedule.getName()); setStartAndRunning(store, pid1, EMPTY_MAP, systemArgs); assertSatisfied(true, concurrencyConstraint.check(schedule, constraintContext)); systemArgs = ImmutableMap.of(ProgramOptionConstants.SCHEDULE_NAME, "not" + schedule.getName()); setStartAndRunning(store, pid2, EMPTY_MAP, systemArgs); assertSatisfied(false, concurrencyConstraint.check(schedule, constraintContext));
private ConstraintResult.SatisfiedState constraintsSatisfied(Job job, long now) { ConstraintResult.SatisfiedState satisfiedState = ConstraintResult.SatisfiedState.SATISFIED; ConstraintContext constraintContext = new ConstraintContext(job, now, store); for (Constraint constraint : job.getSchedule().getConstraints()) { if (!(constraint instanceof CheckableConstraint)) { // this shouldn't happen, since implementation of Constraint in ProgramSchedule // should implement CheckableConstraint throw new IllegalArgumentException("Implementation of Constraint in ProgramSchedule" + " must implement CheckableConstraint"); } CheckableConstraint abstractConstraint = (CheckableConstraint) constraint; ConstraintResult result = abstractConstraint.check(job.getSchedule(), constraintContext); if (result.getSatisfiedState() == ConstraintResult.NEVER_SATISFIED.getSatisfiedState()) { // if any of the constraints are NEVER_SATISFIED, return NEVER_SATISFIED return ConstraintResult.NEVER_SATISFIED.getSatisfiedState(); } if (result.getSatisfiedState() == ConstraintResult.SatisfiedState.NOT_SATISFIED) { satisfiedState = ConstraintResult.SatisfiedState.NOT_SATISFIED; } } return satisfiedState; }
@Override public void deleteSchedules(Iterable<? extends ScheduleId> scheduleIds) throws NotFoundException { checkStarted(); execute((StoreQueueAndProfileTxRunnable<Void, NotFoundException>) (store, queue, profileDataset) -> { long deleteTime = System.currentTimeMillis(); for (ScheduleId scheduleId : scheduleIds) { ProgramSchedule schedule = store.getSchedule(scheduleId); deleteScheduleInScheduler(schedule); queue.markJobsForDeletion(scheduleId, deleteTime); adminEventPublisher.publishScheduleDeletion(scheduleId, schedule); // if the deleted schedule has properties with profile assignment, remove the assignment Optional<ProfileId> profileId = SystemArguments.getProfileIdFromArgs(scheduleId.getNamespaceId(), schedule.getProperties()); if (profileId.isPresent()) { try { profileDataset.removeProfileAssignment(profileId.get(), scheduleId); } catch (NotFoundException e) { // this should not happen since the profile cannot be deleted if there is a schedule who is using it LOG.warn("Unable to find the profile {} when deleting schedule {}, " + "skipping assignment deletion.", profileId.get(), scheduleId); } } } store.deleteSchedules(scheduleIds); return null; }, NotFoundException.class); }
private void doGetSchedule(HttpResponder responder, String namespace, String app, String version, String scheduleName) throws Exception { ScheduleId scheduleId = new ApplicationId(namespace, app, version).schedule(scheduleName); ProgramSchedule schedule = programScheduleService.get(scheduleId); ScheduleDetail detail = schedule.toScheduleDetail(); responder.sendJson(HttpResponseStatus.OK, GSON.toJson(detail, ScheduleDetail.class)); }
if (now - job.getCreationTime() >= job.getSchedule().getTimeoutMillis() + 2 * Schedulers.SUBSCRIBER_TX_TIMEOUT_MILLIS) { LOG.info("Deleted job {}, due to timeout value of {}.", job.getJobKey(), job.getSchedule().getTimeoutMillis()); jobQueue.deleteJob(job); return;
@Override public ConstraintResult check(ProgramSchedule schedule, ConstraintContext context) { int numRunning = context.getProgramRuns(schedule.getProgramId(), ProgramRunStatus.RUNNING, maxConcurrency).size(); if (numRunning >= maxConcurrency) { LOG.debug("Skipping run of program {} from schedule {} because there are at least {} running runs.", schedule.getProgramId(), schedule.getName(), maxConcurrency); return notSatisfied(context); } int numSuspended = context.getProgramRuns(schedule.getProgramId(), ProgramRunStatus.SUSPENDED, maxConcurrency).size(); if (numRunning + numSuspended >= maxConcurrency) { LOG.debug("Skipping run of program {} from schedule {} because there are at least" + "{} running runs and at least {} suspended runs.", schedule.getProgramId(), schedule.getName(), numRunning, numSuspended); return notSatisfied(context); } return ConstraintResult.SATISFIED; }