/** * Get the profile id for the given program, given arguments for a run. All non-workflow program types will use the * native profile. Workflow program types will use whatever profile is specified in its arguments, or the native * profile if none is specified. * * @param programId program to get the profile for * @param args arguments for a program run * @return the profile id for the program run */ public static ProfileId getProfileIdForProgram(ProgramId programId, Map<String, String> args) { if (programId.getType() == ProgramType.WORKFLOW) { return getProfileIdFromArgs(programId.getNamespaceId(), args).orElse(ProfileId.NATIVE); } return ProfileId.NATIVE; }
private Optional<Runnable> getEmitMetricsRunnable(ProgramRunId programRunId, RunRecordMeta recordedRunRecord, String metricName) { Optional<ProfileId> profile = SystemArguments.getProfileIdFromArgs(programRunId.getNamespaceId(), recordedRunRecord.getSystemArgs()); return profile.map(profileId -> () -> emitProfileMetrics(programRunId, profileId, metricName)); }
private Optional<Runnable> getEmitMetricsRunnable(ProgramRunId programRunId, RunRecordMeta recordedRunRecord, String metricName) { Optional<ProfileId> profile = SystemArguments.getProfileIdFromArgs(programRunId.getNamespaceId(), recordedRunRecord.getSystemArgs()); return profile.map(profileId -> () -> emitProfileMetrics(programRunId, profileId, metricName)); }
private void collectScheduleProfileMetadata(ProgramSchedule schedule, ProfileId programProfile, Map<MetadataEntity, Map<String, String>> updates) { ScheduleId scheduleId = schedule.getScheduleId(); LOG.trace("Updating profile metadata for {}", scheduleId); // if we are able to get profile from preferences or schedule properties, use it // otherwise default profile will be used Optional<ProfileId> scheduleProfileId = SystemArguments.getProfileIdFromArgs(scheduleId.getNamespaceId(), schedule.getProperties()); programProfile = scheduleProfileId.orElse(programProfile); addProfileMetadataUpdate(scheduleId, programProfile, updates); }
/** * Creates a {@link ProfileMetricService} for the profile being used for the given program run. */ private ProfileMetricService createProfileMetricsService(ProgramRunId programRunId, ProgramOptions programOptions, Cluster cluster) { Optional<ProfileId> profile = SystemArguments.getProfileIdFromArgs(programRunId.getNamespaceId(), programOptions.getArguments().asMap()); ProfileId profileId = profile.orElseThrow( () -> new IllegalStateException("Missing profile information for program run " + programRunId)); return new ProfileMetricService(metricsCollectionService, programRunId, profileId, cluster.getNodes().size(), monitorScheduler); }
/** * Creates a {@link ProfileMetricService} for the profile being used for the given program run. */ private ProfileMetricService createProfileMetricsService(ProgramRunId programRunId, ProgramOptions programOptions, Cluster cluster) { Optional<ProfileId> profile = SystemArguments.getProfileIdFromArgs(programRunId.getNamespaceId(), programOptions.getArguments().asMap()); ProfileId profileId = profile.orElseThrow( () -> new IllegalStateException("Missing profile information for program run " + programRunId)); return new ProfileMetricService(metricsCollectionService, programRunId, profileId, cluster.getNodes().size(), monitorScheduler); }
/** * Get the profile id for the given program, given arguments for a run. All non-workflow program types will use the * native profile. Workflow program types will use whatever profile is specified in its arguments, or the native * profile if none is specified. * * @param programId program to get the profile for * @param args arguments for a program run * @return the profile id for the program run */ public static ProfileId getProfileIdForProgram(ProgramId programId, Map<String, String> args) { if (isProfileAllowed(programId.getType())) { return getProfileIdFromArgs(programId.getNamespaceId(), args).orElse(ProfileId.NATIVE); } return ProfileId.NATIVE; }
private void updateScheduleProfileMetadata(ProgramSchedule schedule, ProfileId profileId) { ScheduleId scheduleId = schedule.getScheduleId(); Optional<ProfileId> scheduleProfileId = SystemArguments.getProfileIdFromArgs(scheduleId.getNamespaceId(), schedule.getProperties()); profileId = scheduleProfileId.orElse(profileId); setProfileMetadata(scheduleId, profileId); }
/** * Get the profile id for the provided entity id from the resolved preferences from preference dataset, * if no profile is inside, it will return the default profile * * @param entityId entity id to lookup the profile id * @return the profile id which will be used by this entity id, default profile if not find */ // TODO: CDAP-13579 consider preference key starts with [scope].[name].system.profile.name private ProfileId getResolvedProfileId(EntityId entityId) { NamespaceId namespaceId = entityId.getEntityType().equals(EntityType.INSTANCE) ? NamespaceId.SYSTEM : ((NamespacedEntityId) entityId).getNamespaceId(); return SystemArguments.getProfileIdFromArgs( namespaceId, preferencesDataset.getResolvedPreferences(entityId)).orElse(ProfileId.NATIVE); }
Optional<ProfileId> profile = SystemArguments.getProfileIdFromArgs(namespaceId, propertyMap); if (profile.isPresent()) { ProfileId profileId = profile.get(); Optional<ProfileId> oldProfile = SystemArguments.getProfileIdFromArgs(namespaceId, oldProperties); preferencesDataset.setPreferences(entityId, propertyMap);
@Override public void deleteSchedules(Iterable<? extends ScheduleId> scheduleIds) throws NotFoundException { checkStarted(); execute((StoreQueueAndProfileTxRunnable<Void, NotFoundException>) (store, queue, profileDataset) -> { long deleteTime = System.currentTimeMillis(); for (ScheduleId scheduleId : scheduleIds) { ProgramSchedule schedule = store.getSchedule(scheduleId); deleteScheduleInScheduler(schedule); queue.markJobsForDeletion(scheduleId, deleteTime); adminEventPublisher.publishScheduleDeletion(scheduleId, schedule); // if the deleted schedule has properties with profile assignment, remove the assignment Optional<ProfileId> profileId = SystemArguments.getProfileIdFromArgs(scheduleId.getNamespaceId(), schedule.getProperties()); if (profileId.isPresent()) { try { profileDataset.removeProfileAssignment(profileId.get(), scheduleId); } catch (NotFoundException e) { // this should not happen since the profile cannot be deleted if there is a schedule who is using it LOG.warn("Unable to find the profile {} when deleting schedule {}, " + "skipping assignment deletion.", profileId.get(), scheduleId); } } } store.deleteSchedules(scheduleIds); return null; }, NotFoundException.class); }
@Override public void deleteSchedules(Iterable<? extends ScheduleId> scheduleIds) throws NotFoundException { checkStarted(); execute((StoreQueueAndProfileTxRunnable<Void, NotFoundException>) (store, queue, profileDataset) -> { long deleteTime = System.currentTimeMillis(); for (ScheduleId scheduleId : scheduleIds) { ProgramSchedule schedule = store.getSchedule(scheduleId); deleteScheduleInScheduler(schedule); queue.markJobsForDeletion(scheduleId, deleteTime); adminEventPublisher.publishScheduleDeletion(scheduleId, schedule); // if the deleted schedule has properties with profile assignment, remove the assignment Optional<ProfileId> profileId = SystemArguments.getProfileIdFromArgs(scheduleId.getNamespaceId(), schedule.getProperties()); if (profileId.isPresent()) { try { profileDataset.removeProfileAssignment(profileId.get(), scheduleId); } catch (NotFoundException e) { // this should not happen since the profile cannot be deleted if there is a schedule who is using it LOG.warn("Unable to find the profile {} when deleting schedule {}, " + "skipping assignment deletion.", profileId.get(), scheduleId); } } } store.deleteSchedules(scheduleIds); return null; }, NotFoundException.class); }
@Override public void deleteSchedules(ProgramId programId) { checkStarted(); execute((StoreQueueAndProfileTxRunnable<Void, RuntimeException>) (store, queue, profileDataset) -> { long deleteTime = System.currentTimeMillis(); List<ProgramSchedule> schedules = store.listSchedules(programId); deleteSchedulesInScheduler(schedules); List<ScheduleId> deleted = store.deleteSchedules(programId); for (ScheduleId scheduleId : deleted) { queue.markJobsForDeletion(scheduleId, deleteTime); } for (ProgramSchedule programSchedule : schedules) { ScheduleId scheduleId = programSchedule.getScheduleId(); adminEventPublisher.publishScheduleDeletion(scheduleId, programSchedule); // if the deleted schedule has properties with profile assignment, remove the assignment Optional<ProfileId> profileId = SystemArguments.getProfileIdFromArgs(scheduleId.getNamespaceId(), programSchedule.getProperties()); if (profileId.isPresent()) { try { profileDataset.removeProfileAssignment(profileId.get(), scheduleId); } catch (NotFoundException e) { // this should not happen since the profile cannot be deleted if there is a schedule who is using it LOG.warn("Unable to find the profile {} when deleting schedule {}, " + "skipping assignment deletion.", profileId.get(), scheduleId); } } } return null; }, RuntimeException.class); }
@Test public void testGetProfileId() { // should get null profile id if the args is empty Assert.assertFalse(SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, Collections.emptyMap()).isPresent()); Map<String, String> args = new HashMap<>(); args.put("system.log.level", "DEBUG"); args.put("system.log.leveldummyKey", "ERROR"); // Having other unrelated args should also get null profile id Assert.assertFalse(SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, args).isPresent()); // without scope the profile will be considered in user scope ProfileId expected = NamespaceId.DEFAULT.profile("MyProfile"); args.put("system.profile.name", expected.getProfile()); Assert.assertEquals(expected, SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, args).get()); // put a profile with scope SYSTEM, the profile we get should be in system namespace expected = NamespaceId.SYSTEM.profile("MyProfile"); args.put("system.profile.name", expected.getScopedName()); Assert.assertEquals(expected, SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, args).get()); // put a profile with scope USER, the profile we get should be in the user namespace expected = NamespaceId.DEFAULT.profile("MyProfile"); args.put("system.profile.name", expected.getScopedName()); Assert.assertEquals(expected, SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, args).get()); } }
@Override public void deleteSchedules(ApplicationId appId) { checkStarted(); execute((StoreQueueAndProfileTxRunnable<Void, RuntimeException>) (store, queue, profileDataset) -> { long deleteTime = System.currentTimeMillis(); List<ProgramSchedule> schedules = store.listSchedules(appId); deleteSchedulesInScheduler(schedules); List<ScheduleId> deleted = store.deleteSchedules(appId); for (ScheduleId scheduleId : deleted) { queue.markJobsForDeletion(scheduleId, deleteTime); } for (ProgramSchedule programSchedule : schedules) { ScheduleId scheduleId = programSchedule.getScheduleId(); adminEventPublisher.publishScheduleDeletion(scheduleId, programSchedule); // if the deleted schedule has properties with profile assignment, remove the assignment Optional<ProfileId> profileId = SystemArguments.getProfileIdFromArgs(scheduleId.getNamespaceId(), programSchedule.getProperties()); if (profileId.isPresent()) { try { profileDataset.removeProfileAssignment(profileId.get(), scheduleId); } catch (NotFoundException e) { // this should not happen since the profile cannot be deleted if there is a schedule who is using it LOG.warn("Unable to find the profile {} when deleting schedule {}, " + "skipping assignment deletion.", profileId.get(), scheduleId); } } } return null; }, RuntimeException.class); }
@Override public void deleteSchedules(ApplicationId appId) { checkStarted(); execute((StoreQueueAndProfileTxRunnable<Void, RuntimeException>) (store, queue, profileDataset) -> { long deleteTime = System.currentTimeMillis(); List<ProgramSchedule> schedules = store.listSchedules(appId); deleteSchedulesInScheduler(schedules); List<ScheduleId> deleted = store.deleteSchedules(appId); for (ScheduleId scheduleId : deleted) { queue.markJobsForDeletion(scheduleId, deleteTime); } for (ProgramSchedule programSchedule : schedules) { ScheduleId scheduleId = programSchedule.getScheduleId(); adminEventPublisher.publishScheduleDeletion(scheduleId, programSchedule); // if the deleted schedule has properties with profile assignment, remove the assignment Optional<ProfileId> profileId = SystemArguments.getProfileIdFromArgs(scheduleId.getNamespaceId(), programSchedule.getProperties()); if (profileId.isPresent()) { try { profileDataset.removeProfileAssignment(profileId.get(), scheduleId); } catch (NotFoundException e) { // this should not happen since the profile cannot be deleted if there is a schedule who is using it LOG.warn("Unable to find the profile {} when deleting schedule {}, " + "skipping assignment deletion.", profileId.get(), scheduleId); } } } return null; }, RuntimeException.class); }
@Override public void deleteSchedules(ProgramId programId) { checkStarted(); execute((StoreQueueAndProfileTxRunnable<Void, RuntimeException>) (store, queue, profileDataset) -> { long deleteTime = System.currentTimeMillis(); List<ProgramSchedule> schedules = store.listSchedules(programId); deleteSchedulesInScheduler(schedules); List<ScheduleId> deleted = store.deleteSchedules(programId); for (ScheduleId scheduleId : deleted) { queue.markJobsForDeletion(scheduleId, deleteTime); } for (ProgramSchedule programSchedule : schedules) { ScheduleId scheduleId = programSchedule.getScheduleId(); adminEventPublisher.publishScheduleDeletion(scheduleId, programSchedule); // if the deleted schedule has properties with profile assignment, remove the assignment Optional<ProfileId> profileId = SystemArguments.getProfileIdFromArgs(scheduleId.getNamespaceId(), programSchedule.getProperties()); if (profileId.isPresent()) { try { profileDataset.removeProfileAssignment(profileId.get(), scheduleId); } catch (NotFoundException e) { // this should not happen since the profile cannot be deleted if there is a schedule who is using it LOG.warn("Unable to find the profile {} when deleting schedule {}, " + "skipping assignment deletion.", profileId.get(), scheduleId); } } } return null; }, RuntimeException.class); }
private void deleteConfig(EntityId entityId) { Transactionals.execute(transactional, context -> { PreferencesDataset dataset = PreferencesDataset.get(context, datasetFramework); Map<String, String> oldProp = dataset.getPreferences(entityId); NamespaceId namespaceId = entityId.getEntityType().equals(EntityType.INSTANCE) ? NamespaceId.SYSTEM : ((NamespacedEntityId) entityId).getNamespaceId(); Optional<ProfileId> oldProfile = SystemArguments.getProfileIdFromArgs(namespaceId, oldProp); dataset.deleteProperties(entityId); // if there is profile properties, publish the message to update metadata and remove the assignment if (oldProfile.isPresent()) { ProfileDataset.get(context, datasetFramework).removeProfileAssignment(oldProfile.get(), entityId); adminEventPublisher.publishProfileUnAssignment(entityId); } }); }
private void deleteConfig(EntityId entityId) { Transactionals.execute(transactional, context -> { PreferencesDataset dataset = PreferencesDataset.get(context, datasetFramework); Map<String, String> oldProp = dataset.getPreferences(entityId); NamespaceId namespaceId = entityId.getEntityType().equals(EntityType.INSTANCE) ? NamespaceId.SYSTEM : ((NamespacedEntityId) entityId).getNamespaceId(); Optional<ProfileId> oldProfile = SystemArguments.getProfileIdFromArgs(namespaceId, oldProp); dataset.deleteProperties(entityId); // if there is profile properties, publish the message to update metadata and remove the assignment if (oldProfile.isPresent()) { ProfileDataset.get(context, datasetFramework).removeProfileAssignment(oldProfile.get(), entityId); adminEventPublisher.publishProfileUnAssignment(entityId); } }); }
ProgramOptions options = programLifecycleService.createProgramOptions(programId, userArgs, systemArgs, false); Optional<ProfileId> opt = SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, options.getArguments().asMap()); Assert.assertTrue(opt.isPresent());