private void addPrograms(ImmutableMap.Builder<String, String> properties) { addPrograms(ProgramType.MAPREDUCE, appSpec.getMapReduce().values(), properties); addPrograms(ProgramType.SERVICE, appSpec.getServices().values(), properties); addPrograms(ProgramType.SPARK, appSpec.getSpark().values(), properties); addPrograms(ProgramType.WORKER, appSpec.getWorkers().values(), properties); addPrograms(ProgramType.WORKFLOW, appSpec.getWorkflows().values(), properties); }
@Override public Map<String, String> getSystemPropertiesToAdd() { ImmutableMap.Builder<String, String> properties = ImmutableMap.builder(); properties.put(ENTITY_NAME_KEY, appSpec.getName()); properties.put(VERSION_KEY, appId.getVersion()); String description = appSpec.getDescription(); if (!Strings.isNullOrEmpty(description)) { properties.put(DESCRIPTION_KEY, description); } if (!existing) { properties.put(CREATION_TIME_KEY, String.valueOf(System.currentTimeMillis())); } addPrograms(properties); addSchedules(properties); // appSpec.getPlugins() returns all instances of all plugins, so there may be duplicates. // we only store unique plugins right now Set<PluginClass> existingPluginClasses = new HashSet<>(); for (Plugin plugin : appSpec.getPlugins().values()) { if (!existingPluginClasses.contains(plugin.getPluginClass())) { addPlugin(plugin.getPluginClass(), null, properties); existingPluginClasses.add(plugin.getPluginClass()); } } return properties.build(); }
@Override public Map<String, String> getSystemPropertiesToAdd() { ImmutableMap.Builder<String, String> properties = ImmutableMap.builder(); properties.put(ENTITY_NAME_KEY, appSpec.getName()); properties.put(VERSION_KEY, appId.getVersion()); String description = appSpec.getDescription(); if (!Strings.isNullOrEmpty(description)) { properties.put(DESCRIPTION_KEY, description); } properties.put(CREATION_TIME_KEY, creationTime); addPrograms(properties); addSchedules(properties); // appSpec.getPlugins() returns all instances of all plugins, so there may be duplicates. // we only store unique plugins right now Set<PluginClass> existingPluginClasses = new HashSet<>(); for (Plugin plugin : appSpec.getPlugins().values()) { if (!existingPluginClasses.contains(plugin.getPluginClass())) { SystemMetadataProvider.addPlugin(plugin.getPluginClass(), null, properties); existingPluginClasses.add(plugin.getPluginClass()); } } return properties.build(); }
@Override public void process(ApplicationWithPrograms input) { // use current time as creation time for app and all programs creationTime = String.valueOf(System.currentTimeMillis()); // add system metadata for apps ApplicationId appId = input.getApplicationId(); ApplicationSpecification appSpec = input.getSpecification(); new AppSystemMetadataWriter(metadataPublisher, appId, appSpec, creationTime).write(); // add system metadata for programs writeProgramSystemMetadata(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce().values()); writeProgramSystemMetadata(appId, ProgramType.SERVICE, appSpec.getServices().values()); writeProgramSystemMetadata(appId, ProgramType.SPARK, appSpec.getSpark().values()); writeProgramSystemMetadata(appId, ProgramType.WORKER, appSpec.getWorkers().values()); writeProgramSystemMetadata(appId, ProgramType.WORKFLOW, appSpec.getWorkflows().values()); // Emit input to the next stage emit(input); }
@Override public void process(ApplicationWithPrograms input) throws Exception { // add system metadata for apps ApplicationId appId = input.getApplicationId(); ApplicationSpecification appSpec = input.getSpecification(); // only update creation time if this is a new app Map<String, String> properties = metadataStore.getProperties(MetadataScope.SYSTEM, appId.toMetadataEntity()); SystemMetadataWriter appSystemMetadataWriter = new AppSystemMetadataWriter(metadataStore, appId, appSpec, !properties.isEmpty()); appSystemMetadataWriter.write(); // add system metadata for programs writeProgramSystemMetadata(appId, ProgramType.FLOW, appSpec.getFlows().values()); writeProgramSystemMetadata(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce().values()); writeProgramSystemMetadata(appId, ProgramType.SERVICE, appSpec.getServices().values()); writeProgramSystemMetadata(appId, ProgramType.SPARK, appSpec.getSpark().values()); writeProgramSystemMetadata(appId, ProgramType.WORKER, appSpec.getWorkers().values()); writeProgramSystemMetadata(appId, ProgramType.WORKFLOW, appSpec.getWorkflows().values()); // Emit input to the next stage emit(input); }
private void addPrograms(ImmutableMap.Builder<String, String> properties) { addPrograms(ProgramType.FLOW, appSpec.getFlows().values(), properties); addPrograms(ProgramType.MAPREDUCE, appSpec.getMapReduce().values(), properties); addPrograms(ProgramType.SERVICE, appSpec.getServices().values(), properties); addPrograms(ProgramType.SPARK, appSpec.getSpark().values(), properties); addPrograms(ProgramType.WORKER, appSpec.getWorkers().values(), properties); addPrograms(ProgramType.WORKFLOW, appSpec.getWorkflows().values(), properties); }