private void publishMetadata(DatasetId dataset, SystemMetadata metadata) { if (metadata != null && !metadata.isEmpty()) { SystemMetadataWriter metadataWriter = new DelegateSystemMetadataWriter(metadataPublisher, dataset, metadata); metadataWriter.write(); } } }
private void writeSystemMetadata(co.cask.cdap.proto.id.ArtifactId artifactId, ArtifactInfo artifactInfo) { // add system metadata for artifacts ArtifactSystemMetadataWriter writer = new ArtifactSystemMetadataWriter(metadataPublisher, artifactId, artifactInfo); writer.write(); } }
@Override public Map<String, String> getSystemPropertiesToAdd() { return metadataProvider.getSystemPropertiesToAdd(); }
private void assertDatasetSchema(String expected, DatasetProperties properties) { DatasetSystemMetadataProvider metadataWriter = new DatasetSystemMetadataProvider(new DatasetId("ns1", "avro1"), properties, null, null, null); Assert.assertEquals(expected, metadataWriter.getSchemaToAdd()); } }
@Override public void process(ApplicationWithPrograms input) throws Exception { // add system metadata for apps ApplicationId appId = input.getApplicationId(); ApplicationSpecification appSpec = input.getSpecification(); // only update creation time if this is a new app Map<String, String> properties = metadataStore.getProperties(MetadataScope.SYSTEM, appId.toMetadataEntity()); SystemMetadataWriter appSystemMetadataWriter = new AppSystemMetadataWriter(metadataStore, appId, appSpec, !properties.isEmpty()); appSystemMetadataWriter.write(); // add system metadata for programs writeProgramSystemMetadata(appId, ProgramType.FLOW, appSpec.getFlows().values()); writeProgramSystemMetadata(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce().values()); writeProgramSystemMetadata(appId, ProgramType.SERVICE, appSpec.getServices().values()); writeProgramSystemMetadata(appId, ProgramType.SPARK, appSpec.getSpark().values()); writeProgramSystemMetadata(appId, ProgramType.WORKER, appSpec.getWorkers().values()); writeProgramSystemMetadata(appId, ProgramType.WORKFLOW, appSpec.getWorkflows().values()); // Emit input to the next stage emit(input); }
private void writeProgramSystemMetadata(ApplicationId appId, ProgramType programType, Iterable<? extends ProgramSpecification> specs) { for (ProgramSpecification spec : specs) { ProgramId programId = appId.program(programType, spec.getName()); new ProgramSystemMetadataWriter(metadataPublisher, programId, spec, creationTime).write(); } } }
@Override public Set<String> getSystemTagsToAdd() { return ImmutableSet.<String>builder() .add(programId.getType().getPrettyName()) .add(getMode()) .addAll(getWorkflowNodes()) .build(); }
@Nullable @Override public String getSchemaToAdd() { // TODO: fix schema determination after CDAP-2790 is fixed (CDAP-5408) Map<String, String> datasetProperties = dsProperties.getProperties(); String schemaStr = null; if (datasetProperties.containsKey(DatasetProperties.SCHEMA)) { schemaStr = datasetProperties.get(DatasetProperties.SCHEMA); } else if (datasetProperties.containsKey(ObjectMappedTableProperties.OBJECT_SCHEMA)) { // If it is an ObjectMappedTable, the schema is in a property called 'object.schema' schemaStr = datasetProperties.get(ObjectMappedTableProperties.OBJECT_SCHEMA); } else if (datasetProperties.containsKey(getExplorePropName(FILESET_AVRO_SCHEMA_PROPERTY))) { // Fileset with avro schema (CDAP-5322) schemaStr = datasetProperties.get(getExplorePropName(FILESET_AVRO_SCHEMA_PROPERTY)); } else if (datasetProperties.containsKey(getOutputPropName(FILESET_AVRO_SCHEMA_OUTPUT_KEY))) { // Fileset with avro schema defined in output property (CDAP-5322) schemaStr = datasetProperties.get(getOutputPropName(FILESET_AVRO_SCHEMA_OUTPUT_KEY)); } else if (datasetProperties.containsKey(getOutputPropName(FILESET_PARQUET_SCHEMA_OUTPUT_KEY))) { // Fileset with parquet schema defined in output property (CDAP-5322) schemaStr = datasetProperties.get(getOutputPropName(FILESET_PARQUET_SCHEMA_OUTPUT_KEY)); } return schemaStr; }
@Override public Set<String> getSystemTagsToAdd() { return metadataProvider.getSystemTagsToAdd(); }
@Nullable @Override public String getSchemaToAdd() { return metadataProvider.getSchemaToAdd(); } }
private void addPrograms(ImmutableMap.Builder<String, String> properties) { addPrograms(ProgramType.MAPREDUCE, appSpec.getMapReduce().values(), properties); addPrograms(ProgramType.SERVICE, appSpec.getServices().values(), properties); addPrograms(ProgramType.SPARK, appSpec.getSpark().values(), properties); addPrograms(ProgramType.WORKER, appSpec.getWorkers().values(), properties); addPrograms(ProgramType.WORKFLOW, appSpec.getWorkflows().values(), properties); }
@Override public Map<String, String> getSystemPropertiesToAdd() { ImmutableMap.Builder<String, String> properties = ImmutableMap.builder(); properties.put(ENTITY_NAME_KEY, artifactInfo.getName()); ArtifactClasses classes = artifactInfo.getClasses(); for (PluginClass pluginClass : classes.getPlugins()) { SystemMetadataProvider.addPlugin(pluginClass, artifactInfo.getVersion(), properties); } properties.put(CREATION_TIME_KEY, String.valueOf(System.currentTimeMillis())); return properties.build(); } }
private void publishMetadata(DatasetId dataset, SystemMetadata metadata) { if (metadata != null && !metadata.isEmpty()) { SystemMetadataWriter metadataWriter = new DelegateSystemMetadataWriter(metadataStore, dataset, metadata); metadataWriter.write(); } } }
private void writeSystemMetadata(co.cask.cdap.proto.id.ArtifactId artifactId, ArtifactInfo artifactInfo) { // add system metadata for artifacts ArtifactSystemMetadataWriter writer = new ArtifactSystemMetadataWriter(metadataStore, artifactId, artifactInfo); writer.write(); } }
private void writeProgramSystemMetadata(ApplicationId appId, ProgramType programType, Iterable<? extends ProgramSpecification> specs) { for (ProgramSpecification spec : specs) { ProgramId programId = appId.program(programType, spec.getName()); Map<String, String> properties = metadataStore.getProperties(MetadataScope.SYSTEM, programId.toMetadataEntity()); ProgramSystemMetadataWriter writer = new ProgramSystemMetadataWriter(metadataStore, programId, spec, !properties.isEmpty()); writer.write(); } } }
@Override public Set<String> getSystemTagsToAdd() { return ImmutableSet.<String>builder() .add(programId.getType().getPrettyName()) .add(getMode()) .addAll(getWorkflowNodes()) .build(); }
@Override public Map<String, String> getSystemPropertiesToAdd() { return metadataProvider.getSystemPropertiesToAdd(); }
@Override public Set<String> getSystemTagsToAdd() { return metadataProvider.getSystemTagsToAdd(); }
@Nullable @Override public String getSchemaToAdd() { return metadataProvider.getSchemaToAdd(); } }
private void addPrograms(ImmutableMap.Builder<String, String> properties) { addPrograms(ProgramType.FLOW, appSpec.getFlows().values(), properties); addPrograms(ProgramType.MAPREDUCE, appSpec.getMapReduce().values(), properties); addPrograms(ProgramType.SERVICE, appSpec.getServices().values(), properties); addPrograms(ProgramType.SPARK, appSpec.getSpark().values(), properties); addPrograms(ProgramType.WORKER, appSpec.getWorkers().values(), properties); addPrograms(ProgramType.WORKFLOW, appSpec.getWorkflows().values(), properties); }