case PUT: { try { if (metadata != null && metadata.getProperties() != null && !metadata.getProperties().isEmpty()) { metadataAdmin.addProperties(entity, metadata.getProperties()); if (metadata != null && metadata.getTags() != null && !metadata.getTags().isEmpty()) { Set<String> toAdd = metadata.getTags(); metadataAdmin.addTags(entity, toAdd); if (metadata != null && metadata.getProperties() != null && !metadata.getProperties().isEmpty()) { Set<String> toRemove = metadata.getProperties().keySet(); metadataAdmin.removeProperties(entity, toRemove); if (metadata != null && metadata.getTags() != null && !metadata.getTags().isEmpty()) { Set<String> toRemove = metadata.getTags(); metadataAdmin.removeTags(entity, toRemove);
@Override public void addTags(MetadataEntity metadataEntity, Iterable<String> tags) { Set<String> tagsToAdd = new HashSet<>(); tags.forEach(tagsToAdd::add); metadataPublisher.publish(programRunId, new MetadataOperation(metadataEntity, MetadataOperation.Type.PUT, new Metadata(Collections.emptyMap(), tagsToAdd))); }
/************************************************ DELETE *********************************************************/ @DELETE @Path("metadata/{dataset}/tags/{tag}") public void removeTag(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("dataset") String dataset, @PathParam("tag") String tag) { getContext().removeTags(MetadataEntity.ofDataset(getContext().getNamespace(), dataset), tag); // wait till change is recorded to store try { Tasks.waitFor(true, () -> { Map<MetadataScope, Metadata> metadata2 = getContext().getMetadata(MetadataEntity.ofDataset(getContext().getNamespace(), dataset)); return !metadata2.get(MetadataScope.USER).getTags().contains(tag); }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); } catch (Exception e) { responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()); } responder.sendStatus(HttpResponseStatus.OK.code()); }
@PUT @Path("metadata/{dataset}/properties") public void addProperties(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("dataset") String dataset) { String body = StandardCharsets.UTF_8.decode(request.getContent()).toString(); Map<String, String> properties = GSON.fromJson(body, MAP_STRING_STRING_TYPE); getContext().addProperties(MetadataEntity.ofDataset(getContext().getNamespace(), dataset), properties); // wait till change is recorded to store try { Tasks.waitFor(true, () -> { Map<MetadataScope, Metadata> metadata2 = getContext().getMetadata(MetadataEntity.ofDataset(getContext().getNamespace(), dataset)); return metadata2.get(MetadataScope.USER).getProperties().keySet().containsAll(properties.keySet()); }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); } catch (Exception e) { responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()); } responder.sendStatus(HttpResponseStatus.OK.code()); }
/************************************************* PUT ***********************************************************/ @PUT @Path("metadata/{dataset}/tags") public void addTag(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("dataset") String dataset) { String tag = StandardCharsets.UTF_8.decode(request.getContent()).toString(); getContext().addTags(MetadataEntity.ofDataset(getContext().getNamespace(), dataset), tag); // wait till change is recorded to store try { Tasks.waitFor(true, () -> { Map<MetadataScope, Metadata> metadata2 = getContext().getMetadata(MetadataEntity.ofDataset(getContext().getNamespace(), dataset)); return metadata2.get(MetadataScope.USER).getTags().contains(tag); }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); } catch (Exception e) { responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()); } responder.sendStatus(HttpResponseStatus.OK.code()); }
@DELETE @Path("metadata/{dataset}/properties") public void removeAllProperties(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("dataset") String dataset) { getContext().removeProperties(MetadataEntity.ofDataset(getContext().getNamespace(), dataset)); // wait till change is recorded to store try { Tasks.waitFor(true, () -> { Map<MetadataScope, Metadata> metadata2 = getContext().getMetadata(MetadataEntity.ofDataset(getContext().getNamespace(), dataset)); return metadata2.get(MetadataScope.USER).getProperties().isEmpty(); }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); } catch (Exception e) { responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()); } responder.sendStatus(HttpResponseStatus.OK.code()); }
private int addAllSystemMetadata(Set<String> allMetadata) { for (AuditMessage auditMessage : getMetadataUpdateMessages()) { AuditPayload payload = auditMessage.getPayload(); Assert.assertTrue(payload instanceof MetadataPayload); MetadataPayload metadataPayload = (MetadataPayload) payload; Map<MetadataScope, Metadata> additions = metadataPayload.getAdditions(); if (additions.containsKey(MetadataScope.SYSTEM)) { allMetadata.addAll(additions.get(MetadataScope.SYSTEM).getProperties().keySet()); allMetadata.addAll(additions.get(MetadataScope.SYSTEM).getTags()); } Map<MetadataScope, Metadata> deletions = metadataPayload.getDeletions(); if (deletions.containsKey(MetadataScope.SYSTEM)) { allMetadata.addAll(deletions.get(MetadataScope.SYSTEM).getProperties().keySet()); allMetadata.addAll(deletions.get(MetadataScope.SYSTEM).getTags()); } } return allMetadata.size(); }
@Override public void removeProperties(MetadataEntity metadataEntity, String... keys) { Map<String, String> props = new HashMap<>(); for (String key : keys) { props.put(key, ""); } MetadataOperation metadataOperation = new MetadataOperation(metadataEntity, MetadataOperation.Type.DELETE, new Metadata(props, Collections.emptySet())); metadataPublisher.publish(programRunId, metadataOperation); }
@DELETE @Path("metadata/{dataset}/tags") public void removeAllTags(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("dataset") String dataset) { getContext().removeTags(MetadataEntity.ofDataset(getContext().getNamespace(), dataset)); // wait till change is recorded to store try { Tasks.waitFor(true, () -> { Map<MetadataScope, Metadata> metadata2 = getContext().getMetadata(MetadataEntity.ofDataset(getContext().getNamespace(), dataset)); return metadata2.get(MetadataScope.USER).getTags().isEmpty(); }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); } catch (Exception e) { responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()); } responder.sendStatus(HttpResponseStatus.OK.code()); }
@DELETE @Path("metadata/{dataset}/properties/{key}") public void removeProperty(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("dataset") String dataset, @PathParam("key") String key) { getContext().removeProperties(MetadataEntity.ofDataset(getContext().getNamespace(), dataset), key); // wait till change is recorded to store try { Tasks.waitFor(true, () -> { Map<MetadataScope, Metadata> metadata2 = getContext().getMetadata(MetadataEntity.ofDataset(getContext().getNamespace(), dataset)); return !metadata2.get(MetadataScope.USER).getProperties().containsKey(key); }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); } catch (Exception e) { responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()); } responder.sendStatus(HttpResponseStatus.OK.code()); }
Set<MetadataOperation> operations = GSON.fromJson(config.metadataOperations, SET_METADATA_OPERATION_TYPE); if (currentMetadata.get(MetadataScope.SYSTEM).getProperties().isEmpty() || currentMetadata.get(MetadataScope.SYSTEM).getProperties().isEmpty()) { throw new IllegalArgumentException(String.format("System properties or tags for '%s' is empty. " + "Expected to have system metadata.", metadataEntity)); case PUT: context.addTags(curOperation.getEntity(), curOperation.getMetadata().getTags()); context.addProperties(curOperation.getEntity(), curOperation.getMetadata().getProperties()); break; case DELETE: curOperation.getMetadata().getTags().toArray(new String[0])); context.removeProperties(curOperation.getEntity(), curOperation.getMetadata().getProperties().keySet().toArray(new String[0])); break; case DELETE_ALL:
@Override public void removeTags(MetadataEntity metadataEntity, String... tags) { MetadataOperation metadataOperation = new MetadataOperation(metadataEntity, MetadataOperation.Type.DELETE, new Metadata(Collections.emptyMap(), new HashSet<>(Arrays.asList(tags)))); metadataPublisher.publish(programRunId, metadataOperation); }
Metadata metadata = null; for (Map.Entry<MetadataScope, co.cask.cdap.api.metadata.Metadata> entry : record.getMetadata().entrySet()) { Metadata toAdd = new Metadata(entry.getKey(), entry.getValue().getTags(), entry.getValue().getProperties()); metadata = metadata == null ? toAdd : mergeDisjointMetadata(metadata, toAdd);
@Override public void addProperties(MetadataEntity metadataEntity, Map<String, String> properties) { metadataPublisher.publish(programRunId, new MetadataOperation(metadataEntity, MetadataOperation.Type.PUT, new Metadata(properties, Collections.emptySet()))); }
@DELETE @Path("metadata/{dataset}") public void removeMetadata(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("dataset") String dataset) { getContext().removeMetadata(MetadataEntity.ofDataset(getContext().getNamespace(), dataset)); // wait till change is recorded to store try { Tasks.waitFor(true, () -> { Map<MetadataScope, Metadata> metadata2 = getContext().getMetadata(MetadataEntity.ofDataset(getContext().getNamespace(), dataset)); return metadata2.get(MetadataScope.USER).getProperties().isEmpty() && metadata2.get(MetadataScope.USER) .getTags().isEmpty(); }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); } catch (Exception e) { responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()); } responder.sendStatus(HttpResponseStatus.OK.code()); } }
@Override public Map<MetadataScope, Metadata> getMetadata(MetadataEntity metadataEntity) { Map<MetadataScope, Metadata> scopeMetadata = new HashMap<>(); Set<MetadataRecord> metadata = metadataAdmin.getMetadata(metadataEntity); metadata.forEach(record -> scopeMetadata.put(record.getScope(), new Metadata(record.getProperties(), record.getTags()))); return scopeMetadata; }
Assert.assertFalse(scopeMetadata.get(MetadataScope.SYSTEM).getProperties().isEmpty()); Assert.assertFalse(scopeMetadata.get(MetadataScope.SYSTEM).getTags().isEmpty()); Assert.assertTrue(scopeMetadata.get(MetadataScope.SYSTEM).getProperties().containsKey("entity-name")); Assert.assertEquals(AppWithMetadataPrograms.METADATA_SERVICE_DATASET, scopeMetadata.get(MetadataScope.SYSTEM).getProperties().get("entity-name")); Assert.assertTrue(scopeMetadata.get(MetadataScope.SYSTEM).getTags() .containsAll(Arrays.asList("explore", "batch"))); Assert.assertFalse(scopeMetadata.get(MetadataScope.USER).getProperties().isEmpty()); Assert.assertFalse(scopeMetadata.get(MetadataScope.USER).getTags().isEmpty()); Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getTags().containsAll(Arrays.asList("tag1", "tag2"))); Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getProperties().containsKey("k1")); Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getProperties().containsKey("k2")); Assert.assertEquals("v1", scopeMetadata.get(MetadataScope.USER).getProperties().get("k1")); Assert.assertEquals("v2", scopeMetadata.get(MetadataScope.USER).getProperties().get("k2")); AppWithMetadataPrograms.METADATA_SERVICE_DATASET); scopeMetadata = GSON.fromJson(result, MAP_METADATASCOPE_METADATA_TYPE); Assert.assertEquals(1, scopeMetadata.get(MetadataScope.USER).getTags().size()); Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getTags().contains("tag2")); Assert.assertEquals(1, scopeMetadata.get(MetadataScope.USER).getProperties().size()); Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getProperties().containsKey("k2")); Assert.assertEquals("v2", scopeMetadata.get(MetadataScope.USER).getProperties().get("k2")); AppWithMetadataPrograms.METADATA_SERVICE_DATASET); scopeMetadata = GSON.fromJson(result, MAP_METADATASCOPE_METADATA_TYPE); Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getTags().isEmpty()); Assert.assertFalse(scopeMetadata.get(MetadataScope.USER).getProperties().isEmpty());
@Override public Map<MetadataScope, Metadata> getMetadata(MetadataEntity metadataEntity) { Map<MetadataScope, Metadata> scopeMetadata = new HashMap<>(); Set<MetadataRecordV2> metadata = metadataAdmin.getMetadata(metadataEntity); metadata.forEach(record -> scopeMetadata.put(record.getScope(), new Metadata(record.getProperties(), record.getTags()))); return scopeMetadata; }
private Set<MetadataSearchResultRecord> addMetadataToEntities(Set<MetadataEntity> entities, Map<MetadataEntity, MetadataDataset.Record> systemMetadata, Map<MetadataEntity, MetadataDataset.Record> userMetadata) { Set<MetadataSearchResultRecord> result = new LinkedHashSet<>(); for (MetadataEntity entity : entities) { ImmutableMap.Builder<MetadataScope, Metadata> builder = ImmutableMap.builder(); // Add system metadata MetadataDataset.Record metadata = systemMetadata.get(entity); if (metadata != null) { builder.put(SYSTEM, new Metadata(metadata.getProperties(), metadata.getTags())); } // Add user metadata metadata = userMetadata.get(entity); if (metadata != null) { builder.put(MetadataScope.USER, new Metadata(metadata.getProperties(), metadata.getTags())); } // Create result result.add(new MetadataSearchResultRecord(entity, builder.build())); } return result; }
private Set<MetadataSearchResultRecord> addMetadataToEntities(Set<MetadataEntity> entities, Map<MetadataEntity, MetadataDataset.Record> systemMetadata, Map<MetadataEntity, MetadataDataset.Record> userMetadata) { Set<MetadataSearchResultRecord> result = new LinkedHashSet<>(); for (MetadataEntity entity : entities) { ImmutableMap.Builder<MetadataScope, Metadata> builder = ImmutableMap.builder(); // Add system metadata MetadataDataset.Record metadata = systemMetadata.get(entity); if (metadata != null) { builder.put(SYSTEM, new Metadata(metadata.getProperties(), metadata.getTags())); } // Add user metadata metadata = userMetadata.get(entity); if (metadata != null) { builder.put(MetadataScope.USER, new Metadata(metadata.getProperties(), metadata.getTags())); } // Create result result.add(new MetadataSearchResultRecord(entity, builder.build())); } return result; } }