/** * If the MetadataEntity Builder key-value represent an application or artifact which is versioned in CDAP i.e. their * metadata entity representation ends with 'version' rather than 'application' or 'artifact' and the type is also * set to 'version' then for backward compatibility of rest end points return an updated builder which has the * correct type. This is only needed in 5.0 for backward compatibility of the rest endpoint. * From 5.0 and later the rest end point must be called with a query parameter which specify the type of the * metadata entity if the type is not the last key. (CDAP-13678) */ @VisibleForTesting static MetadataEntity.Builder makeBackwardCompatible(MetadataEntity.Builder builder) { MetadataEntity entity = builder.build(); List<MetadataEntity.KeyValue> entityKeyValues = StreamSupport.stream(entity.spliterator(), false) .collect(Collectors.toList()); if (entityKeyValues.size() == 3 && entity.getType().equals(MetadataEntity.VERSION) && (entityKeyValues.get(1).getKey().equals(MetadataEntity.ARTIFACT) || entityKeyValues.get(1).getKey().equals(MetadataEntity.APPLICATION))) { // this is artifact or application so update the builder MetadataEntity.Builder actualEntityBuilder = MetadataEntity.builder(); // namespace actualEntityBuilder.append(entityKeyValues.get(0).getKey(), entityKeyValues.get(0).getValue()); // application or artifact (so append as type) actualEntityBuilder.appendAsType(entityKeyValues.get(1).getKey(), entityKeyValues.get(1).getValue()); // version detail actualEntityBuilder.append(entityKeyValues.get(2).getKey(), entityKeyValues.get(2).getValue()); return actualEntityBuilder; } return builder; }
/** * Builds a {@link MetadataEntity} from the builder. * * @return {@link MetadataEntity} from the builder * @throws IllegalArgumentException if the key is a CDAP entity and the MetadataEntity is not correct to represent * the CDAP entity */ public MetadataEntity build() { if (parts.isEmpty()) { throw new IllegalArgumentException("key-value pair must be specified"); } if (type == null) { // traverse till the last key and make that the type parts.keySet().forEach(x -> type = x); } validateHierarchy(); return new MetadataEntity(new LinkedHashMap<>(parts), type); }
final MetadataEntity dataset1 = ns1Id.dataset("ds1").toMetadataEntity(); final MetadataEntity dataset2 = ns1Id.dataset("ds2").toMetadataEntity(); final MetadataEntity file1 = MetadataEntity.builder(dataset1).appendAsType("file", "f1").build();
private MetadataEntity getMetadataEntityFromPath(String uri, @Nullable String entityType, String suffix) { String[] parts = uri.substring((uri.indexOf(Constants.Gateway.API_VERSION_3) + Constants.Gateway.API_VERSION_3.length() + 1), uri.lastIndexOf(suffix)).split("/"); MetadataEntity.Builder builder = MetadataEntity.builder(); int curIndex = 0; while (curIndex < parts.length - 1) { // the api part which we get for program does not have 'type' keyword as part of the uri. It goes like // ../apps/appName/programType/ProgramName so handle that correctly if (curIndex >= 2 && parts[curIndex - 2].equalsIgnoreCase("apps") && !parts[curIndex].equalsIgnoreCase ("versions")) { builder = appendHelper(builder, entityType, MetadataEntity.TYPE, ProgramType.valueOfCategoryName(parts[curIndex]).name()); builder = appendHelper(builder, entityType, MetadataEntity.PROGRAM, parts[curIndex + 1]); } else { if (MetadataClient.ENTITY_TYPE_TO_API_PART.inverse().containsKey(parts[curIndex])) { builder = appendHelper(builder, entityType, MetadataClient.ENTITY_TYPE_TO_API_PART.inverse().get(parts[curIndex]), parts[curIndex + 1]); } else { builder = appendHelper(builder, entityType, parts[curIndex], parts[curIndex + 1]); } } curIndex += 2; } builder = makeBackwardCompatible(builder); return builder.build(); }
/** * If the MetadataEntity Builder key-value represent an application or artifact which is versioned in CDAP i.e. their * metadata entity representation ends with 'version' rather than 'application' or 'artifact' and the type is also * set to 'version' then for backward compatibility of rest end points return an updated builder which has the * correct type. This is only needed in 5.0 for backward compatibility of the rest endpoint. * From 5.0 and later the rest end point must be called with a query parameter which specify the type of the * metadata entity if the type is not the last key. (CDAP-13678) */ @VisibleForTesting static MetadataEntity.Builder makeBackwardCompatible(MetadataEntity.Builder builder) { MetadataEntity entity = builder.build(); List<MetadataEntity.KeyValue> entityKeyValues = StreamSupport.stream(entity.spliterator(), false) .collect(Collectors.toList()); if (entityKeyValues.size() == 3 && entity.getType().equals(MetadataEntity.VERSION) && (entityKeyValues.get(1).getKey().equals(MetadataEntity.ARTIFACT) || entityKeyValues.get(1).getKey().equals(MetadataEntity.APPLICATION))) { // this is artifact or application so update the builder MetadataEntity.Builder actualEntityBuilder = MetadataEntity.builder(); // namespace actualEntityBuilder.append(entityKeyValues.get(0).getKey(), entityKeyValues.get(0).getValue()); // application or artifact (so append as type) actualEntityBuilder.appendAsType(entityKeyValues.get(1).getKey(), entityKeyValues.get(1).getValue()); // version detail actualEntityBuilder.append(entityKeyValues.get(2).getKey(), entityKeyValues.get(2).getValue()); return actualEntityBuilder; } return builder; }
private MetadataEntity getMetadataEntityFromPath(String uri, @Nullable String entityType, String suffix) { String[] parts = uri.substring((uri.indexOf(Constants.Gateway.API_VERSION_3) + Constants.Gateway.API_VERSION_3.length() + 1), uri.lastIndexOf(suffix)).split("/"); MetadataEntity.Builder builder = MetadataEntity.builder(); int curIndex = 0; while (curIndex < parts.length - 1) { // the api part which we get for program does not have 'type' keyword as part of the uri. It goes like // ../apps/appName/programType/ProgramName so handle that correctly if (curIndex >= 2 && parts[curIndex - 2].equalsIgnoreCase("apps") && !parts[curIndex].equalsIgnoreCase ("versions")) { builder = appendHelper(builder, entityType, MetadataEntity.TYPE, ProgramType.valueOfCategoryName(parts[curIndex]).name()); builder = appendHelper(builder, entityType, MetadataEntity.PROGRAM, parts[curIndex + 1]); } else { if (MetadataClient.ENTITY_TYPE_TO_API_PART.inverse().containsKey(parts[curIndex])) { builder = appendHelper(builder, entityType, MetadataClient.ENTITY_TYPE_TO_API_PART.inverse().get(parts[curIndex]), parts[curIndex + 1]); } else { builder = appendHelper(builder, entityType, parts[curIndex], parts[curIndex + 1]); } } curIndex += 2; } builder = makeBackwardCompatible(builder); return builder.build(); }
builder = builder.appendAsType(key, value); } else { builder = builder.append(key, value); MetadataEntity metadataEntity = builder.build(); if (VERSIONED_ENTITIES.contains(metadataEntity.getType())) {
builder = builder.appendAsType(key, value); } else { builder = builder.append(key, value); MetadataEntity metadataEntity = builder.build(); if (VERSIONED_ENTITIES.contains(metadataEntity.getType())) {
@Test public void testMakeBackwardCompatible() { MetadataEntity.Builder entity = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "ns") .append(MetadataEntity.APPLICATION, "app") .append(MetadataEntity.VERSION, "ver"); MetadataEntity expected = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "ns") .appendAsType(MetadataEntity.APPLICATION, "app") .append(MetadataEntity.VERSION, "ver").build(); Assert.assertEquals(expected, actual.build()); entity = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "ns") .append(MetadataEntity.ARTIFACT, "art") .append(MetadataEntity.VERSION, "ver"); expected = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "ns") .appendAsType(MetadataEntity.ARTIFACT, "art") .append(MetadataEntity.VERSION, "ver").build(); Assert.assertEquals(expected, actual.build()); entity = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "ns") .append(MetadataEntity.APPLICATION, "app"); actual = MetadataHttpHandler.makeBackwardCompatible(entity); expected = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "ns") .appendAsType(MetadataEntity.APPLICATION, "app").build(); Assert.assertEquals(expected, actual.build());
@Test public void testSearchOnTypes() throws Exception { MetadataEntity myField1 = MetadataEntity.builder(MetadataEntity.ofDataset(NamespaceId.DEFAULT.getEntityName(), "myDs")) .appendAsType("field", "myField1").build(); MetadataEntity myField2 = MetadataEntity.builder(MetadataEntity.ofDataset(NamespaceId.DEFAULT.getEntityName(), "myDs")) .appendAsType("field", "myField2").build(); final MetadataEntry myFieldEntry1 = new MetadataEntry(myField1, "testKey1", "testValue1"); final MetadataEntry myFieldEntry2 = new MetadataEntry(myField2, "testKey2", "testValue2"); txnl.execute(() -> { dataset.addProperty(myField1, "testKey1", "testValue1"); dataset.addProperty(myField2, "testKey2", "testValue2"); }); // Search for it based on value txnl.execute(() -> { List<MetadataEntry> results = searchByDefaultIndex("default", "field:myField1", ImmutableSet.of(EntityTypeSimpleName.ALL)); Assert.assertEquals(ImmutableList.of(myFieldEntry1), results); // should return both fields results = searchByDefaultIndex("default", "field:myFie*", ImmutableSet.of(EntityTypeSimpleName.ALL)); Assert.assertEquals(ImmutableList.of(myFieldEntry1, myFieldEntry2), results); results = searchByDefaultIndex("default", "field*", ImmutableSet.of(EntityTypeSimpleName.ALL)); Assert.assertEquals(ImmutableList.of(myFieldEntry1, myFieldEntry2), results); }); }
@Test public void testMesssages() { // test dataset InvalidMetadataException invalidMetadataException = new InvalidMetadataException(NamespaceId.DEFAULT.dataset("ds").toMetadataEntity(), "error"); String expectedMessage = "Unable to set metadata for dataset: ds " + "which exists in namespace: default. error"; Assert.assertEquals(expectedMessage, invalidMetadataException.getMessage()); // test program invalidMetadataException = new InvalidMetadataException(NamespaceId.DEFAULT.app("app").program(ProgramType.WORKER, "wk") .toMetadataEntity(), "error"); expectedMessage = "Unable to set metadata for worker: wk in application: app of version: -SNAPSHOT deployed in " + "namespace: default. error"; Assert.assertEquals(expectedMessage, invalidMetadataException.getMessage()); // test custom entity MetadataEntity customEntity = MetadataEntity.builder(NamespaceId.DEFAULT.dataset("ds").toMetadataEntity()) .appendAsType("field", "empName").build(); invalidMetadataException = new InvalidMetadataException(customEntity, "error"); expectedMessage = "Unable to set metadata for namespace=default,dataset=ds,field=empName of type 'field'. error"; Assert.assertEquals(expectedMessage, invalidMetadataException.getMessage()); } }
private static MetadataEntity fromCliString(String cliString) { String[] keyValues = cliString.split(METADATA_ENTITY_PARTS_SEPARATOR); int lastKeyValueIndex = keyValues.length - 1; MetadataEntity.Builder builder = MetadataEntity.builder(); String customType = null; if (getKeyValue(keyValues[lastKeyValueIndex])[0].equalsIgnoreCase(METADATA_ENTITY_TYPE)) { // if a type is specified then store it to call appendAsType later customType = getKeyValue(keyValues[lastKeyValueIndex])[1]; lastKeyValueIndex -= 1; } for (int i = 0; i <= lastKeyValueIndex; i++) { String[] part = getKeyValue(keyValues[i]); if (part[0].equals(customType)) { builder.appendAsType(part[0], part[1]); } else { builder.append(part[0], part[1]); } } return builder.build(); }
private static MetadataEntity fromCliString(String cliString) { String[] keyValues = cliString.split(METADATA_ENTITY_PARTS_SEPARATOR); int lastKeyValueIndex = keyValues.length - 1; MetadataEntity.Builder builder = MetadataEntity.builder(); String customType = null; if (getKeyValue(keyValues[lastKeyValueIndex])[0].equalsIgnoreCase(METADATA_ENTITY_TYPE)) { // if a type is specified then store it to call appendAsType later customType = getKeyValue(keyValues[lastKeyValueIndex])[1]; lastKeyValueIndex -= 1; } for (int i = 0; i <= lastKeyValueIndex; i++) { String[] part = getKeyValue(keyValues[i]); if (part[0].equals(customType)) { builder.appendAsType(part[0], part[1]); } else { builder.append(part[0], part[1]); } } return builder.build(); }
@Test public void testToEntityId() { // should be able to get get an EntityId if Metadata belong to a cdap entity id DatasetId myDs = NamespaceId.DEFAULT.dataset("myDs"); MetadataDataset.Record metadata1 = new MetadataDataset.Record(myDs); Assert.assertEquals(myDs, metadata1.getEntityId()); MetadataEntity metadataEntity = MetadataEntity.builder(MetadataEntity.ofDataset(NamespaceId.DEFAULT.getEntityName(), "myDs")) .appendAsType("field", "myField").build(); MetadataDataset.Record metadata2 = new MetadataDataset.Record(metadataEntity); try { metadata2.getEntityId(); Assert.fail(); } catch (IllegalArgumentException e) { // expected } } }
@Test public void testSearchOnTypes() throws Exception { MetadataStorage mds = getMetadataStorage(); MetadataEntity myDs = NamespaceId.DEFAULT.dataset("myDs").toMetadataEntity(); MetadataEntity myField1 = MetadataEntity.builder(myDs).appendAsType("field", "myField1").build(); MetadataEntity myField2 = MetadataEntity.builder(myDs).appendAsType("field", "myField2").build(); MetadataRecord record1 = new MetadataRecord(myField1, new Metadata(USER, props("testKey1", "testValue1"))); MetadataRecord record2 = new MetadataRecord(myField2, new Metadata(USER, props("testKey2", "testValue2"))); mds.apply(new Update(myField1, record1.getMetadata())); mds.apply(new Update(myField2, record2.getMetadata())); // Search for it based on value assertResults(mds, SearchRequest.of("field:myField1").build(), record1); // should return both fields assertResults(mds, SearchRequest.of("field:myFie*").build(), record1, record2); assertResults(mds, SearchRequest.of("field*").build(), record1, record2); // clean up mds.batch(ImmutableList.of(new Drop(myField1), new Drop(myField2))); }
private MetadataEntity.Builder appendHelper(MetadataEntity.Builder builder, @Nullable String entityType, String key, String value) { if (entityType == null || entityType.isEmpty()) { // if a type is not provided then keep appending as type to update the type on every append return builder.appendAsType(key, value); } else { if (entityType.equalsIgnoreCase(key)) { // if a type was provided and this key is the type then appendAsType return builder.appendAsType(key, value); } else { return builder.append(key, value); } } }
public static Builder builder(MetadataEntity metadataEntity) { return new Builder(metadataEntity); }
public static Builder builder() { return new Builder(); }
private MetadataEntity.Builder appendHelper(MetadataEntity.Builder builder, @Nullable String entityType, String key, String value) { if (entityType == null || entityType.isEmpty()) { // if a type is not provided then keep appending as type to update the type on every append return builder.appendAsType(key, value); } else { if (entityType.equalsIgnoreCase(key)) { // if a type was provided and this key is the type then appendAsType return builder.appendAsType(key, value); } else { return builder.append(key, value); } } }
@Override public MetadataEntity toMetadataEntity() { return MetadataEntity.builder().append(MetadataEntity.NAMESPACE, namespace).append(MetadataEntity.STREAM, stream) .appendAsType(MetadataEntity.VIEW, view) .build(); }