@Override public String toString() { int max = getMaxCardinality(); int min = getMinCardinality(); List<String> targetExprList = getTargetExpressions().stream() .map(TargetExpression::toString).collect(Collectors.toList()); List<String> targetConstraints = new ArrayList<>(); .append(getScope()).append(",") .append(targetExpr) .toString()); .append(getScope()).append(",") .append(targetExpr) .toString()); .append(getScope()).append(",") .append(getNodeAttributeOpCode()).append(",") .append(targetExpr) .toString()); .append(getScope()).append(",") .append(targetExpr).append(",") .append(min).append(",")
private SingleConstraint convert(SimplePlacementConstraintProto proto) { Set<TargetExpression> targets = new HashSet<>(); for (PlacementConstraintTargetProto tp : proto.getTargetExpressionsList()) { targets.add(convert(tp)); } return new SingleConstraint(proto.getScope(), proto.getMinCardinality(), proto.getMaxCardinality(), convertFromProtoFormat(proto.getAttributeOpCode()), targets); }
Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; Assert.assertEquals("node", single.getScope()); Assert.assertEquals(0, single.getMinCardinality()); Assert.assertEquals(1, single.getMaxCardinality()); Assert.assertEquals(1, single.getTargetExpressions().size()); TargetExpression exp = single.getTargetExpressions().iterator().next(); Assert.assertEquals("ALLOCATION_TAG", exp.getTargetType().toString()); Assert.assertEquals(1, exp.getTargetValues().size()); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; Assert.assertEquals("rack", single.getScope()); Assert.assertEquals(0, single.getMinCardinality()); Assert.assertEquals(1, single.getMaxCardinality()); Assert.assertEquals(1, single.getTargetExpressions().size()); exp = single.getTargetExpressions().iterator().next(); Assert.assertEquals("ALLOCATION_TAG", exp.getTargetType().toString()); Assert.assertEquals(3, exp.getTargetValues().size());
Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; Assert.assertEquals("node", single.getScope()); Assert.assertEquals(0, single.getMinCardinality()); Assert.assertEquals(0, single.getMaxCardinality()); verifyConstraintToString(expressionStr, constraint); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; Assert.assertEquals("node", single.getScope()); Assert.assertEquals(0, single.getMinCardinality()); Assert.assertEquals(0, single.getMaxCardinality()); verifyConstraintToString(expressionStr, constraint); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; Assert.assertEquals("node", single.getScope()); Assert.assertEquals(1, single.getMinCardinality()); Assert.assertEquals(Integer.MAX_VALUE, single.getMaxCardinality()); verifyConstraintToString(expressionStr, constraint); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; Assert.assertEquals("node", single.getScope()); Assert.assertEquals(0, single.getMinCardinality()); Assert.assertEquals(0, single.getMaxCardinality()); Assert.assertEquals(1, single.getTargetExpressions().size()); TargetExpression exp = single.getTargetExpressions().iterator().next();
int desiredMinCardinality = sc.getMinCardinality(); int desiredMaxCardinality = sc.getMaxCardinality(); boolean checkMinCardinality = desiredMinCardinality > 0; boolean checkMaxCardinality = desiredMaxCardinality < Integer.MAX_VALUE; if (sc.getScope().equals(PlacementConstraints.NODE)) { if (checkMinCardinality) { minScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), allocationTags, Long::max); } else if (sc.getScope().equals(PlacementConstraints.RACK)) { if (checkMinCardinality) { minScopeCardinality = tm.getRackCardinalityByOp(node.getRackName(),
SimplePlacementConstraintProto sProto = protoConstraint.getSimpleConstraint(); Assert.assertEquals(single.getScope(), sProto.getScope()); Assert.assertEquals(single.getMinCardinality(), sProto.getMinCardinality()); Assert.assertEquals(single.getMaxCardinality(), sProto.getMaxCardinality()); Assert.assertEquals(single.getTargetExpressions().size(), sProto.getTargetExpressionsList().size()); Assert.assertTrue(newConstraintExpr instanceof SingleConstraint); SingleConstraint newSingle = (SingleConstraint) newConstraintExpr; Assert.assertEquals(single.getScope(), newSingle.getScope()); Assert.assertEquals(single.getMinCardinality(), newSingle.getMinCardinality()); Assert.assertEquals(single.getMaxCardinality(), newSingle.getMaxCardinality()); Assert.assertEquals(single.getTargetExpressions(), newSingle.getTargetExpressions());
(PlacementConstraint.SingleConstraint) ac; for (PlacementConstraint.TargetExpression targetExpression : singleConstraint.getTargetExpressions()) {
Assert.assertEquals(single.toString(), target.toString()); Assert.assertEquals(single.getScope(), target.getScope()); Assert.assertEquals(TargetOperator.IN, target.getOp()); Assert.assertEquals(single.getTargetExpressions(), target.getTargetExpressions()); Assert.assertEquals(target.getScope(), single.getScope()); Assert.assertEquals(1, single.getMinCardinality()); Assert.assertEquals(Integer.MAX_VALUE, single.getMaxCardinality()); Assert.assertEquals(single.getTargetExpressions(), target.getTargetExpressions());
/** * This constraint generalizes the cardinality and target constraints. * * Consider a set of nodes N that belongs to the scope specified in the * constraint. If the target expressions are satisfied at least minCardinality * times and at most maxCardinality times in the node set N, then the * constraint is satisfied. * * For example, {@code targetCardinality(RACK, 2, 10, allocationTag("zk"))}, * requires an allocation to be placed within a rack that has at least 2 and * at most 10 other allocations with tag "zk". * * @param scope the scope of the constraint * @param minCardinality the minimum number of times the target expressions * have to be satisfied with the given scope * @param maxCardinality the maximum number of times the target expressions * have to be satisfied with the given scope * @param targetExpressions the target expressions * @return the resulting placement constraint */ public static AbstractConstraint targetCardinality(String scope, int minCardinality, int maxCardinality, TargetExpression... targetExpressions) { return new SingleConstraint(scope, minCardinality, maxCardinality, targetExpressions); }
private static boolean canSatisfySingleConstraint(ApplicationId applicationId, SingleConstraint singleConstraint, SchedulerNode schedulerNode, AllocationTagsManager tagsManager) throws InvalidAllocationTagsQueryException { // Iterate through TargetExpressions Iterator<TargetExpression> expIt = singleConstraint.getTargetExpressions().iterator(); while (expIt.hasNext()) { TargetExpression currentExp = expIt.next(); // Supporting AllocationTag Expressions for now if (currentExp.getTargetType().equals(TargetType.ALLOCATION_TAG)) { // Check if conditions are met if (!canSatisfySingleConstraintExpression(applicationId, singleConstraint, currentExp, schedulerNode, tagsManager)) { return false; } } else if (currentExp.getTargetType().equals(TargetType.NODE_ATTRIBUTE)) { // This is a node attribute expression, check it. if (!canSatisfyNodeConstraintExpression(singleConstraint, currentExp, schedulerNode)) { return false; } } } // return true if all targetExpressions are satisfied return true; }
private static boolean canSatisfyNodeConstraintExpression( SingleConstraint sc, TargetExpression targetExpression, SchedulerNode schedulerNode) { Set<String> values = targetExpression.getTargetValues(); if (targetExpression.getTargetKey().equals(NODE_PARTITION)) { if (values == null || values.isEmpty()) { return schedulerNode.getPartition() .equals(RMNodeLabelsManager.NO_LABEL); } else { String nodePartition = values.iterator().next(); if (!nodePartition.equals(schedulerNode.getPartition())) { return false; } } } else { NodeAttributeOpCode opCode = sc.getNodeAttributeOpCode(); // compare attributes. String inputAttribute = values.iterator().next(); NodeAttribute requestAttribute = getNodeConstraintFromRequest( targetExpression.getTargetKey(), inputAttribute); if (requestAttribute == null) { return true; } return getNodeConstraintEvaluatedResult(schedulerNode, opCode, requestAttribute); } return true; }
@Override public GeneratedMessage visit(SingleConstraint constraint) { SimplePlacementConstraintProto.Builder sb = SimplePlacementConstraintProto.newBuilder(); if (constraint.getScope() != null) { sb.setScope(constraint.getScope()); } sb.setMinCardinality(constraint.getMinCardinality()); sb.setMaxCardinality(constraint.getMaxCardinality()); if (constraint.getNodeAttributeOpCode() != null) { sb.setAttributeOpCode( convertToProtoFormat(constraint.getNodeAttributeOpCode())); } if (constraint.getTargetExpressions() != null) { for (TargetExpression target : constraint.getTargetExpressions()) { sb.addTargetExpressions( (PlacementConstraintTargetProto) target.accept(this)); } } SimplePlacementConstraintProto sProto = sb.build(); // Wrap around PlacementConstraintProto object. PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder(); pb.setSimpleConstraint(sProto); return pb.build(); }
@Test public void testCardinalityConstraint() { CardinalityConstraint cardinality = new CardinalityConstraint(RACK, 3, 10, new HashSet<>(Arrays.asList("hb"))); PlacementConstraint cConstraint = PlacementConstraints.build(cardinality); // Transform from specialized CardinalityConstraint to SimpleConstraint SingleConstraintTransformer singleTransformer = new SingleConstraintTransformer(cConstraint); PlacementConstraint sConstraint = singleTransformer.transform(); AbstractConstraint sConstraintExpr = sConstraint.getConstraintExpr(); Assert.assertTrue(sConstraintExpr instanceof SingleConstraint); SingleConstraint single = (SingleConstraint) sConstraintExpr; // Make sure the consistent expression string is consistent // before and after transforming Assert.assertEquals(single.toString(), cardinality.toString()); Assert.assertEquals(cardinality.getScope(), single.getScope()); Assert.assertEquals(cardinality.getMinCardinality(), single.getMinCardinality()); Assert.assertEquals(cardinality.getMaxCardinality(), single.getMaxCardinality()); Assert.assertEquals( new HashSet<>(Arrays.asList(PlacementTargets.allocationTag("hb"))), single.getTargetExpressions()); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof SingleConstraint)) { return false; } SingleConstraint that = (SingleConstraint) o; if (getMinCardinality() != that.getMinCardinality()) { return false; } if (getMaxCardinality() != that.getMaxCardinality()) { return false; } if (!getScope().equals(that.getScope())) { return false; } if (getNodeAttributeOpCode() != null && !getNodeAttributeOpCode() .equals(that.getNodeAttributeOpCode())) { return false; } return getTargetExpressions().equals(that.getTargetExpressions()); }
/** * Creates a constraint that restricts the number of allocations within a * given scope (e.g., node or rack). * * For example, {@code cardinality(NODE, 3, 10, "zk")} is satisfied on nodes * where there are no less than 3 allocations with tag "zk" and no more than * 10. * * @param scope the scope of the constraint * @param minCardinality determines the minimum number of allocations within * the scope * @param maxCardinality determines the maximum number of allocations within * the scope * @param allocationTags the constraint targets allocations with these tags * @return the resulting placement constraint */ public static AbstractConstraint cardinality(String scope, int minCardinality, int maxCardinality, String... allocationTags) { return new SingleConstraint(scope, minCardinality, maxCardinality, PlacementTargets.allocationTag(allocationTags)); }
@Test public void testAndConstraint() { AbstractConstraint constraintExpr = and(targetIn(RACK, allocationTag("spark")), maxCardinality(NODE, 3, "spark"), targetCardinality(RACK, 2, 10, allocationTag("zk"))); And andExpr = (And) constraintExpr; Assert.assertEquals(3, andExpr.getChildren().size()); SingleConstraint sConstr = (SingleConstraint) andExpr.getChildren().get(0); TargetExpression tExpr = sConstr.getTargetExpressions().iterator().next(); Assert.assertEquals("spark", tExpr.getTargetValues().iterator().next()); sConstr = (SingleConstraint) andExpr.getChildren().get(1); Assert.assertEquals(0, sConstr.getMinCardinality()); Assert.assertEquals(3, sConstr.getMaxCardinality()); sConstr = (SingleConstraint) andExpr.getChildren().get(2); Assert.assertEquals(2, sConstr.getMinCardinality()); Assert.assertEquals(10, sConstr.getMaxCardinality()); } }
/** * Creates a constraint that requires allocations to be placed on nodes that * satisfy all target expressions within the given scope (e.g., node or rack). * * For example, {@code targetIn(RACK, allocationTag("hbase-m"))}, allows * allocations on nodes that belong to a rack that has at least one tag with * value "hbase-m". * * @param scope the scope within which the target expressions should be * satisfied * @param targetExpressions the expressions that need to be satisfied within * the scope * @return the resulting placement constraint */ public static AbstractConstraint targetIn(String scope, TargetExpression... targetExpressions) { return new SingleConstraint(scope, 1, Integer.MAX_VALUE, targetExpressions); }
@Test public void testNodeAffinityToTag() { AbstractConstraint constraintExpr = targetIn(NODE, allocationTag("hbase-m")); SingleConstraint sConstraint = (SingleConstraint) constraintExpr; Assert.assertEquals(NODE, sConstraint.getScope()); Assert.assertEquals(1, sConstraint.getMinCardinality()); Assert.assertEquals(Integer.MAX_VALUE, sConstraint.getMaxCardinality()); Assert.assertEquals(1, sConstraint.getTargetExpressions().size()); TargetExpression tExpr = sConstraint.getTargetExpressions().iterator().next(); Assert.assertEquals(AllocationTagNamespaceType.SELF.toString(), tExpr.getTargetKey()); Assert.assertEquals(TargetType.ALLOCATION_TAG, tExpr.getTargetType()); Assert.assertEquals(1, tExpr.getTargetValues().size()); Assert.assertEquals("hbase-m", tExpr.getTargetValues().iterator().next()); PlacementConstraint constraint = PlacementConstraints.build(constraintExpr); Assert.assertNotNull(constraint.getConstraintExpr()); }
/** * Similar to {@link #cardinality(String, int, int, String...)}, but let you * attach a namespace to the given allocation tags. * * @param scope the scope of the constraint * @param namespace the namespace of the allocation tags * @param minCardinality determines the minimum number of allocations within * the scope * @param maxCardinality determines the maximum number of allocations within * the scope * @param allocationTags allocation tags * @return the resulting placement constraint */ public static AbstractConstraint cardinality(String scope, String namespace, int minCardinality, int maxCardinality, String... allocationTags) { return new SingleConstraint(scope, minCardinality, maxCardinality, PlacementTargets.allocationTagWithNamespace(namespace, allocationTags)); }
/** * Creates a constraint that requires allocations to be placed on nodes that * belong to a scope (e.g., node or rack) that satisfy any of the * target expressions based on node attribute op code. * * @param scope the scope within which the target expressions should not be * true * @param opCode Node Attribute code which could be equals, not equals. * @param targetExpressions the expressions that need to not be true within * the scope * @return the resulting placement constraint */ public static AbstractConstraint targetNodeAttribute(String scope, NodeAttributeOpCode opCode, TargetExpression... targetExpressions) { return new SingleConstraint(scope, -1, -1, opCode, targetExpressions); }