@Override public FlowletSpecification configure() { return FlowletSpecification.Builder.with() .setName(getName()) .setDescription(getDescription()) .build(); }
/** * Creates a {@link Builder} for building instance of this class. * * @return A new builder instance. */ public static NameSetter with() { return new Builder().new NameSetter(); }
private Callback createCallback(Flowlet flowlet, FlowletSpecification flowletSpec) { if (flowlet instanceof Callback) { return (Callback) flowlet; } final FailurePolicy failurePolicy = flowletSpec.getFailurePolicy(); return new Callback() { @Override public void onSuccess(Object input, InputContext inputContext) { // No-op } @Override public FailurePolicy onFailure(Object input, InputContext inputContext, FailureReason reason) { return failurePolicy; } }; }
@Override public JsonElement serialize(FlowletSpecification src, Type typeOfSrc, JsonSerializationContext context) { JsonObject jsonObj = new JsonObject(); jsonObj.add("className", new JsonPrimitive(src.getClassName())); jsonObj.add("name", new JsonPrimitive(src.getName())); jsonObj.add("description", new JsonPrimitive(src.getDescription())); jsonObj.add("failurePolicy", new JsonPrimitive(src.getFailurePolicy().name())); jsonObj.add("properties", serializeMap(src.getProperties(), context, String.class)); jsonObj.add("resources", context.serialize(src.getResources(), new TypeToken<ResourceSpecification>() { }.getType())); jsonObj.add("maxInstances", new JsonPrimitive(src.getMaxInstances())); return jsonObj; }
FlowletDefinition(String flowletName, Flowlet flowlet, int instances) { FlowletSpecification flowletSpec = flowlet.configure(); this.instances = instances; Map<String, Set<Type>> inputTypes = Maps.newHashMap(); Map<String, Set<Type>> outputTypes = Maps.newHashMap(); Map<String, String> properties = Maps.newHashMap(flowletSpec.getProperties()); Reflections.visit(flowlet, TypeToken.of(flowlet.getClass()), new PropertyFieldExtractor(properties), new OutputEmitterFieldExtractor(outputTypes), new ProcessMethodExtractor(inputTypes)); this.inputTypes = immutableCopyOf(inputTypes); this.outputTypes = immutableCopyOf(outputTypes); this.flowletSpec = new DefaultFlowletSpecification(flowlet.getClass().getName(), flowletName == null ? flowletSpec.getName() : flowletName, flowletSpec.getDescription(), flowletSpec.getFailurePolicy(), properties, flowletSpec.getResources(), flowletSpec.getMaxInstances()); }
(FlowletProgramController) Iterables.getFirst(liveFlowlets.values(), null); FlowletSpecification flowletSpecification = flowletProgramController.getFlowletContext().getSpecification(); int flowletMaxInstances = flowletSpecification.getMaxInstances(); Preconditions.checkArgument(newInstanceCount <= flowletMaxInstances, "Flowlet %s can have a maximum of %s instances", flowletSpecification.getName(), flowletMaxInstances);
@Override public FailurePolicy onFailure(@Nullable Object input, @Nullable InputContext inputContext, FailureReason reason) { // Return the policy as specified in the spec return flowletContext.getSpecification().getFailurePolicy(); }
InputAcknowledger inputAcknowledger) { LOG.warn("Process failure: {}, {}, input: {}", flowletContext, reason.getMessage(), input, reason.getCause()); FailurePolicy failurePolicy; try { flowletContext.getProgramMetrics().gauge("process.errors", 1); failurePolicy = txCallback.onFailure(inputObject, inputContext, reason); if (failurePolicy == null) { failurePolicy = FailurePolicy.RETRY;
@Override public MoreFlowlet add(String name, Flowlet flowlet, int instances) { Preconditions.checkArgument(flowlet != null, UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NULL)); FlowletDefinition flowletDef = new FlowletDefinition(name, flowlet, instances); String flowletName = flowletDef.getFlowletSpec().getName(); Preconditions.checkArgument(instances > 0, String.format(UserMessages.getMessage(UserErrors.INVALID_INSTANCES), flowletName, instances)); Preconditions.checkArgument(!flowlets.containsKey(flowletName), UserMessages.getMessage(UserErrors.INVALID_FLOWLET_EXISTS), flowletName); flowlets.put(flowletName, flowletDef); return this; }
void update(String flowletId, int newInstanceCount, int oldInstanceCount) throws Exception { FlowletDefinition flowletDefinition = program.getSpecification().getFlowlets().get(flowletId); int maxInstances = flowletDefinition.getFlowletSpec().getMaxInstances(); Preconditions.checkArgument(newInstanceCount <= maxInstances, "Flowlet %s can have a maximum of %s instances", flowletId, maxInstances); waitForInstances(flowletId, oldInstanceCount); twillController.sendCommand(flowletId, ProgramCommands.SUSPEND).get(); FlowUtils.reconfigure(consumerQueues.get(flowletId), FlowUtils.generateConsumerGroupId(program, flowletId), newInstanceCount, queueAdmin); twillController.changeInstances(flowletId, newInstanceCount).get(); twillController.sendCommand(flowletId, ProgramCommands.RESUME).get(); }
/** * Sets the name of a flowlet. * @param name Name of the flowlet. * @return An instance of {@link DescriptionSetter} */ public DescriptionSetter setName(String name) { Preconditions.checkArgument(name != null, "Name cannot be null."); Builder.this.name = name; return new DescriptionSetter(); } }
/** * Sets the description of the flowlet. * @param description Descripition to be associated with flowlet. * @return An instance of what needs to be done after description {@link AfterDescription} */ public AfterDescription setDescription(String description) { Preconditions.checkArgument(description != null, "Description cannot be null."); Builder.this.description = description; return new AfterDescription(); } }
/** * @return A descriptive message about this {@link Flowlet}. */ protected String getDescription() { return String.format("Flowlet of %s.", getName()); } }
@Override public void onSuccess(Object object, InputContext inputContext) { try { gaugeEventProcessed(input.getQueueName()); txCallback.onSuccess(object, inputContext); } catch (Throwable t) { LOG.error("Exception on onSuccess call: {}", flowletContext, t); } finally { enqueueEntry(); inflight.decrementAndGet(); } }
@Override public FlowletSpecification deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { JsonObject jsonObj = json.getAsJsonObject(); String className = jsonObj.get("className").getAsString(); String name = jsonObj.get("name").getAsString(); String description = jsonObj.get("description").getAsString(); FailurePolicy policy = FailurePolicy.valueOf(jsonObj.get("failurePolicy").getAsString()); Map<String, String> properties = deserializeMap(jsonObj.get("properties"), context, String.class); ResourceSpecification resources = context.deserialize(jsonObj.get("resources"), new TypeToken<ResourceSpecification>() { }.getType()); int maxInstances = jsonObj.get("maxInstances").getAsInt(); return new DefaultFlowletSpecification(className, name, description, policy, properties, resources, maxInstances); } }
private void initFlowlet() throws InterruptedException { final TransactionContext txContext = flowletContext.createTransactionContext(); try { txContext.start(); try { LOG.info("Initializing flowlet: " + flowletContext); flowlet.initialize(flowletContext); LOG.info("Flowlet initialized: " + flowletContext); } catch (Throwable t) { LOG.error("User code exception. Aborting transaction.", t); txContext.abort(new TransactionFailureException("User code exception. Aborting transaction", t)); throw Throwables.propagate(t); } txContext.finish(); } catch (TransactionFailureException e) { LOG.error("Flowlet throws exception during flowlet initialize: " + flowletContext, e); throw Throwables.propagate(e); } }
private void destroyFlowlet() { final TransactionContext txContext = flowletContext.createTransactionContext(); try { txContext.start(); try { LOG.info("Destroying flowlet: " + flowletContext); flowlet.destroy(); LOG.info("Flowlet destroyed: " + flowletContext); } catch (Throwable t) { LOG.error("User code exception. Aborting transaction.", t); txContext.abort(new TransactionFailureException("User code exception. Aborting transaction", t)); // No need to propagate, as it is shutting down. } txContext.finish(); } catch (TransactionFailureException e) { LOG.error("Flowlet throws exception during flowlet destroy: " + flowletContext, e); // No need to propagate, as it is shutting down. } }
@Override public MoreConnect to(String flowlet) { Preconditions.checkArgument(flowlet != null, UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NULL)); Preconditions.checkArgument(flowlets.containsKey(flowlet), UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NAME), flowlet); FlowletConnection.Type sourceType; String sourceName; sourceType = FlowletConnection.Type.FLOWLET; sourceName = fromFlowlet.getFlowletSpec().getName(); connections.add(new FlowletConnection(sourceType, sourceName, flowlet)); return this; }