@Override public void setConf(AbstractConfig config) { this.config = (ForkOperationConfig) config; List<List<OperationProcessor>> processors = new ArrayList<List<OperationProcessor>>(); OperationFactoryFactory off = new OperationFactoryFactory(); for (Fork fork : this.config.getForks()) { List<OperationProcessor> processorsInFork = new ArrayList<OperationProcessor>(); for (OperationConfig opConfig : fork.getOperations()) { try { processorsInFork.add(new OperationProcessor(off.getFactory(opConfig))); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } processors.add(processorsInFork); } this.processors = processors; }
conditionInput = proc.perform(conditionInput);
this.getRuntimeStat().start(); try { InternalEvent i = ((EventOperation) op).perform(ievent); this.getSuccessCountStat().increment(); return i; } catch (OperationException e) { this.getErrorCountStat().increment(); logger.warn(e); return null; } finally { this.getRuntimeStat().stop(); this.getRuntimeStat().start(); try { Stream<InternalEvent> s = ((MultiplexOperation) op).perform(ievent).stream(); this.getSuccessCountStat().increment(); return s; } catch (OperationException e) { this.getErrorCountStat().increment(); logger.warn(e); return Stream.empty(); } finally { this.getRuntimeStat().stop(); return ((FilterOperation) this.op).test(ievent); } catch (OperationException e) { this.getErrorCountStat().increment(); logger.warn(e); return false;
@Test public void testNullInternalEventFiltering() throws JsonSyntaxException, UnsupportedEncodingException, IOException, OperationException { /* * Setup mocks for test */ DummyOperation op = spy(new DummyOperation()); when(op.perform(any(InternalEvent.class))).thenReturn(null); DummyOperationFactory operationFactory = new DummyOperationFactory(op); OperationProcessor processor = new OperationProcessor(operationFactory); /* * Do call */ Stream<InternalEvent> stream = processor.perform(Stream.of(new InternalEvent("foo", null, 1))); List<InternalEvent> output = stream.collect(Collectors.toList()); /* * Verify nothing came out */ assertEquals(0, output.size()); }
@Test public void testStatsLogging() throws JsonSyntaxException, UnsupportedEncodingException, IOException, OperationException { DummyOperationFactory mutatorFactory = new DummyOperationFactory(); OperationProcessor processor = new OperationProcessor(mutatorFactory); /* * Mock the Stat object */ Stat runtimeStat = mock(Stat.class); Stat successStat = mock(Stat.class); Stat errorStat = mock(Stat.class); processor.setRuntimeStat(runtimeStat); processor.setSuccessCountStat(successStat); processor.setErrorCountStat(errorStat); InternalEvent ievent = new InternalEvent("foo", null, 1); ievent.setEventObj(new DummyDeserializerHelper.DummyStringEvent("test")); Stream<InternalEvent> stream = processor.perform(Stream.of(ievent)); List<InternalEvent> output = stream.collect(Collectors.toList()); /* * Verify start, stop, increment success count, and never increment error count. */ verify(runtimeStat, times(1)).start(); verify(runtimeStat, times(1)).stop(); verify(successStat, times(1)).increment(); verify(errorStat, never()).increment(); /* * Verify contents of output stream */ assertEquals(1, output.size()); }
@Test public void testOperationException() throws HandlerException { BaseHandler.CONFIG_FILE = "/config/handler_config.json"; handler.skipWriteStats = true; List<DummyEvent> events = new ArrayList<DummyEvent>(1); events.add(new DummyEvent("foo", 0)); TestContext context = new TestContext(); context.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test:tag"); handler.init(context); List<OperationProcessor> operationProcessors = handler.sources.get(0).getOperationProcessors(); for (OperationProcessor operationProcessor : operationProcessors) { EventOperation operation = (EventOperation) spy(operationProcessor.getOperation()); doThrow(new OperationException("expected")).when(operation).perform(any()); operationProcessor.setOperation(operation); } handler.handler(events, context); assertEquals(1, operationProcessors.get(0).getErrorCountStat().getValue()); }
public String toString() { String patterns = this.regexPatterns.stream().map(c -> { return c.toString(); }).collect(Collectors.joining(", ")); String operations = this.operationProcessors.stream().map(c -> { return c.getOperation().getClass().getSimpleName(); }).collect(Collectors.joining(", ")); return this.sourceName + "[" + "sourceRegex=" + this.sourceRegex + ", containsStrings=[" + StringUtils.join(this.containsStrings, ',') + "], regexPatterns=[" + patterns + "]" + "], deserializers=[" + this.deserProcessor + "]" + "], operations=[" + operations + "]]"; } }
@Test public void testNullDeserializedEventFiltering() throws JsonSyntaxException, UnsupportedEncodingException, IOException, OperationException { /* * Setup mocks for test */ DummyOperation op = spy(new DummyOperation()); InternalEvent retEvent = new InternalEvent("foo", null, 1); retEvent.setEventObj(null); when(op.perform(any(InternalEvent.class))).thenReturn(retEvent); DummyOperationFactory operationFactory = new DummyOperationFactory(op); OperationProcessor processor = new OperationProcessor(operationFactory); /* * Do call */ Stream<InternalEvent> stream = processor.perform(Stream.of(new InternalEvent("foo", null, 1))); List<InternalEvent> output = stream.collect(Collectors.toList()); /* * Verify nothing came out */ assertEquals(0, output.size()); }
@Test public void testStatsLoggingOnError() { DummyOperation operation = mock(DummyOperation.class); DummyOperationFactory mutatorFactory = new DummyOperationFactory(operation); OperationProcessor processor = new OperationProcessor(mutatorFactory); InternalEvent ievent = new InternalEvent("a", null, 1); doThrow(new OperationException("Expceted")).when(operation).perform(ievent); /* * Mock the Stat object */ Stat runtimeStat = mock(Stat.class); Stat successStat = mock(Stat.class); Stat errorStat = mock(Stat.class); processor.setRuntimeStat(runtimeStat); processor.setSuccessCountStat(successStat); processor.setErrorCountStat(errorStat); Stream<InternalEvent> stream = processor.perform(Stream.of(ievent)); List<InternalEvent> output = stream.collect(Collectors.toList()); /* * Verify start, stop are called, increment error count and never increment success count. */ verify(runtimeStat, times(1)).start(); verify(runtimeStat, times(1)).stop(); verify(successStat, never()).increment(); verify(errorStat, times(1)).increment(); /* * Verify contents of output stream */ assertEquals(0, output.size()); }
public String toString() { String patterns = this.regexPatterns.stream().map(c -> { return c.toString(); }).collect(Collectors.joining(", ")); String operations = this.operationProcessors.stream().map(c -> { return c.getOperation().getClass().getSimpleName(); }).collect(Collectors.joining(", ")); return this.sourceName + "[" + "sourceRegex=" + this.sourceRegex + ", containsStrings=[" + StringUtils.join(this.containsStrings, ',') + "], regexPatterns=[" + patterns + "]" + "], deserializers=[" + this.deserProcessor + "]" + "], operations=[" + operations + "]]"; } }
@Test public void testOperationThroughProcessor() throws FieldNotFoundException { List<PartitionSpec> partitionSpecs = new ArrayList<PartitionSpec>(1); List<String> sources = Arrays.asList("foo"); PartitionSpec spec = new PartitionSpec("foo", sources, PartitionSpec.Interpreter.STRING); partitionSpecs.add(spec); PartitionOperation op = new PartitionOperation(partitionSpecs); InternalEvent ievent = new InternalEvent("foo", null, 1); DummyStringEvent devent = spy(new DummyStringEvent("")); ievent.setEventObj(devent); doReturn("baz").when(devent).getFieldAsString("foo"); DummyOperationFactory opFact = new DummyOperationFactory(op); OperationProcessor opProc = new OperationProcessor(opFact); opProc.perform(Stream.of(ievent)).count(); LinkedHashMap<String, String> actual = ievent.getPartitions(); LinkedHashMap<String, String> expected = new LinkedHashMap<String, String>(1); expected.put("foo", "baz"); assertEquals(expected, actual); } }
forkInput = opProcInFork.perform(forkInput);
this.getRuntimeStat().start(); try { InternalEvent i = ((EventOperation) op).perform(ievent); this.getSuccessCountStat().increment(); return i; } catch (OperationException e) { this.getErrorCountStat().increment(); logger.warn(e); return null; } finally { this.getRuntimeStat().stop(); this.getRuntimeStat().start(); try { Stream<InternalEvent> s = ((MultiplexOperation) op).perform(ievent).stream(); this.getSuccessCountStat().increment(); return s; } catch (OperationException e) { this.getErrorCountStat().increment(); logger.warn(e); return Stream.empty(); } finally { this.getRuntimeStat().stop(); return ((FilterOperation) this.op).test(ievent); } catch (OperationException e) { this.getErrorCountStat().increment(); logger.warn(e); return false;
@Override public void setConf(AbstractConfig config) { this.config = (ForkOperationConfig) config; List<List<OperationProcessor>> processors = new ArrayList<List<OperationProcessor>>(); OperationFactoryFactory off = new OperationFactoryFactory(); for (Fork fork : this.config.getForks()) { List<OperationProcessor> processorsInFork = new ArrayList<OperationProcessor>(); for (OperationConfig opConfig : fork.getOperations()) { try { processorsInFork.add(new OperationProcessor(off.getFactory(opConfig))); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } processors.add(processorsInFork); } this.processors = processors; }
@Test public void testNullPayloadFiltering() throws JsonSyntaxException, UnsupportedEncodingException, IOException, OperationException { /* * Setup mocks for test */ DummyOperation op = spy(new DummyOperation()); InternalEvent retEvent = new InternalEvent("foo", null, 1); retEvent.setEventObj(new DummyDeserializerHelper.DummyStringEvent(null)); when(op.perform(any(InternalEvent.class))).thenReturn(retEvent); DummyOperationFactory operationFactory = new DummyOperationFactory(op); OperationProcessor processor = new OperationProcessor(operationFactory); /* * Do call */ Stream<InternalEvent> stream = processor.perform(Stream.of(new InternalEvent("foo", null, 1))); List<InternalEvent> output = stream.collect(Collectors.toList()); /* * Verify nothing came out */ assertEquals(0, output.size()); } }
forkInput = opProcInFork.perform(forkInput);
@Override public void setConf(AbstractConfig config) { this.config = (ConditionalOperationConfig) config; List<Pair<FilterOperation, List<OperationProcessor>>> cases = new ArrayList<Pair<FilterOperation, List<OperationProcessor>>>(); OperationFactoryFactory off = new OperationFactoryFactory(); for (Condition caze : this.config.getConditions()) { List<OperationProcessor> processorsInCase = new ArrayList<OperationProcessor>(); /* * Create {@OperationProcessor}s from configs */ for (OperationConfig opConfig : caze.getOperations()) { try { processorsInCase.add(new OperationProcessor(off.getFactory(opConfig))); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } FilterOperation filter; try { filter = (FilterOperation) off.getFactory(caze.getCondition()).newInstance(); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } cases.add( new ImmutablePair<FilterOperation, List<OperationProcessor>>(filter, processorsInCase)); } this.cases = cases; }
conditionInput = proc.perform(conditionInput);
@Override public void setConf(AbstractConfig config) { this.config = (ConditionalOperationConfig) config; List<Pair<FilterOperation, List<OperationProcessor>>> cases = new ArrayList<Pair<FilterOperation, List<OperationProcessor>>>(); OperationFactoryFactory off = new OperationFactoryFactory(); for (Condition caze : this.config.getConditions()) { List<OperationProcessor> processorsInCase = new ArrayList<OperationProcessor>(); /* * Create {@OperationProcessor}s from configs */ for (OperationConfig opConfig : caze.getOperations()) { try { processorsInCase.add(new OperationProcessor(off.getFactory(opConfig))); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } FilterOperation filter; try { filter = (FilterOperation) off.getFactory(caze.getCondition()).newInstance(); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } cases.add( new ImmutablePair<FilterOperation, List<OperationProcessor>>(filter, processorsInCase)); } this.cases = cases; }
@Test public void testMultipleOperationsConfig() throws HandlerException { BaseHandler.CONFIG_FILE = "/config/handler_config_two_operations.json"; List<DummyEvent> events = new ArrayList<DummyEvent>(1); events.add(new DummyEvent("foo", 0)); TestContext context = new TestContext(); context.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test:tag"); handler.init(context); List<OperationProcessor> operationProcessores = handler.sources.get(0).getOperationProcessors(); for (int i = 0; i < operationProcessores.size(); i++) { OperationProcessor operationProcessor = spy(operationProcessores.get(i)); operationProcessores.set(i, operationProcessor); } handler.handler(events, context); /* * 2 operations specified in the config file */ verify(operationProcessores.get(0), times(1)).perform(any()); verify(operationProcessores.get(1), times(1)).perform(any()); }