public static BenderConfig load(String filename, String data) { /* * Configure Mapper and register polymorphic types */ ObjectMapper mapper = BenderConfig.getObjectMapper(filename); mapper.setPropertyNamingStrategy( PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); /* * Optionally don't validate the config. Assume user has already * done this. */ String v = System.getenv("BENDER_SKIP_VALIDATE"); if (v != null && v.equals("true")) { return BenderConfig.load(filename, data, mapper, false); } else { return BenderConfig.load(filename, data, mapper, true); } }
@Override public void onException(Exception e) { /* * Always close the iterator to prevent connection leaking */ try { if (this.recordIterator != null) { this.recordIterator.close(); } } catch (IOException e1) { logger.error("unable to close record iterator", e); } if (this.config == null || this.config.getHandlerConfig() == null) { return; } /* * Notify SNS topic */ SNSS3HandlerConfig handlerConfig = (SNSS3HandlerConfig) this.config.getHandlerConfig(); if (handlerConfig.getSnsNotificationArn() != null) { AmazonSNSClient snsClient = this.snsClientFactory.newInstance(); snsClient.publish(handlerConfig.getSnsNotificationArn(), this.inputFiles.stream().map(Object::toString).collect(Collectors.joining(",")), "SNSS3Handler Failed"); } }
public HandlerResources(BenderConfig config) throws ClassNotFoundException { setSources(config.getSources()); setSerializerProcessor(config.getSerializerConfig()); setTransportFactory(config.getTransportConfig()); setWrapperFactory(config.getWrapperConfig()); setReporters(config.getReporters()); }
public static BenderConfig load(String filename, String data, ObjectMapper mapper, boolean validate) { String swappedData = swapEnvironmentVariables(data); if (validate) { BenderConfig.validate(swappedData, mapper); } BenderConfig config = null; try { config = mapper.readValue(swappedData, BenderConfig.class); } catch (IOException e) { throw new ConfigurationException("invalid config file", e); } return config; }
@Test public void testYaml() throws ConfigurationException, ClassNotFoundException { envVars.set("CUSTOM_SOURCE_NAME", "TestSourceName"); BenderConfig config = BenderConfig.load("/config/config_with_env.yaml"); List<SourceConfig> sources = config.getSources(); assertEquals(sources.size(), 1); assertEquals(sources.get(0).getName(), "TestSourceName"); } }
public static BenderConfig load(AmazonS3ClientFactory s3ClientFactory, AmazonS3URI s3Uri) { AmazonS3Client s3 = s3ClientFactory.newInstance(); S3Object s3object = s3.getObject(s3Uri.getBucket(), s3Uri.getKey()); StringWriter writer = new StringWriter(); try { IOUtils.copy(s3object.getObjectContent(), writer, "UTF-8"); } catch (IOException e) { throw new ConfigurationException("Unable to read file from s3", e); } BenderConfig config = load(s3Uri.getKey().toString(), writer.toString()); config.setConfigFile(s3Uri.getURI().toString()); return config; }
config = BenderConfig.load(s3ClientFactory, new AmazonS3URI(configFile)); } else if (configFile.startsWith("file://")) { File file = new File(configFile.replaceFirst("file://", "")); String string = FileUtils.readFileToString(file); config = BenderConfig.load(configFile, string); } else { config = BenderConfig.load(configFile); monitor.addTags(config.getHandlerConfig().getMetricTags()); if (config.getHandlerConfig().getIncludeFunctionTags()) { AWSLambda lambda = this.lambdaClientFactory.newInstance(); ListTagsResult res = lambda.listTags(new ListTagsRequest().withResource(ctx.getInvokedFunctionArn())); setIpcService(new IpcSenderService(handlerResources.getTransportFactory())); sources = new ArrayList<Source>(handlerResources.getSources().values()); queueSize = config.getHandlerConfig().getQueueSize(); initialized = true;
@Test(expected = ConfigurationException.class) public void testMissingStringTemplatesInConfig() throws ConfigurationException, ClassNotFoundException { BenderConfig.load("/config/config_with_env.json"); }
@Test public void testS3Config() throws HandlerException { BaseHandler handler = new DummyHandler(); handler.CONFIG_FILE = null; handler.s3ClientFactory = this.clientFactory; List<DummyEvent> events = new ArrayList<DummyEvent>(1); TestContext context = new TestContext(); context.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test"); handler.handler(events, context); assertEquals("s3://" + S3_BUCKET + "/bender/config.json", handler.config.getConfigFile()); assertEquals("Test Events", handler.config.getSources().get(0).getName()); } }
@Test public void testStringTemplatesInConfig() throws ConfigurationException, ClassNotFoundException { envVars.set("CUSTOM_SOURCE_NAME", "TestSourceName"); BenderConfig config = BenderConfig.load("/config/config_with_env.json"); List<SourceConfig> sources = config.getSources(); assertEquals(sources.size(), 1); assertEquals(sources.get(0).getName(), "TestSourceName"); }
BenderConfig config = load(resource, data); config.setConfigFile(resource);
config = BenderConfig.load(s3ClientFactory, new AmazonS3URI(configFile)); } else if (configFile.startsWith("file://")) { File file = new File(configFile.replaceFirst("file://", "")); String string = FileUtils.readFileToString(file); config = BenderConfig.load(configFile, string); } else { config = BenderConfig.load(configFile); monitor.addTags(config.getHandlerConfig().getMetricTags()); if (config.getHandlerConfig().getIncludeFunctionTags()) { AWSLambda lambda = this.lambdaClientFactory.newInstance(); ListTagsResult res = lambda.listTags(new ListTagsRequest().withResource(ctx.getInvokedFunctionArn())); setIpcService(new IpcSenderService(handlerResources.getTransportFactory())); sources = new ArrayList<Source>(handlerResources.getSources().values()); queueSize = config.getHandlerConfig().getQueueSize(); initialized = true;
public static BenderConfig load(String filename, String data, ObjectMapper mapper, boolean validate) { String swappedData = swapEnvironmentVariables(data); if (validate) { BenderConfig.validate(swappedData, mapper); } BenderConfig config = null; try { config = mapper.readValue(swappedData, BenderConfig.class); } catch (IOException e) { throw new ConfigurationException("invalid config file", e); } return config; }
@Test public void testLoadConfg() { BenderConfig config = BenderConfig.load("/com/nextdoor/bender/partition/partition_config.json"); PartitionOperationConfig op = (PartitionOperationConfig) config.getSources().get(0).getOperationConfigs().get(0); PartitionSpec spec = op.getPartitionSpecs().get(0); assertEquals("type", spec.getName()); assertEquals(Interpreter.STRING, spec.getInterpreter()); List<String> expected = Arrays.asList("one", "two", "three"); assertEquals(expected, spec.getSources()); }
public HandlerResources(BenderConfig config) throws ClassNotFoundException { setSources(config.getSources()); setSerializerProcessor(config.getSerializerConfig()); setTransportFactory(config.getTransportConfig()); setWrapperFactory(config.getWrapperConfig()); setReporters(config.getReporters()); }
public static BenderConfig load(String filename, String data) { /* * Configure Mapper and register polymorphic types */ ObjectMapper mapper = BenderConfig.getObjectMapper(filename); mapper.setPropertyNamingStrategy( PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); /* * Optionally don't validate the config. Assume user has already * done this. */ String v = System.getenv("BENDER_SKIP_VALIDATE"); if (v != null && v.equals("true")) { return BenderConfig.load(filename, data, mapper, false); } else { return BenderConfig.load(filename, data, mapper, true); } }
BenderConfig config = load(resource, data); config.setConfigFile(resource);
@Override public void onException(Exception e) { /* * Always close the iterator to prevent connection leaking */ try { if (this.recordIterator != null) { this.recordIterator.close(); } } catch (IOException e1) { logger.error("unable to close record iterator", e); } if (this.config == null || this.config.getHandlerConfig() == null) { return; } /* * Notify SNS topic */ SNSS3HandlerConfig handlerConfig = (SNSS3HandlerConfig) this.config.getHandlerConfig(); if (handlerConfig.getSnsNotificationArn() != null) { AmazonSNSClient snsClient = this.snsClientFactory.newInstance(); snsClient.publish(handlerConfig.getSnsNotificationArn(), this.inputFiles.stream().map(Object::toString).collect(Collectors.joining(",")), "SNSS3Handler Failed"); } }
@Test public void testLoadConfgTime() { BenderConfig config = BenderConfig.load("/com/nextdoor/bender/partition/partition_config_time.json"); PartitionOperationConfig op = (PartitionOperationConfig) config.getSources().get(0).getOperationConfigs().get(0); PartitionSpec spec = op.getPartitionSpecs().get(0); assertEquals("dt", spec.getName()); assertEquals(Interpreter.SECONDS, spec.getInterpreter()); List<String> expected = Arrays.asList("epoch"); assertEquals(expected, spec.getSources()); assertEquals("2017-01-19 05:05:59", spec.interpret("1484802359")); }
ObjectMapper mapper = BenderConfig.getObjectMapper(configFilename); mapper.setPropertyNamingStrategy( PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); BenderConfig.load(configFilename, sb.toString(), mapper, true); System.out.println("Valid"); BenderConfig config = BenderConfig.load(configFilename, sb.toString()); } catch (ConfigurationException e) { System.out.println("Invalid");