new File(morphlineFile), morphlineId, morphlineContext, finalChild, override); this.mappingTimer = morphlineContext.getMetricRegistry().timer( MetricRegistry.name("morphline.app", Metrics.ELAPSED_TIME)); this.numRecords = morphlineContext.getMetricRegistry().meter( MetricRegistry.name("morphline.app", Metrics.NUM_RECORDS)); this.numFailedRecords = morphlineContext.getMetricRegistry().meter( MetricRegistry.name("morphline.app", "numFailedRecords")); this.numExceptionRecords = morphlineContext.getMetricRegistry().meter( MetricRegistry.name("morphline.app", "numExceptionRecords"));
@Override public void process(Event event) { numRecords.mark(); Timer.Context timerContext = mappingTimer.time(); try { Record record = new Record(); for (Entry<String, String> entry : event.getHeaders().entrySet()) { record.put(entry.getKey(), entry.getValue()); } byte[] bytes = event.getBody(); if (bytes != null && bytes.length > 0) { record.put(Fields.ATTACHMENT_BODY, bytes); } try { Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { numFailedRecords.mark(); LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record); } } catch (RuntimeException t) { numExceptionRecords.mark(); morphlineContext.getExceptionHandler().handleException(t, record); } } finally { timerContext.stop(); } }
.build(); assertSame(settings, ctx.getSettings()); assertSame(ex, ctx.getExceptionHandler()); assertSame(metricRegistry, ctx.getMetricRegistry()); assertSame(healthChecks, ctx.getHealthCheckRegistry()); ctx.getHealthCheckRegistry().runHealthChecks(); assertEquals(0, new MorphlineContext.Builder().build().getSettings().size());
public RegisterJVMMetrics(CommandBuilder builder, Config config, Command parent, Command child, final MorphlineContext context) { super(builder, config, parent, child, context); validateArguments(); MetricRegistry registry = context.getMetricRegistry(); BufferPoolMetricSet bufferPoolMetrics = new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()); registerAll("jvm.buffers", bufferPoolMetrics, registry); registerAll("jvm.gc", new GarbageCollectorMetricSet(), registry); registerAll("jvm.memory", new MemoryUsageGaugeSet(), registry); registerAll("jvm.threads", new ThreadStatesGaugeSet(), registry); register("jvm.fileDescriptorCountRatio", new FileDescriptorRatioGauge(), registry); context.getHealthCheckRegistry().register("deadlocks", new ThreadDeadlockHealthCheck()); }
public LoadSolr(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); Config solrLocatorConfig = getConfigs().getConfig(config, SOLR_LOCATOR_PARAM); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); RetryPolicyFactory retryPolicyFactory = parseRetryPolicyFactory( getConfigs().getConfig(config, "retryPolicy", null)); this.loader = locator.getLoader(retryPolicyFactory, new CodahaleMetricsFacade(context.getMetricRegistry())); Config boostsConfig = getConfigs().getConfig(config, "boosts", ConfigFactory.empty()); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(boostsConfig)) { String fieldName = entry.getKey(); float boost = Float.parseFloat(entry.getValue().toString().trim()); boosts.put(fieldName, boost); } this.rateLimiter = RateLimiter.create(getConfigs().getDouble(config, "maxRecordsPerSecond", Double.MAX_VALUE)); this.isDryRun = context.getTypedSettings().getBoolean(TypedSettings.DRY_RUN_SETTING_NAME, false); validateArguments(); this.elapsedTime = getTimer(Metrics.ELAPSED_TIME); }
@Test public void testImportSpecs() { List<String> importSpecs = Arrays.asList("org.kitesdk.**", "org.apache.solr.**", "net.*", getClass().getName()); for (Class clazz : new MorphlineContext().getTopLevelClasses(importSpecs, CommandBuilder.class)) { //System.out.println("found " + clazz); } MorphlineContext ctx = new MorphlineContext.Builder().build(); ctx.importCommandBuilders(importSpecs); ctx.importCommandBuilders(importSpecs); }
@Test @Ignore public void testHugeImportSpecs() { long start = System.currentTimeMillis(); List<String> importSpecs = Arrays.asList("com.**", "org.**", "net.*", getClass().getName()); for (Class clazz : new MorphlineContext().getTopLevelClasses(importSpecs, CommandBuilder.class)) { System.out.println("found " + clazz); } float secs = (System.currentTimeMillis() - start) / 1000.0f; System.out.println("secs=" + secs); }
@Override protected HealthCheckRegistry getHealthCheckRegistry() { return context.getHealthCheckRegistry(); } });
public LoadSolr(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); Config solrLocatorConfig = getConfigs().getConfig(config, SOLR_LOCATOR_PARAM); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); this.loader = locator.getLoader(); Config boostsConfig = getConfigs().getConfig(config, "boosts", ConfigFactory.empty()); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(boostsConfig)) { String fieldName = entry.getKey(); float boost = Float.parseFloat(entry.getValue().toString().trim()); boosts.put(fieldName, boost); } this.isDryRun = context.getTypedSettings().getBoolean(TypedSettings.DRY_RUN_SETTING_NAME, false); validateArguments(); this.elapsedTime = getTimer(Metrics.ELAPSED_TIME); }
protected MorphlineContext create() { return new MorphlineContext(); }
public TypedSettings getTypedSettings() { return new TypedSettings(getSettings()); }
public Pipe(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.id = getConfigs().getString(config, "id"); List<String> importCommandSpecs = getConfigs().getStringList(config, "importCommands", Arrays.asList("com.**", "org.**", "net.**")); context.importCommandBuilders(importCommandSpecs); getConfigs().getConfigList(config, "commands", null); List<Command> childCommands = buildCommandChain(config, "commands", child, false); if (childCommands.size() > 0) { this.realChild = childCommands.get(0); } else { this.realChild = child; } validateArguments(); }
public void importCommandBuilders(Collection<String> importSpecs) { if (commandBuilders == Collections.EMPTY_MAP) { // intentionally effective no more than once commandBuilders = Maps.newHashMap(); if (LOG.isDebugEnabled()) { LOG.debug("Importing commands from Java classpath: {}", System.getProperty("java.class.path")); } else { LOG.info("Importing commands"); } Collection<Class<CommandBuilder>> builderClasses = getTopLevelClasses(importSpecs, CommandBuilder.class); for (Class<CommandBuilder> builderClass : builderClasses) { try { CommandBuilder builder = builderClass.newInstance(); for (String builderName : builder.getNames()) { LOG.debug("Importing command: {} from class: {}", builderName, builderClass.getName()); if (builderName.contains(".")) { LOG.warn("Command name should not contain a period character: " + builderName); } commandBuilders.put(builderName, builderClass); } } catch (Exception e) { throw new MorphlineRuntimeException(e); } } LOG.info("Done importing commands"); } }
public RegisterJVMMetrics(CommandBuilder builder, Config config, Command parent, Command child, final MorphlineContext context) { super(builder, config, parent, child, context); validateArguments(); MetricRegistry registry = context.getMetricRegistry(); BufferPoolMetricSet bufferPoolMetrics = new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()); registerAll("jvm.buffers", bufferPoolMetrics, registry); registerAll("jvm.gc", new GarbageCollectorMetricSet(), registry); registerAll("jvm.memory", new MemoryUsageGaugeSet(), registry); registerAll("jvm.threads", new ThreadStatesGaugeSet(), registry); register("jvm.fileDescriptorCountRatio", new FileDescriptorRatioGauge(), registry); context.getHealthCheckRegistry().register("deadlocks", new ThreadDeadlockHealthCheck()); }
@Test public void testImportSpecsWithOnlyFQCNs() { List<String> importSpecs = Arrays.asList(getClass().getName()); for (Class clazz : new MorphlineContext().getTopLevelClasses(importSpecs, CommandBuilder.class)) { //System.out.println("found " + clazz); } MorphlineContext ctx = new MorphlineContext.Builder().build(); ctx.importCommandBuilders(importSpecs); ctx.importCommandBuilders(importSpecs); }
@Override protected HealthCheckRegistry getHealthCheckRegistry() { return context.getHealthCheckRegistry(); } });
@Before public void setUp() { map = new HashMap<String, Object>(); settings = new MorphlineContext.Builder().setSettings(map).build().getTypedSettings(); }
protected MorphlineContext create() { return new MorphlineContext(); }
public TypedSettings getTypedSettings() { return new TypedSettings(getSettings()); }
public Pipe(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.id = getConfigs().getString(config, "id"); List<String> importCommandSpecs = getConfigs().getStringList(config, "importCommands", Arrays.asList("com.**", "org.**", "net.**")); context.importCommandBuilders(importCommandSpecs); getConfigs().getConfigList(config, "commands", null); List<Command> childCommands = buildCommandChain(config, "commands", child, false); if (childCommands.size() > 0) { this.realChild = childCommands.get(0); } else { this.realChild = child; } validateArguments(); }