context.getString(FaultTolerance.RECOVERABLE_EXCEPTION_CLASSES)); morphlineContext = new MorphlineContext.Builder() .setExceptionHandler(faultTolerance) .setMetricRegistry(SharedMetricRegistries.getOrCreate(morphlineFileAndId)) .build();
.setMetricRegistry(metricRegistry) .build();
.setDocumentLoader(loader) .setExceptionHandler(faultTolerance) .setMetricRegistry(new MetricRegistry()) .build(); .setDocumentLoader(loader) .setExceptionHandler(faultTolerance) .setMetricRegistry(morphlineContext.getMetricRegistry()) .build();
.setDocumentLoader(loader) .setExceptionHandler(faultTolerance) .setMetricRegistry(new MetricRegistry()) .build(); .setDocumentLoader(loader) .setExceptionHandler(faultTolerance) .setMetricRegistry(morphlineContext.getMetricRegistry()) .build();
.setMetricRegistry(metricRegistry) .build();
mySettings.put(TypedSettings.TASK_CONTEXT_SETTING_NAME, getContext()); morphlineContext = new MorphlineContext.Builder() .setExceptionHandler(faultTolerance) .setMetricRegistry(SharedMetricRegistries.getOrCreate(morphlineFileAndId)) .setSettings(mySettings) .build();
MorphlineContext context = new MorphlineContext.Builder() .setExceptionHandler(new FaultTolerance(isProduction, false)) .build();
context.getString(FaultTolerance.RECOVERABLE_EXCEPTION_CLASSES)); morphlineContext = new MorphlineContext.Builder() .setExceptionHandler(faultTolerance) .setMetricRegistry(SharedMetricRegistries.getOrCreate(morphlineFileAndId)) .build();
MorphlineContext ctx = new MorphlineContext.Builder() .setSettings(settings) .setExceptionHandler(ex) .setHealthCheckRegistry(healthChecks) .setMetricRegistry(metricRegistry) .build(); ctx.getHealthCheckRegistry().runHealthChecks(); assertEquals(0, new MorphlineContext.Builder().build().getSettings().size());
/** Usage: java ... <morphline.conf> <dataFile1> ... <dataFileN> */ public static void main(String[] args) throws IOException { // compile morphline.conf file on the fly File morphlineFile = new File(args[0]); String morphlineId = null; MorphlineContext morphlineContext = new MorphlineContext.Builder().build(); Command morphline = new Compiler().compile(morphlineFile, morphlineId, morphlineContext, null); // process each input data file Notifications.notifyBeginTransaction(morphline); try { for (int i = 1; i < args.length; i++) { InputStream in = new BufferedInputStream(new FileInputStream(new File(args[i]))); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); Notifications.notifyStartSession(morphline); boolean success = morphline.process(record); if (!success) { System.out.println("Morphline failed to process record: " + record); } in.close(); } Notifications.notifyCommitTransaction(morphline); } catch (RuntimeException e) { Notifications.notifyRollbackTransaction(morphline); morphlineContext.getExceptionHandler().handleException(e, null); } Notifications.notifyShutdown(morphline); } }
public void run() { try { int iters = 0; MorphlineContext ctx = new MorphlineContext.Builder().build(); Config config = parse("test-morphlines/convertHTML"); Collector myCollector = new Collector(); Command myMorphline = new PipeBuilder().build(config, null, myCollector, ctx); long start = System.currentTimeMillis(); while (System.currentTimeMillis() < start + durationMillis) { Record record = new Record(); record.put("id", "123"); record.put(Fields.ATTACHMENT_BODY, bytes); for (int i = 0; i < 3; i++) { processAndVerifySuccess(myMorphline, myCollector, record.copy(), ImmutableMultimap.of("id", "123", Fields.MESSAGE, expected) ); } iters++; //break; } totalIters.addAndGet(iters); } catch (Exception e) { hasException.countDown(); throw new RuntimeException(e); } } });
@Test public void testLoadManagedSchema() throws Exception { // Copy the collection1 config files, so we don't have to keep multiple // copies of the auxiliary files in source File solrHomeDir = Files.createTempDir(); solrHomeDir.deleteOnExit(); File collection1Dir = new File(SOLR_INSTANCE_DIR, "collection1"); FileUtils.copyDirectory(collection1Dir, solrHomeDir); // Copy in the managed collection files, remove the schema.xml since the // managed schema uses a generated one File managedCollectionDir = new File(SOLR_INSTANCE_DIR, "managedSchemaCollection"); FileUtils.copyDirectory(managedCollectionDir, solrHomeDir); File oldSchemaXml = new File(solrHomeDir + File.separator + "conf" + File.separator + "schema.xml"); oldSchemaXml.delete(); assertFalse(oldSchemaXml.exists()); SolrLocator locator = new SolrLocator(new MorphlineContext.Builder().build()); locator.setCollectionName("managedSchemaCollection"); locator.setSolrHomeDir(solrHomeDir.getAbsolutePath()); IndexSchema schema = locator.getIndexSchema(); assertNotNull(schema); schema.getField("test-managed-morphline-field"); }
@Test public void testCompileWithUnknownMorphlineId() throws Exception { String file = "test-morphlines/pipeWithTwoBasicCommands"; try { new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "morphline2", new MorphlineContext.Builder().build(), null); fail(); } catch (MorphlineCompilationException e) { ; // expected } }
private MorphlineContext createMorphlineContext() { return new MorphlineContext.Builder().setMetricRegistry(new MetricRegistry()).build(); }
private MorphlineContext createMorphlineContext() { return new MorphlineContext.Builder().setMetricRegistry(new MetricRegistry()).build(); }
@Test public void testCompileWithMissingMorphline() throws Exception { String file = "test-morphlines/compileWithMissingMorphline"; try { new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "morphline1", new MorphlineContext.Builder().build(), null); fail(); } catch (MorphlineCompilationException e) { ; // expected } }
@Override public HBaseMorphlineContext build() { return (HBaseMorphlineContext)super.build(); }
protected Command createMorphline(Config config) { morphContext = new MorphlineContext.Builder().setMetricRegistry(new MetricRegistry()).build(); return new PipeBuilder().build(config, null, collector, morphContext); }
@Override public HBaseMorphlineContext build() { return (HBaseMorphlineContext)super.build(); }
@Override protected void setup(Context context) throws IOException, InterruptedException { File morphLineFile = new File(context.getConfiguration().get(MORPHLINE_FILE)); String morphLineId = context.getConfiguration().get(MORPHLINE_ID); RecordEmitter recordEmitter = new RecordEmitter(context); MorphlineContext morphlineContext = new MorphlineContext.Builder().build(); morphline = new org.kitesdk.morphline.base.Compiler() .compile(morphLineFile, morphLineId, morphlineContext, recordEmitter); }