private AutoCloseableHdfsSpout makeSpout(String readerType, String[] outputFields) { HdfsSpout spout = new HdfsSpout().withOutputFields(outputFields) .setReaderType(readerType) .setHdfsUri(DFS_CLUSTER_RULE.getDfscluster().getURI().toString()) .setSourceDir(source.toString()) .setArchiveDir(archive.toString()) .setBadFilesDir(badfiles.toString()); return new AutoCloseableHdfsSpout(spout); }
private void commitProgress(FileOffset position) { if (position == null) { return; } if (lock != null && canCommitNow()) { try { String pos = position.toString(); lock.heartbeat(pos); LOG.debug("{} Committed progress. {}", spoutId, pos); acksSinceLastCommit = 0; commitTimeElapsed.set(false); setupCommitElapseTimer(); } catch (IOException e) { LOG.error("Unable to commit progress Will retry later. Spout ID = " + spoutId, e); } } }
/** * Execute a sequence of calls on HdfsSpout. * * @param cmds: set of commands to run, e.g. "r,r,r,r,a1,f2,...". The commands are: r[N] - receive() called N times aN - ack, item * number: N fN - fail, item number: N */ private List<String> runSpout(HdfsSpout spout, String... cmds) { MockCollector collector = (MockCollector) spout.getCollector(); for (String cmd : cmds) { if (cmd.startsWith("r")) { int count = 1; if (cmd.length() > 1) { count = Integer.parseInt(cmd.substring(1)); } for (int i = 0; i < count; ++i) { spout.nextTuple(); } } else if (cmd.startsWith("a")) { int n = Integer.parseInt(cmd.substring(1)); Pair<HdfsSpout.MessageId, List<Object>> item = collector.items.get(n); spout.ack(item.getKey()); } else if (cmd.startsWith("f")) { int n = Integer.parseInt(cmd.substring(1)); Pair<HdfsSpout.MessageId, List<Object>> item = collector.items.get(n); spout.fail(item.getKey()); } } return collector.lines; }
static StormTopology getTopology(Map<String, Object> config) { final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); final int boltNum = Helper.getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM); final String fileFormat = Helper.getStr(config, "text"); final String hdfsUri = Helper.getStr(config, HDFS_URI); final String sourceDir = Helper.getStr(config, SOURCE_DIR); final String archiveDir = Helper.getStr(config, ARCHIVE_DIR); final String badDir = Helper.getStr(config, BAD_DIR); // 1 - Setup Hdfs Spout -------- HdfsSpout spout = new HdfsSpout() .setReaderType(fileFormat) .setHdfsUri(hdfsUri) .setSourceDir(sourceDir) .setArchiveDir(archiveDir) .setBadFilesDir(badDir) .withOutputFields(TextFileReader.defaultFields); // 2 - DevNull Bolt -------- DevNullBolt bolt = new DevNullBolt(); // 3 - Setup Topology -------- TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, spout, spoutNum); builder.setBolt(BOLT_ID, bolt, boltNum) .localOrShuffleGrouping(SPOUT_ID); return builder.createTopology(); }
private void commitProgress(FileOffset position) { if (position == null) { return; } if (lock != null && canCommitNow()) { try { String pos = position.toString(); lock.heartbeat(pos); LOG.debug("{} Committed progress. {}", spoutId, pos); acksSinceLastCommit = 0; commitTimeElapsed.set(false); setupCommitElapseTimer(); } catch (IOException e) { LOG.error("Unable to commit progress Will retry later. Spout ID = " + spoutId, e); } } }
HdfsSpout spout = new HdfsSpout().withOutputFields(TextFileReader.defaultFields) .setReaderType(fileFormat) .setHdfsUri(hdfsUri) .setSourceDir(sourceDir) .setArchiveDir(archiveDir) .setBadFilesDir(badDir);