@Override public void configure(Configuration parameters) { super.configure(parameters); // get the charset for the decoding String charsetName = parameters.getString(CHARSET_NAME, DEFAULT_CHARSET_NAME); if (charsetName == null || !Charset.isSupported(charsetName)) { throw new RuntimeException("Unsupported charset: " + charsetName); } if (charsetName.equals("ISO-8859-1") || charsetName.equalsIgnoreCase("ASCII")) { this.ascii = true; } else { this.decoder = Charset.forName(charsetName).newDecoder(); this.byteWrapper = ByteBuffer.allocate(1); } // get the field position to write in the record this.pos = parameters.getInteger(FIELD_POS, 0); if (this.pos < 0) { throw new RuntimeException("Illegal configuration value for the target position: " + this.pos); } }
@SuppressWarnings({ "deprecation", "unchecked" }) @Override protected Plan getTestJob() { String input1Path = config.getString("UnionTest#Input1Path", "").equals("empty") ? emptyInPath : inPath; String input2Path = config.getString("UnionTest#Input2Path", "").equals("empty") ? emptyInPath : inPath; FileDataSource input1 = new FileDataSource( new ContractITCaseInputFormat(), input1Path); DelimitedInputFormat.configureDelimitedFormat(input1) .recordDelimiter('\n'); input1.setDegreeOfParallelism(config.getInteger("UnionTest#NoSubtasks", 1)); FileDataSource input2 = new FileDataSource( new ContractITCaseInputFormat(), input2Path); DelimitedInputFormat.configureDelimitedFormat(input2) .recordDelimiter('\n'); input2.setDegreeOfParallelism(config.getInteger("UnionTest#NoSubtasks", 1)); MapOperator testMapper = MapOperator.builder(new TestMapper()).build(); testMapper.setDegreeOfParallelism(config.getInteger("UnionTest#NoSubtasks", 1)); FileDataSink output = new FileDataSink( new ContractITCaseOutputFormat(), resultPath); output.setDegreeOfParallelism(1); output.setInput(testMapper); testMapper.addInput(input1); testMapper.addInput(input2); return new Plan(output); }
@Override protected Plan getTestJob() { FileDataSource input_left = new FileDataSource(new CoGroupTestInFormat(), leftInPath); DelimitedInputFormat.configureDelimitedFormat(input_left) .recordDelimiter('\n'); input_left.setDegreeOfParallelism(config.getInteger("CoGroupTest#NoSubtasks", 1)); FileDataSource input_right = new FileDataSource(new CoGroupTestInFormat(), rightInPath); DelimitedInputFormat.configureDelimitedFormat(input_right) .recordDelimiter('\n'); input_right.setDegreeOfParallelism(config.getInteger("CoGroupTest#NoSubtasks", 1)); CoGroupOperator testCoGrouper = CoGroupOperator.builder(new TestCoGrouper(), StringValue.class, 0, 0) .build(); testCoGrouper.setDegreeOfParallelism(config.getInteger("CoGroupTest#NoSubtasks", 1)); testCoGrouper.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY, config.getString("CoGroupTest#LocalStrategy", "")); testCoGrouper.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY, config.getString("CoGroupTest#ShipStrategy", "")); FileDataSink output = new FileDataSink(new CoGroupOutFormat(), resultPath); output.setDegreeOfParallelism(1); output.setInput(testCoGrouper); testCoGrouper.setFirstInput(input_left); testCoGrouper.setSecondInput(input_right); return new Plan(output); }
@Override public void configure(Configuration parameters) { super.configure(parameters); this.delimiter = (char) parameters.getInteger(ID_DELIMITER_CHAR, ','); } }
@Override protected Plan getTestJob() { FileDataSource input = new FileDataSource( new ContractITCaseInputFormat(), inPath); DelimitedInputFormat.configureDelimitedFormat(input) .recordDelimiter('\n'); input.setDegreeOfParallelism(config.getInteger("MapTest#NoSubtasks", 1)); MapOperator testMapper = MapOperator.builder(new TestMapper()).build(); testMapper.setDegreeOfParallelism(config.getInteger("MapTest#NoSubtasks", 1)); FileDataSink output = new FileDataSink( new ContractITCaseOutputFormat(), resultPath); output.setDegreeOfParallelism(1); output.setInput(testMapper); testMapper.setInput(input); return new Plan(output); }
@Override public void configure(Configuration parameters) { long numVertices = ConfigUtils.asLong("pageRank.numVertices", parameters); initialRank = new DoubleValue(1 / (double) numVertices); super.configure(parameters); }
DelimitedInputFormat.configureDelimitedFormat(input_left) .recordDelimiter('\n'); input_left.setDegreeOfParallelism(config.getInteger("CrossTest#NoSubtasks", 1)); DelimitedInputFormat.configureDelimitedFormat(input_right) .recordDelimiter('\n'); input_right.setDegreeOfParallelism(config.getInteger("CrossTest#NoSubtasks", 1));
@Override public void configure(Configuration parameters) { super.configure(parameters); this.vertexDelimiter = (char) parameters.getInteger(VERTEX_DELIMITER_CHAR, '|'); this.degreeDelimiter = (char) parameters.getInteger(DEGREE_DELIMITER_CHAR, ','); } }
FileDataSource input_left = new FileDataSource( new ContractITCaseInputFormat(), leftInPath); DelimitedInputFormat.configureDelimitedFormat(input_left) .recordDelimiter('\n'); input_left.setDegreeOfParallelism(config.getInteger("MatchTest#NoSubtasks", 1)); DelimitedInputFormat.configureDelimitedFormat(input_right) .recordDelimiter('\n'); input_right.setDegreeOfParallelism(config.getInteger("MatchTest#NoSubtasks", 1));
@Override protected JobGraph getJobGraph() throws Exception { FileDataSource input = new FileDataSource( new ContractITCaseInputFormat(), inPath); DelimitedInputFormat.configureDelimitedFormat(input) .recordDelimiter('\n'); input.setDegreeOfParallelism(config.getInteger("ReduceTest#NoSubtasks", 1)); ReduceOperator testReducer = ReduceOperator.builder(new TestReducer(), StringValue.class, 0) .build(); testReducer.setDegreeOfParallelism(config.getInteger("ReduceTest#NoSubtasks", 1)); testReducer.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY, config.getString("ReduceTest#LocalStrategy", "")); testReducer.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY, config.getString("ReduceTest#ShipStrategy", "")); FileDataSink output = new FileDataSink( new ContractITCaseOutputFormat(), resultPath); output.setDegreeOfParallelism(1); output.setInput(testReducer); testReducer.setInput(input); Plan plan = new Plan(output); PactCompiler pc = new PactCompiler(new DataStatistics()); OptimizedPlan op = pc.compile(plan); NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator(); return jgg.compileJobGraph(op); }