@Override public void readFields(DataInput in) throws IOException { this.setMaster(in.readBoolean()); if(!isMaster()) { int len = in.readInt(); FileSplit[] splits = new FileSplit[len]; for(int i = 0; i < len; i++) { splits[i] = new FileSplit(null, 0, 0, (String[]) null); splits[i].readFields(in); } this.setFileSplits(splits); } }
@Override public void readFields(DataInput in) throws IOException { this.setMaster(in.readBoolean()); if(!isMaster()) { int len = in.readInt(); FileSplit[] splits = new FileSplit[len]; for(int i = 0; i < len; i++) { splits[i] = new FileSplit(null, 0, 0, (String[]) null); splits[i].readFields(in); } this.setFileSplits(splits); } }
splits[i].readFields(in); this.setFileSplits(splits); int extLen = in.readInt(); if(extLen > 0) {
protected void addCrossValidationDataset(List<InputSplit> trainingSplit, JobContext context) throws IOException { List<InputSplit> trainingNoMaster = new ArrayList<InputSplit>(); for(InputSplit split: trainingSplit) { GuaguaInputSplit guaguaInput = (GuaguaInputSplit) split; if(guaguaInput.isMaster()) { continue; } trainingNoMaster.add(guaguaInput); } List<List<FileSplit>> csSplits = this.getCrossValidationSplits(context, trainingNoMaster.size()); for(int i = 0; i < csSplits.size(); i++) { List<FileSplit> oneInput = csSplits.get(i); GuaguaInputSplit guaguaInput = (GuaguaInputSplit) trainingNoMaster.get(i); int trainingSize = guaguaInput.getFileSplits().length; FileSplit[] finalSplits = (FileSplit[]) ArrayUtils.addAll(guaguaInput.getFileSplits(), oneInput.toArray(new FileSplit[0])); guaguaInput.setFileSplits(finalSplits); Boolean[] validationFlags = new Boolean[finalSplits.length]; for(int j = 0; j < finalSplits.length; j++) { validationFlags[j] = j < trainingSize ? false : true; } guaguaInput.setExtensions(validationFlags); } LOG.info("Training input split size is: {}.", trainingSplit.size()); LOG.info("Validation input split size is {}.", csSplits.size()); }
splits[i].readFields(in); this.setFileSplits(splits); int extLen = in.readInt(); if(extLen > 0) {