/** * Get output file setting and write final sum value to HDFS file. */ @Override public void postApplication(final MasterContext<LogisticRegressionParams, LogisticRegressionParams> context) { LOG.info("Starts to write final value to file."); Path out = new Path(context.getProps().getProperty("lr.model.output")); LOG.info("Writing results to {}", out.toString()); PrintWriter pw = null; try { FSDataOutputStream fos = FileSystem.get(new Configuration()).create(out); pw = new PrintWriter(fos); pw.println(Arrays.toString(context.getMasterResult().getParameters())); pw.flush(); } catch (IOException e) { LOG.error("Error in writing output.", e); } finally { IOUtils.closeStream(pw); } } }
@Override public void postApplication(MasterContext<KMeansMasterParams, KMeansWorkerParams> context) { LOG.info("KMeansCentersOutput starts to write k centers to file."); Path out = new Path(context.getProps().getProperty(KMeansContants.KMEANS_CENTERS_OUTPUT)); PrintWriter pw = null; try { FSDataOutputStream fos = FileSystem.get(new Configuration()).create(out); LOG.info("Writing results to {}", out.toString()); pw = new PrintWriter(fos); KMeansMasterParams masterResult = context.getMasterResult(); for(double[] center: masterResult.getPointList()) { pw.println(Arrays.toString(center)); } pw.flush(); } catch (IOException e) { LOG.error("Error in writing output.", e); } finally { IOUtils.closeStream(pw); } }
@Override public void postApplication(MasterContext<KMeansMasterParams, KMeansWorkerParams> context) { LOG.info("KMeansCentersOutput starts to write k centers to file."); Path out = new Path(context.getProps().getProperty(KMeansContants.KMEANS_CENTERS_OUTPUT)); PrintWriter pw = null; try { FSDataOutputStream fos = FileSystem.get(new Configuration()).create(out); LOG.info("Writing results to {}", out.toString()); pw = new PrintWriter(fos); KMeansMasterParams masterResult = context.getMasterResult(); for(double[] center: masterResult.getPointList()) { pw.println(Arrays.toString(center)); } pw.flush(); } catch (IOException e) { LOG.error("Error in writing output.", e); } finally { IOUtils.closeStream(pw); } }
/** * Get output file setting and write final sum value to HDFS file. */ @Override public void postApplication(final MasterContext<LinearRegressionParams, LinearRegressionParams> context) { LOG.info("Starts to write final value to file."); Path out = new Path(context.getProps().getProperty("lr.model.output")); LOG.info("Writing results to {}", out.toString()); PrintWriter pw = null; try { FSDataOutputStream fos = FileSystem.get(new Configuration()).create(out); pw = new PrintWriter(fos); pw.println(Arrays.toString(context.getMasterResult().getParameters())); pw.flush(); } catch (IOException e) { LOG.error("Error in writing output.", e); } finally { IOUtils.closeStream(pw); } } }
/** * Get output file setting and write final sum value to HDFS file. */ @Override public void postApplication(final MasterContext<LogisticRegressionParams, LogisticRegressionParams> context) { LOG.info("Starts to write final value to file."); Path out = new Path(context.getProps().getProperty("lr.model.output")); LOG.info("Writing results to {}", out.toString()); PrintWriter pw = null; try { FSDataOutputStream fos = FileSystem.get(new Configuration()).create(out); pw = new PrintWriter(fos); pw.println(Arrays.toString(context.getMasterResult().getParameters())); pw.flush(); } catch (IOException e) { LOG.error("Error in writing output.", e); } finally { IOUtils.closeStream(pw); } } }
/** * Get output file setting and write final sum value to HDFS file. */ @Override public void postApplication(final MasterContext<LinearRegressionParams, LinearRegressionParams> context) { LOG.info("Starts to write final value to file."); Path out = new Path(context.getProps().getProperty("lr.model.output")); LOG.info("Writing results to {}", out.toString()); PrintWriter pw = null; try { FSDataOutputStream fos = FileSystem.get(new Configuration()).create(out); pw = new PrintWriter(fos); pw.println(Arrays.toString(context.getMasterResult().getParameters())); pw.flush(); } catch (IOException e) { LOG.error("Error in writing output.", e); } finally { IOUtils.closeStream(pw); } } }
/** * Get output file setting and write final sum value to HDFS file. */ @Override public void postApplication( final MasterContext<GuaguaWritableAdapter<LongWritable>, GuaguaWritableAdapter<LongWritable>> context) { LOG.info("SumOutput starts to write final sum value to file."); Path out = new Path(context.getProps().getProperty("guagua.sum.output")); LOG.info("Writing results to {}", out.toString()); PrintWriter pw = null; try { FSDataOutputStream fos = FileSystem.get(new Configuration()).create(out); pw = new PrintWriter(fos); pw.println(context.getMasterResult().getWritable().get()); pw.flush(); } catch (IOException e) { LOG.error("Error in writing output.", e); } finally { IOUtils.closeStream(pw); } } }
/** * Get output file setting and write final sum value to HDFS file. */ @Override public void postApplication( final MasterContext<GuaguaWritableAdapter<LongWritable>, GuaguaWritableAdapter<LongWritable>> context) { LOG.info("SumOutput starts to write final sum value to file."); Path out = new Path(context.getProps().getProperty("guagua.sum.output")); LOG.info("Writing results to {}", out.toString()); PrintWriter pw = null; try { FSDataOutputStream fos = FileSystem.get(new Configuration()).create(out); pw = new PrintWriter(fos); pw.println(context.getMasterResult().getWritable().get()); pw.flush(); } catch (IOException e) { LOG.error("Error in writing output.", e); } catch (Exception e) { LOG.error("Error in writing output.", e); } finally { IOUtils.closeStream(pw); } } }
@SuppressWarnings("deprecation") private void updateProgressLog(final MasterContext<LogisticRegressionParams, LogisticRegressionParams> context) { int currentIteration = context.getCurrentIteration(); if(currentIteration == 1) { // first iteration is used for training preparation return; } String progress = new StringBuilder(200).append(" Trainer ").append(this.trainerId).append(" Epoch #") .append(currentIteration - 1).append(" Training Error:") .append(context.getMasterResult().getTrainError()).append(" Validation Error:") .append(context.getMasterResult().getTestError()).append("\n").toString(); try { LOG.debug("Writing progress results to {} {}", context.getCurrentIteration(), progress.toString()); this.progressOutput.write(progress.getBytes("UTF-8")); this.progressOutput.flush(); this.progressOutput.sync(); } catch (IOException e) { LOG.error("Error in write progress log:", e); } }
@SuppressWarnings("deprecation") private void updateProgressLog(final MasterContext<NNParams, NNParams> context) { int currentIteration = context.getCurrentIteration(); if(context.isFirstIteration()) { // first iteration is used for training preparation return; } String progress = new StringBuilder(200).append(" Trainer ").append(this.trainerId).append(" Epoch #") .append(currentIteration - 1).append(" Training Error:") .append(String.format("%.10f", context.getMasterResult().getTrainError())).append(" Validation Error:") .append(String.format("%.10f", context.getMasterResult().getTestError())).append("\n").toString(); try { LOG.debug("Writing progress results to {} {}", context.getCurrentIteration(), progress.toString()); this.progressOutput.write(progress.getBytes("UTF-8")); this.progressOutput.flush(); this.progressOutput.sync(); } catch (IOException e) { LOG.error("Error in write progress log:", e); } }
@Override public void postIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { this.coordinator.signalWorkers(context.getCurrentIteration(), context.getMasterResult()); }
@Override public void postApplication(MasterContext<DTMasterParams, DTWorkerParams> context) { List<TreeNode> trees = context.getMasterResult().getTrees(); if(this.isGBDT) { trees = context.getMasterResult().getTmpTrees(); } if(LOG.isDebugEnabled()) { LOG.debug("final trees", trees.toString()); } Path out = new Path(context.getProps().getProperty(CommonConstants.GUAGUA_OUTPUT)); writeModelToFileSystem(trees, out); if(this.isGsMode || this.isKFoldCV) { Path valErrOutput = new Path(context.getProps().getProperty(CommonConstants.GS_VALIDATION_ERROR)); writeValErrorToFileSystem(context.getMasterResult().getValidationError() / context.getMasterResult().getValidationCount(), valErrOutput); } IOUtils.closeStream(this.progressOutput); }
@Override public void postApplication(MasterContext<VarSelMasterResult, VarSelWorkerResult> context) { VarSelMasterResult varSelMasterResult = context.getMasterResult(); LOG.info("Results is - {}", varSelMasterResult.getBestSeed()); String out = context.getProps().getProperty(Constants.VAR_SEL_COLUMN_IDS_OUPUT); writeColumnIdsIntoHDFS(out, varSelMasterResult.getBestSeed().getColumnIdList()); }
/** * Check whether GuaguaConstants.GUAGUA_WORKER_HALT_ENABLE) is enabled, if yes, check whether all workers are halted * and update master status. */ protected void updateMasterHaltStatus(final MasterContext<MASTER_RESULT, WORKER_RESULT> context) { MASTER_RESULT result = context.getMasterResult(); // a switch to make all workers have the right to terminate the application if(Boolean.TRUE.toString().equalsIgnoreCase( context.getProps().getProperty(GuaguaConstants.GUAGUA_WORKER_HALT_ENABLE, GuaguaConstants.GUAGUA_WORKER_DEFAULT_HALT_ENABLE))) { if(isAllWorkersHalt(context.getWorkerResults()) && result instanceof HaltBytable) { ((HaltBytable) result).setHalt(true); context.setMasterResult(result); } } }
@Override public void postApplication(MasterContext<LogisticRegressionParams, LogisticRegressionParams> context) { IOUtils.closeStream(this.progressOutput); // for dry mode, we don't save models files. if(this.isDry) { return; } if(optimizedWeights == null) { optimizedWeights = context.getMasterResult().getParameters(); } Path out = new Path(context.getProps().getProperty(CommonConstants.GUAGUA_OUTPUT)); writeModelWeightsToFileSystem(optimizedWeights, out); if(this.isKFoldCV || this.isGsMode) { Path valErrOutput = new Path(context.getProps().getProperty(CommonConstants.GS_VALIDATION_ERROR)); writeValErrorToFileSystem(context.getMasterResult().getTestError(), valErrOutput); } IOUtils.closeStream(this.progressOutput); }
@Override public void postApplication(MasterContext<NNParams, NNParams> context) { LOG.info("NNOutput starts to write model to files."); int inputs = NumberFormatUtils.getInt(context.getProps().getProperty(NNConstants.GUAGUA_NN_INPUT_NODES), NNConstants.GUAGUA_NN_DEFAULT_INPUT_NODES); int hiddens = NumberFormatUtils.getInt(context.getProps().getProperty(NNConstants.GUAGUA_NN_HIDDEN_NODES), NNConstants.GUAGUA_NN_DEFAULT_HIDDEN_NODES); int outputs = NumberFormatUtils.getInt(context.getProps().getProperty(NNConstants.GUAGUA_NN_OUTPUT_NODES), NNConstants.GUAGUA_NN_DEFAULT_OUTPUT_NODES); BasicNetwork network = NNUtils.generateNetwork(inputs, hiddens, outputs); Path out = new Path(context.getProps().getProperty(NNConstants.GUAGUA_NN_OUTPUT)); FSDataOutputStream fos = null; try { fos = FileSystem.get(new Configuration()).create(out); LOG.info("Writing results to {}", out.toString()); network.getFlat().setWeights(context.getMasterResult().getWeights()); EncogDirectoryPersistence.saveObject(fos, network); } catch (IOException e) { LOG.error("Error in writing output.", e); } finally { IOUtils.closeStream(fos); } }
@Override public void postApplication(MasterContext<NNParams, NNParams> context) { IOUtils.closeStream(this.progressOutput); // for dry mode, we don't save models files. if(this.isDry) { return; } if(optimizedWeights != null) { Path out = new Path(context.getProps().getProperty(CommonConstants.GUAGUA_OUTPUT)); // TODO do we need to check IOException and retry again to make sure such important model is saved // successfully. writeModelWeightsToFileSystem(optimizedWeights, out, true); } if(this.gridSearch.hasHyperParam() || this.isKFoldCV) { Path valErrOutput = new Path(context.getProps().getProperty(CommonConstants.GS_VALIDATION_ERROR)); writeValErrorToFileSystem(context.getMasterResult().getTestError(), valErrOutput); } }
@Override public void postApplication(MasterContext<NNParams, NNParams> context) { LOG.info("NNOutput starts to write model to files."); int inputs = NumberFormatUtils.getInt(context.getProps().getProperty(NNConstants.GUAGUA_NN_INPUT_NODES), NNConstants.GUAGUA_NN_DEFAULT_INPUT_NODES); int hiddens = NumberFormatUtils.getInt(context.getProps().getProperty(NNConstants.GUAGUA_NN_HIDDEN_NODES), NNConstants.GUAGUA_NN_DEFAULT_HIDDEN_NODES); int outputs = NumberFormatUtils.getInt(context.getProps().getProperty(NNConstants.GUAGUA_NN_OUTPUT_NODES), NNConstants.GUAGUA_NN_DEFAULT_OUTPUT_NODES); BasicNetwork network = NNUtils.generateNetwork(inputs, hiddens, outputs); Path out = new Path(context.getProps().getProperty(NNConstants.GUAGUA_NN_OUTPUT)); FSDataOutputStream fos = null; try { fos = FileSystem.get(new Configuration()).create(out); LOG.info("Writing results to {}", out.toString()); network.getFlat().setWeights(context.getMasterResult().getWeights()); EncogDirectoryPersistence.saveObject(fos, network); } catch (IOException e) { LOG.error("Error in writing output.", e); } finally { IOUtils.closeStream(fos); } }
@Override public void init(MasterContext<LogisticRegressionParams, LogisticRegressionParams> context) { this.inputNum = NumberFormatUtils.getInt(LogisticRegressionContants.LR_INPUT_NUM, LogisticRegressionContants.LR_INPUT_DEFAULT_NUM); this.learnRate = NumberFormatUtils.getDouble(LogisticRegressionContants.LR_LEARNING_RATE, LogisticRegressionContants.LR_LEARNING_DEFAULT_RATE); // if not first iteration, means this is fail-over and should be recovered for state in master. if(!context.isFirstIteration()) { LogisticRegressionParams masterResult = context.getMasterResult(); if(masterResult != null && masterResult.getParameters() != null) { this.weights = masterResult.getParameters(); } else { initWeights(); } } }
@Override public void init(MasterContext<LinearRegressionParams, LinearRegressionParams> context) { this.inputNum = NumberFormatUtils.getInt(LinearRegressionContants.LR_INPUT_NUM, LinearRegressionContants.LR_INPUT_DEFAULT_NUM); this.learnRate = NumberFormatUtils.getDouble(LinearRegressionContants.LR_LEARNING_RATE, LinearRegressionContants.LR_LEARNING_DEFAULT_RATE); // not initialized and not first iteration, should be fault tolerence, recover state in LogisticRegressionMaster if(!context.isFirstIteration()) { LinearRegressionParams lastMasterResult = context.getMasterResult(); if(lastMasterResult != null && lastMasterResult.getParameters() != null) { // recover state in current master computable and return to workers this.weights = lastMasterResult.getParameters(); } else { // no weights, restarted from the very beginning, this may not happen initWeights(); } } }