ml.shifu.guagua.hadoop.io.GuaguaInputSplit.isMaster java code examples

Add the Codota plugin to your IDE and get smart completions

private void myMethod () {
 Dictionary d =new Hashtable()
Bundle bundle;bundle.getHeaders()
new Properties()
Smart code suggestions by Tabnine
}

@Override
public void write(DataOutput out) throws IOException {
  out.writeBoolean(this.isMaster());
  if(!this.isMaster()) {
    int length = this.getFileSplits().length;
    out.writeInt(length);
    for(int i = 0; i < length; i++) {
      this.getFileSplits()[i].write(out);
    }
  }
}

@Override
public void write(DataOutput out) throws IOException {
  out.writeBoolean(this.isMaster());
  if(!this.isMaster()) {
    int length = this.getFileSplits().length;
    out.writeInt(length);
    for(int i = 0; i < length; i++) {
      this.getFileSplits()[i].write(out);
    }
  }
}

@Override
public void write(DataOutput out) throws IOException {
  out.writeBoolean(this.isMaster());
  if(!this.isMaster()) {
    int length = this.getFileSplits().length;
    out.writeInt(length);
    for(int i = 0; i < length; i++) {
      this.getFileSplits()[i].write(out);
    }
    if(this.extensions != null) {
      out.writeInt(extensions.length);
      for(int i = 0; i < extensions.length; i++) {
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        ObjectOutput ext = null;
        try {
          ext = new ObjectOutputStream(bos);
          ext.writeObject(extensions[i]);
          byte[] bytes = bos.toByteArray();
          out.writeInt(bytes.length);
          out.write(bytes);
        } finally {
          IOUtils.closeQuietly(bos);
        }
      }
    }else{
      out.writeInt(0);
    }
  }
}

@Override
public void write(DataOutput out) throws IOException {
  out.writeBoolean(this.isMaster());
  if(!this.isMaster()) {
    int length = this.getFileSplits().length;
    out.writeInt(length);
    for(int i = 0; i < length; i++) {
      this.getFileSplits()[i].write(out);
    }
    if(this.extensions != null) {
      out.writeInt(extensions.length);
      for(int i = 0; i < extensions.length; i++) {
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        ObjectOutput ext = null;
        try {
          ext = new ObjectOutputStream(bos);
          ext.writeObject(extensions[i]);
          byte[] bytes = bos.toByteArray();
          out.writeInt(bytes.length);
          out.write(bytes);
        } finally {
          IOUtils.closeQuietly(bos);
        }
      }
    }else{
      out.writeInt(0);
    }
  }
}

/**
 * For master split, use <code>Long.MAX_VALUE</code> as its length to make it is the first task for Hadoop job. It
 * is convenient for users to check master in Hadoop UI.
 */
@Override
public long getLength() throws IOException, InterruptedException {
  if(isMaster()) {
    return Long.MAX_VALUE;
  }
  long len = 0;
  for(FileSplit split: this.getFileSplits()) {
    len += split.getLength();
  }
  return len;
}

/**
 * For master split, use <code>Long.MAX_VALUE</code> as its length to make it is the first task for Hadoop job. It
 * is convenient for users to check master in Hadoop UI.
 */
@Override
public long getLength() throws IOException, InterruptedException {
  if(isMaster()) {
    return Long.MAX_VALUE;
  }
  long len = 0;
  for(FileSplit split: this.getFileSplits()) {
    len += split.getLength();
  }
  return len;
}

/**
 * For master split, use <code>Long.MAX_VALUE</code> as its length to make it is the first task for Hadoop job. It
 * is convenient for users to check master in Hadoop UI.
 */
@Override
public long getLength() throws IOException, InterruptedException {
  if(isMaster()) {
    return Long.MAX_VALUE;
  }
  long len = 0;
  for(FileSplit split: this.getFileSplits()) {
    len += split.getLength();
  }
  return len;
}

/**
 * For master split, use <code>Long.MAX_VALUE</code> as its length to make it is the first task for Hadoop job. It
 * is convenient for users to check master in Hadoop UI.
 */
@Override
public long getLength() throws IOException, InterruptedException {
  if(isMaster()) {
    return Long.MAX_VALUE;
  }
  long len = 0;
  for(FileSplit split: this.getFileSplits()) {
    len += split.getLength();
  }
  return len;
}

@Override
public void readFields(DataInput in) throws IOException {
  this.setMaster(in.readBoolean());
  if(!isMaster()) {
    int len = in.readInt();
    FileSplit[] splits = new FileSplit[len];
    for(int i = 0; i < len; i++) {
      splits[i] = new FileSplit(null, 0, 0, (String[]) null);
      splits[i].readFields(in);
    }
    this.setFileSplits(splits);
  }
}

@Override
public void readFields(DataInput in) throws IOException {
  this.setMaster(in.readBoolean());
  if(!isMaster()) {
    int len = in.readInt();
    FileSplit[] splits = new FileSplit[len];

@Override
public void readFields(DataInput in) throws IOException {
  this.setMaster(in.readBoolean());
  if(!isMaster()) {
    int len = in.readInt();
    FileSplit[] splits = new FileSplit[len];
    for(int i = 0; i < len; i++) {
      splits[i] = new FileSplit(null, 0, 0, (String[]) null);
      splits[i].readFields(in);
    }
    this.setFileSplits(splits);
  }
}

protected void addCrossValidationDataset(List<InputSplit> trainingSplit, JobContext context) throws IOException {
  List<InputSplit> trainingNoMaster = new ArrayList<InputSplit>();
  for(InputSplit split: trainingSplit) {
    GuaguaInputSplit guaguaInput = (GuaguaInputSplit) split;
    if(guaguaInput.isMaster()) {
      continue;
    }
    trainingNoMaster.add(guaguaInput);
  }
  List<List<FileSplit>> csSplits = this.getCrossValidationSplits(context, trainingNoMaster.size());
  for(int i = 0; i < csSplits.size(); i++) {
    List<FileSplit> oneInput = csSplits.get(i);
    GuaguaInputSplit guaguaInput = (GuaguaInputSplit) trainingNoMaster.get(i);
    int trainingSize = guaguaInput.getFileSplits().length;
    FileSplit[] finalSplits = (FileSplit[]) ArrayUtils.addAll(guaguaInput.getFileSplits(),
        oneInput.toArray(new FileSplit[0]));
    guaguaInput.setFileSplits(finalSplits);
    Boolean[] validationFlags = new Boolean[finalSplits.length];
    for(int j = 0; j < finalSplits.length; j++) {
      validationFlags[j] = j < trainingSize ? false : true;
    }
    guaguaInput.setExtensions(validationFlags);
  }
  LOG.info("Training input split size is: {}.", trainingSplit.size());
  LOG.info("Validation input split size is {}.", csSplits.size());
}

@Override
public void readFields(DataInput in) throws IOException {
  this.setMaster(in.readBoolean());
  if(!isMaster()) {
    int len = in.readInt();
    FileSplit[] splits = new FileSplit[len];

@Override
protected void setup(Context context) throws java.io.IOException, InterruptedException {
  GuaguaInputSplit inputSplit = (GuaguaInputSplit) context.getInputSplit();
  this.setMaster(inputSplit.isMaster());
  if(this.isMaster()) {
    context.setStatus("Master initializing ...");
    this.setGuaguaService(new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>());
  } else {
    context.setStatus("Worker initializing ...");
    this.setGuaguaService(new GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>());
    List<GuaguaFileSplit> splits = new LinkedList<GuaguaFileSplit>();
    for(int i = 0; i < inputSplit.getFileSplits().length; i++) {
      FileSplit fs = inputSplit.getFileSplits()[i];
      GuaguaFileSplit gfs = new GuaguaFileSplit(fs.getPath().toString(), fs.getStart(), fs.getLength());
      if(inputSplit.getExtensions() != null && i < inputSplit.getExtensions().length) {
        gfs.setExtension(inputSplit.getExtensions()[i]);
      }
      splits.add(gfs);
    }
    this.getGuaguaService().setSplits(splits);
  }
  Properties props = replaceConfToProps(context.getConfiguration());
  this.getGuaguaService().setAppId(context.getConfiguration().get(GuaguaMapReduceConstants.MAPRED_JOB_ID));
  this.getGuaguaService().setContainerId(
      context.getConfiguration().get(GuaguaMapReduceConstants.MAPRED_TASK_PARTITION));
  this.getGuaguaService().init(props);
  this.getGuaguaService().start();
}

@Override
protected void setup(Context context) throws java.io.IOException, InterruptedException {
  GuaguaInputSplit inputSplit = (GuaguaInputSplit) context.getInputSplit();
  this.setMaster(inputSplit.isMaster());
  if(this.isMaster()) {
    context.setStatus("Master initializing ...");
    this.setGuaguaService(new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>());
  } else {
    context.setStatus("Worker initializing ...");
    this.setGuaguaService(new GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>());
    List<GuaguaFileSplit> splits = new LinkedList<GuaguaFileSplit>();
    for(int i = 0; i < inputSplit.getFileSplits().length; i++) {
      FileSplit fs = inputSplit.getFileSplits()[i];
      GuaguaFileSplit gfs = new GuaguaFileSplit(fs.getPath().toString(), fs.getStart(), fs.getLength());
      if(inputSplit.getExtensions() != null && i < inputSplit.getExtensions().length) {
        gfs.setExtension(inputSplit.getExtensions()[i]);
      }
      splits.add(gfs);
    }
    this.getGuaguaService().setSplits(splits);
  }
  Properties props = replaceConfToProps(context.getConfiguration());
  this.getGuaguaService().setAppId(context.getConfiguration().get(GuaguaMapReduceConstants.MAPRED_JOB_ID));
  this.getGuaguaService().setContainerId(
      context.getConfiguration().get(GuaguaMapReduceConstants.MAPRED_TASK_PARTITION));
  this.getGuaguaService().init(props);
  this.getGuaguaService().start();
}

/**
 * Set up guagua service
 */
protected void setup() {
  this.setMaster(this.getInputSplit().isMaster());
  if(this.isMaster()) {
    this.setGuaguaService(new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>());
  } else {
    this.setGuaguaService(new GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>());
    List<GuaguaFileSplit> splits = new LinkedList<GuaguaFileSplit>();
    for(FileSplit fileSplit: getInputSplit().getFileSplits()) {
      splits.add(new GuaguaFileSplit(fileSplit.getPath().toString(), fileSplit.getStart(), fileSplit
          .getLength()));
    }
    this.getGuaguaService().setSplits(splits);
  }
  Properties props = replaceConfToProps();
  this.getGuaguaService().setAppId(this.getAppId().toString());
  this.getGuaguaService().setContainerId(this.getPartition() + "");
  this.getGuaguaService().init(props);
  this.getGuaguaService().start();
  initRPCClient();
}

/**
 * Set up guagua service
 */
protected void setup() {
  this.setMaster(this.getInputSplit().isMaster());
  if(this.isMaster()) {
    this.setGuaguaService(new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>());
  } else {
    this.setGuaguaService(new GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>());
    List<GuaguaFileSplit> splits = new LinkedList<GuaguaFileSplit>();
    for(FileSplit fileSplit: getInputSplit().getFileSplits()) {
      splits.add(new GuaguaFileSplit(fileSplit.getPath().toString(), fileSplit.getStart(), fileSplit
          .getLength()));
    }
    this.getGuaguaService().setSplits(splits);
  }
  Properties props = replaceConfToProps();
  this.getGuaguaService().setAppId(this.getAppId().toString());
  this.getGuaguaService().setContainerId(this.getPartition() + "");
  this.getGuaguaService().init(props);
  this.getGuaguaService().start();
  initRPCClient();
}

Javadoc

Whether the input split is master split.

Popular methods of GuaguaInputSplit

<init>
Constructor with #isMaster and #fileSplits settings.
getFileSplits
setFileSplits
setExtensions
setMaster
getExtensions

Popular in Java

Updating database using SQL prepared statement
getContentResolver (Context)
requestLocationUpdates (LocationManager)
runOnUiThread (Activity)
FileInputStream (java.io)
An input stream that reads bytes from a file. File file = ...finally if (in != null) in.clos
ServerSocket (java.net)
This class represents a server-side socket that waits for incoming client connections. A ServerSocke
KeyStore (java.security)
KeyStore is responsible for maintaining cryptographic keys and their owners. The type of the syste
Semaphore (java.util.concurrent)
A counting semaphore. Conceptually, a semaphore maintains a set of permits. Each #acquire blocks if
Servlet (javax.servlet)
Defines methods that all servlets must implement. A servlet is a small Java program that runs within
Logger (org.slf4j)
The org.slf4j.Logger interface is the main user entry point of SLF4J API. It is expected that loggin
Top 17 Free Sublime Text Plugins

How to use isMastermethodin ml.shifu.guagua.hadoop.io.GuaguaInputSplit

Best Java code snippets using ml.shifu.guagua.hadoop.io.GuaguaInputSplit.isMaster (Showing top 17 results out of 315)

How to use
isMaster
method
in
ml.shifu.guagua.hadoop.io.GuaguaInputSplit