/** * Returns total number of examples in the dataset (all k folds) * * @return total number of examples in the dataset including all k folds */ @Override public int totalExamples() { // FIXME: int cast return (int) singleFold.getLabels().size(0); }
@Override public int hashCode() { int result = getFeatures() != null ? getFeatures().hashCode() : 0; result = 31 * result + (getLabels() != null ? getLabels().hashCode() : 0); result = 31 * result + (getFeaturesMaskArray() != null ? getFeaturesMaskArray().hashCode() : 0); result = 31 * result + (getLabelsMaskArray() != null ? getLabelsMaskArray().hashCode() : 0); return result; }
@Override public int numOutcomes() { // FIXME: int cast return (int) getLabels().size(1); }
@Override public String toString() { StringBuilder builder = new StringBuilder(); if (features != null && labels != null) { builder.append("===========INPUT===================\n") .append(getFeatures().toString().replaceAll(";", "\n")) .append("\n=================OUTPUT==================\n") .append(getLabels().toString().replaceAll(";", "\n")); if (featuresMask != null) { builder.append("\n===========INPUT MASK===================\n") .append(getFeaturesMaskArray().toString().replaceAll(";", "\n")); } if (labelsMask != null) { builder.append("\n===========OUTPUT MASK===================\n") .append(getLabelsMaskArray().toString().replaceAll(";", "\n")); } return builder.toString(); } else { log.info("Features or labels are null values"); return ""; } }
@Override public int totalOutcomes() { // FIXME: int cast return (int) singleFold.getLabels().size(1); }
/** * Clone the dataset * * @return a clone of the dataset */ @Override public DataSet copy() { DataSet ret = new DataSet(getFeatures().dup(), getLabels().dup()); if (getLabelsMaskArray() != null) ret.setLabelsMaskArray(getLabelsMaskArray().dup()); if (getFeaturesMaskArray() != null) ret.setFeaturesMaskArray(getFeaturesMaskArray().dup()); ret.setColumnNames(getColumnNames()); ret.setLabelNames(getLabelNames()); return ret; }
@Override public int totalOutcomes() { // FIXME: int cast return (int) list.get(0).getLabels().columns(); }
private int getLabel(DataSet data) { Float f = data.getLabels().maxNumber().floatValue(); return f.intValue(); }
private String[] writeData(DataSet write) throws IOException { String[] ret = new String[2]; String dataSetId = UUID.randomUUID().toString(); BufferedOutputStream dataOut = new BufferedOutputStream(new FileOutputStream(new File(rootDir, dataSetId + ".bin"))); DataOutputStream dos = new DataOutputStream(dataOut); Nd4j.write(write.getFeatureMatrix(), dos); dos.flush(); dos.close(); BufferedOutputStream dataOutLabels = new BufferedOutputStream(new FileOutputStream(new File(rootDir, dataSetId + ".labels.bin"))); DataOutputStream dosLabels = new DataOutputStream(dataOutLabels); Nd4j.write(write.getLabels(), dosLabels); dosLabels.flush(); dosLabels.close(); ret[0] = new File(rootDir, dataSetId + ".bin").getAbsolutePath(); ret[1] = new File(rootDir, dataSetId + ".labels.bin").getAbsolutePath(); return ret; }
@Override public int outcome() { return Nd4j.getBlasWrapper().iamax(getLabels()); }
@Override public void validate() { if (getFeatures().size(0) != getLabels().size(0)) throw new IllegalStateException("Invalid dataset"); }
/** * Initializes this data transform fetcher from the passed in datasets * * @param examples the examples to use */ protected void initializeCurrFromList(List<DataSet> examples) { if (examples.isEmpty()) log.warn("Warning: empty dataset from the fetcher"); INDArray inputs = createInputMatrix(examples.size()); INDArray labels = createOutputMatrix(examples.size()); for (int i = 0; i < examples.size(); i++) { inputs.putRow(i, examples.get(i).getFeatureMatrix()); labels.putRow(i, examples.get(i).getLabels()); } curr = new DataSet(inputs, labels); }
/** * Reshapes the input in to the given rows and columns * * @param rows the row size * @param cols the column size * @return a copy of this data op with the input resized */ @Override public DataSet reshape(int rows, int cols) { DataSet ret = new DataSet(getFeatures().reshape(new long[] {rows, cols}), getLabels()); return ret; }
/** * Sets the outcome of a particular example * * @param example the example to transform * @param label the label of the outcome */ @Override public void setOutcome(int example, int label) { if (example > numExamples()) throw new IllegalArgumentException("No example at " + example); if (label > numOutcomes() || label < 0) throw new IllegalArgumentException("Illegal label"); INDArray outcome = FeatureUtil.toOutcomeVector(label, numOutcomes()); getLabels().putRow(example, outcome); }
private static void evaluatePerformance(MultiLayerNetwork net, int testStartIdx, int nExamples, String outputDirectory) throws Exception { //Assuming here that the full test data set doesn't fit in memory -> load 10 examples at a time Map<Integer, String> labelMap = new HashMap<>(); labelMap.put(0, "circle"); labelMap.put(1, "square"); labelMap.put(2, "arc"); labelMap.put(3, "line"); Evaluation evaluation = new Evaluation(labelMap); DataSetIterator testData = getDataSetIterator(outputDirectory, testStartIdx, nExamples, 1000); while(testData.hasNext()) { DataSet dsTest = testData.next(); INDArray predicted = net.output(dsTest.getFeatures(), false); INDArray actual = dsTest.getLabels(); evaluation.evalTimeSeries(actual, predicted); } System.out.println(evaluation.stats()); }
/** * Gets a copy of example i * * @param i the example to getFromOrigin * @return the example at i (one example) */ @Override public DataSet get(int[] i) { return new DataSet(getFeatures().getRows(i), getLabels().getRows(i)); }
/** * Gets a copy of example i * * @param i the example to getFromOrigin * @return the example at i (one example) */ @Override public DataSet get(int i) { if (i > numExamples() || i < 0) throw new IllegalArgumentException("invalid example number"); if (i == 0 && numExamples() == 1) return this; if (getFeatureMatrix().rank() == 4) { //ensure rank is preserved INDArray slice = getFeatureMatrix().slice(i); return new DataSet(slice.reshape(ArrayUtil.combine(new long[] {1}, slice.shape())), getLabels().slice(i)); } return new DataSet(getFeatures().slice(i), getLabels().slice(i)); }
/** * Sample a dataset * * @param numSamples the number of samples to getFromOrigin * @param rng the rng to use * @param withReplacement whether to allow duplicates (only tracked by example row number) * @return the sample dataset */ @Override public DataSet sample(int numSamples, org.nd4j.linalg.api.rng.Random rng, boolean withReplacement) { INDArray examples = Nd4j.create(numSamples, getFeatures().columns()); INDArray outcomes = Nd4j.create(numSamples, numOutcomes()); Set<Integer> added = new HashSet<>(); for (int i = 0; i < numSamples; i++) { int picked = rng.nextInt(numExamples()); if (!withReplacement) while (added.contains(picked)) picked = rng.nextInt(numExamples()); examples.putRow(i, get(picked).getFeatures()); outcomes.putRow(i, get(picked).getLabels()); } return new DataSet(examples, outcomes); }