public RecordReader getRecordReader(int batchSize, int numExamples, PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) { return getRecordReader(numExamples, batchSize, new int[] {height, width, channels}, useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator, train, splitTrainTest, rng); }
/** * Create LFW data specific iterator * @param batchSize the batch size of the examples * @param numExamples the overall number of examples * @param imgDim an array of height, width and channels * @param numLabels the overall number of examples * @param useSubset use a subset of the LFWDataSet * @param labelGenerator path label generator to use * @param train true if use train value * @param splitTrainTest the percentage to split data for train and remainder goes to test * @param imageTransform how to transform the image * @param rng random number to lock in batch shuffling * */ public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset, PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, ImageTransform imageTransform, Random rng) { super(new LFWLoader(imgDim, imageTransform, useSubset).getRecordReader(batchSize, numExamples, imgDim, numLabels, labelGenerator, train, splitTrainTest, rng), batchSize, 1, numLabels); }
public void load() { load(NUM_IMAGES, NUM_IMAGES, NUM_LABELS, LABEL_PATTERN, 1, rng); }
public void load(int batchSize, int numExamples, int numLabels, PathLabelGenerator labelGenerator, double splitTrainTest, Random rng) { if (!imageFilesExist()) { if (!fullDir.exists() || fullDir.listFiles() == null || fullDir.listFiles().length == 0) { fullDir.mkdir(); if (useSubset) { log.info("Downloading {} subset...", localDir); downloadAndUntar(lfwSubsetData, fullDir); } else { log.info("Downloading {}...", localDir); downloadAndUntar(lfwData, fullDir); downloadAndUntar(lfwLabel, fullDir); } } } FileSplit fileSplit = new FileSplit(fullDir, ALLOWED_FORMATS, rng); BalancedPathFilter pathFilter = new BalancedPathFilter(rng, ALLOWED_FORMATS, labelGenerator, numExamples, numLabels, 0, batchSize, null); inputSplit = fileSplit.sample(pathFilter, numExamples * splitTrainTest, numExamples * (1 - splitTrainTest)); }
public LFWLoader(int[] imgDim, ImageTransform imgTransform, boolean useSubset) { this.height = imgDim[0]; this.width = imgDim[1]; this.channels = imgDim[2]; this.imageTransform = imgTransform; this.useSubset = useSubset; this.localDir = useSubset ? localSubDir : localDir; this.fullDir = new File(BASE_DIR, localDir); generateLfwMaps(); }
public RecordReader getRecordReader(int batchSize, int numExamples, int[] imgDim, boolean train, double splitTrainTest, Random rng) { return getRecordReader(numExamples, batchSize, imgDim, useSubset ? SUB_NUM_LABELS : NUM_LABELS, LABEL_PATTERN, train, splitTrainTest, rng); }
/** * Create LFW data specific iterator * @param batchSize the batch size of the examples * @param numExamples the overall number of examples * @param imgDim an array of height, width and channels * @param numLabels the overall number of examples * @param useSubset use a subset of the LFWDataSet * @param labelGenerator path label generator to use * @param train true if use train value * @param splitTrainTest the percentage to split data for train and remainder goes to test * @param imageTransform how to transform the image * @param rng random number to lock in batch shuffling * */ public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset, PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, ImageTransform imageTransform, Random rng) { super(new LFWLoader(imgDim, imageTransform, useSubset).getRecordReader(batchSize, numExamples, imgDim, numLabels, labelGenerator, train, splitTrainTest, rng), batchSize, 1, numLabels); }
public RecordReader getRecordReader(int batchSize, int numExamples, int[] imgDim, int numLabels, PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) { load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng); RecordReader recordReader = new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform); try { InputSplit data = train ? inputSplit[0] : inputSplit[1]; recordReader.initialize(data); } catch (IOException | InterruptedException e) { e.printStackTrace(); } return recordReader; }
public RecordReader getRecordReader(int batchSize, int numExamples, int numLabels, Random rng) { return getRecordReader(numExamples, batchSize, new int[] {height, width, channels}, numLabels, LABEL_PATTERN, true, 1, rng); }
public RecordReader getRecordReader(int batchSize, int numExamples, int[] imgDim, PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) { return getRecordReader(numExamples, batchSize, imgDim, useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator, train, splitTrainTest, rng); }
public RecordReader getRecordReader(int numExamples) { return getRecordReader(numExamples, numExamples, new int[] {height, width, channels}, useSubset ? SUB_NUM_LABELS : NUM_LABELS, LABEL_PATTERN, true, 1, new Random(System.currentTimeMillis())); }
public RecordReader getRecordReader(int batchSize, int numExamples, boolean train, double splitTrainTest) { return getRecordReader(numExamples, batchSize, new int[] {height, width, channels}, useSubset ? SUB_NUM_LABELS : NUM_LABELS, LABEL_PATTERN, train, splitTrainTest, new Random(System.currentTimeMillis())); }