@Override public void save(File to) { try (FileOutputStream fos = new FileOutputStream(to, false); BufferedOutputStream bos = new BufferedOutputStream(fos)) { save(bos); } catch (IOException e) { throw new RuntimeException(e); } }
private static void saveToDisk(DataSet currentFeaturized, int iterNum, boolean isTrain) { File fileFolder = isTrain ? new File("trainFolder"): new File("testFolder"); if (iterNum == 0) { fileFolder.mkdirs(); } String fileName = "flowers-" + featurizeExtractionLayer + "-"; fileName += isTrain ? "train-" : "test-"; fileName += iterNum + ".bin"; currentFeaturized.save(new File(fileFolder,fileName)); LOGGER.info("Saved " + (isTrain?"train ":"test ") + "dataset #"+ iterNum); } }
@Override public void put(String key, DataSet dataSet) { if (cache.containsKey(key)) { log.debug("evicting key %s from data set cache", key); cache.remove(key); } ByteArrayOutputStream os = new ByteArrayOutputStream(); dataSet.save(os); cache.put(key, os.toByteArray()); }
private static void saveToDisk(DataSet currentFeaturized, int iterNum, boolean isTrain) { File fileFolder = isTrain ? new File("trainFolder"): new File("testFolder"); if (iterNum == 0) { fileFolder.mkdirs(); } String fileName = "flowers-" + featurizeExtractionLayer + "-"; fileName += isTrain ? "train-" : "test-"; fileName += iterNum + ".bin"; currentFeaturized.save(new File(fileFolder,fileName)); LOGGER.info("Saved " + (isTrain?"train ":"test ") + "dataset #"+ iterNum); } }
@Override public void put(String key, DataSet dataSet) { File file = resolveKey(key); File parentDir = file.getParentFile(); if (!parentDir.exists()) { if (!parentDir.mkdirs()) { throw new IllegalStateException("ERROR: cannot create parent directory: " + parentDir); } } if (file.exists()) { file.delete(); } dataSet.save(file); }
protected void saveDatasets(DataSetIterator iterator, String dir) { AtomicInteger counter = new AtomicInteger(0); new File(dir).mkdirs(); while (iterator.hasNext()) { String path = FilenameUtils.concat(dir, "dataset-" + (counter.getAndIncrement()) + ".bin"); iterator.next().save(new File(path)); if (counter.get() % 500 == 0) log.info("{} datasets saved so far...", counter.get()); } }
while(trainIter.hasNext()) { OutputStream os = fs.create(new Path(hdfsRoot + "/" + "train","dataset" + trainCountWrote++)); trainIter.next().save(os); os.close(); while(testIter.hasNext()) { OutputStream os = fs.create(new Path(testDir,"dataset" + testCountWrote++)); testIter.next().save(os); os.close();
@Override public void save(File to) { try (FileOutputStream fos = new FileOutputStream(to, false); BufferedOutputStream bos = new BufferedOutputStream(fos)) { save(bos); } catch (IOException e) { throw new RuntimeException(e); } }
@Override public void put(String key, DataSet dataSet) { if (cache.containsKey(key)) { log.debug("evicting key %s from data set cache", key); cache.remove(key); } ByteArrayOutputStream os = new ByteArrayOutputStream(); dataSet.save(os); cache.put(key, os.toByteArray()); }
@Override public void put(String key, DataSet dataSet) { File file = resolveKey(key); File parentDir = file.getParentFile(); if (!parentDir.exists()) { if (!parentDir.mkdirs()) { throw new IllegalStateException("ERROR: cannot create parent directory: " + parentDir); } } if (file.exists()) { file.delete(); } dataSet.save(file); }
result.get(i).setFeatures(newFeatures); result.save(fileName);
DataSet ds = inputs.next(); baos.reset(); ds.save(baos);