@VisibleForTesting public Appender getAppender(TableMeta meta, Schema schema, Path filePath) throws IOException { return getAppender(null, null, meta, schema, filePath); }
public Scanner getFileScanner(TableMeta meta, Schema schema, Path path) throws IOException { FileStatus status = fs.getFileStatus(path); return getFileScanner(meta, schema, path, status); }
@Override public List<Fragment> getSplits(String inputSourceId, TableDesc table, @Nullable EvalNode filterCondition) throws IOException { return getSplits(inputSourceId, table.getMeta(), table.getSchema(), new Path(table.getUri())); }
Appender appender = sm.getAppender(meta, schema, tablePath); appender.init(); Scanner scanner = sm.getScanner(meta, schema, fragment, null); scanner.init();
FileTablespace space = new FileTablespace("testGetSplit", fs.getUri(), null); space.init(conf); assertEquals(fs.getUri(), space.getUri()); splits.addAll(space.getSplits("data", meta, schema, false, partitions.toArray(new Path[partitions.size()]))); assertEquals(testCount, splits.size()); splits.addAll(space.getSplits("data", meta, schema, false, partitions.subList(0, partitions.size() / 2).toArray(new Path[partitions.size() / 2]))); assertEquals(testCount / 2, splits.size());
Appender appender = sm.getAppender(meta, schema, tablePath); appender.enableStats(); appender.init(); List<Fragment> splits = sm.getSplits("testZeroRows", meta, schema, false, testDir); int tupleCnt = 0; for (Fragment fragment : splits) { Scanner scanner = sm.getScanner(meta, schema, fragment, schema); scanner.init(); while (scanner.next() != null) {
localFs.mkdirs(path.getParent()); FileTablespace fileStorageManager = TablespaceManager.getLocalFs(); assertEquals(localFs.getUri(), fileStorageManager.getFileSystem().getUri()); Appender appender = fileStorageManager.getAppender(meta, schema, path); appender.init(); for(Tuple t : tuples) { Scanner scanner = fileStorageManager.getFileScanner(meta, schema, path); scanner.init(); int i=0;
fs.mkdirs(path.getParent()); Appender appender = sm.getAppender(meta, schema, path); appender.init(); for (Tuple t : tuples) { FileStatus fileStatus = fs.getFileStatus(path); List<Fragment> splits = sm.getSplits("table", meta, schema, false, path); int splitSize = (int) Math.ceil(fileStatus.getLen() / (double) fileStatus.getBlockSize()); assertEquals(splitSize, splits.size());
@Test(timeout = 120000) public void testGetFileTablespace() throws Exception { final Configuration hdfsConf = getTestHdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf) .numDataNodes(1).format(true).build(); URI uri = URI.create(cluster.getFileSystem().getUri() + "/tajo"); try { /* Local FileSystem */ FileTablespace space = TablespaceManager.getLocalFs(); assertEquals(localFs.getUri(), space.getFileSystem().getUri()); FileTablespace distTablespace = new FileTablespace("testGetFileTablespace", uri, null); distTablespace.init(conf); TablespaceManager.addTableSpaceForTest(distTablespace); /* Distributed FileSystem */ space = TablespaceManager.get(uri); assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri()); space = TablespaceManager.getByName("testGetFileTablespace"); assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri()); } finally { cluster.shutdown(); } } }
sm.getFileSystem().mkdirs(tablePath.getParent()); .getAppender(employeeMeta, schema, tablePath); appender.init(); VTuple tuple = new VTuple(schema.size());
assertTrue("Checking meaningfulness of test", stats.getNumBytes() > StorageUnit.MB); FileFragment[] frags = FileTablespace.splitNG(conf, "default.score_large", largeScore.getMeta(), new Path(largeScore.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testStorePlanWithMaxOutputFileSize"); Scanner scanner = ((FileTablespace) TablespaceManager.getLocalFs()).getFileScanner( CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf), rootNode.getOutSchema(),
new FileTablespace(TablespaceManager.DEFAULT_TABLESPACE_NAME, defaultTsUri, null); defaultTableSpace.init(conf); TablespaceManager.addTableSpaceForTest(defaultTableSpace);
@Test public final void testCreateIndex() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testCreateIndex"); Path indexPath = StorageUtil.concatPath(TajoConf.getWarehouseDir(conf), "default/idx_employee"); if (sm.getFileSystem().exists(indexPath)) { sm.getFileSystem().delete(indexPath, true); } TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(createIndexStmt[0]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); while (exec.next() != null) { } exec.close(); FileStatus[] list = sm.getFileSystem().listStatus(indexPath); assertEquals(2, list.length); }
public FileFragment[] split(String tableName, long fragmentSize) throws IOException { Path tablePath = new Path(spacePath, tableName); return split(tableName, tablePath, fragmentSize); }
@Override public void init(TajoConf tajoConf) throws IOException { super.init(tajoConf);
public FileAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema schema, TableMeta meta, Path workDir) { this.conf = conf; this.meta = meta; this.schema = schema; this.workDir = workDir; this.taskAttemptId = taskAttemptId; if (taskAttemptId != null) { if (!(conf instanceof TajoConf)) { throw new IllegalArgumentException("Configuration must be an instance of TajoConf"); } FileTablespace space = TablespaceManager.get(workDir.toUri()); this.path = space.getAppenderFilePath(taskAttemptId, workDir); } else { this.path = workDir; } }
public Scanner getFileScanner(TableMeta meta, Schema schema, Path path, FileStatus status) throws IOException { Fragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); return getScanner(meta, schema, fragment, null); }
Appender appender = sm.getAppender(meta, schema, tablePath); appender.init(); Scanner scanner = sm.getScanner(meta, schema, fragment, null); scanner.init();
FileTablespace space = new FileTablespace("testZeroLengthSplit", fs.getUri(), new JSONObject()); space.init(conf); TablespaceManager.addTableSpaceForTest(space); assertEquals(fs.getUri(), space.getUri()); splits.addAll(space.getSplits("data", meta, schema, false, partitions.toArray(new Path[partitions.size()]))); assertEquals(0, splits.size());
@Test public final void testStorePlan() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testStorePlan"); .getFileScanner(outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple;