@VisibleForTesting public Appender getAppender(TableMeta meta, Schema schema, Path filePath) throws IOException { return getAppender(null, null, meta, schema, filePath); }
@VisibleForTesting public Appender getAppender(TableMeta meta, Schema schema, Path filePath) throws IOException { return getAppender(null, null, meta, schema, filePath); }
public void openAppender(int suffixId) throws IOException { Path actualFilePath = lastFileName; if (suffixId > 0) { actualFilePath = new Path(lastFileName + "_" + suffixId); } appender = ((FileTablespace) TablespaceManager.get(lastFileName.toUri())) .getAppender(meta, outSchema, actualFilePath); appender.enableStats(); appender.init(); }
public void openAppender(int suffixId) throws IOException { Path actualFilePath = lastFileName; if (suffixId > 0) { actualFilePath = new Path(lastFileName + "_" + suffixId); } appender = ((FileTablespace) TablespaceManager.get(lastFileName.toUri())) .getAppender(meta, outSchema, actualFilePath); appender.enableStats(); appender.init(); }
public void init() throws IOException { keySchema = PlannerUtil.sortSpecsToSchema(sortSpecs); keyProjector = new KeyProjector(inSchema, keySchema.toArray()); BSTIndex bst = new BSTIndex(context.getConf()); this.comp = new BaseTupleComparator(keySchema, sortSpecs); Path storeTablePath = new Path(context.getWorkDir(), "output"); LOG.info("Output data directory: " + storeTablePath); FileSystem fs = new RawLocalFileSystem(); fs.mkdirs(storeTablePath); this.appender = (FileAppender) ((FileTablespace) TablespaceManager.getDefault()) .getAppender(meta, outSchema, new Path(storeTablePath, "output")); this.appender.enableStats(keySchema.getAllColumns()); this.appender.init(); this.indexWriter = bst.getIndexWriter(new Path(storeTablePath, "index"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp, true); this.indexWriter.init(); super.init(); }
public void init() throws IOException { keySchema = PlannerUtil.sortSpecsToSchema(sortSpecs); keyProjector = new KeyProjector(inSchema, keySchema.toArray()); BSTIndex bst = new BSTIndex(context.getConf()); this.comp = new BaseTupleComparator(keySchema, sortSpecs); Path storeTablePath = new Path(context.getWorkDir(), "output"); LOG.info("Output data directory: " + storeTablePath); FileSystem fs = new RawLocalFileSystem(); fs.mkdirs(storeTablePath); this.appender = (FileAppender) ((FileTablespace) TablespaceManager.getDefault()) .getAppender(meta, outSchema, new Path(storeTablePath, "output")); this.appender.enableStats(keySchema.getAllColumns()); this.appender.init(); this.indexWriter = bst.getIndexWriter(new Path(storeTablePath, "index"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp, true); this.indexWriter.init(); super.init(); }
public void openNewFile(int suffixId) throws IOException { Schema appenderSchema = (plan instanceof InsertNode) ? ((InsertNode) plan).getTableSchema() : outSchema; if (PlannerUtil.isFileStorageType(meta.getDataFormat())) { String prevFile = null; lastFileName = context.getOutputPath(); if (suffixId > 0) { prevFile = lastFileName.toString(); lastFileName = new Path(lastFileName + "_" + suffixId); } FileTablespace space = TablespaceManager.get(lastFileName.toUri()); appender = space.getAppender(meta, appenderSchema, lastFileName); if (suffixId > 0) { LOG.info(prevFile + " exceeds " + SessionVars.MAX_OUTPUT_FILE_SIZE.keyname() + " (" + maxPerFileSize + " MB), " + "The remain output will be written into " + lastFileName.toString()); } } else { Path stagingDir = context.getQueryContext().getStagingDir(); appender = TablespaceManager.get(stagingDir.toUri()).getAppender( context.getQueryContext(), context.getTaskId(), meta, appenderSchema, stagingDir); } appender.enableStats(); appender.init(); }
public void openNewFile(int suffixId) throws IOException { Schema appenderSchema = (plan instanceof InsertNode) ? ((InsertNode) plan).getTableSchema() : outSchema; if (PlannerUtil.isFileStorageType(meta.getDataFormat())) { String prevFile = null; lastFileName = context.getOutputPath(); if (suffixId > 0) { prevFile = lastFileName.toString(); lastFileName = new Path(lastFileName + "_" + suffixId); } FileTablespace space = TablespaceManager.get(lastFileName.toUri()); appender = space.getAppender(meta, appenderSchema, lastFileName); if (suffixId > 0) { LOG.info(prevFile + " exceeds " + SessionVars.MAX_OUTPUT_FILE_SIZE.keyname() + " (" + maxPerFileSize + " MB), " + "The remain output will be written into " + lastFileName.toString()); } } else { Path stagingDir = context.getQueryContext().getStagingDir(); appender = TablespaceManager.get(stagingDir.toUri()).getAppender( context.getQueryContext(), context.getTaskId(), meta, appenderSchema, stagingDir); } appender.enableStats(); appender.init(); }
public static void writeTmpTable(TajoConf conf, Path tablePath) throws IOException { FileTablespace sm = TablespaceManager.getDefault(); Appender appender; Path filePath = new Path(tablePath, "table.csv"); FileSystem fs = filePath.getFileSystem(conf); if (fs.exists(tablePath)) { fs.delete(tablePath, true); } fs.mkdirs(tablePath); appender = sm.getAppender(mockupMeta, mockupSchema, filePath); appender.init(); int deptSize = 10000; int tupleNum = 100; Tuple tuple; for (int i = 0; i < tupleNum; i++) { tuple = new VTuple(2); String key = "test" + (i % deptSize); tuple.put(0, DatumFactory.createText(key)); tuple.put(1, DatumFactory.createInt4(i + 1)); appender.addTuple(tuple); } appender.close(); } }
@Test public void testFileAlreadyExists() throws IOException { if (internalType) return; Schema schema = SchemaBuilder.builder() .add("id", Type.INT4) .add("age", Type.INT8) .add("score", Type.FLOAT4) .build(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf); if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) { meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_PROJECTION_AVRO_SCHEMA); } FileTablespace sm = TablespaceManager.getLocalFs(); Path tablePath = new Path(testDir, "testFileAlreadyExists.data"); Appender appender = sm.getAppender(meta, schema, tablePath); appender.init(); appender.close(); try { appender = sm.getAppender(meta, schema, tablePath); appender.init(); if (BuiltinStorages.ORC.equals(dataFormat)) { appender.close(); } fail(dataFormat); } catch (IOException e) { } finally { IOUtils.cleanup(null, appender); } }
fileIndex++; appender = (((FileTablespace) TablespaceManager.getLocalFs())) .getAppender(tableMeta, schema, dataPath); appender.init();
fs.mkdirs(path.getParent()); Appender appender = sm.getAppender(meta, schema, path); appender.init(); for (Tuple t : tuples) {
Appender appender = sm.getAppender(meta, schema, tablePath); appender.enableStats(); appender.init();
Appender appender = sm.getAppender(meta, dataSchema, tablePath); appender.init();
assertEquals(localFs.getUri(), fileStorageManager.getFileSystem().getUri()); Appender appender = fileStorageManager.getAppender(meta, schema, path); appender.init(); for(Tuple t : tuples) {
Appender appender = sm.getAppender(meta, schema, tablePath); appender.init();
@Test public void testDateTextHandling() throws Exception { if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO) || internalType) { return; } Schema schema = SchemaBuilder.builder().add("col1", Type.TEXT).build(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf); FileTablespace sm = TablespaceManager.getLocalFs(); Path tablePath = new Path(testDir, "testTextHandling.data"); Appender appender = sm.getAppender(meta, schema, tablePath); appender.init(); VTuple tuple = new VTuple(1); tuple.put(0, DatumFactory.createDate(1994,7,30)); appender.addTuple(tuple); appender.flush(); appender.close(); FileStatus status = fs.getFileStatus(tablePath); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); Scanner scanner = sm.getScanner(meta, schema, fragment, null); scanner.init(); Tuple retrieved; while ((retrieved = scanner.next()) != null) { assertEquals(tuple.get(0).asChars(), retrieved.asDatum(0).asChars()); } scanner.close(); if (internalType){ OldStorageManager.clearCache(); } }
Appender appender = sm.getAppender(meta, schema, tablePath); appender.init();
TableMeta scoreLargeMeta = CatalogUtil.newTableMeta("RAW", new KeyValueSet()); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(scoreLargeMeta, scoreSchmea, scoreLargePath); appender.enableStats(); appender.init();
Appender appender = sm.getAppender(meta, schema, tablePath); appender.init();