protected FileFragment makeSplit(String fragmentId, Path file, long start, long length, String[] hosts) { return new FileFragment(fragmentId, file, start, length, hosts); }
protected FileFragment makeSplit(String fragmentId, Path file, BlockLocation blockLocation) throws IOException { return new FileFragment(fragmentId, file, blockLocation); }
/** * A factory that makes the split for this class. It can be overridden * by sub-classes to make sub-types */ protected FileFragment makeSplit(String fragmentId, Path file, long start, long length) { return new FileFragment(fragmentId, file, start, length); }
/** * A factory that makes the split for this class. It can be overridden * by sub-classes to make sub-types */ protected FileFragment makeSplit(String fragmentId, Path file, long start, long length) { return new FileFragment(fragmentId, file, start, length); }
/** * A factory that makes the split for this class. It can be overridden * by sub-classes to make sub-types */ protected FileFragment makeSplit(String fragmentId, Path file, long start, long length) { return new FileFragment(fragmentId, file, start, length); }
protected FileFragment makeSplit(String fragmentId, Path file, long start, long length, String[] hosts) { return new FileFragment(fragmentId, file, start, length, hosts); }
protected FileFragment makeSplit(String fragmentId, Path file, BlockLocation blockLocation) throws IOException { return new FileFragment(fragmentId, file, blockLocation); }
protected FileFragment makeSplit(String fragmentId, Path file, BlockLocation blockLocation) throws IOException { return new FileFragment(fragmentId, file, blockLocation); }
protected FileFragment makeSplit(String fragmentId, Path file, long start, long length, String[] hosts) { return new FileFragment(fragmentId, file, start, length, hosts); }
public Scanner getFileScanner(TableMeta meta, Schema schema, Path path, FileStatus status) throws IOException { Fragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); return getScanner(meta, schema, fragment, null); }
public FileFragment[] splitBroadcastTable(Path tablePath) throws IOException { FileSystem fs = tablePath.getFileSystem(conf); List<FileFragment> listTablets = new ArrayList<FileFragment>(); FileFragment tablet; FileStatus[] fileLists = fs.listStatus(tablePath); for (FileStatus file : fileLists) { tablet = new FileFragment(tablePath.getName(), file.getPath(), 0, file.getLen()); listTablets.add(tablet); } FileFragment[] tablets = new FileFragment[listTablets.size()]; listTablets.toArray(tablets); return tablets; }
public static synchronized SeekableScanner getSeekableScanner( TajoConf conf, TableMeta meta, Schema schema, Path path) throws IOException { FileSystem fs = path.getFileSystem(conf); FileStatus status = fs.getFileStatus(path); FileFragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); return getSeekableScanner(conf, meta, schema, fragment, schema); } }
public Scanner getFileScanner(TableMeta meta, Schema schema, Path path, FileStatus status) throws IOException { Fragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); return getScanner(meta, schema, fragment, null); }
public Scanner getFileScanner(TableMeta meta, Schema schema, Path path, FileStatus status) throws IOException { FileFragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); return getScanner(meta, schema, fragment); }
@Override public FileFragment deserialize(FileFragmentProto proto) { return new FileFragment( proto.getId(), new Path(proto.getPath()), proto.getStartOffset(), proto.getLength(), proto.getHostsList().toArray(new String[proto.getHostsCount()]), proto.getDiskIdsList().toArray(new Integer[proto.getDiskIdsCount()])); } }
@Test public final void testCompareTo2() { final int num = 1860; FileFragment[] tablets = new FileFragment[num]; for (int i = num - 1; i >= 0; i--) { tablets[i] = new FileFragment("tablet1_"+i, new Path(path, "tablet0"), (long)i * 6553500, (long)(i+1) * 6553500); } SortedSet sortedSet = Sets.newTreeSet(); Collections.addAll(sortedSet, tablets); assertEquals(num, sortedSet.size()); } }
private static final FileFragment getFileFragment(String fileName) throws IOException { TajoConf conf = new TajoConf(); Path tablePath = new Path(getResourcePath("dataset", "TestDelimitedTextFile"), fileName); FileSystem fs = FileSystem.getLocal(conf); FileStatus status = fs.getFileStatus(tablePath); return new FileFragment("table", tablePath, 0, status.getLen()); }
@Test public final void testCompareTo() { final int num = 10; FileFragment[] tablets = new FileFragment[num]; for (int i = num - 1; i >= 0; i--) { tablets[i] = new FileFragment("tablet1_"+i, new Path(path, "tablet0"), i * 500, (i+1) * 500); } Arrays.sort(tablets); for(int i = 0; i < num; i++) { assertEquals("tablet1_"+i, tablets[i].getInputSourceId()); } }
@Test public final void testGetAndSetFields() { FileFragment fragment1 = new FileFragment("table1_1", new Path(path, "table0"), 0, 500); assertEquals("table1_1", fragment1.getInputSourceId()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); assertTrue(500 == fragment1.getLength()); }
@Test public final void testGetProtoAndRestore() { FileFragment fragment = new FileFragment("table1_1", new Path(path, "table0"), 0, 500); FileFragment fragment1 = FragmentConvertor.convert(conf, BuiltinFragmentKinds.FILE, FragmentConvertor.toFragmentProto(conf, fragment)); assertEquals("table1_1", fragment1.getInputSourceId()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); assertTrue(500 == fragment1.getLength()); }