public SimpleDatasetHierarchicalPrioritizer(State state) throws IOException { super(SimpleDatasetHierarchicalPrioritizer .createRequestorComparator(state), new AllEqualComparator()); }
public SimpleHiveDatasetTieringPrioritizer(Properties properties) throws IOException { super(SimpleHiveDatasetTieringPrioritizer.createRequestorComparator(properties), new AllEqualComparator<FileSet<CopyEntity>>()); }
/** * Creates a new {@link ConcurrentBoundedWorkUnitList}. * @param maxSize Maximum number of {@link WorkUnit}s to contain. * @param comparator {@link Comparator} for {@link org.apache.gobblin.data.management.partition.FileSet}s to use for {@link org.apache.gobblin.data.management.partition.FileSet} priority. * @param strictLimitMultiplier the list will only start rejecting {@link WorkUnit}s if its capacity exceeds * maxSize * strictLimitMultiplier. If this parameter is < 1, it will be auto-set to 1. */ @Builder public ConcurrentBoundedWorkUnitList(int maxSize, final Comparator<FileSet<CopyEntity>> comparator, double strictLimitMultiplier) { this.currentSize = 0; this.maxSize = maxSize; double actualStrictLimitMultiplier = Math.min((Integer.MAX_VALUE / (double) this.maxSize), Math.max(1.0, strictLimitMultiplier)); this.strictMaxSize = (int) (this.maxSize * actualStrictLimitMultiplier); this.comparator = comparator == null ? new AllEqualComparator<FileSet<CopyEntity>>() : comparator; this.workUnitsMap = new TreeMap<>(new AugmentedComparator(this.comparator)); this.rejectedFileSet = false; }
@Test public void testBoundedAdd() throws IOException { ConcurrentBoundedWorkUnitList list = new ConcurrentBoundedWorkUnitList(10, new AllEqualComparator<FileSet<CopyEntity>>(), 1); Assert.assertTrue(addFiles(list, "fs", 6)); Assert.assertFalse(list.hasRejectedFileSet()); Assert.assertFalse(addFiles(list, "fs", 5)); Assert.assertTrue(list.hasRejectedFileSet()); Assert.assertTrue(addFiles(list, "fs", 4)); Assert.assertFalse(addFiles(list, "fs", 1)); }
@Test public void testNonStrictBoundAdd() throws IOException { ConcurrentBoundedWorkUnitList list = new ConcurrentBoundedWorkUnitList(10, new AllEqualComparator<FileSet<CopyEntity>>(), 2.0); Assert.assertTrue(addFiles(list, "fs", 6)); Assert.assertFalse(list.hasRejectedFileSet()); Assert.assertFalse(list.isFull()); Assert.assertTrue(addFiles(list, "fs", 5)); Assert.assertFalse(list.hasRejectedFileSet()); Assert.assertTrue(list.isFull()); Assert.assertTrue(addFiles(list, "fs", 4)); Assert.assertFalse(list.hasRejectedFileSet()); Assert.assertFalse(addFiles(list, "fs", 6)); Assert.assertTrue(list.hasRejectedFileSet()); Assert.assertTrue(list.isFull()); }
public SimpleHiveDatasetTieringPrioritizer(Properties properties) throws IOException { super(SimpleHiveDatasetTieringPrioritizer.createRequestorComparator(properties), new AllEqualComparator<FileSet<CopyEntity>>()); }
public SimpleDatasetHierarchicalPrioritizer(State state) throws IOException { super(SimpleDatasetHierarchicalPrioritizer .createRequestorComparator(state), new AllEqualComparator()); }
/** * Creates a new {@link ConcurrentBoundedWorkUnitList}. * @param maxSize Maximum number of {@link WorkUnit}s to contain. * @param comparator {@link Comparator} for {@link org.apache.gobblin.data.management.partition.FileSet}s to use for {@link org.apache.gobblin.data.management.partition.FileSet} priority. * @param strictLimitMultiplier the list will only start rejecting {@link WorkUnit}s if its capacity exceeds * maxSize * strictLimitMultiplier. If this parameter is < 1, it will be auto-set to 1. */ @Builder public ConcurrentBoundedWorkUnitList(int maxSize, final Comparator<FileSet<CopyEntity>> comparator, double strictLimitMultiplier) { this.currentSize = 0; this.maxSize = maxSize; double actualStrictLimitMultiplier = Math.min((Integer.MAX_VALUE / (double) this.maxSize), Math.max(1.0, strictLimitMultiplier)); this.strictMaxSize = (int) (this.maxSize * actualStrictLimitMultiplier); this.comparator = comparator == null ? new AllEqualComparator<FileSet<CopyEntity>>() : comparator; this.workUnitsMap = new TreeMap<>(new AugmentedComparator(this.comparator)); this.rejectedFileSet = false; }