VersionSelectionPolicy<T> selectionPolicy = versionFinderAndPolicy.getVersionSelectionPolicy(); VersionFinder<? extends T> versionFinder = versionFinderAndPolicy.getVersionFinder(); allVersions.add(ver); for (RetentionAction retentionAction : versionFinderAndPolicy.getRetentionActions()) { try { retentionAction.execute(allVersions);
@SuppressWarnings("unchecked") public VersionFinderAndPolicy<T> build() { VersionSelectionPolicy<T> localVersionSelectionPolicy; List<RetentionAction> localRetentionActions; if (this.versionSelectionPolicy == null) { localVersionSelectionPolicy = (VersionSelectionPolicy<T>) new SelectNothingPolicy(new Properties()); } else { localVersionSelectionPolicy = this.versionSelectionPolicy; } if (this.retentionActions == null) { localRetentionActions = Lists.newArrayList(); } else { localRetentionActions = Lists.newArrayList(this.retentionActions); } return new VersionFinderAndPolicy<T>(localVersionSelectionPolicy, this.versionFinder, localRetentionActions); } }
private void initWithSelectionPolicy(Config config, Properties jobProps) { String selectionPolicyKey = StringUtils.substringAfter(SELECTION_POLICY_CLASS_KEY, CONFIGURATION_KEY_PREFIX); String versionFinderKey = StringUtils.substringAfter(VERSION_FINDER_CLASS_KEY, CONFIGURATION_KEY_PREFIX); Preconditions.checkArgument( config.hasPath(versionFinderKey), String.format("Version finder class is required at %s in config %s", versionFinderKey, config.root().render(ConfigRenderOptions.concise()))); VersionFinderAndPolicyBuilder<T> builder = VersionFinderAndPolicy.builder(); builder.versionFinder(createVersionFinder(config.getString(versionFinderKey), config, jobProps)); if (config.hasPath(selectionPolicyKey)) { builder.versionSelectionPolicy(createSelectionPolicy( ConfigUtils.getString(config, selectionPolicyKey, SelectNothingPolicy.class.getName()), config, jobProps)); } for (Class<? extends RetentionActionFactory> factoryClass : RETENTION_ACTION_TYPES) { try { RetentionActionFactory factory = factoryClass.newInstance(); if (factory.canCreateWithConfig(config)) { builder.retentionAction((RetentionAction) factory.createRetentionAction(config, this.fs, ConfigUtils.propertiesToConfig(jobProps))); } } catch (InstantiationException | IllegalAccessException e) { Throwables.propagate(e); } } this.versionFindersAndPolicies.add(builder.build()); }
VersionSelectionPolicy<T> selectionPolicy = versionFinderAndPolicy.getVersionSelectionPolicy(); VersionFinder<? extends T> versionFinder = versionFinderAndPolicy.getVersionFinder(); allVersions.add(ver); for (RetentionAction retentionAction : versionFinderAndPolicy.getRetentionActions()) { try { retentionAction.execute(allVersions);
private void initWithSelectionPolicy(Config config, Properties jobProps) { String selectionPolicyKey = StringUtils.substringAfter(SELECTION_POLICY_CLASS_KEY, CONFIGURATION_KEY_PREFIX); String versionFinderKey = StringUtils.substringAfter(VERSION_FINDER_CLASS_KEY, CONFIGURATION_KEY_PREFIX); Preconditions.checkArgument( config.hasPath(versionFinderKey), String.format("Version finder class is required at %s in config %s", versionFinderKey, config.root().render(ConfigRenderOptions.concise()))); VersionFinderAndPolicyBuilder<T> builder = VersionFinderAndPolicy.builder(); builder.versionFinder(createVersionFinder(config.getString(versionFinderKey), config, jobProps)); if (config.hasPath(selectionPolicyKey)) { builder.versionSelectionPolicy(createSelectionPolicy( ConfigUtils.getString(config, selectionPolicyKey, SelectNothingPolicy.class.getName()), config, jobProps)); } for (Class<? extends RetentionActionFactory> factoryClass : RETENTION_ACTION_TYPES) { try { RetentionActionFactory factory = factoryClass.newInstance(); if (factory.canCreateWithConfig(config)) { builder.retentionAction((RetentionAction) factory.createRetentionAction(config, this.fs, ConfigUtils.propertiesToConfig(jobProps))); } } catch (InstantiationException | IllegalAccessException e) { Throwables.propagate(e); } } this.versionFindersAndPolicies.add(builder.build()); }
@SuppressWarnings("unchecked") public VersionFinderAndPolicy<T> build() { VersionSelectionPolicy<T> localVersionSelectionPolicy; List<RetentionAction> localRetentionActions; if (this.versionSelectionPolicy == null) { localVersionSelectionPolicy = (VersionSelectionPolicy<T>) new SelectNothingPolicy(new Properties()); } else { localVersionSelectionPolicy = this.versionSelectionPolicy; } if (this.retentionActions == null) { localRetentionActions = Lists.newArrayList(); } else { localRetentionActions = Lists.newArrayList(this.retentionActions); } return new VersionFinderAndPolicy<T>(localVersionSelectionPolicy, this.versionFinder, localRetentionActions); } }
@Override public List<VersionFinderAndPolicy<T>> getVersionFindersAndPolicies() { return ImmutableList .<VersionFinderAndPolicy<T>> of(new VersionFinderAndPolicy<>(getRetentionPolicy(), getVersionFinder())); } }
private void initWithRetentionPolicy(Config config, Properties jobProps, String retentionPolicyKey, String versionFinderKey) { this.versionFindersAndPolicies .add(new VersionFinderAndPolicy<>(createRetentionPolicy(config.getString(retentionPolicyKey), config, jobProps), createVersionFinder(config.getString(versionFinderKey), config, jobProps))); }
@Test public void testConfigureWithMulitplePolicies() throws Exception { Map<String, String> partitionConf = ImmutableMap.<String, String> of("version.finder.class", "org.apache.gobblin.data.management.version.finder.WatermarkDatasetVersionFinder", "selection.policy.class", "org.apache.gobblin.data.management.policy.NewestKSelectionPolicy", "selection.newestK.versionsSelected", "2"); Config conf = ConfigFactory.parseMap(ImmutableMap.<String, List<Map<String, String>>> of("gobblin.retention.dataset.partitions", ImmutableList.of(partitionConf, partitionConf))); ConfigurableCleanableDataset<FileSystemDatasetVersion> dataset = new ConfigurableCleanableDataset<FileSystemDatasetVersion>(FileSystem.get(new URI(ConfigurationKeys.LOCAL_FS_URI), new Configuration()), new Properties(), new Path("/someroot"), conf, LoggerFactory.getLogger(ConfigurableCleanableDatasetTest.class)); Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(0).getVersionSelectionPolicy().getClass(), NewestKSelectionPolicy.class); Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(0).getVersionFinder().getClass(), WatermarkDatasetVersionFinder.class); Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(1).getVersionSelectionPolicy().getClass(), NewestKSelectionPolicy.class); Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(1).getVersionFinder().getClass(), WatermarkDatasetVersionFinder.class); Assert.assertEquals(dataset.isDatasetBlacklisted(), false); }
@Test public void testConfigureWithSelectionPolicy() throws Exception { Config conf = ConfigFactory.parseMap(ImmutableMap.<String, String> of("gobblin.retention.version.finder.class", "org.apache.gobblin.data.management.version.finder.WatermarkDatasetVersionFinder", "gobblin.retention.selection.policy.class", "org.apache.gobblin.data.management.policy.NewestKSelectionPolicy", "gobblin.retention.selection.newestK.versionsSelected", "2")); ConfigurableCleanableDataset<FileSystemDatasetVersion> dataset = new ConfigurableCleanableDataset<FileSystemDatasetVersion>(FileSystem.get(new URI(ConfigurationKeys.LOCAL_FS_URI), new Configuration()), new Properties(), new Path("/someroot"), conf, LoggerFactory.getLogger(ConfigurableCleanableDatasetTest.class)); Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(0).getVersionSelectionPolicy().getClass(), NewestKSelectionPolicy.class); Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(0).getVersionFinder().getClass(), WatermarkDatasetVersionFinder.class); Assert.assertEquals(dataset.isDatasetBlacklisted(), false); }
@Test public void testConfigureWithRetentionPolicy() throws Exception { Config conf = ConfigFactory.parseMap(ImmutableMap.<String, String> of("gobblin.retention.version.finder.class", "org.apache.gobblin.data.management.version.finder.WatermarkDatasetVersionFinder", "gobblin.retention.retention.policy.class", "org.apache.gobblin.data.management.retention.policy.NewestKRetentionPolicy", "gobblin.retention.newestK.versions.retained", "2")); ConfigurableCleanableDataset<FileSystemDatasetVersion> dataset = new ConfigurableCleanableDataset<FileSystemDatasetVersion>(FileSystem.get(new URI(ConfigurationKeys.LOCAL_FS_URI), new Configuration()), new Properties(), new Path("/someroot"), conf, LoggerFactory.getLogger(ConfigurableCleanableDatasetTest.class)); Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(0).getVersionSelectionPolicy().getClass(), EmbeddedRetentionSelectionPolicy.class); Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(0).getVersionFinder().getClass(), WatermarkDatasetVersionFinder.class); Assert.assertEquals(dataset.isDatasetBlacklisted(), false); }
@Override public List<VersionFinderAndPolicy<T>> getVersionFindersAndPolicies() { return ImmutableList .<VersionFinderAndPolicy<T>> of(new VersionFinderAndPolicy<>(getRetentionPolicy(), getVersionFinder())); } }
private void initWithRetentionPolicy(Config config, Properties jobProps, String retentionPolicyKey, String versionFinderKey) { this.versionFindersAndPolicies .add(new VersionFinderAndPolicy<>(createRetentionPolicy(config.getString(retentionPolicyKey), config, jobProps), createVersionFinder(config.getString(versionFinderKey), config, jobProps))); }