@Override public void onDatasetCompactionCompletion(Dataset dataset) throws Exception { for (HiveSpec spec : this.hiveRegistrationPolicy.getHiveSpecs(dataset.outputPath())) { this.hiveRegister.register(spec); } } }
/** * Register the given {@link Path}s. * * @param paths The {@link Path}s to be registered. * @param state A {@link State} which will be used to instantiate a {@link HiveRegister} and a * {@link HiveRegistrationPolicy} for registering the given The {@link Path}s. */ public static void register(Iterable<String> paths, State state) throws IOException { try (HiveRegister hiveRegister = HiveRegister.get(state)) { HiveRegistrationPolicy policy = HiveRegistrationPolicyBase.getPolicy(state); for (String path : paths) { for (HiveSpec spec : policy.getHiveSpecs(new Path(path))) { hiveRegister.register(spec); } } } } }
public void onCompactionJobComplete(FileSystemDataset dataset) throws IOException { if (state.contains(ConfigurationKeys.HIVE_REGISTRATION_POLICY)) { HiveRegister hiveRegister = HiveRegister.get(state); HiveRegistrationPolicy hiveRegistrationPolicy = HiveRegistrationPolicyBase.getPolicy(state); CompactionPathParser.CompactionParserResult result = new CompactionPathParser(state).parse(dataset); List<String> paths = new ArrayList<>(); for (HiveSpec spec : hiveRegistrationPolicy.getHiveSpecs(new Path(result.getDstAbsoluteDir()))) { hiveRegister.register(spec); paths.add(spec.getPath().toUri().toASCIIString()); log.info("Hive registration is done for {}", result.getDstAbsoluteDir()); } // submit events for hive registration if (eventSubmitter != null) { Map<String, String> eventMetadataMap = ImmutableMap.of(CompactionSlaEventHelper.DATASET_URN, dataset.datasetURN(), CompactionSlaEventHelper.HIVE_REGISTRATION_PATHS, Joiner.on(',').join(paths)); this.eventSubmitter.submit(CompactionSlaEventHelper.COMPACTION_HIVE_REGISTRATION_EVENT, eventMetadataMap); } } }
@Override public void execute() throws IOException { if (this.verifyBeforeRegistering) { if (!this.hiveSpec.getTable().getLocation().isPresent()) { throw getException("Table does not have a location parameter."); } Path tablePath = new Path(this.hiveSpec.getTable().getLocation().get()); FileSystem fs = this.hiveSpec.getPath().getFileSystem(new Configuration()); if (!fs.exists(tablePath)) { throw getException(String.format("Table location %s does not exist.", tablePath)); } if (this.hiveSpec.getPartition().isPresent()) { if (!this.hiveSpec.getPartition().get().getLocation().isPresent()) { throw getException("Partition does not have a location parameter."); } Path partitionPath = new Path(this.hiveSpec.getPartition().get().getLocation().get()); if (!fs.exists(this.hiveSpec.getPath())) { throw getException(String.format("Partition location %s does not exist.", partitionPath)); } } } try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { log.info("Registering Hive Spec " + this.hiveSpec); ListenableFuture<Void> future = hiveRegister.register(this.hiveSpec); future.get(); } catch (InterruptedException | ExecutionException ie) { throw new IOException("Hive registration was interrupted.", ie); } }
try { for (HiveSpec spec : completionService.take().get()) { this.hiveRegister.register(spec);
@Override public void onDatasetCompactionCompletion(Dataset dataset) throws Exception { for (HiveSpec spec : this.hiveRegistrationPolicy.getHiveSpecs(dataset.outputPath())) { this.hiveRegister.register(spec); } } }
/** * Register the given {@link Path}s. * * @param paths The {@link Path}s to be registered. * @param state A {@link State} which will be used to instantiate a {@link HiveRegister} and a * {@link HiveRegistrationPolicy} for registering the given The {@link Path}s. */ public static void register(Iterable<String> paths, State state) throws IOException { try (HiveRegister hiveRegister = HiveRegister.get(state)) { HiveRegistrationPolicy policy = HiveRegistrationPolicyBase.getPolicy(state); for (String path : paths) { for (HiveSpec spec : policy.getHiveSpecs(new Path(path))) { hiveRegister.register(spec); } } } } }
public void onCompactionJobComplete(FileSystemDataset dataset) throws IOException { if (state.contains(ConfigurationKeys.HIVE_REGISTRATION_POLICY)) { HiveRegister hiveRegister = HiveRegister.get(state); HiveRegistrationPolicy hiveRegistrationPolicy = HiveRegistrationPolicyBase.getPolicy(state); CompactionPathParser.CompactionParserResult result = new CompactionPathParser(state).parse(dataset); List<String> paths = new ArrayList<>(); for (HiveSpec spec : hiveRegistrationPolicy.getHiveSpecs(new Path(result.getDstAbsoluteDir()))) { hiveRegister.register(spec); paths.add(spec.getPath().toUri().toASCIIString()); log.info("Hive registration is done for {}", result.getDstAbsoluteDir()); } // submit events for hive registration if (eventSubmitter != null) { Map<String, String> eventMetadataMap = ImmutableMap.of(CompactionSlaEventHelper.DATASET_URN, dataset.datasetURN(), CompactionSlaEventHelper.HIVE_REGISTRATION_PATHS, Joiner.on(',').join(paths)); this.eventSubmitter.submit(CompactionSlaEventHelper.COMPACTION_HIVE_REGISTRATION_EVENT, eventMetadataMap); } } }
@Override public void execute() throws IOException { if (this.verifyBeforeRegistering) { if (!this.hiveSpec.getTable().getLocation().isPresent()) { throw getException("Table does not have a location parameter."); } Path tablePath = new Path(this.hiveSpec.getTable().getLocation().get()); FileSystem fs = this.hiveSpec.getPath().getFileSystem(new Configuration()); if (!fs.exists(tablePath)) { throw getException(String.format("Table location %s does not exist.", tablePath)); } if (this.hiveSpec.getPartition().isPresent()) { if (!this.hiveSpec.getPartition().get().getLocation().isPresent()) { throw getException("Partition does not have a location parameter."); } Path partitionPath = new Path(this.hiveSpec.getPartition().get().getLocation().get()); if (!fs.exists(this.hiveSpec.getPath())) { throw getException(String.format("Partition location %s does not exist.", partitionPath)); } } } try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { log.info("Registering Hive Spec " + this.hiveSpec); ListenableFuture<Void> future = hiveRegister.register(this.hiveSpec); future.get(); } catch (InterruptedException | ExecutionException ie) { throw new IOException("Hive registration was interrupted.", ie); } }
try { for (HiveSpec spec : completionService.take().get()) { this.hiveRegister.register(spec);