/** * Returns the transaction ID. * @return the transaction ID */ public String getId() { return Optional.ofNullable(transactionId) .orElseThrow(() -> new CommandConfigurationException( "transaction ID (execution ID of the target jobflow) must be specified")); } }
private Optional<org.apache.hadoop.fs.FileStatus> stat(org.apache.hadoop.fs.Path path) { try { return Optional.of(dataSourceParameter.getHadoopFileSystem(path).getFileStatus(path)); } catch (FileNotFoundException e) { LOG.trace("not found: {}", path, e); return Optional.empty(); } catch (IOException e) { throw new CommandConfigurationException(MessageFormat.format( "error occurred while resolving Hadoop path: {0}", path), e); } }
private Optional<FileStatus> stat(Path path) { try { return Optional.of(dataSourceParameter.getHadoopFileSystem(path).getFileStatus(path)); } catch (FileNotFoundException e) { LOG.trace("not found: {}", path, e); return Optional.empty(); } catch (IOException e) { throw new CommandConfigurationException(MessageFormat.format( "error occurred while resolving Hadoop path: {0}", path), e); } }
/** * Returns the path. * @return the path * @throws CommandConfigurationException if there is no available batch application base directory */ public Path getPath() { if (path == null) { throw new CommandConfigurationException(MessageFormat.format( "please set environment variable \"{0}\", or specify {1} /path/to/batchapps to command line", ENV_ASAKUSA_HOME, OPT_BATCHAPPS)); } Path result = LocalPath.of(path); if (Files.isDirectory(result) == false) { throw new CommandConfigurationException(MessageFormat.format( "batch applications directory is not found: {0}", result)); } return result; }
/** * Returns the Hadoop file system for the path. * @param path the target path * @return the corresponded file system object */ public org.apache.hadoop.fs.FileSystem getHadoopFileSystem(org.apache.hadoop.fs.Path path) { try { return path.getFileSystem(getConfiguration()); } catch (IOException e) { throw new CommandConfigurationException(MessageFormat.format( "error occurred while resolving Hadoop path: {0} ({1})", path, Optional.ofNullable(e.getMessage()).orElseGet(() -> e.toString())), e); } }
private java.nio.file.Path getDestination() { java.nio.file.Path destination = localPathParameter.resolve(paths.get(paths.size() - 1)); if (overwriteParameter.isEnabled() == false && Files.isRegularFile(destination)) { throw new CommandConfigurationException(MessageFormat.format( "destination file already exists: {0}", destination)); } return destination; }
/** * Returns the target jobflow, and must be it is unique in the current context. * @return the target jobflow * @throws CommandConfigurationException if there are no available jobflows, or they are ambiguous */ public JobflowInfo getUniqueJobflow() { List<JobflowInfo> candidates = getJobflows(); if (candidates.size() != 1) { throw new CommandConfigurationException(MessageFormat.format( "target jobflow is ambiguous, please specify \"--jobflow <flow-ID>\": '{'{0}'}'", candidates.stream() .map(JobflowInfo::getId) .collect(Collectors.joining(", ")))); } return candidates.get(0); } }
@Override public void run() { LOG.debug("starting {}", getClass().getSimpleName()); if (paths.isEmpty()) { throw new CommandConfigurationException("no target paths are specified"); } List<Path> hpaths = paths.stream() .map(dataSourceParameter::resolveAsHadoopPath) .collect(Collectors.toList()); try (PrintWriter writer = outputParameter.open()) { executorParameter.execute(hpaths.stream() .map(path -> (Task) context -> mkdir(writer, path)) .collect(Collectors.toList())); } }
@Override public void run() { LOG.debug("starting {}", getClass().getSimpleName()); String id = transactionIdParameter.getId(); TransactionInfo spec = transactionEditorParameter.find(id) .orElseThrow(() -> new CommandConfigurationException(MessageFormat.format( "transaction \"{0}\" is not found (may be already completed)", id))); try (PrintWriter writer = outputParameter.open()) { print(writer, spec, 0); } }
/** * Loads the given information file. * @param path the target information file path * @return the loaded information * @throws CommandConfigurationException if error occurred while loading the information file */ public static BatchInfo load(Path path) { ObjectMapper mapper = new ObjectMapper(); try { return mapper.readValue(path.toFile(), BatchInfo.class); } catch (IOException e) { throw new CommandConfigurationException(MessageFormat.format( "failed to load DSL information file: {0}", path), e); } }
private void validate(List<ResourceInfo> files, Path destination) { Set<Path> ancestors = new HashSet<>(); for (Path path = qualify(destination); path != null; path = path.getParent()) { ancestors.add(path); } for (ResourceInfo file : files) { Path source = qualify(asHadoopPath(file.getPath())); LOG.debug("validate: {} -> {}", source, destination); if (ancestors.contains(source)) { throw new CommandConfigurationException(MessageFormat.format( "cannot copy directory into its sub-directories: {0} -> {1}", source, destination)); } } }
/** * Returns the information file path. * @return the information file path * @throws CommandConfigurationException if it is not found */ public Path getPath() { if (location == null) { throw new CommandConfigurationException(MessageFormat.format( "target batch ID must be specified ({0})", getAvailableApplicationsMessage())); } Path base = batchappsParameter.findPath().orElse(null); return findInfo(base, location) .orElseThrow(() -> new CommandConfigurationException(MessageFormat.format( "batch application \"{0}\" is not found ({1})", location, getAvailableApplicationsMessage()))); }
/** * Returns the target tables and their locations. * @return target tables and locations */ public Map<TableInfo, Set<LocationInfo>> collect() { Map<TableInfo, Set<LocationInfo>> candidates = collect0(direction) .filter(it -> getTableNamePattern().matcher(it.getTable().getName()).matches()) .collect(Collectors.groupingBy( TableLocationInfo::getTable, Collectors.mapping(TableLocationInfo::getLocation, Collectors.toSet()))); if (candidates.isEmpty()) { if (collect0(direction).findAny().isPresent() == false) { throw new CommandConfigurationException("there are no available tables for pattern"); } else { throw new CommandConfigurationException(MessageFormat.format( "there are no available tables for pattern: {0}", tableNamePatternString)); } } return candidates; }
/** * Returns the target flow-part operator view. * @param jobflow the owner jobflow * @param graph the target graph * @return the target operator view * @throws CommandConfigurationException if there is no such a flow-part */ public OperatorView select(JobflowInfo jobflow, OperatorGraphView graph) { if (name == null) { throw new IllegalStateException(); } return findFlowPart(graph, name) .orElseThrow(() -> new CommandConfigurationException(MessageFormat.format( "there are no flow part named \"{1}\" in jobflow {0}", Optional.ofNullable(jobflow.getDescriptionClass()) .orElse(jobflow.getId()), name))); }
private List<ResourceInfo> list() { if (paths.isEmpty()) { List<DataSourceInfo> all = dataSourceParameter.getDataSourceInfo() .map(Collections::singletonList) .orElseGet(dataSourceParameter::getAllDataSourceInfo); if (all.isEmpty()) { throw new CommandConfigurationException("there are no available Direct I/O data sources"); } return all.stream() .map(info -> new DirectIoPath(info, info.getPath(), ALL)) .flatMap(it -> list(it).stream()) .collect(Collectors.toList()); } else { return paths.stream() .flatMap(path -> list(dataSourceParameter.resolve(path)).stream()) .collect(Collectors.toList()); } }
@Override public void run() { LOG.debug("starting {}", getClass().getSimpleName()); List<HiveCreateTable> statements = getStatements(); if (statements.isEmpty()) { throw new CommandConfigurationException("there are no available tables to generate DDL"); } try (PrintWriter writer = outputParameter.open()) { for (HiveCreateTable statement : statements) { HiveQlEmitter.emit(statement, writer); writer.printf("%s%n%n", STATEMENT_SEPARATOR); } } catch (IOException e) { throw new CommandException("error occurred while generating Hive DDL", e); } }
private List<DirectIoPath> getSources() { List<DirectIoPath> dpaths = paths.subList(0, paths.size() - 1).stream() .map(dataSourceParameter::resolve) .peek(it -> { if (it.isComponentRoot()) { throw new CommandConfigurationException(MessageFormat.format( "cannot copy data source root \"{0}\"", it)); } if (it.getSource().getEntity().findProperty(HadoopDataSourceCore.class).isPresent() == false) { throw new CommandConfigurationException(MessageFormat.format( "unsupported data source \"{0}\" (type: {1}): {2}", it.getSource().getId(), it.getSource().getEntity().getClass().getName(), it)); } }) .collect(Collectors.toList()); return dpaths; }
private List<DirectIoPath> getSources() { List<DirectIoPath> dpaths = paths.subList(0, paths.size() - 1).stream() .map(dataSourceParameter::resolve) .peek(it -> { if (it.isComponentRoot()) { throw new CommandConfigurationException(MessageFormat.format( "cannot copy data source root \"{0}\"", it)); } if (it.getSource().getEntity().findProperty(HadoopDataSourceCore.class).isPresent() == false) { throw new CommandConfigurationException(MessageFormat.format( "unsupported data source \"{0}\" (type: {1}): {2}", it.getSource().getId(), it.getSource().getEntity().getClass().getName(), it)); } }) .collect(Collectors.toList()); return dpaths; }
/** * Returns the target vertex. * @param jobflow the owner jobflow * @param plan the execution plan * @return the target vertex * @throws CommandConfigurationException if there is no such a vertex */ public OperatorView select(JobflowInfo jobflow, OperatorGraphView plan) { if (name == null) { throw new IllegalStateException(); } return plan.getOperators(OperatorKind.PLAN_VERTEX).stream() .filter(it -> ((PlanVertexSpec) it.getSpec()).getName().equals(name)) .findFirst() .orElseThrow(() -> new CommandConfigurationException(MessageFormat.format( "there are no vertex named \"{1}\" in jobflow {0}", Optional.ofNullable(jobflow.getDescriptionClass()) .orElse(jobflow.getId()), name))); } }
@Override public void run() { LOG.debug("start run command"); ExecutionContext context = contextParameter.getExecutionContext(); BatchInfo workflow = workflowParameter.getBatchInfo(context); BatchExecutor executor = executorParameter.getBatchExecutor(context); Map<String, String> arguments = applicationParameter.getBatchArguments(); try { executor.execute(context, workflow, arguments); } catch (ExecutionConditionException e) { throw new CommandConfigurationException(MessageFormat.format( "cannot execute batch \"{0}\" ({1})", workflow.getId(), e.getMessage()), e); } catch (IOException | InterruptedException e) { throw new CommandExecutionException(MessageFormat.format( "executing batch \"{0}\" was failed", workflow.getId()), e); } }