How to use

ApplicationFile

in

com.yahoo.config.application.api

Map<String, ExpressionFunction> readExpressions() {
  Map<String, ExpressionFunction> expressions = new HashMap<>();
  ApplicationFile expressionPath = application.getFile(modelFiles.expressionsPath());
  if ( ! expressionPath.exists() || ! expressionPath.isDirectory()) return Collections.emptyMap();
  for (ApplicationFile expressionFile : expressionPath.listFiles()) {
    try (BufferedReader reader = new BufferedReader(expressionFile.createReader())){
      String name = expressionFile.getPath().getName();
      expressions.put(name, readExpression(name, reader));
    }
    catch (IOException e) {
      throw new UncheckedIOException("Failed reading " + expressionFile.getPath(), e);
    }
    catch (ParseException e) {
      throw new IllegalStateException("Invalid stored expression in " + expressionFile, e);
    }
  }
  return expressions;
}

/**
 * List the files in this directory, optionally list files for subdirectories recursively as well.
 *
 * @param recurse Set to true if all files in the directory tree should be returned.
 * @return a list of files in this directory.
 */
public List<ApplicationFile> listFiles(boolean recurse) {
  List<ApplicationFile> ret = new ArrayList<>();
  List<ApplicationFile> files = listFiles();
  ret.addAll(files);
  if (recurse) {
    for (ApplicationFile file : files) {
      if (file.isDirectory()) {
        ret.addAll(file.listFiles(recurse));
      }
    }
  }
  return ret;
}

private Collection<Element> getPermanentServices(DeployState deployState) throws IOException, SAXException {
  List<Element> permanentServices = new ArrayList<>();
  Optional<ApplicationPackage> applicationPackage = deployState.getPermanentApplicationPackage();
  if (applicationPackage.isPresent()) {
    ApplicationFile file = applicationPackage.get().getFile(Path.fromString(ApplicationPackage.PERMANENT_SERVICES));
    if (file.exists()) {
      try (Reader reader = file.createReader()) {
        Element permanentServicesRoot = getServicesFromReader(reader);
        permanentServices.addAll(getServiceElements(permanentServicesRoot));
      }
    }
  }
  return permanentServices;
}

/**
 * Reads the information about all the large (aka ranking) constants stored in the application package
 * (the constant value itself is replicated with file distribution).
 */
List<RankingConstant> readLargeConstants() {
  try {
    List<RankingConstant> constants = new ArrayList<>();
    for (ApplicationFile constantFile : application.getFile(modelFiles.largeConstantsInfoPath()).listFiles()) {
      String[] parts = IOUtils.readAll(constantFile.createReader()).split(":");
      constants.add(new RankingConstant(parts[0], TensorType.fromSpec(parts[1]), parts[2]));
    }
    return constants;
  }
  catch (IOException e) {
    throw new UncheckedIOException(e);
  }
}

/**
 * Creates a rank profile not attached to any search definition, for each imported model in the application package,
 * and adds it to the given rank profile registry.
 */
private void createGlobalRankProfiles(DeployLogger deployLogger, ImportedMlModels importedModels,
                   RankProfileRegistry rankProfileRegistry,
                   QueryProfiles queryProfiles) {
  if ( ! importedModels.all().isEmpty()) { // models/ directory is available
    for (ImportedMlModel model : importedModels.all()) {
      RankProfile profile = new RankProfile(model.name(), this, rankProfileRegistry);
      rankProfileRegistry.add(profile);
      ConvertedModel convertedModel = ConvertedModel.fromSource(new ModelName(model.name()),
                                   model.name(), profile, queryProfiles.getRegistry(), model);
      convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false));
    }
  }
  else { // generated and stored model information may be available instead
    ApplicationFile generatedModelsDir = applicationPackage.getFile(ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR);
    for (ApplicationFile generatedModelDir : generatedModelsDir.listFiles()) {
      String modelName = generatedModelDir.getPath().last();
      if (modelName.contains(".")) continue; // Name space: Not a global profile
      RankProfile profile = new RankProfile(modelName, this, rankProfileRegistry);
      rankProfileRegistry.add(profile);
      ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile);
      convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false));
    }
  }
  new Processing().processRankProfiles(deployLogger, rankProfileRegistry, queryProfiles, true, false);
}

private Optional<Reader> optionalFile(Path filePath) {
  try {
    return Optional.of(getFile(filePath).createReader());
  } catch (FileNotFoundException e) {
    return Optional.empty();
  }
}

@Override
public int compareTo(ApplicationFile other) {
  return this.getPath().getName().compareTo((other).getPath().getName());
}

/**
 * List the files under this directory. If this is file, an empty list is returned.
 * Only immediate files/subdirectories are returned.
 *
 * @return a list of files in this directory.
 */
public List<ApplicationFile> listFiles() {
  return listFiles(defaultFilter);
}

/**
 * Adds this expression to the application package, such that it can be read later.
 *
 * @param name the name of this ranking expression - may have 1-3 parts separated by dot where the first part
 *             is always the model name
 */
void writeExpression(String name, ExpressionFunction expression) {
  StringBuilder b = new StringBuilder(expression.getBody().getRoot().toString());
  for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet())
    b.append('\n').append(input.getKey()).append('\t').append(input.getValue());
  application.getFile(modelFiles.expressionPath(name)).writeFile(new StringReader(b.toString()));
}

/** Adds this function expression to the application package so it can be read later. */
public void writeFunction(String name, RankingExpression expression) {
  application.getFile(modelFiles.functionsPath()).appendFile(name + "\t" +
                                expression.getRoot().toString() + "\n");
}

/** Returns whether a model store for this application and model name exists */
public boolean exists() {
  return application.getFile(modelFiles.storedModelReplicatedPath()).exists();
}

/** Reads the previously stored function expressions for these arguments */
List<Pair<String, RankingExpression>> readFunctions() {
  try {
    ApplicationFile file = application.getFile(modelFiles.functionsPath());
    if ( ! file.exists()) return Collections.emptyList();
    List<Pair<String, RankingExpression>> functions = new ArrayList<>();
    try (BufferedReader reader = new BufferedReader(file.createReader())) {
      String line;
      while (null != (line = reader.readLine())) {
        String[] parts = line.split("\t");
        String name = parts[0];
        try {
          RankingExpression expression = new RankingExpression(parts[0], parts[1]);
          functions.add(new Pair<>(name, expression));
        } catch (ParseException e) {
          throw new IllegalStateException("Could not parse " + name, e);
        }
      }
      return functions;
    }
  }
  catch (IOException e) {
    throw new UncheckedIOException(e);
  }
}

private void validateRankingConstant(RankingConstant rankingConstant, ApplicationPackage application) throws FileNotFoundException {
  // TODO: Handle validation of URI soon too.
  if (rankingConstant.getPathType() == RankingConstant.PathType.FILE) {
    String constantFile = rankingConstant.getFileName();
    if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) &&
        constantFile.startsWith(FilesApplicationPackage.preprocessed)) {
      constantFile = constantFile.substring(FilesApplicationPackage.preprocessed.length());
    }
    ApplicationFile tensorApplicationFile = application.getFile(Path.fromString(constantFile));
    new ConstantTensorJsonValidator().validate(constantFile,
        rankingConstant.getTensorType(),
        tensorApplicationFile.createReader());
  }
}

@Override
public int compareTo(ApplicationFile other) {
  if (other == this) return 0;
  return this.getPath().getName().compareTo((other).getPath().getName());
}

/**
 * Adds this constant to the application package as a file,
 * such that it can be distributed using file distribution.
 *
 * @return the path to the stored constant, relative to the application package root
 */
Path writeLargeConstant(String name, Tensor constant) {
  Path constantsPath = modelFiles.largeConstantsContentPath();
  // "tbf" ending for "typed binary format" - recognized by the nodes receiving the file:
  Path constantPath = constantsPath.append(name + ".tbf");
  // Remember the constant in a file we replicate in ZooKeeper
  application.getFile(modelFiles.largeConstantsInfoPath().append(name + ".constant"))
        .writeFile(new StringReader(name + ":" + constant.type() + ":" + correct(constantPath)));
  // Write content explicitly as a file on the file system as this is distributed using file distribution
  // - but only if this is a global model to avoid writing the same constants for each rank profile
  //   where they are used
  if (modelFiles.modelName.isGlobal()) {
    createIfNeeded(constantsPath);
    IOUtils.writeFile(application.getFileReference(constantPath), TypedBinaryFormat.encode(constant));
  }
  return correct(constantPath);
}

/**
 * Append this constant to the single file used for small constants distributed as config
 */
public void writeSmallConstant(String name, Tensor constant) {
  // Secret file format for remembering constants:
  application.getFile(modelFiles.smallConstantsPath()).appendFile(name + "\t" +
                                  constant.type().toString() + "\t" +
                                  constant.toString() + "\n");
}

private List<Pair<String, Tensor>> readSmallConstants() {
  try {
    ApplicationFile file = application.getFile(modelFiles.smallConstantsPath());
    if ( ! file.exists()) return Collections.emptyList();
    List<Pair<String, Tensor>> constants = new ArrayList<>();
    BufferedReader reader = new BufferedReader(file.createReader());
    String line;
    while (null != (line = reader.readLine())) {
      String[] parts = line.split("\t");
      String name = parts[0];
      TensorType type = TensorType.fromSpec(parts[1]);
      Tensor tensor = Tensor.from(type, parts[2]);
      constants.add(new Pair<>(name, tensor));
    }
    return constants;
  }
  catch (IOException e) {
    throw new UncheckedIOException(e);
  }
}