How to use

filter

method

in

org.apache.crunch.PCollection

/**
 * Writes out HFiles from the provided <code>cells</code> and <code>table</code>. <code>limitToAffectedRegions</code>
 * is used to indicate that the regions the <code>cells</code> will be loaded into should be identified prior to writing
 * HFiles. Identifying the regions ahead of time will reduce the number of reducers needed when writing. This is
 * beneficial if the data to be loaded only touches a small enough subset of the total regions in the table. If set to
 * false, the number of reducers will equal the number of regions in the table.
 *
 * @see <a href='https://issues.apache.org/jira/browse/CRUNCH-588'>CRUNCH-588</a>
 */
public static <C extends Cell> void writeToHFilesForIncrementalLoad(
  PCollection<C> cells,
  HTable table,
  Path outputPath,
  boolean limitToAffectedRegions) throws IOException {
 HColumnDescriptor[] families = table.getTableDescriptor().getColumnFamilies();
 if (families.length == 0) {
  LOG.warn("{} has no column families", table);
  return;
 }
 PCollection<C> partitioned = sortAndPartition(cells, table, limitToAffectedRegions);
 for (HColumnDescriptor f : families) {
  byte[] family = f.getName();
  partitioned
    .filter(new FilterByFamilyFn<C>(family))
    .write(new HFileTarget(new Path(outputPath, Bytes.toString(family)), f));
 }
}

cells = cells.filter(new StartRowFilterFn<C>(scan.getStartRow()));
cells = cells.filter(new StopRowFilterFn<C>(scan.getStopRow()));
cells = cells.filter(new FamilyMapFilterFn<C>(scan.getFamilyMap()));
cells = cells.filter(new TimeRangeFilterFn<C>(timeRange));

cells = cells.filter(new StartRowFilterFn<C>(scan.getStartRow()));
cells = cells.filter(new StopRowFilterFn<C>(scan.getStopRow()));
cells = cells.filter(new FamilyMapFilterFn<C>(scan.getFamilyMap()));
cells = cells.filter(new TimeRangeFilterFn<C>(timeRange));

hfileTarget.outputConf(RegionLocationTable.REGION_LOCATION_TABLE_PATH, regionLocationFilePath.toString());
partitioned
  .filter(new FilterByFamilyFn<C>(family))
  .write(hfileTarget);