/** * Main entry point. * @param args The command line parameters. * @throws Exception When running the job fails. */ public static void main(String[] args) throws Exception { int errCode = ToolRunner.run(HBaseConfiguration.create(), new Import(), args); System.exit(errCode); }
@Override public int run(String[] args) throws Exception { if (args.length < 2) { usage("Wrong number of arguments: " + args.length); return -1; } String inputVersionString = System.getProperty(ResultSerialization.IMPORT_FORMAT_VER); if (inputVersionString != null) { getConf().set(ResultSerialization.IMPORT_FORMAT_VER, inputVersionString); } Job job = createSubmittableJob(getConf(), args); boolean isJobSuccessful = job.waitForCompletion(true); if(isJobSuccessful){ // Flush all the regions of the table flushRegionsIfNecessary(getConf()); } long inputRecords = job.getCounters().findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue(); long outputRecords = job.getCounters().findCounter(TaskCounter.MAP_OUTPUT_RECORDS).getValue(); if (outputRecords < inputRecords) { System.err.println("Warning, not all records were imported (maybe filtered out)."); if (outputRecords == 0) { System.err.println("If the data was exported from HBase 0.94 "+ "consider using -Dhbase.import.version=0.94."); } } return (isJobSuccessful ? 0 : 1); }
Import.configureCfRenaming(job.getConfiguration(), cfRenameMap);
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length < 2) { usage("Wrong number of arguments: " + otherArgs.length); System.exit(-1); conf.set(ResultSerialization.IMPORT_FORMAT_VER, inputVersionString); Job job = createSubmittableJob(conf, otherArgs); boolean isJobSuccessful = job.waitForCompletion(true); if(isJobSuccessful){ flushRegionsIfNecessary(conf);
@Override public void setup(Context context) { cfRenameMap = createCfRenameMap(context.getConfiguration()); filter = instantiateFilter(context.getConfiguration()); } }
/** * @param row The current table row key. * @param value The columns. * @param context The current context. * @throws IOException When something is broken with the data. */ @Override public void map(ImmutableBytesWritable row, Result value, Context context) throws IOException { try { if (LOG.isTraceEnabled()) { LOG.trace("Considering the row." + Bytes.toString(row.get(), row.getOffset(), row.getLength())); } if (filter == null || !filter.filterRowKey( PrivateCellUtil.createFirstOnRow(row.get(), row.getOffset(), (short) row.getLength()))) { for (Cell kv : value.rawCells()) { kv = filterKv(filter, kv); // skip if we filtered it out if (kv == null) continue; Cell ret = convertKv(kv, cfRenameMap); context.write(new CellWritableComparable(ret), ret); } } } catch (InterruptedException e) { LOG.error("Interrupted while emitting Cell", e); Thread.currentThread().interrupt(); } }
/** * Main entry point. * * @param args The command line parameters. * @throws Exception When running the job fails. */ public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length < 2) { usage("Wrong number of arguments: " + otherArgs.length); System.exit(-1); } Job job = createSubmittableJob(conf, otherArgs); System.exit(job.waitForCompletion(true) ? 0 : 1); } }
private static boolean runImport(String[] args, Configuration configuration) throws IOException, InterruptedException, ClassNotFoundException { // need to make a copy of the configuration because to make sure different temp dirs are used. GenericOptionsParser opts = new GenericOptionsParser(new Configuration(configuration), args); Configuration newConf = opts.getConfiguration(); args = opts.getRemainingArgs(); Job job = Import.createSubmittableJob(newConf, args); job.waitForCompletion(false); return job.isSuccessful(); }
ArrayList<byte[]> quotedArgs = toQuotedByteArrays(filterArgs); try { Method m = filterClass.getMethod("createFilterFromArguments", ArrayList.class);
/** * Test addFilterAndArguments method of Import This method set couple * parameters into Configuration */ @Test public void testAddFilterAndArguments() throws IOException { Configuration configuration = new Configuration(); List<String> args = new ArrayList<>(); args.add("param1"); args.add("param2"); Import.addFilterAndArguments(configuration, FilterBase.class, args); assertEquals("org.apache.hadoop.hbase.filter.FilterBase", configuration.get(Import.FILTER_CLASS_CONF_KEY)); assertEquals("param1,param2", configuration.get(Import.FILTER_ARGS_CONF_KEY)); }
Configuration conf = context.getConfiguration(); cfRenameMap = createCfRenameMap(conf); filter = instantiateFilter(conf); String durabilityStr = conf.get(WAL_DURABILITY); if(durabilityStr != null){
/** * @param row The current table row key. * @param value The columns. * @param context The current context. * @throws IOException When something is broken with the data. */ @Override public void map(ImmutableBytesWritable row, Result value, Context context) throws IOException { try { if (LOG.isTraceEnabled()) { LOG.trace("Considering the row." + Bytes.toString(row.get(), row.getOffset(), row.getLength())); } if (filter == null || !filter.filterRowKey(PrivateCellUtil.createFirstOnRow(row.get(), row.getOffset(), (short) row.getLength()))) { for (Cell kv : value.rawCells()) { kv = filterKv(filter, kv); // skip if we filtered it out if (kv == null) continue; context.write(row, new MapReduceExtendedCell(convertKv(kv, cfRenameMap))); } } } catch (InterruptedException e) { LOG.error("Interrupted while emitting Cell", e); Thread.currentThread().interrupt(); } }
ArrayList<byte[]> quotedArgs = toQuotedByteArrays(filterArgs); try { Method m = filterClass.getMethod("createFilterFromArguments", ArrayList.class);
/** * Test addFilterAndArguments method of Import This method set couple * parameters into Configuration */ @Test public void testAddFilterAndArguments() throws IOException { Configuration configuration = new Configuration(); List<String> args = new ArrayList<>(); args.add("param1"); args.add("param2"); Import.addFilterAndArguments(configuration, FilterBase.class, args); assertEquals("org.apache.hadoop.hbase.filter.FilterBase", configuration.get(Import.FILTER_CLASS_CONF_KEY)); assertEquals("param1,param2", configuration.get(Import.FILTER_ARGS_CONF_KEY)); }
@Override public int run(String[] args) throws Exception { if (args.length < 2) { usage("Wrong number of arguments: " + args.length); return -1; } String inputVersionString = System.getProperty(ResultSerialization.IMPORT_FORMAT_VER); if (inputVersionString != null) { getConf().set(ResultSerialization.IMPORT_FORMAT_VER, inputVersionString); } Job job = createSubmittableJob(getConf(), args); boolean isJobSuccessful = job.waitForCompletion(true); if(isJobSuccessful){ // Flush all the regions of the table flushRegionsIfNecessary(getConf()); } long inputRecords = job.getCounters().findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue(); long outputRecords = job.getCounters().findCounter(TaskCounter.MAP_OUTPUT_RECORDS).getValue(); if (outputRecords < inputRecords) { System.err.println("Warning, not all records were imported (maybe filtered out)."); if (outputRecords == 0) { System.err.println("If the data was exported from HBase 0.94 "+ "consider using -Dhbase.import.version=0.94."); } } return (isJobSuccessful ? 0 : 1); }
/** * Runs an import job with the specified command line args * @param args * @return true if job completed successfully * @throws IOException * @throws InterruptedException * @throws ClassNotFoundException */ boolean runImport(String[] args) throws Throwable { // need to make a copy of the configuration because to make sure different temp dirs are used. int status = ToolRunner.run(new Configuration(UTIL.getConfiguration()), new Import(), args); return status == 0; }
@Override public void setup(Context context) throws IOException { cfRenameMap = createCfRenameMap(context.getConfiguration()); filter = instantiateFilter(context.getConfiguration()); int reduceNum = context.getNumReduceTasks(); Configuration conf = context.getConfiguration(); TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME)); try (Connection conn = ConnectionFactory.createConnection(conf); RegionLocator regionLocator = conn.getRegionLocator(tableName)) { byte[][] startKeys = regionLocator.getStartKeys(); if (startKeys.length != reduceNum) { throw new IOException("Region split after job initialization"); } CellWritableComparable[] startKeyWraps = new CellWritableComparable[startKeys.length - 1]; for (int i = 1; i < startKeys.length; ++i) { startKeyWraps[i - 1] = new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i])); } CellWritableComparablePartitioner.START_KEYS = startKeyWraps; } } }
protected void processKV(ImmutableBytesWritable key, Result result, Context context, Put put, Delete delete) throws IOException, InterruptedException { for (Cell kv : result.rawCells()) { kv = filterKv(filter, kv);
Import.configureCfRenaming(job.getConfiguration(), cfRenameMap);
ArrayList<byte[]> quotedArgs = toQuotedByteArrays(filterArgs); try { Method m = filterClass.getMethod("createFilterFromArguments", ArrayList.class);