private void completeManifest() throws IOException { Path tempManifestPath = new Path(destPath, TMP_MANIFEST_FILE_NAME); Path manifestPath = new Path(destPath, MANIFEST_FILE_NAME); FileSystem fs = tempManifestPath.getFileSystem(getConf()); fs.rename(tempManifestPath, manifestPath); }
private void writeTempManifestFile() throws IOException { Path tempManifestPath = new Path(destPath, TMP_MANIFEST_FILE_NAME); FileSystem fs = tempManifestPath.getFileSystem(getConf()); tableHash.writePropertiesFile(fs, tempManifestPath); }
public Job createSubmittableJob(String[] args) throws IOException { Path partitionsPath = new Path(destPath, PARTITIONS_FILE_NAME); generatePartitions(partitionsPath); Job job = Job.getInstance(getConf(), getConf().get("mapreduce.job.name", "hashTable_" + tableHash.tableName)); Configuration jobConf = job.getConfiguration(); jobConf.setLong(HASH_BATCH_SIZE_CONF_KEY, tableHash.batchSize); job.setJarByClass(HashTable.class); TableMapReduceUtil.initTableMapperJob(tableHash.tableName, tableHash.initScan(), HashMapper.class, ImmutableBytesWritable.class, ImmutableBytesWritable.class, job); // use a TotalOrderPartitioner and reducers to group region output into hash files job.setPartitionerClass(TotalOrderPartitioner.class); TotalOrderPartitioner.setPartitionFile(jobConf, partitionsPath); job.setReducerClass(Reducer.class); // identity reducer job.setNumReduceTasks(tableHash.numHashFiles); job.setOutputKeyClass(ImmutableBytesWritable.class); job.setOutputValueClass(ImmutableBytesWritable.class); job.setOutputFormatClass(MapFileOutputFormat.class); FileOutputFormat.setOutputPath(job, new Path(destPath, HASH_DATA_DIR)); return job; }
private void generatePartitions(Path partitionsPath) throws IOException { Connection connection = ConnectionFactory.createConnection(getConf()); Pair<byte[][], byte[][]> regionKeys = connection.getRegionLocator(TableName.valueOf(tableHash.tableName)).getStartEndKeys(); connection.close(); tableHash.selectPartitions(regionKeys); LOG.info("Writing " + tableHash.partitions.size() + " partition keys to " + partitionsPath); tableHash.writePartitionFile(getConf(), partitionsPath); }
@Override public int run(String[] args) throws Exception { String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs(); if (!doCommandLine(otherArgs)) { return 1; } Job job = createSubmittableJob(otherArgs); writeTempManifestFile(); if (!job.waitForCompletion(true)) { LOG.info("Map-reduce job failed!"); return 1; } completeManifest(); return 0; }
private void completeManifest() throws IOException { Path tempManifestPath = new Path(destPath, TMP_MANIFEST_FILE_NAME); Path manifestPath = new Path(destPath, MANIFEST_FILE_NAME); FileSystem fs = tempManifestPath.getFileSystem(getConf()); fs.rename(tempManifestPath, manifestPath); }
private void completeManifest() throws IOException { Path tempManifestPath = new Path(destPath, TMP_MANIFEST_FILE_NAME); Path manifestPath = new Path(destPath, MANIFEST_FILE_NAME); FileSystem fs = tempManifestPath.getFileSystem(getConf()); fs.rename(tempManifestPath, manifestPath); }
private void completeManifest() throws IOException { Path tempManifestPath = new Path(destPath, TMP_MANIFEST_FILE_NAME); Path manifestPath = new Path(destPath, MANIFEST_FILE_NAME); FileSystem fs = tempManifestPath.getFileSystem(getConf()); fs.rename(tempManifestPath, manifestPath); }
private void writeTempManifestFile() throws IOException { Path tempManifestPath = new Path(destPath, TMP_MANIFEST_FILE_NAME); FileSystem fs = tempManifestPath.getFileSystem(getConf()); tableHash.writePropertiesFile(fs, tempManifestPath); }
private void writeTempManifestFile() throws IOException { Path tempManifestPath = new Path(destPath, TMP_MANIFEST_FILE_NAME); FileSystem fs = tempManifestPath.getFileSystem(getConf()); tableHash.writePropertiesFile(fs, tempManifestPath); }
private void writeTempManifestFile() throws IOException { Path tempManifestPath = new Path(destPath, TMP_MANIFEST_FILE_NAME); FileSystem fs = tempManifestPath.getFileSystem(getConf()); tableHash.writePropertiesFile(fs, tempManifestPath); }
public Job createSubmittableJob(String[] args) throws IOException { Path partitionsPath = new Path(destPath, PARTITIONS_FILE_NAME); generatePartitions(partitionsPath); Job job = Job.getInstance(getConf(), getConf().get("mapreduce.job.name", "hashTable_" + tableHash.tableName)); Configuration jobConf = job.getConfiguration(); jobConf.setLong(HASH_BATCH_SIZE_CONF_KEY, tableHash.batchSize); job.setJarByClass(HashTable.class); TableMapReduceUtil.initTableMapperJob(tableHash.tableName, tableHash.initScan(), HashMapper.class, ImmutableBytesWritable.class, ImmutableBytesWritable.class, job); // use a TotalOrderPartitioner and reducers to group region output into hash files job.setPartitionerClass(TotalOrderPartitioner.class); TotalOrderPartitioner.setPartitionFile(jobConf, partitionsPath); job.setReducerClass(Reducer.class); // identity reducer job.setNumReduceTasks(tableHash.numHashFiles); job.setOutputKeyClass(ImmutableBytesWritable.class); job.setOutputValueClass(ImmutableBytesWritable.class); job.setOutputFormatClass(MapFileOutputFormat.class); FileOutputFormat.setOutputPath(job, new Path(destPath, HASH_DATA_DIR)); return job; }
public Job createSubmittableJob(String[] args) throws IOException { Path partitionsPath = new Path(destPath, PARTITIONS_FILE_NAME); generatePartitions(partitionsPath); Job job = Job.getInstance(getConf(), getConf().get("mapreduce.job.name", "hashTable_" + tableHash.tableName)); Configuration jobConf = job.getConfiguration(); jobConf.setLong(HASH_BATCH_SIZE_CONF_KEY, tableHash.batchSize); job.setJarByClass(HashTable.class); TableMapReduceUtil.initTableMapperJob(tableHash.tableName, tableHash.initScan(), HashMapper.class, ImmutableBytesWritable.class, ImmutableBytesWritable.class, job); // use a TotalOrderPartitioner and reducers to group region output into hash files job.setPartitionerClass(TotalOrderPartitioner.class); TotalOrderPartitioner.setPartitionFile(jobConf, partitionsPath); job.setReducerClass(Reducer.class); // identity reducer job.setNumReduceTasks(tableHash.numHashFiles); job.setOutputKeyClass(ImmutableBytesWritable.class); job.setOutputValueClass(ImmutableBytesWritable.class); job.setOutputFormatClass(MapFileOutputFormat.class); FileOutputFormat.setOutputPath(job, new Path(destPath, HASH_DATA_DIR)); return job; }
private void generatePartitions(Path partitionsPath) throws IOException { Connection connection = ConnectionFactory.createConnection(getConf()); Pair<byte[][], byte[][]> regionKeys = connection.getRegionLocator(TableName.valueOf(tableHash.tableName)).getStartEndKeys(); connection.close(); tableHash.selectPartitions(regionKeys); LOG.info("Writing " + tableHash.partitions.size() + " partition keys to " + partitionsPath); tableHash.writePartitionFile(getConf(), partitionsPath); }
public Job createSubmittableJob(String[] args) throws IOException { Path partitionsPath = new Path(destPath, PARTITIONS_FILE_NAME); generatePartitions(partitionsPath); Job job = Job.getInstance(getConf(), getConf().get("mapreduce.job.name", "hashTable_" + tableHash.tableName)); Configuration jobConf = job.getConfiguration(); jobConf.setLong(HASH_BATCH_SIZE_CONF_KEY, tableHash.batchSize); job.setJarByClass(HashTable.class); TableMapReduceUtil.initTableMapperJob(tableHash.tableName, tableHash.initScan(), HashMapper.class, ImmutableBytesWritable.class, ImmutableBytesWritable.class, job); // use a TotalOrderPartitioner and reducers to group region output into hash files job.setPartitionerClass(TotalOrderPartitioner.class); TotalOrderPartitioner.setPartitionFile(jobConf, partitionsPath); job.setReducerClass(Reducer.class); // identity reducer job.setNumReduceTasks(tableHash.numHashFiles); job.setOutputKeyClass(ImmutableBytesWritable.class); job.setOutputValueClass(ImmutableBytesWritable.class); job.setOutputFormatClass(MapFileOutputFormat.class); FileOutputFormat.setOutputPath(job, new Path(destPath, HASH_DATA_DIR)); return job; }
private void generatePartitions(Path partitionsPath) throws IOException { Connection connection = ConnectionFactory.createConnection(getConf()); Pair<byte[][], byte[][]> regionKeys = connection.getRegionLocator(TableName.valueOf(tableHash.tableName)).getStartEndKeys(); connection.close(); tableHash.selectPartitions(regionKeys); LOG.info("Writing " + tableHash.partitions.size() + " partition keys to " + partitionsPath); tableHash.writePartitionFile(getConf(), partitionsPath); }
private void generatePartitions(Path partitionsPath) throws IOException { Connection connection = ConnectionFactory.createConnection(getConf()); Pair<byte[][], byte[][]> regionKeys = connection.getRegionLocator(TableName.valueOf(tableHash.tableName)).getStartEndKeys(); connection.close(); tableHash.selectPartitions(regionKeys); LOG.info("Writing " + tableHash.partitions.size() + " partition keys to " + partitionsPath); tableHash.writePartitionFile(getConf(), partitionsPath); }
@Override public int run(String[] args) throws Exception { String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs(); if (!doCommandLine(otherArgs)) { return 1; } Job job = createSubmittableJob(otherArgs); writeTempManifestFile(); if (!job.waitForCompletion(true)) { LOG.info("Map-reduce job failed!"); return 1; } completeManifest(); return 0; }
@Override public int run(String[] args) throws Exception { String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs(); if (!doCommandLine(otherArgs)) { return 1; } Job job = createSubmittableJob(otherArgs); writeTempManifestFile(); if (!job.waitForCompletion(true)) { LOG.info("Map-reduce job failed!"); return 1; } completeManifest(); return 0; }
@Override public int run(String[] args) throws Exception { String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs(); if (!doCommandLine(otherArgs)) { return 1; } Job job = createSubmittableJob(otherArgs); writeTempManifestFile(); if (!job.waitForCompletion(true)) { LOG.info("Map-reduce job failed!"); return 1; } completeManifest(); return 0; }