int n1 = WritableUtils.decodeVIntSize(b1[s1]); int n2 = WritableUtils.decodeVIntSize(b2[s2]); List <KeyDescription> allKeySpecs = keyFieldHelper.keySpecs(); keyFieldHelper.getWordLengths(b1, s1 + n1, s1 + l1); int []lengthIndicesSecond = keyFieldHelper.getWordLengths(b2, s2 + n2, s2 + l2); int startCharFirst = keyFieldHelper.getStartOffset(b1, s1 + n1, s1 + l1, lengthIndicesFirst, keySpec); int endCharFirst = keyFieldHelper.getEndOffset(b1, s1 + n1, s1 + l1, lengthIndicesFirst, keySpec); int startCharSecond = keyFieldHelper.getStartOffset(b2, s2 + n2, s2 + l2, lengthIndicesSecond, keySpec); int endCharSecond = keyFieldHelper.getEndOffset(b2, s2 + n2, s2 + l2, lengthIndicesSecond, keySpec); int result;
public void setConf(Configuration conf) { this.conf = conf; keyFieldHelper = new KeyFieldHelper(); String keyFieldSeparator = conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, "\t"); keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator); if (conf.get("num.key.fields.for.partition") != null) { LOG.warn("Using deprecated num.key.fields.for.partition. " + "Use mapreduce.partition.keypartitioner.options instead"); this.numOfPartitionFields = conf.getInt("num.key.fields.for.partition",0); keyFieldHelper.setKeyFieldSpec(1,numOfPartitionFields); } else { String option = conf.get(PARTITIONER_OPTIONS); keyFieldHelper.parseOption(option); } }
public void setConf(Configuration conf) { this.conf = conf; String option = conf.get(COMPARATOR_OPTIONS); String keyFieldSeparator = conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR,"\t"); keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator); keyFieldHelper.parseOption(option); }
public void setConf(Configuration conf) { this.conf = conf; String keyFieldSeparator = conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, "\t"); keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator); if (conf.get("num.key.fields.for.partition") != null) { LOG.warn("Using deprecated num.key.fields.for.partition. " + "Use mapreduce.partition.keypartitioner.options instead"); this.numOfPartitionFields = conf.getInt("num.key.fields.for.partition",0); keyFieldHelper.setKeyFieldSpec(1,numOfPartitionFields); } else { String option = conf.get(PARTITIONER_OPTIONS); keyFieldHelper.parseOption(option); } }
KeyFieldHelper helper = new KeyFieldHelper(); helper.setKeyFieldSeparator("\t"); int[] result = helper.getWordLengths(input.getBytes(), 0, 2); assertTrue(equals(result, new int[] {1})); helper.setKeyFieldSpec(1, 2); result = helper.getWordLengths(input.getBytes(), 0, input.length()); assertTrue(equals(result, new int[] {2, 2, 11})); helper.setKeyFieldSeparator(" "); input = "hi hello\tthere you"; result = helper.getWordLengths(input.getBytes(), 0, input.length()); assertTrue(equals(result, new int[] {3, 2, 11, 3})); result = helper.getWordLengths(input.getBytes(), 10, 33); assertTrue(equals(result, new int[] {5, 4, 3, 5, 2, 3})); result = helper.getWordLengths(input.getBytes(), 10, input.length()); assertTrue(equals(result, new int[] {5, 4, 3, 5, 2, 0})); result = helper.getWordLengths(input.getBytes(), 0, 0); assertTrue(equals(result, new int[] {1, 0})); result = helper.getWordLengths(input.getBytes(), 0, 5); assertTrue(equals(result, new int[] {3, 0, 0, 3})); result = helper.getWordLengths(input.getBytes(), 0, 2);
KeyFieldHelper helper = new KeyFieldHelper(); helper.setKeyFieldSeparator("\t"); String keySpecs = "-k1.2,3.4"; String eKeySpecs = keySpecs; helper.parseOption(keySpecs); String actKeySpecs = helper.keySpecs().get(0).toString(); assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs); helper = new KeyFieldHelper(); helper.parseOption(keySpecs); actKeySpecs = helper.keySpecs().get(0).toString(); assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs); helper = new KeyFieldHelper(); helper.parseOption(keySpecs); actKeySpecs = helper.keySpecs().get(0).toString(); assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs); helper = new KeyFieldHelper(); helper.parseOption(keySpecs); actKeySpecs = helper.keySpecs().get(0).toString(); assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs); helper = new KeyFieldHelper(); helper.parseOption(keySpecs); actKeySpecs = helper.keySpecs().get(0).toString(); assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs); helper = new KeyFieldHelper();
KeyDescription k = parseKey(arg, args); if (k != null) { allKeySpecs.add(k);
KeyFieldHelper helper = new KeyFieldHelper(); helper.setKeyFieldSeparator("\t"); int[] result = helper.getWordLengths(input.getBytes(), 0, 2); assertTrue(equals(result, new int[] {1})); helper.setKeyFieldSpec(1, 2); result = helper.getWordLengths(input.getBytes(), 0, input.length()); assertTrue(equals(result, new int[] {2, 2, 11})); helper.setKeyFieldSeparator(" "); input = "hi hello\tthere you"; result = helper.getWordLengths(input.getBytes(), 0, input.length()); assertTrue(equals(result, new int[] {3, 2, 11, 3})); result = helper.getWordLengths(input.getBytes(), 10, 33); assertTrue(equals(result, new int[] {5, 4, 3, 5, 2, 3})); result = helper.getWordLengths(input.getBytes(), 10, input.length()); assertTrue(equals(result, new int[] {5, 4, 3, 5, 2, 0})); result = helper.getWordLengths(input.getBytes(), 0, 0); assertTrue(equals(result, new int[] {1, 0})); result = helper.getWordLengths(input.getBytes(), 0, 5); assertTrue(equals(result, new int[] {3, 0, 0, 3})); result = helper.getWordLengths(input.getBytes(), 0, 2);
KeyFieldHelper helper = new KeyFieldHelper(); helper.setKeyFieldSeparator("\t"); String keySpecs = "-k1.2,3.4"; String eKeySpecs = keySpecs; helper.parseOption(keySpecs); String actKeySpecs = helper.keySpecs().get(0).toString(); assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs); helper = new KeyFieldHelper(); helper.parseOption(keySpecs); actKeySpecs = helper.keySpecs().get(0).toString(); assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs); helper = new KeyFieldHelper(); helper.parseOption(keySpecs); actKeySpecs = helper.keySpecs().get(0).toString(); assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs); helper = new KeyFieldHelper(); helper.parseOption(keySpecs); actKeySpecs = helper.keySpecs().get(0).toString(); assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs); helper = new KeyFieldHelper(); helper.parseOption(keySpecs); actKeySpecs = helper.keySpecs().get(0).toString(); assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs); helper = new KeyFieldHelper();
public void setConf(Configuration conf) { this.conf = conf; String option = conf.get(COMPARATOR_OPTIONS); String keyFieldSeparator = conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR,"\t"); keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator); keyFieldHelper.parseOption(option); }
KeyDescription k = parseKey(arg, args); if (k != null) { allKeySpecs.add(k);
int n1 = WritableUtils.decodeVIntSize(b1[s1]); int n2 = WritableUtils.decodeVIntSize(b2[s2]); List <KeyDescription> allKeySpecs = keyFieldHelper.keySpecs(); keyFieldHelper.getWordLengths(b1, s1 + n1, s1 + l1); int []lengthIndicesSecond = keyFieldHelper.getWordLengths(b2, s2 + n2, s2 + l2); int startCharFirst = keyFieldHelper.getStartOffset(b1, s1 + n1, s1 + l1, lengthIndicesFirst, keySpec); int endCharFirst = keyFieldHelper.getEndOffset(b1, s1 + n1, s1 + l1, lengthIndicesFirst, keySpec); int startCharSecond = keyFieldHelper.getStartOffset(b2, s2 + n2, s2 + l2, lengthIndicesSecond, keySpec); int endCharSecond = keyFieldHelper.getEndOffset(b2, s2 + n2, s2 + l2, lengthIndicesSecond, keySpec); int result;
public void setConf(Configuration conf) { this.conf = conf; keyFieldHelper = new KeyFieldHelper(); String keyFieldSeparator = conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, "\t"); keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator); if (conf.get("num.key.fields.for.partition") != null) { LOG.warn("Using deprecated num.key.fields.for.partition. " + "Use mapreduce.partition.keypartitioner.options instead"); this.numOfPartitionFields = conf.getInt("num.key.fields.for.partition",0); keyFieldHelper.setKeyFieldSpec(1,numOfPartitionFields); } else { String option = conf.get(PARTITIONER_OPTIONS); keyFieldHelper.parseOption(option); } }
public void setConf(Configuration conf) { this.conf = conf; String option = conf.get(COMPARATOR_OPTIONS); String keyFieldSeparator = conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR,"\t"); keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator); keyFieldHelper.parseOption(option); }
KeyDescription k = parseKey(arg, args); if (k != null) { allKeySpecs.add(k);
int n1 = WritableUtils.decodeVIntSize(b1[s1]); int n2 = WritableUtils.decodeVIntSize(b2[s2]); List <KeyDescription> allKeySpecs = keyFieldHelper.keySpecs(); keyFieldHelper.getWordLengths(b1, s1 + n1, s1 + l1); int []lengthIndicesSecond = keyFieldHelper.getWordLengths(b2, s2 + n2, s2 + l2); int startCharFirst = keyFieldHelper.getStartOffset(b1, s1 + n1, s1 + l1, lengthIndicesFirst, keySpec); int endCharFirst = keyFieldHelper.getEndOffset(b1, s1 + n1, s1 + l1, lengthIndicesFirst, keySpec); int startCharSecond = keyFieldHelper.getStartOffset(b2, s2 + n2, s2 + l2, lengthIndicesSecond, keySpec); int endCharSecond = keyFieldHelper.getEndOffset(b2, s2 + n2, s2 + l2, lengthIndicesSecond, keySpec); int result;
public void setConf(Configuration conf) { this.conf = conf; keyFieldHelper = new KeyFieldHelper(); String keyFieldSeparator = conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, "\t"); keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator); if (conf.get("num.key.fields.for.partition") != null) { LOG.warn("Using deprecated num.key.fields.for.partition. " + "Use mapreduce.partition.keypartitioner.options instead"); this.numOfPartitionFields = conf.getInt("num.key.fields.for.partition",0); keyFieldHelper.setKeyFieldSpec(1,numOfPartitionFields); } else { String option = conf.get(PARTITIONER_OPTIONS); keyFieldHelper.parseOption(option); } }
public void setConf(Configuration conf) { this.conf = conf; String option = conf.get(COMPARATOR_OPTIONS); String keyFieldSeparator = conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR,"\t"); keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator); keyFieldHelper.parseOption(option); }
KeyDescription k = parseKey(arg, args); if (k != null) { allKeySpecs.add(k);
int n1 = WritableUtils.decodeVIntSize(b1[s1]); int n2 = WritableUtils.decodeVIntSize(b2[s2]); List <KeyDescription> allKeySpecs = keyFieldHelper.keySpecs(); keyFieldHelper.getWordLengths(b1, s1 + n1, s1 + l1); int []lengthIndicesSecond = keyFieldHelper.getWordLengths(b2, s2 + n2, s2 + l2); int startCharFirst = keyFieldHelper.getStartOffset(b1, s1 + n1, s1 + l1, lengthIndicesFirst, keySpec); int endCharFirst = keyFieldHelper.getEndOffset(b1, s1 + n1, s1 + l1, lengthIndicesFirst, keySpec); int startCharSecond = keyFieldHelper.getStartOffset(b2, s2 + n2, s2 + l2, lengthIndicesSecond, keySpec); int endCharSecond = keyFieldHelper.getEndOffset(b2, s2 + n2, s2 + l2, lengthIndicesSecond, keySpec); int result;