private void nextRandomRow(byte[][] row, BytesRefArrayWritable bytes) { bytes.resetValid(row.length); for (int i = 0; i < row.length; i++) { int len = Math.abs(randColLenGenerator.nextInt(columnMaxSize)); row[i] = new byte[len]; for (int j = 0; j < len; j++) { row[i][j] = getRandomChar(randomCharGenerator); } bytes.get(i).set(row[i], 0, len); } }
int size = columns.size(); for (int i = 0; i < size; i++) { BytesRefWritable cu = columns.get(i); int plainLen = cu.getLength(); columnBufferSize += plainLen;
int size = columns.size(); for (int i = 0; i < size; i++) { BytesRefWritable cu = columns.get(i); int plainLen = cu.getLength(); columnBufferSize += plainLen;
public int performRCFileReadFirstColumnTest(FileSystem fs, Path file, int allColumnsNumber, boolean chechCorrect) throws IOException { byte[][] checkBytes = null; BytesRefArrayWritable checkRow = new BytesRefArrayWritable(allColumnsNumber); if (chechCorrect) { resetRandomGenerators(); checkBytes = new byte[allColumnsNumber][]; } int actualReadCount = 0; java.util.ArrayList<Integer> readCols = new java.util.ArrayList<Integer>(); readCols.add(Integer.valueOf(0)); ColumnProjectionUtils.appendReadColumns(conf, readCols); RCFile.Reader reader = new RCFile.Reader(fs, file, conf); LongWritable rowID = new LongWritable(); BytesRefArrayWritable cols = new BytesRefArrayWritable(); while (reader.next(rowID)) { reader.getCurrentRow(cols); boolean ok = true; if (chechCorrect) { nextRandomRow(checkBytes, checkRow); ok = ok && (checkRow.get(0).equals(cols.get(0))); } if (!ok) { throw new IllegalStateException("Compare read and write error."); } actualReadCount++; } return actualReadCount; }
@Test public void testReadOldFileHeader() throws IOException { String[] row = new String[]{"Tester", "Bart", "333 X St.", "Reno", "NV", "USA"}; RCFile.Reader reader = new RCFile.Reader(fs, new Path(HiveTestUtils.getFileFromClasspath("rc-file-v0.rc")), conf); LongWritable rowID = new LongWritable(); BytesRefArrayWritable cols = new BytesRefArrayWritable(); assertTrue("old file reader first row", reader.next(rowID)); reader.getCurrentRow(cols); assertEquals(row.length, cols.size()); for (int i=0; i < cols.size(); ++i) { assertEquals(row[i], new String(cols.get(i).getBytesCopy())); } assertFalse("old file reader end", reader.next(rowID)); reader.close(); }
brw = result.get(row); int start = brw.getStart(); int len = brw.getLength();
int length = selCol.prvLength; BytesRefWritable currentCell = rest.get(i);
int length = selCol.prvLength; BytesRefWritable currentCell = rest.get(i);
txt.clear(); for (int i = 0; i < value.size(); i++) { BytesRefWritable v = value.get(i); txt.set(v.getData(), v.getStart(), v.getLength()); sourceLine[i] = txt.toString();
int size = columns.size(); for (int i = 0; i < size; i++) { BytesRefWritable cu = columns.get(i); int plainLen = cu.getLength(); columnBufferSize += plainLen;
int size = columns.size(); for (int i = 0; i < size; i++) { BytesRefWritable cu = columns.get(i); int plainLen = cu.getLength(); columnBufferSize += plainLen;
int size = columns.size(); for (int i = 0; i < size; i++) { BytesRefWritable cu = columns.get(i); int plainLen = cu.getLength(); columnBufferSize += plainLen;
BytesRefWritable columnBytes = rowBatchBytes.get(rcColumn.getInputField()); outputRecord.put(rcColumn.getOutputField(), updateColumnValue(rcColumn, columnBytes));
Record outputRecord = record.copy(); for (int rowIndex = 0; rowIndex < rowBatchBytes.size(); rowIndex++) { BytesRefWritable rowBytes = rowBatchBytes.get(rowIndex); rowValues.add(updateColumnValue(rcColumn, rowBytes));
BytesRefWritable columnBytes = rowBatchBytes.get(rcColumn.getInputField()); outputRecord.put(rcColumn.getOutputField(), updateColumnValue(rcColumn, columnBytes));
Record outputRecord = record.copy(); for (int rowIndex = 0; rowIndex < rowBatchBytes.size(); rowIndex++) { BytesRefWritable rowBytes = rowBatchBytes.get(rowIndex); rowValues.add(updateColumnValue(rcColumn, rowBytes));
int length = getColumnNextValueLength(columnID); BytesRefWritable currentCell = rest.get(i); if (currentValue.decompressedFlag[columnID]) { currentCell.set(currentValue.loadedColumnsValueBuffer[columnID]
int length = selCol.prvLength; BytesRefWritable currentCell = rest.get(i);
int length = selCol.prvLength; BytesRefWritable currentCell = rest.get(i);