List<Cell> cells = tableEntry.getEdit().getCells();
@Override public long append(HTableDescriptor htd, HRegionInfo info, WALKey key, WALEdit edits, boolean inMemstore) { if (!this.listeners.isEmpty()) { final long start = System.nanoTime(); long len = 0; for (Cell cell : edits.getCells()) { len += CellUtil.estimatedSerializedSizeOf(cell); } final long elapsed = (System.nanoTime() - start)/1000000l; for (WALActionsListener listener : this.listeners) { listener.postAppend(len, elapsed); } } return -1; }
/** * Count the number of different row keys in the given edit because of mini-batching. We assume * that there's at least one Cell in the WALEdit. * @param edit edit to count row keys from * @return number of different row keys */ private int countDistinctRowKeys(WALEdit edit) { List<Cell> cells = edit.getCells(); int distinctRowKeys = 1; Cell lastCell = cells.get(0); for (int i = 0; i < edit.size(); i++) { if (!CellUtil.matchingRow(cells.get(i), lastCell)) { distinctRowKeys++; } } return distinctRowKeys; }
private long postAppend(final Entry e, final long elapsedTime) { long len = 0; if (!listeners.isEmpty()) { for (Cell cell : e.getEdit().getCells()) { len += CellUtil.estimatedSerializedSizeOf(cell); } for (WALActionsListener listener : listeners) { listener.postAppend(len, elapsedTime); } } return len; }
@Override public void preBatchMutate(HRegion region, WALEdit walEdit) throws IOException { // TODO we should return back the status of this hook run to HRegion so that those Mutations // with OperationStatus as SUCCESS or FAILURE should not get applied to memstore. RegionCoprocessorHost coprocessorHost = region.getCoprocessorHost(); OperationStatus[] opStatus = new OperationStatus[mutations.size()]; Arrays.fill(opStatus, OperationStatus.NOT_RUN); WALEdit[] walEditsFromCP = new WALEdit[mutations.size()]; if (coprocessorHost != null) { miniBatch = new MiniBatchOperationInProgress<Mutation>( mutations.toArray(new Mutation[mutations.size()]), opStatus, walEditsFromCP, 0, mutations.size()); coprocessorHost.preBatchMutate(miniBatch); } // Apply edits to a single WALEdit for (int i = 0; i < mutations.size(); i++) { if (opStatus[i] == OperationStatus.NOT_RUN) { // Other OperationStatusCode means that Mutation is already succeeded or failed in CP hook // itself. No need to apply again to region if (walEditsFromCP[i] != null) { // Add the WALEdit created by CP hook for (Cell walCell : walEditsFromCP[i].getCells()) { walEdit.add(walCell); } } } } }
@Override public void append(RegionEntryBuffer buffer) throws IOException { List<Entry> entries = buffer.getEntryBuffer(); if (entries.isEmpty() || entries.get(0).getEdit().getCells().isEmpty()) { return; } // meta edits (e.g. flush) are always replicated. // data edits (e.g. put) are replicated if the table requires them. if (!requiresReplication(buffer.getTableName(), entries)) { return; } sinkWriter.append(buffer.getTableName(), buffer.getEncodedRegionName(), entries.get(0).getEdit().getCells().get(0).getRow(), entries); }
@Override public Entry filter(Entry entry) { NavigableMap<byte[], Integer> scopes = entry.getKey().getScopes(); if (scopes == null || scopes.isEmpty()) { return null; } ArrayList<Cell> cells = entry.getEdit().getCells(); int size = cells.size(); for (int i = size - 1; i >= 0; i--) { Cell cell = cells.get(i); // The scope will be null or empty if // there's nothing to replicate in that WALEdit if (!scopes.containsKey(cell.getFamily()) || scopes.get(cell.getFamily()) == HConstants.REPLICATION_SCOPE_LOCAL) { cells.remove(i); } } if (cells.size() < size / 2) { cells.trimToSize(); } return entry; }
private void filterCellByStore(Entry logEntry) { Map<byte[], Long> maxSeqIdInStores = regionMaxSeqIdInStores.get(Bytes.toString(logEntry.getKey().getEncodedRegionName())); if (maxSeqIdInStores == null || maxSeqIdInStores.isEmpty()) { return; } // Create the array list for the cells that aren't filtered. // We make the assumption that most cells will be kept. ArrayList<Cell> keptCells = new ArrayList<Cell>(logEntry.getEdit().getCells().size()); for (Cell cell : logEntry.getEdit().getCells()) { if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) { keptCells.add(cell); } else { byte[] family = CellUtil.cloneFamily(cell); Long maxSeqId = maxSeqIdInStores.get(family); // Do not skip cell even if maxSeqId is null. Maybe we are in a rolling upgrade, // or the master was crashed before and we can not get the information. if (maxSeqId == null || maxSeqId.longValue() < logEntry.getKey().getLogSeqNum()) { keptCells.add(cell); } } } // Anything in the keptCells array list is still live. // So rather than removing the cells from the array list // which would be an O(n^2) operation, we just replace the list logEntry.getEdit().setCells(keptCells); }
FSWALEntry(final long sequence, final WALKey key, final WALEdit edit, final HTableDescriptor htd, final HRegionInfo hri, final boolean inMemstore) { super(key, edit); this.inMemstore = inMemstore; this.htd = htd; this.hri = hri; this.sequence = sequence; if (inMemstore) { // construct familyNames here to reduce the work of log sinker. ArrayList<Cell> cells = this.getEdit().getCells(); if (CollectionUtils.isEmpty(cells)) { this.familyNames = Collections.<byte[]> emptySet(); } else { Set<byte[]> familySet = Sets.newTreeSet(Bytes.BYTES_COMPARATOR); for (Cell cell : cells) { if (!CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) { familySet.add(CellUtil.cloneFamily(cell)); } } this.familyNames = Collections.unmodifiableSet(familySet); } } else { this.familyNames = Collections.<byte[]> emptySet(); } }
@Override public Entry filter(Entry entry) { TableName tabName = entry.getKey().getTablename(); ArrayList<Cell> cells = entry.getEdit().getCells(); Map<TableName, List<String>> tableCFs = null;
for (Cell cell : edit.getCells()) {
/** * Utility method used to set the correct scopes on each log key. Doesn't set a scope on keys * from compaction WAL edits and if the scope is local. * @param htd Descriptor used to find the scope to use * @param logKey Key that may get scoped according to its edits * @param logEdit Edits used to lookup the scopes */ public static void scopeWALEdits(HTableDescriptor htd, WALKey logKey, WALEdit logEdit) { NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR); byte[] family; for (Cell cell : logEdit.getCells()) { family = cell.getFamily(); // This is expected and the KV should not be replicated if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) continue; // Unexpected, has a tendency to happen in unit tests assert htd.getFamily(family) != null; int scope = htd.getFamily(family).getScope(); if (scope != REPLICATION_SCOPE_LOCAL && !scopes.containsKey(family)) { scopes.put(family, scope); } } if (!scopes.isEmpty()) { logKey.setScopes(scopes); } }
@Override public void prepare(boolean reload) throws IOException { if (!reload) return; // relocate regions in case we have a new dead server or network hiccup // if not due to connection issue, the following code should run fast because it uses // cached location boolean skip = false; for (Entry entry : this.entries) { WALEdit edit = entry.getEdit(); List<Cell> cells = edit.getCells(); for (Cell cell : cells) { // filtering WAL meta entries setLocation(conn.locateRegion(tableName, cell.getRow())); skip = true; break; } // use first log entry to relocate region because all entries are for one region if (skip) break; } } }
@Override public void map(WALKey key, WALEdit value, Context context) throws IOException { try { // skip all other tables if (Bytes.equals(table, key.getTablename().getName())) { for (Cell cell : value.getCells()) { KeyValue kv = KeyValueUtil.ensureKeyValueTypeForMR(cell); if (WALEdit.isMetaEditFamily(kv.getFamily())) continue; context.write(new ImmutableBytesWritable(kv.getRow()), kv); } } } catch (InterruptedException e) { e.printStackTrace(); } }
Delete del = null; Cell lastCell = null; for (Cell cell : value.getCells()) {
for (Entry entry : entries) { WALEdit newEdit = new WALEdit(); ArrayList<Cell> cells = entry.getEdit().getCells(); for (Cell cell : cells) { if (cell.getTagsLength() > 0) {
@Override public void append(Entry entry) throws IOException { entry.setCompressionContext(compressionContext); entry.getKey().getBuilder(compressor). setFollowingKvCount(entry.getEdit().size()).build().writeDelimitedTo(output); for (Cell cell : entry.getEdit().getCells()) { // cellEncoder must assume little about the stream, since we write PB and cells in turn. cellEncoder.write(cell); } }
walEdits.getCells().addAll(results);
/** * Here is where a WAL edit gets its sequenceid. * @return The sequenceid we stamped on this edit. * @throws IOException */ long stampRegionSequenceId() throws IOException { long regionSequenceId = WALKey.NO_SEQUENCE_ID; MultiVersionConcurrencyControl mvcc = getKey().getMvcc(); MultiVersionConcurrencyControl.WriteEntry we = null; if (mvcc != null) { we = mvcc.begin(); regionSequenceId = we.getWriteNumber(); } if (!this.getEdit().isReplay() && inMemstore) { for (Cell c:getEdit().getCells()) { CellUtil.setSequenceId(c, regionSequenceId); } } // This has to stay in this order WALKey key = getKey(); key.setLogSeqNum(regionSequenceId); key.setWriteEntry(we); return regionSequenceId; }
List<Cell> cells = edit.getCells();