p.add(FAMILY, null, Bytes.toBytes("v1")); WALEdit withPut = new WALEdit(); addMutation(withPut, p, FAMILY); edits.add(withPut); WALEdit withDelete = new WALEdit(); addMutation(withDelete, d, FAMILY); edits.add(withDelete); WALEdit withPutsAndDeletes = new WALEdit(); addMutation(withPutsAndDeletes, d, FAMILY); addMutation(withPutsAndDeletes, p, FAMILY); edits.add(withPutsAndDeletes); WALEdit justIndexUpdates = new WALEdit(); byte[] table = Bytes.toBytes("targetTable"); IndexedKeyValue ikv = new IndexedKeyValue(table, p); justIndexUpdates.add(ikv); edits.add(justIndexUpdates); WALEdit mixed = new WALEdit(); addMutation(mixed, d, FAMILY); mixed.add(ikv); addMutation(mixed, p, FAMILY); edits.add(mixed);
edit.setCodec(codec); edit.write(out); List<WALEdit> read = new ArrayList<WALEdit>(); for (int i = 0; i < edits.size(); i++) { WALEdit edit = new WALEdit(); edit.setCodec(codec); edit.readFields(in); read.add(edit); WALEdit expected = edits.get(i); WALEdit found = read.get(i); for(int j=0; j< expected.getKeyValues().size(); j++){ KeyValue fkv = found.getKeyValues().get(j); KeyValue ekv = expected.getKeyValues().get(j); assertEquals("KV mismatch for edit! Expected: "+expected+", but found: "+found, ekv, fkv);
List<Cell> cells = tableEntry.getEdit().getCells();
/** * Copy-constructor. Only does a surface copy of the delegates fields - no actual data is copied, only referenced. * @param delegate to copy */ @SuppressWarnings("deprecation") public IndexedWALEdit(WALEdit delegate) { this.delegate = delegate; // reset the delegate's fields this.delegate.getKeyValues().clear(); if (this.delegate.getScopes() != null) { this.delegate.getScopes().clear(); } }
@SuppressWarnings("deprecation") @Override public void readFields(DataInput in) throws IOException { delegate.getKeyValues().clear(); if (delegate.getScopes() != null) { delegate.getScopes().clear(); delegate.getKeyValues().addAll(kvs); NavigableMap<byte[], Integer> scopes = delegate.getScopes(); if (numFamilies > 0) { if (scopes == null) { scopes.put(fam, scope); delegate.setScopes(scopes);
public static WALEdit createFlushWALEdit(HRegionInfo hri, FlushDescriptor f) { KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, FLUSH, EnvironmentEdgeManager.currentTime(), f.toByteArray()); return new WALEdit().add(kv); }
long startTimeMs = EnvironmentEdgeManager.currentTimeMillis(); WALEdit walEdit = new WALEdit(); for (KeyValue kv : fromCP.getKeyValues()) { walEdit.add(kv); if (walEdit.size() > 0) { syncOrDefer(txid);
/** * Search the {@link WALEdit} for the first {@link IndexedKeyValue} present * @param edit {@link WALEdit} * @return the first {@link IndexedKeyValue} in the {@link WALEdit} or <tt>null</tt> if not * present */ private IndexedKeyValue getFirstIndexedKeyValue(WALEdit edit) { for (KeyValue kv : edit.getKeyValues()) { if (kv instanceof IndexedKeyValue) { return (IndexedKeyValue) kv; } } return null; }
@Override public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c, final Put put, final WALEdit edit, final boolean writeToWAL) throws IOException { if (this.disabled) { super.prePut(c, put, edit, writeToWAL); return; } // just have to add a batch marker to the WALEdit so we get the edit again in the batch // processing step. We let it throw an exception here because something terrible has happened. edit.add(BATCH_MARKER); }
WALEdit walEdit = new WALEdit(isInReplay); MultiVersionConcurrencyControl.WriteEntry writeEntry = null; long txid = 0; if (walEdit.size() > 0) { assert isInReplay; if (!isInReplay) { txid = this.wal.append(this.htableDescriptor, this.getRegionInfo(), walKey, walEdit, true); walEdit = new WALEdit(isInReplay); walKey = null; for (Cell cell : fromCP.getCells()) { walEdit.add(cell); walKey.setOrigLogSeqNum(replaySeqId); if (walEdit.size() > 0) { if (!isInReplay) {
WALEdit val = new WALEdit(); e = new HLog.Entry(key, val);
private void doPreMutationHook(BatchOperationInProgress<Pair<Mutation, Integer>> batchOp) throws IOException { WALEdit walEdit = new WALEdit(); if (coprocessorHost != null) { for (int i = 0; i < batchOp.operations.length; i++) { "Put/Delete mutations only supported in batchMutate() now"); if (!walEdit.isEmpty()) { batchOp.walEditsFromCoprocessors[i] = walEdit; walEdit = new WALEdit();
List<Entry> newEntries = new ArrayList<Entry>(entries.size()); for (Entry entry : entries) { WALEdit newEdit = new WALEdit(); ArrayList<Cell> cells = entry.getEdit().getCells(); for (Cell cell : cells) { if (cell.getTagsLength() > 0) { + cell + ".", ioe); newEdit.add(cell); continue; newEdit.add(newCell); } else { newEdit.add(cell); newEdit.add(cell);
/** * Count the number of different row keys in the given edit because of * mini-batching. We assume that there's at least one KV in the WALEdit. * @param edit edit to count row keys from * @return number of different row keys */ private int countDistinctRowKeys(WALEdit edit) { List<KeyValue> kvs = edit.getKeyValues(); int distinctRowKeys = 1; KeyValue lastKV = kvs.get(0); for (int i = 0; i < edit.size(); i++) { if (!kvs.get(i).matchingRow(lastKV)) { distinctRowKeys++; } } return distinctRowKeys; }
/** * We only want KVs that are scoped other than local * @param edit The KV to check for replication */ protected void removeNonReplicableEdits(WALEdit edit) { NavigableMap<byte[], Integer> scopes = edit.getScopes(); List<KeyValue> kvs = edit.getKeyValues(); for (int i = edit.size()-1; i >= 0; i--) { KeyValue kv = kvs.get(i); // The scope will be null or empty if // there's nothing to replicate in that WALEdit if (scopes == null || !scopes.containsKey(kv.getFamily())) { kvs.remove(i); } } }
final WALEdit edit = new WALEdit(); key.readFields(in); edit.readFields(in); instance = new Entry(key, edit); } else {
@Override public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEdit logEdit) { NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR); byte[] family; for (KeyValue kv : logEdit.getKeyValues()) { family = kv.getFamily(); int scope = htd.getFamily(family).getScope(); if (scope != REPLICATION_SCOPE_LOCAL && !scopes.containsKey(family)) { scopes.put(family, scope); } } if (!scopes.isEmpty()) { logEdit.setScopes(scopes); } }
@Override public void preBatchMutate(HRegion region, WALEdit walEdit) throws IOException { // TODO we should return back the status of this hook run to HRegion so that those Mutations // with OperationStatus as SUCCESS or FAILURE should not get applied to memstore. RegionCoprocessorHost coprocessorHost = region.getCoprocessorHost(); OperationStatus[] opStatus = new OperationStatus[mutations.size()]; Arrays.fill(opStatus, OperationStatus.NOT_RUN); WALEdit[] walEditsFromCP = new WALEdit[mutations.size()]; if (coprocessorHost != null) { miniBatch = new MiniBatchOperationInProgress<Mutation>( mutations.toArray(new Mutation[mutations.size()]), opStatus, walEditsFromCP, 0, mutations.size()); coprocessorHost.preBatchMutate(miniBatch); } // Apply edits to a single WALEdit for (int i = 0; i < mutations.size(); i++) { if (opStatus[i] == OperationStatus.NOT_RUN) { // Other OperationStatusCode means that Mutation is already succeeded or failed in CP hook // itself. No need to apply again to region if (walEditsFromCP[i] != null) { // Add the WALEdit created by CP hook for (Cell walCell : walEditsFromCP[i].getCells()) { walEdit.add(walCell); } } } } }