/** * Gets an estimate of the amount of memory used by this mutation. The estimate includes data * sizes and object overhead. * * @return memory usage estimate */ public long estimatedMemoryUsed() { return numBytes() + 238; }
/** * Log a single mutation. This method expects mutations that have a durability other than NONE. */ public void log(final CommitSession commitSession, final Mutation m, final Durability durability) throws IOException { if (durability == Durability.DEFAULT || durability == Durability.NONE) { throw new IllegalArgumentException("Unexpected durability " + durability); } write(singletonList(commitSession), false, logger -> logger.log(commitSession, m, durability), writeRetryFactory.createRetry()); logSizeEstimate.addAndGet(m.numBytes()); }
/** * Log mutations. This method expects mutations that have a durability other than NONE. */ public void logManyTablets(Map<CommitSession,TabletMutations> loggables) throws IOException { if (loggables.size() == 0) return; write(loggables.keySet(), false, logger -> logger.logManyTablets(loggables.values()), writeRetryFactory.createRetry()); for (TabletMutations entry : loggables.values()) { if (entry.getMutations().size() < 1) { throw new IllegalArgumentException("logManyTablets: logging empty mutation list"); } for (Mutation m : entry.getMutations()) { logSizeEstimate.addAndGet(m.numBytes()); } } }
@Override public List<Short> check(Environment env, Mutation mutation) { // fast size check if (mutation.numBytes() < maxSize) return NO_VIOLATIONS; List<Short> violations = new ArrayList<>(); for (ColumnUpdate cu : mutation.getUpdates()) { int size = mutation.getRow().length; size += cu.getColumnFamily().length; size += cu.getColumnQualifier().length; size += cu.getColumnVisibility().length; if (size > maxSize) violations.add(MAX__KEY_SIZE_EXCEEDED_VIOLATION); } return violations; } }
public void commit(CommitSession commitSession, List<Mutation> mutations) { int totalCount = 0; long totalBytes = 0; // write the mutation to the in memory table for (Mutation mutation : mutations) { totalCount += mutation.size(); totalBytes += mutation.numBytes(); } getTabletMemory().mutate(commitSession, mutations, totalCount); synchronized (this) { if (writesInProgress < 1) { throw new IllegalStateException( "commiting mutations after logging, but not waiting for any log messages"); } if (isCloseComplete()) { throw new IllegalStateException("tablet closed with outstanding messages to the logger"); } getTabletMemory().updateMemoryUsageStats(); // decrement here in case an exception is thrown below writesInProgress--; if (writesInProgress == 0) this.notifyAll(); commitSession.decrementCommitsInProgress(); numEntries += totalCount; numEntriesInMemory += totalCount; ingestCount += totalCount; ingestBytes += totalBytes; } }
Mutation mutation = iter.next(); updates.add(mutation.toThrift()); size += mutation.numBytes();
Mutation mutation = new ServerMutation(tmutation); mutations.add(mutation); additionalMutationSize += mutation.numBytes();
/** * Gets an estimate of the amount of memory used by this mutation. The estimate includes data * sizes and object overhead. * * @return memory usage estimate */ public long estimatedMemoryUsed() { return numBytes() + 238; }
@Override public void addMutation(Mutation m) throws MutationsRejectedException { if (mutationsAdded == 0) { startTime = System.currentTimeMillis(); } mutationsAdded++; m.numBytes(); }
public void log(final CommitSession commitSession, final long tabletSeq, final Mutation m, final Durability durability) throws IOException { if (durability == Durability.NONE) { return; } if (durability == Durability.DEFAULT) { throw new IllegalArgumentException("Unexpected durability " + durability); } write(commitSession, false, new Writer() { @Override public LoggerOperation write(DfsLogger logger) throws Exception { return logger.log(tabletSeq, commitSession.getLogId(), m, durability); } }); logSizeEstimate.addAndGet(m.numBytes()); }
public int log(final CommitSession commitSession, final int tabletSeq, final Mutation m) throws IOException { if (!enabled(commitSession)) return -1; int seq = write(commitSession, false, new Writer() { @Override public LoggerOperation write(DfsLogger logger, int ignored) throws Exception { return logger.log(tabletSeq, commitSession.getLogId(), m); } }); logSizeEstimate.addAndGet(m.numBytes()); return seq; }
public int logManyTablets(Map<CommitSession,List<Mutation>> mutations) throws IOException { final Map<CommitSession,List<Mutation>> loggables = new HashMap<CommitSession,List<Mutation>>(mutations); for (CommitSession t : mutations.keySet()) { if (!enabled(t)) loggables.remove(t); } if (loggables.size() == 0) return -1; int seq = write(loggables.keySet(), false, new Writer() { @Override public LoggerOperation write(DfsLogger logger, int ignored) throws Exception { List<TabletMutations> copy = new ArrayList<TabletMutations>(loggables.size()); for (Entry<CommitSession,List<Mutation>> entry : loggables.entrySet()) { CommitSession cs = entry.getKey(); copy.add(new TabletMutations(cs.getLogId(), cs.getWALogSeq(), entry.getValue())); } return logger.logManyTablets(copy); } }); for (List<Mutation> entry : loggables.values()) { if (entry.size() < 1) throw new IllegalArgumentException("logManyTablets: logging empty mutation list"); for (Mutation m : entry) { logSizeEstimate.addAndGet(m.numBytes()); } } return seq; }
logSizeEstimate.addAndGet(m.numBytes());
public void commit(CommitSession commitSession, List<Mutation> mutations) { int totalCount = 0; long totalBytes = 0; // write the mutation to the in memory table for (Mutation mutation : mutations) { totalCount += mutation.size(); totalBytes += mutation.numBytes(); } tabletMemory.mutate(commitSession, mutations); synchronized (this) { if (writesInProgress < 1) { throw new IllegalStateException("commiting mutations after logging, but not waiting for any log messages"); } if (closed && closeComplete) { throw new IllegalStateException("tablet closed with outstanding messages to the logger"); } tabletMemory.updateMemoryUsageStats(); // decrement here in case an exception is thrown below writesInProgress--; if (writesInProgress == 0) this.notifyAll(); commitSession.decrementCommitsInProgress(); numEntries += totalCount; numEntriesInMemory += totalCount; ingestCount += totalCount; ingestBytes += totalBytes; } }
@Override public List<Short> check(Environment env, Mutation mutation) { // fast size check if (mutation.numBytes() < maxSize) return NO_VIOLATIONS; List<Short> violations = new ArrayList<>(); for (ColumnUpdate cu : mutation.getUpdates()) { int size = mutation.getRow().length; size += cu.getColumnFamily().length; size += cu.getColumnQualifier().length; size += cu.getColumnVisibility().length; if (size > maxSize) violations.add(MAX__KEY_SIZE_EXCEEDED_VIOLATION); } return violations; } }
@Override public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) { UpdateSession us = (UpdateSession) sessionManager.reserveSession(updateID); if (us == null) { throw new RuntimeException("No Such SessionID"); } try { KeyExtent keyExtent = new KeyExtent(tkeyExtent); setUpdateTablet(us, keyExtent); if (us.currentTablet != null) { List<Mutation> mutations = us.queuedMutations.get(us.currentTablet); for (TMutation tmutation : tmutations) { Mutation mutation = new ServerMutation(tmutation); mutations.add(mutation); us.queuedMutationSize += mutation.numBytes(); } if (us.queuedMutationSize > getSystemConfiguration().getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX)) flush(us); } } finally { sessionManager.unreserveSession(us); } }
@Override public void commit(CommitSession commitSession, List<Mutation> mutations) { int totalCount = 0; long totalBytes = 0; // write the mutation to the in memory table for (Mutation mutation : mutations) { totalCount += mutation.size(); totalBytes += mutation.numBytes(); } getTabletMemory().mutate(commitSession, mutations); synchronized (this) { if (writesInProgress < 1) { throw new IllegalStateException( "commiting mutations after logging, but not waiting for any log messages"); } if (isCloseComplete()) { throw new IllegalStateException("tablet closed with outstanding messages to the logger"); } getTabletMemory().updateMemoryUsageStats(); // decrement here in case an exception is thrown below writesInProgress--; if (writesInProgress == 0) this.notifyAll(); commitSession.decrementCommitsInProgress(); numEntries += totalCount; numEntriesInMemory += totalCount; ingestCount += totalCount; ingestBytes += totalBytes; } }
Mutation m = ContinuousIngest.genMutation(rowId, r.nextInt(maxColF), r.nextInt(maxColQ), cv, ingestInstanceId.getBytes(UTF_8), count, null, r, false); totalBytes += m.numBytes(); try { bw.addMutation(m);
Mutation mutation = iter.next(); updates.add(mutation.toThrift()); size += mutation.numBytes();
Mutation mutation = new ServerMutation(tmutation); mutations.add(mutation); additionalMutationSize += mutation.numBytes();