@Override protected boolean shouldQueueWork(ReplicationTarget target) { Map<Table.ID,String> queuedWorkForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (queuedWorkForPeer == null) { return true; } String queuedWork = queuedWorkForPeer.get(target.getSourceTableId()); // If we have no work for the local table to the given peer, submit some! return queuedWork == null; }
protected String getPassword(AccumuloConfiguration localConf, ReplicationTarget target) { requireNonNull(localConf); requireNonNull(target); Map<String,String> peerPasswords = localConf .getAllPropertiesWithPrefix(Property.REPLICATION_PEER_PASSWORD); String password = peerPasswords .get(Property.REPLICATION_PEER_PASSWORD.getKey() + target.getPeerName()); if (password == null) { throw new IllegalArgumentException("Cannot get password for " + target.getPeerName()); } return password; }
protected String getKeytab(AccumuloConfiguration localConf, ReplicationTarget target) { requireNonNull(localConf); requireNonNull(target); Map<String,String> peerKeytabs = localConf .getAllPropertiesWithPrefix(Property.REPLICATION_PEER_KEYTAB); String keytab = peerKeytabs .get(Property.REPLICATION_PEER_KEYTAB.getKey() + target.getPeerName()); if (keytab == null) { throw new IllegalArgumentException("Cannot get keytab for " + target.getPeerName()); } return keytab; }
protected String getPrincipal(AccumuloConfiguration localConf, ReplicationTarget target) { requireNonNull(localConf); requireNonNull(target); String peerName = target.getPeerName(); String userKey = Property.REPLICATION_PEER_USER.getKey() + peerName; Map<String,String> peerUsers = localConf .getAllPropertiesWithPrefix(Property.REPLICATION_PEER_USER); String user = peerUsers.get(userKey); if (user == null) { throw new IllegalArgumentException("Cannot get user for " + target.getPeerName()); } return user; }
@Override protected Set<String> getQueuedWork(ReplicationTarget target) { Map<Table.ID,String> queuedWorkForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (queuedWorkForPeer == null) { return Collections.emptySet(); } String queuedWork = queuedWorkForPeer.get(target.getSourceTableId()); if (queuedWork == null) { return Collections.emptySet(); } else { return Collections.singleton(queuedWork); } }
@Override protected void removeQueuedWork(ReplicationTarget target, String queueKey) { Map<Table.ID,String> queuedWorkForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (queuedWorkForPeer == null) { log.warn("removeQueuedWork called when no work was queued for {}", target.getPeerName()); return; } String queuedWork = queuedWorkForPeer.get(target.getSourceTableId()); if (queuedWork.equals(queueKey)) { queuedWorkForPeer.remove(target.getSourceTableId()); } else { log.warn("removeQueuedWork called on {} with differing queueKeys, expected {} but was {}", target, queueKey, queuedWork); return; } } }
@Override protected Set<String> getQueuedWork(ReplicationTarget target) { String desiredQueueKeySuffix = DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR + target.getPeerName() + DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR + target.getRemoteIdentifier() + DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR + target.getSourceTableId(); Set<String> queuedWorkForTarget = new HashSet<>(); for (String queuedWork : this.queuedWork) { if (queuedWork.endsWith(desiredQueueKeySuffix)) { queuedWorkForTarget.add(queuedWork); } } return queuedWorkForTarget; }
protected ReplicaSystem getReplicaSystem(ReplicationTarget target) { // Find the configured replication peer so we know how to replicate to it // Classname,Configuration String peerType = getPeerType(target.getPeerName()); // Get the peer that we're replicating to return factory.get(context, peerType); }
@Override protected boolean queueWork(Path path, ReplicationTarget target) { String queueKey = DistributedWorkQueueWorkAssignerHelper.getQueueKey(path.getName(), target); Map<Table.ID,String> workForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (workForPeer == null) { workForPeer = new HashMap<>(); this.queuedWorkByPeerName.put(target.getPeerName(), workForPeer); } String queuedWork = workForPeer.get(target.getSourceTableId()); if (queuedWork == null) { try { workQueue.addWork(queueKey, path.toString()); workForPeer.put(target.getSourceTableId(), queueKey); } catch (KeeperException | InterruptedException e) { log.warn("Could not queue work for {} to {}", path, target, e); return false; } return true; } else if (queuedWork.startsWith(path.getName())) { log.debug("Not re-queueing work for {} as it has already been queued for replication to {}", path, target); return false; } else { log.debug("Not queueing {} for work as {} must be replicated to {} first", path, queuedWork, target.getPeerName()); return false; } }
/** * Serialize a filename and a {@link ReplicationTarget} into the expected key format for use with * the {@link DistributedWorkQueue} * * @param filename * Filename for data to be replicated * @param replTarget * Information about replication peer * @return Key for identifying work in queue */ public static String getQueueKey(String filename, ReplicationTarget replTarget) { return filename + KEY_SEPARATOR + replTarget.getPeerName() + KEY_SEPARATOR + replTarget.getRemoteIdentifier() + KEY_SEPARATOR + replTarget.getSourceTableId(); }
.fromQueueKey(work); String filename = entry.getKey(); String peerName = entry.getValue().getPeerName(); Table.ID sourceTableId = entry.getValue().getSourceTableId();
if (!m.getReplicationSources().contains(target.getPeerName())) { mutationsToSend++; if (mutationsRemoved > 0) { log.debug("Removing {} mutations from WAL entry as they have already been replicated to {}", mutationsRemoved, target.getPeerName()); for (Mutation m : value.mutations) { if (!m.getReplicationSources().contains(target.getPeerName())) { m.addReplicationSource(name);
String replicaSystemClass = peers.get(configuredTarget.getPeerName()); if (replicaSystemClass == null) { log.trace("Could not determine configured ReplicaSystem for {}", configuredTarget.getPeerName()); continue; configuredTarget.getPeerName(), configuredTarget.getRemoteIdentifier(), replicaSystemClass, (numFiles == null) ? 0 : numFiles));
if (!isWorkRequired(status) && keysBeingReplicated.contains(key)) { log.debug("Removing {} from replication state to {} because replication is complete", key, target.getPeerName()); this.removeQueuedWork(target, key); if (keysBeingReplicated.contains(key)) { log.debug("Removing {} from replication state to {} because replication is complete", key, target.getPeerName()); this.removeQueuedWork(target, key);
@Override protected boolean shouldQueueWork(ReplicationTarget target) { Map<String,String> queuedWorkForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (null == queuedWorkForPeer) { return true; } String queuedWork = queuedWorkForPeer.get(target.getSourceTableId()); // If we have no work for the local table to the given peer, submit some! return null == queuedWork; }
protected String getPassword(AccumuloConfiguration localConf, ReplicationTarget target) { requireNonNull(localConf); requireNonNull(target); Map<String,String> peerPasswords = localConf .getAllPropertiesWithPrefix(Property.REPLICATION_PEER_PASSWORD); String password = peerPasswords .get(Property.REPLICATION_PEER_PASSWORD.getKey() + target.getPeerName()); if (null == password) { throw new IllegalArgumentException("Cannot get password for " + target.getPeerName()); } return password; }
protected String getKeytab(AccumuloConfiguration localConf, ReplicationTarget target) { requireNonNull(localConf); requireNonNull(target); Map<String,String> peerKeytabs = localConf .getAllPropertiesWithPrefix(Property.REPLICATION_PEER_KEYTAB); String keytab = peerKeytabs .get(Property.REPLICATION_PEER_KEYTAB.getKey() + target.getPeerName()); if (null == keytab) { throw new IllegalArgumentException("Cannot get keytab for " + target.getPeerName()); } return keytab; }
protected String getPrincipal(AccumuloConfiguration localConf, ReplicationTarget target) { requireNonNull(localConf); requireNonNull(target); String peerName = target.getPeerName(); String userKey = Property.REPLICATION_PEER_USER.getKey() + peerName; Map<String,String> peerUsers = localConf .getAllPropertiesWithPrefix(Property.REPLICATION_PEER_USER); String user = peerUsers.get(userKey); if (null == user) { throw new IllegalArgumentException("Cannot get user for " + target.getPeerName()); } return user; }
@Override protected Set<String> getQueuedWork(ReplicationTarget target) { Map<String,String> queuedWorkForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (null == queuedWorkForPeer) { return Collections.emptySet(); } String queuedWork = queuedWorkForPeer.get(target.getSourceTableId()); if (null == queuedWork) { return Collections.emptySet(); } else { return Collections.singleton(queuedWork); } }
protected ReplicaSystem getReplicaSystem(ReplicationTarget target) { // Find the configured replication peer so we know how to replicate to it // Classname,Configuration String peerType = getPeerType(target.getPeerName()); // Get the peer that we're replicating to return factory.get(peerType); }