/** * Serialize a filename and a {@link ReplicationTarget} into the expected key format for use with * the {@link DistributedWorkQueue} * * @param filename * Filename for data to be replicated * @param replTarget * Information about replication peer * @return Key for identifying work in queue */ public static String getQueueKey(String filename, ReplicationTarget replTarget) { return filename + KEY_SEPARATOR + replTarget.getPeerName() + KEY_SEPARATOR + replTarget.getRemoteIdentifier() + KEY_SEPARATOR + replTarget.getSourceTableId(); }
@Override protected boolean shouldQueueWork(ReplicationTarget target) { Map<Table.ID,String> queuedWorkForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (queuedWorkForPeer == null) { return true; } String queuedWork = queuedWorkForPeer.get(target.getSourceTableId()); // If we have no work for the local table to the given peer, submit some! return queuedWork == null; }
public static String getFile(Key k) { Text buff = new Text(); return getFile(k, buff); }
protected String getPassword(AccumuloConfiguration localConf, ReplicationTarget target) { requireNonNull(localConf); requireNonNull(target); Map<String,String> peerPasswords = localConf .getAllPropertiesWithPrefix(Property.REPLICATION_PEER_PASSWORD); String password = peerPasswords .get(Property.REPLICATION_PEER_PASSWORD.getKey() + target.getPeerName()); if (password == null) { throw new IllegalArgumentException("Cannot get password for " + target.getPeerName()); } return password; }
protected Status getStatus(String file, ReplicationTarget target) throws ReplicationTableOfflineException, InvalidProtocolBufferException { Scanner s = ReplicationTable.getScanner(context); s.setRange(Range.exact(file)); s.fetchColumn(WorkSection.NAME, target.toText()); return Status.parseFrom(Iterables.getOnlyElement(s).getValue().get()); } }
/** * Deserialize a ReplicationTarget * * @param s * Serialized copy * @return the deserialized version */ public static ReplicationTarget from(String s) { ReplicationTarget target = new ReplicationTarget(); DataInputBuffer buffer = new DataInputBuffer(); buffer.reset(s.getBytes(UTF_8), s.length()); try { target.readFields(buffer); } catch (IOException e) { throw new RuntimeException(e); } return target; }
/** * Record the updated Status for this file and target * * @param filePath * Path to file being replicated * @param status * Updated Status after replication * @param target * Peer that was replicated to */ public void recordNewStatus(Path filePath, Status status, ReplicationTarget target) throws AccumuloException, TableNotFoundException { try (BatchWriter bw = context.createBatchWriter(ReplicationTable.NAME, new BatchWriterConfig())) { log.debug("Recording new status for {}, {}", filePath, ProtobufUtil.toString(status)); Mutation m = new Mutation(filePath.toString()); WorkSection.add(m, target.toText(), ProtobufUtil.toValue(status)); bw.addMutation(m); } } }
public static ReplicationTarget getTarget(Key k, Text buff) { checkArgument(BYTE_SEQ_NAME.equals(k.getColumnFamilyData()), "Given replication work key with incorrect colfam"); k.getColumnQualifier(buff); return ReplicationTarget.from(buff); }
public static long getTimeClosed(Key k) { return getTimeClosed(k, new Text()); }
/** * Convenience method to serialize a ReplicationTarget to {@link Text} using the {@link Writable} * methods without caring about performance penalties due to excessive object creation * * @return The serialized representation of the object */ public Text toText() { DataOutputBuffer buffer = new DataOutputBuffer(); try { this.write(buffer); } catch (IOException e) { throw new RuntimeException(e); } Text t = new Text(); // Throw it in a text for the mutation t.set(buffer.getData(), 0, buffer.getLength()); return t; } }
public static Scanner getScanner(AccumuloClient client) throws ReplicationTableOfflineException { try { return client.createScanner(NAME, Authorizations.EMPTY); } catch (TableNotFoundException e) { throw new AssertionError(NAME + " should exist, but doesn't."); } catch (TableOfflineException e) { throw new ReplicationTableOfflineException(e); } }
@Override protected Set<String> getQueuedWork(ReplicationTarget target) { String desiredQueueKeySuffix = DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR + target.getPeerName() + DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR + target.getRemoteIdentifier() + DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR + target.getSourceTableId(); Set<String> queuedWorkForTarget = new HashSet<>(); for (String queuedWork : this.queuedWork) { if (queuedWork.endsWith(desiredQueueKeySuffix)) { queuedWorkForTarget.add(queuedWork); } } return queuedWorkForTarget; }
@Override protected Set<String> getQueuedWork(ReplicationTarget target) { Map<Table.ID,String> queuedWorkForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (queuedWorkForPeer == null) { return Collections.emptySet(); } String queuedWork = queuedWorkForPeer.get(target.getSourceTableId()); if (queuedWork == null) { return Collections.emptySet(); } else { return Collections.singleton(queuedWork); } }
protected String getKeytab(AccumuloConfiguration localConf, ReplicationTarget target) { requireNonNull(localConf); requireNonNull(target); Map<String,String> peerKeytabs = localConf .getAllPropertiesWithPrefix(Property.REPLICATION_PEER_KEYTAB); String keytab = peerKeytabs .get(Property.REPLICATION_PEER_KEYTAB.getKey() + target.getPeerName()); if (keytab == null) { throw new IllegalArgumentException("Cannot get keytab for " + target.getPeerName()); } return keytab; }
/** * Deserialize a ReplicationTarget * * @param t * Serialized copy * @return the deserialized version */ public static ReplicationTarget from(Text t) { ReplicationTarget target = new ReplicationTarget(); DataInputBuffer buffer = new DataInputBuffer(); buffer.reset(t.getBytes(), t.getLength()); try { target.readFields(buffer); } catch (IOException e) { throw new RuntimeException(e); } return target; }
public static BatchScanner getBatchScanner(AccumuloClient client, int queryThreads) throws ReplicationTableOfflineException { try { return client.createBatchScanner(NAME, Authorizations.EMPTY, queryThreads); } catch (TableNotFoundException e) { throw new AssertionError(NAME + " should exist, but doesn't."); } catch (TableOfflineException e) { throw new ReplicationTableOfflineException(e); } }
@Override protected void removeQueuedWork(ReplicationTarget target, String queueKey) { Map<Table.ID,String> queuedWorkForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (queuedWorkForPeer == null) { log.warn("removeQueuedWork called when no work was queued for {}", target.getPeerName()); return; } String queuedWork = queuedWorkForPeer.get(target.getSourceTableId()); if (queuedWork.equals(queueKey)) { queuedWorkForPeer.remove(target.getSourceTableId()); } else { log.warn("removeQueuedWork called on {} with differing queueKeys, expected {} but was {}", target, queueKey, queuedWork); return; } } }
protected String getPrincipal(AccumuloConfiguration localConf, ReplicationTarget target) { requireNonNull(localConf); requireNonNull(target); String peerName = target.getPeerName(); String userKey = Property.REPLICATION_PEER_USER.getKey() + peerName; Map<String,String> peerUsers = localConf .getAllPropertiesWithPrefix(Property.REPLICATION_PEER_USER); String user = peerUsers.get(userKey); if (user == null) { throw new IllegalArgumentException("Cannot get user for " + target.getPeerName()); } return user; }
public static BatchWriter getBatchWriter(AccumuloClient client) throws ReplicationTableOfflineException { try { return client.createBatchWriter(NAME, new BatchWriterConfig()); } catch (TableNotFoundException e) { throw new AssertionError(NAME + " should exist, but doesn't."); } catch (TableOfflineException e) { throw new ReplicationTableOfflineException(e); } }
protected ReplicaSystem getReplicaSystem(ReplicationTarget target) { // Find the configured replication peer so we know how to replicate to it // Classname,Configuration String peerType = getPeerType(target.getPeerName()); // Get the peer that we're replicating to return factory.get(context, peerType); }