protected Status getStatus(String file, ReplicationTarget target) throws ReplicationTableOfflineException, InvalidProtocolBufferException { Scanner s = ReplicationTable.getScanner(context); s.setRange(Range.exact(file)); s.fetchColumn(WorkSection.NAME, target.toText()); return Status.parseFrom(Iterables.getOnlyElement(s).getValue().get()); } }
/** * Record the updated Status for this file and target * * @param filePath * Path to file being replicated * @param status * Updated Status after replication * @param target * Peer that was replicated to */ public void recordNewStatus(Path filePath, Status status, ReplicationTarget target) throws AccumuloException, TableNotFoundException { try (BatchWriter bw = context.createBatchWriter(ReplicationTable.NAME, new BatchWriterConfig())) { log.debug("Recording new status for {}, {}", filePath, ProtobufUtil.toString(status)); Mutation m = new Mutation(filePath.toString()); WorkSection.add(m, target.toText(), ProtobufUtil.toValue(status)); bw.addMutation(m); } } }
protected Status getStatus(String file, ReplicationTarget target) throws ReplicationTableOfflineException, AccumuloException, AccumuloSecurityException, InvalidProtocolBufferException { Scanner s = ReplicationTable.getScanner(context.getConnector()); s.setRange(Range.exact(file)); s.fetchColumn(WorkSection.NAME, target.toText()); return Status.parseFrom(Iterables.getOnlyElement(s).getValue().get()); } }
@Test public void doNotCreateWorkForFilesNotNeedingIt() throws Exception { ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1", "1"), target2 = new ReplicationTarget("cluster1", "table2", "2"); Text serializedTarget1 = target1.toText(), serializedTarget2 = target2.toText(); // Create two mutations, both of which need replication work done BatchWriter bw = ReplicationTable.getBatchWriter(conn); String filename1 = UUID.randomUUID().toString(), filename2 = UUID.randomUUID().toString(); String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2; Mutation m = new Mutation(file1); WorkSection.add(m, serializedTarget1, StatusUtil.fileCreatedValue(5)); bw.addMutation(m); m = new Mutation(file2); WorkSection.add(m, serializedTarget2, StatusUtil.fileCreatedValue(10)); bw.addMutation(m); bw.close(); DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class); HashSet<String> queuedWork = new HashSet<>(); assigner.setQueuedWork(queuedWork); assigner.setMaxQueueSize(Integer.MAX_VALUE); replay(workQueue); assigner.createWork(); verify(workQueue); }
/** * Record the updated Status for this file and target * * @param filePath * Path to file being replicated * @param status * Updated Status after replication * @param target * Peer that was replicated to */ public void recordNewStatus(Path filePath, Status status, ReplicationTarget target) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { BatchWriter bw = context.getConnector().createBatchWriter(ReplicationTable.NAME, new BatchWriterConfig()); try { log.debug("Recording new status for {}, {}", filePath.toString(), ProtobufUtil.toString(status)); Mutation m = new Mutation(filePath.toString()); WorkSection.add(m, target.toText(), ProtobufUtil.toValue(status)); bw.addMutation(m); } finally { bw.close(); } } }
String file1 = "/accumulo/wal/tserver+port/wal1"; Mutation m = new Mutation(file1); WorkSection.add(m, target.toText(), StatusUtil.openWithUnknownLengthValue()); bw.addMutation(m);
s.fetchColumn(WorkSection.NAME, target.toText());
Entry<Key,Value> e = Iterables.getOnlyElement(s); log.info("Found entry: " + e.getKey().toStringNoTruncate()); Text expectedColqual = new ReplicationTarget("cluster1", "4", tableId).toText(); Assert.assertEquals(expectedColqual, e.getKey().getColumnQualifier()); notFound = false;
try { Entry<Key,Value> e = Iterables.getOnlyElement(s); Text expectedColqual = new ReplicationTarget("cluster1", "4", tableId).toText(); Assert.assertEquals(expectedColqual, e.getKey().getColumnQualifier()); notFound = false;
.toValue(builder.setBegin(10000).setEnd(10000).setClosed(true).build()); StatusSection.add(m, "5", value); WorkSection.add(m, target.toText(), value); target = new ReplicationTarget("peer2", "5", "5"); WorkSection.add(m, target.toText(), value); target = new ReplicationTarget("peer3", "5", "5"); WorkSection.add(m, target.toText(), ProtobufUtil.toValue(builder.setClosed(false).build())); replBw.addMutation(m);
@Test public void recordsWithProgressUpdateBothTables() throws Exception { conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.READ); conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.WRITE); ReplicationTable.setOnline(conn); String file = "/accumulo/wals/tserver+port/" + UUID.randomUUID(); Status stat = Status.newBuilder().setBegin(100).setEnd(200).setClosed(true) .setInfiniteEnd(false).build(); ReplicationTarget target = new ReplicationTarget("peer", "table1", "1"); // Create a single work record for a file to some peer BatchWriter bw = ReplicationTable.getBatchWriter(conn); Mutation m = new Mutation(file); WorkSection.add(m, target.toText(), ProtobufUtil.toValue(stat)); bw.addMutation(m); bw.close(); updater.run(); Scanner s = ReplicationTable.getScanner(conn); s.setRange(Range.exact(file)); StatusSection.limit(s); Entry<Key,Value> entry = Iterables.getOnlyElement(s); Assert.assertEquals(entry.getKey().getColumnFamily(), StatusSection.NAME); Assert.assertEquals(entry.getKey().getColumnQualifier().toString(), target.getSourceTableId()); // We should only rely on the correct begin attribute being returned Status actual = Status.parseFrom(entry.getValue().get()); Assert.assertEquals(stat.getBegin(), actual.getBegin()); }
builder.setBegin(10000).setEnd(10000).setClosed(true).setCreatedTime(time).build()); StatusSection.add(m, "5", value); WorkSection.add(m, target.toText(), value); replBw.addMutation(m); target = new ReplicationTarget("peer1", "6", "6"); StatusSection.add(m, "6", value); WorkSection.add(m, target.toText(), value); replBw.addMutation(m);
@Test public void reprocessingOfCompletedWorkRemovesWork() throws Exception { ReplicationTarget target = new ReplicationTarget("cluster1", "table1", "1"); Text serializedTarget = target.toText();
ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1", "1"), target2 = new ReplicationTarget("cluster1", "table2", "2"); Text serializedTarget1 = target1.toText(), serializedTarget2 = target2.toText(); String keyTarget1 = target1.getPeerName() + DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR + target1.getRemoteIdentifier() + DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR
@Test public void createWorkForFilesInCorrectOrder() throws Exception { ReplicationTarget target = new ReplicationTarget("cluster1", "table1", "1"); Text serializedTarget = target.toText();