private KetchLeader startLeader(String key, Repository repo) throws URISyntaxException { startLock.lock(); try { KetchLeader leader = leaders.get(key); if (leader != null) { return leader; } leader = system.createLeader(repo); leaders.put(key, leader); return leader; } finally { startLock.unlock(); } }
/** * Construct the KetchLeader instance of a repository. * * @param repo * local repository stored by the leader. * @return leader instance. * @throws java.net.URISyntaxException * a follower configuration contains an unsupported URI. */ public KetchLeader createLeader(Repository repo) throws URISyntaxException { KetchLeader leader = new KetchLeader(this) { @Override protected Repository openRepository() { repo.incrementOpen(); return repo; } }; leader.setReplicas(createReplicas(leader, repo)); return leader; }
/** * Create a default system with a thread pool of 1 thread per CPU. */ public KetchSystem() { this(defaultExecutor(), new MonotonicSystemClock(), DEFAULT_TXN_NAMESPACE); }
/** * Update the leader's view of the replica after a poll. * <p> * Called with {@link KetchLeader#lock} held by caller. * * @param refs * map of refs from the replica. */ void initialize(Map<String, Ref> refs) { if (txnAccepted == null) { txnAccepted = getId(refs.get(getSystem().getTxnAccepted())); } if (txnCommitted == null) { txnCommitted = getId(refs.get(getSystem().getTxnCommitted())); } }
private void retryLater(ReplicaPushRequest req) { Collection<ReceiveCommand> cmds = req.getCommands(); for (ReceiveCommand cmd : cmds) { cmd.setResult(NOT_ATTEMPTED, null); if (!waiting.containsKey(cmd.getRefName())) { waiting.put(cmd.getRefName(), cmd); } } queued.add(0, new ReplicaPushRequest(this, cmds)); if (!waitingForRetry()) { long delay = KetchSystem.delay( lastRetryMillis, minRetryMillis, maxRetryMillis); if (log.isDebugEnabled()) { log.debug("Retrying {} after {} ms", //$NON-NLS-1$ describeForLog(), Long.valueOf(delay)); } lastRetryMillis = delay; retryAtMillis = SystemReader.getInstance().getCurrentTime() + delay; retryFuture = getSystem().getExecutor() .schedule(new WeakRetryPush(this), delay, MILLISECONDS); } }
/** * Initializes local replica by reading accepted and committed references. * <p> * Loads accepted and committed references from the reference database of * the local replica and stores their current ObjectIds in memory. * * @param repo * repository to initialize state from. * @throws IOException * cannot read repository state. */ void initialize(Repository repo) throws IOException { RefDatabase refdb = repo.getRefDatabase(); if (refdb instanceof RefTreeDatabase) { RefTreeDatabase treeDb = (RefTreeDatabase) refdb; String txnNamespace = getSystem().getTxnNamespace(); if (!txnNamespace.equals(treeDb.getTxnNamespace())) { throw new IOException(MessageFormat.format( KetchText.get().mismatchedTxnNamespace, txnNamespace, treeDb.getTxnNamespace())); } refdb = treeDb.getBootstrap(); } initialize(refdb.exactRef( getSystem().getTxnAccepted(), getSystem().getTxnCommitted())); }
batch.setRefLogIdent(getSystem().newCommitter(ts)); batch.setRefLogMessage("ketch", false); //$NON-NLS-1$ batch.setAllowNonFastForwards(true); for (ReceiveCommand cmd : req.getCommands()) { String name = cmd.getRefName(); if (name.equals(getSystem().getTxnAccepted())) { accepted = cmd; } else if (name.equals(getSystem().getTxnCommitted())) { committed = cmd; } else {
if (name.equals(getSystem().getTxnAccepted())) { acceptCmd = cmd; } else if (name.equals(getSystem().getTxnCommitted())) { commitCmd = cmd; } else if (cmd.getResult() == OK && cmd.getType() == CREATE && name.startsWith(getSystem().getTxnStage())) { if (stages == null) { stages = new ArrayList<>();
List<KetchReplica> replicas = new ArrayList<>(); Config cfg = repo.getConfig(); String localName = getLocalName(cfg); for (String name : cfg.getSubsections(CONFIG_KEY_REMOTE)) { if (!hasParticipation(cfg, name)) { continue;
private void scheduleLeader() { idle = false; system.getExecutor().execute(new Runnable() { @Override public void run() { runLeader(); } }); }
boolean canDelete(Ref ref) { String name = ref.getName(); if (HEAD.equals(name)) { return false; } if (name.startsWith(getSystem().getTxnNamespace())) { return false; } // TODO(sop) Do not delete precious names from replica. return true; }
private void prepareTxnCommitted(List<ReceiveCommand> cmds, ObjectId committed) { removeStaged(cmds, committed); cmds.add(new ReceiveCommand( txnCommitted, committed, getSystem().getTxnCommitted())); }
private List<ReceiveCommand> makeStageList(Repository git, ObjectInserter inserter) throws IOException { // For each branch, collapse consecutive updates to only most recent, // avoiding sending multiple objects in a rapid fast-forward chain, or // rewritten content. Map<String, ObjectId> byRef = new HashMap<>(); for (Proposal p : todo) { for (Command c : p.getCommands()) { Ref n = c.getNewRef(); if (n != null && !n.isSymbolic()) { byRef.put(n.getName(), n.getObjectId()); } } } if (byRef.isEmpty()) { return Collections.emptyList(); } Set<ObjectId> newObjs = new HashSet<>(byRef.values()); StageBuilder b = new StageBuilder( leader.getSystem().getTxnStage(), acceptedNewIndex); return b.makeStageList(newObjs, git, inserter); }
/** * Schedule a proposal round with the replica. * <p> * Called with {@link KetchLeader#lock} held by caller. * * @param round * current round being run by the leader. */ void pushTxnAcceptedAsync(Round round) { List<ReceiveCommand> cmds = new ArrayList<>(); if (commitSpeed == BATCHED) { LogIndex committedIndex = leader.getCommitted(); if (equals(txnAccepted, committedIndex) && !equals(txnCommitted, committedIndex)) { prepareTxnCommitted(cmds, committedIndex); } } // TODO(sop) Lagging replicas should build accept on the fly. if (round.stageCommands != null) { for (ReceiveCommand cmd : round.stageCommands) { // TODO(sop): Do not send certain object graphs to replica. cmds.add(copy(cmd)); } } cmds.add(new ReceiveCommand( round.acceptedOldIndex, round.acceptedNewIndex, getSystem().getTxnAccepted())); pushAsync(new ReplicaPushRequest(this, cmds)); }
/** * Initializes local replica by reading accepted and committed references. * <p> * Loads accepted and committed references from the reference database of * the local replica and stores their current ObjectIds in memory. * * @param repo * repository to initialize state from. * @throws IOException * cannot read repository state. */ void initialize(Repository repo) throws IOException { RefDatabase refdb = repo.getRefDatabase(); if (refdb instanceof RefTreeDatabase) { RefTreeDatabase treeDb = (RefTreeDatabase) refdb; String txnNamespace = getSystem().getTxnNamespace(); if (!txnNamespace.equals(treeDb.getTxnNamespace())) { throw new IOException(MessageFormat.format( KetchText.get().mismatchedTxnNamespace, txnNamespace, treeDb.getTxnNamespace())); } refdb = treeDb.getBootstrap(); } initialize(refdb.exactRef( getSystem().getTxnAccepted(), getSystem().getTxnCommitted())); }
batch.setRefLogIdent(getSystem().newCommitter()); batch.setRefLogMessage("ketch", false); //$NON-NLS-1$ batch.setAllowNonFastForwards(true); for (ReceiveCommand cmd : req.getCommands()) { String name = cmd.getRefName(); if (name.equals(getSystem().getTxnAccepted())) { accepted = cmd; } else if (name.equals(getSystem().getTxnCommitted())) { committed = cmd; } else {
if (name.equals(getSystem().getTxnAccepted())) { acceptCmd = cmd; } else if (name.equals(getSystem().getTxnCommitted())) { commitCmd = cmd; } else if (cmd.getResult() == OK && cmd.getType() == CREATE && name.startsWith(getSystem().getTxnStage())) { if (stages == null) { stages = new ArrayList<>();
private Map<String, Ref> push(Repository git, Transport transport, List<RemoteCommand> cmds) throws IOException { Map<String, RemoteRefUpdate> updates = asUpdateMap(cmds); try (PushConnection connection = transport.openPush()) { Map<String, Ref> adv = connection.getRefsMap(); RemoteRefUpdate accepted = updates.get(getSystem().getTxnAccepted()); if (accepted != null && !isExpectedValue(adv, accepted)) { abort(cmds); return adv; } RemoteRefUpdate committed = updates.get(getSystem().getTxnCommitted()); if (committed != null && !isExpectedValue(adv, committed)) { abort(cmds); return adv; } if (committed != null && getCommitMethod() == ALL_REFS) { prepareCommit(git, cmds, updates, adv, committed.getNewObjectId()); } connection.push(NullProgressMonitor.INSTANCE, updates); return adv; } }
private void retryLater(ReplicaPushRequest req) { Collection<ReceiveCommand> cmds = req.getCommands(); for (ReceiveCommand cmd : cmds) { cmd.setResult(NOT_ATTEMPTED, null); if (!waiting.containsKey(cmd.getRefName())) { waiting.put(cmd.getRefName(), cmd); } } queued.add(0, new ReplicaPushRequest(this, cmds)); if (!waitingForRetry()) { long delay = KetchSystem.delay( lastRetryMillis, minRetryMillis, maxRetryMillis); if (log.isDebugEnabled()) { log.debug("Retrying {} after {} ms", //$NON-NLS-1$ describeForLog(), Long.valueOf(delay)); } lastRetryMillis = delay; retryAtMillis = SystemReader.getInstance().getCurrentTime() + delay; retryFuture = getSystem().getExecutor() .schedule(new WeakRetryPush(this), delay, MILLISECONDS); } }
List<KetchReplica> replicas = new ArrayList<>(); Config cfg = repo.getConfig(); String localName = getLocalName(cfg); for (String name : cfg.getSubsections(CONFIG_KEY_REMOTE)) { if (!hasParticipation(cfg, name)) { continue;