@Override public void execute(final CuratorTransactionFinal curatorTransactionFinal) throws Exception { for (Map.Entry<JobInstance, List<Integer>> entry : shardingResults.entrySet()) { for (int shardingItem : entry.getValue()) { curatorTransactionFinal.create().forPath(jobNodePath.getFullPath(ShardingNode.getInstanceNode(shardingItem)), entry.getKey().getJobInstanceId().getBytes()).and(); } } curatorTransactionFinal.delete().forPath(jobNodePath.getFullPath(ShardingNode.NECESSARY)).and(); curatorTransactionFinal.delete().forPath(jobNodePath.getFullPath(ShardingNode.PROCESSING)).and(); } }
public void create(String path, byte[] data, List<ACL> acl, CreateMode mode) throws Exception { transactionFinal = transactionFinal.create() .withMode(mode).withACL(acl).forPath(path, data).and(); }
private void register(String stringType, String value) { try { //can not switch the order of sysStat and appStat.because of the zk sequential consistency Stat sysStat = client.checkExists().creatingParentContainersIfNeeded().forPath(getSystemTypePath(stringType)); Stat appStringStat = client.checkExists().forPath(getAppBaseItemPath(stringType, value)); if(appStringStat != null) { return; } if(sysStat == null) { createNodeIfNotExists(getSystemTypePath(stringType)); } createNodeIfNotExists(getAppTypePath(stringType)); int numChildren = 0; if(sysStat != null) { numChildren = sysStat.getNumChildren(); } client.inTransaction() .create().forPath(getSystemBaseItemPath(stringType, numChildren + 1),value.getBytes()).and() .create().forPath(getAppBaseItemPath(stringType, value),ByteBuffer.allocate(4).putInt(numChildren + 1).array()).and() .commit(); } catch (Exception e) { LOG.warn("register string in zk failed", e); } }
private void createTaskAndDeletePendingTaskPrivate(SingularityTask task) throws Exception { // TODO: Should more of the below be done within a transaction? deletePendingTask(task.getTaskRequest().getPendingTask().getPendingTaskId()); final long now = System.currentTimeMillis(); String msg = String.format("Task launched because of %s", task.getTaskRequest().getPendingTask().getPendingTaskId().getPendingType().name()); if (task.getTaskRequest().getPendingTask().getUser().isPresent()) { msg = String.format("%s by %s", msg, task.getTaskRequest().getPendingTask().getUser().get()); } if (task.getTaskRequest().getPendingTask().getMessage().isPresent()) { msg = String.format("%s (%s)", msg, task.getTaskRequest().getPendingTask().getMessage().get()); } saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(task.getTaskId(), now, ExtendedTaskState.TASK_LAUNCHED, Optional.of(msg), Optional.<String>absent())); saveLastActiveTaskStatus(new SingularityTaskStatusHolder(task.getTaskId(), Optional.absent(), now, serverId, Optional.of(task.getAgentId().getValue()))); try { final String path = getTaskPath(task.getTaskId()); CuratorTransactionFinal transaction = curator.inTransaction().create().forPath(path, taskTranscoder.toBytes(task)).and(); transaction.create().forPath(getActivePath(task.getTaskId().getId())).and().commit(); leaderCache.putActiveTask(task); taskCache.set(path, task); } catch (KeeperException.NodeExistsException nee) { LOG.error("Task or active path already existed for {}", task.getTaskId()); } }
@Override public TransactionCreateBuilder create() { return new TransactionCreateBuilderDecorator(inner.create()); }
/** * Updates and returns a transaction which can be used to create missing parents of the provided * path, if any. */ private static CuratorTransactionFinal createParentsOf( CuratorFramework client, String path, CuratorTransactionFinal curatorTransactionFinal, Set<String> existingAndPendingCreatePaths) throws Exception { for (String parentPath : PersisterUtils.getParentPaths(path)) { if (!existingAndPendingCreatePaths.contains(parentPath) && client.checkExists().forPath(parentPath) == null) { // SUPPRESS CHECKSTYLE ParameterAssignment curatorTransactionFinal = curatorTransactionFinal.create().forPath(parentPath).and(); } existingAndPendingCreatePaths.add(parentPath); } return curatorTransactionFinal; }
public void addFinishedApplication(String appId, String queue, String yarnState, String yarnStatus, String user, String name) { String path = zkRoot + "/jobs/" + appId; try { if (curator.checkExists().forPath(path) != null) { curator.delete().deletingChildrenIfNeeded().forPath(path); } name = name.replace("/", "_"); if (name.length() > 50) { name = name.substring(0, 50); } CuratorTransactionBridge result = curator.inTransaction().create().withMode(CreateMode.PERSISTENT).forPath(path, ZKStateConstant.AppStatus.INIT.toString().getBytes("UTF-8")); result = result.and().create().withMode(CreateMode.PERSISTENT).forPath(path + "/info", String.format("%s/%s/%s/%s/%s", queue, yarnState, yarnStatus, user, name).getBytes("UTF-8")); result.and().commit(); } catch (Exception e) { LOG.error("fail adding finished application", e); throw new RuntimeException(e); } }
public CuratorTransactionFinal build( CuratorFramework client, String serviceRootPath ) throws Exception { // List of paths that are known to exist, or which are about to be created by the transaction // Includes "known to exist" in order to avoid repeated lookups for the same path Set<String> existingAndPendingCreatePaths = new HashSet<>(); CuratorTransactionFinal transaction = client .inTransaction() .check() .forPath(serviceRootPath) .and(); for (Map.Entry<String, byte[]> entry : pathBytesMap.entrySet()) { String path = entry.getKey(); if (!existingAndPendingCreatePaths.contains(path) && client.checkExists().forPath(path) == null) { // Path does not exist and is not being created: Create value (and any parents as needed). transaction = createParentsOf(client, path, transaction, existingAndPendingCreatePaths) .create().forPath(path, entry.getValue()).and(); existingAndPendingCreatePaths.add(path); } else { // Path exists (or will exist): Update existing value. transaction = transaction.setData().forPath(path, entry.getValue()).and(); } } return transaction; } }
@Override public void execute(final CuratorTransactionFinal curatorTransactionFinal) throws Exception { for (Map.Entry<JobInstance, List<Integer>> entry : shardingResults.entrySet()) { for (int shardingItem : entry.getValue()) { curatorTransactionFinal.create().forPath(jobNodePath.getFullPath(ShardingNode.getInstanceNode(shardingItem)), entry.getKey().getJobInstanceId().getBytes()).and(); } } curatorTransactionFinal.delete().forPath(jobNodePath.getFullPath(ShardingNode.NECESSARY)).and(); curatorTransactionFinal.delete().forPath(jobNodePath.getFullPath(ShardingNode.PROCESSING)).and(); } }
@Override public void execute(final CuratorTransactionFinal curatorTransactionFinal) throws Exception { for (Map.Entry<JobInstance, List<Integer>> entry : shardingResults.entrySet()) { for (int shardingItem : entry.getValue()) { curatorTransactionFinal.create().forPath(jobNodePath.getFullPath(ShardingNode.getInstanceNode(shardingItem)), entry.getKey().getJobInstanceId().getBytes()).and(); } } curatorTransactionFinal.delete().forPath(jobNodePath.getFullPath(ShardingNode.NECESSARY)).and(); curatorTransactionFinal.delete().forPath(jobNodePath.getFullPath(ShardingNode.PROCESSING)).and(); } }
@Override public void applyMigration() { try { for (String service : baragonStateDatastore.getServices()) { Collection<String> upstreams = curatorFramework.getChildren().forPath(ZKPaths.makePath(BaragonStateDatastore.SERVICES_FORMAT, service)); for (String upstream : upstreams) { Optional<UpstreamInfo> maybeUpstream = baragonStateDatastore.getUpstreamInfo(service, upstream); UpstreamInfo mergedInfo; if (maybeUpstream.isPresent()) { UpstreamInfo fromPath = UpstreamInfo.fromString(upstream); mergedInfo = new UpstreamInfo(fromPath.getUpstream(), maybeUpstream.get().getRequestId().or(fromPath.getRequestId()), maybeUpstream.get().getRackId().or(fromPath.getRackId())); } else { mergedInfo = UpstreamInfo.fromString(upstream); } curatorFramework.inTransaction() .delete().forPath(String.format(BaragonStateDatastore.UPSTREAM_FORMAT, service, upstream)).and() .create().forPath(String.format(BaragonStateDatastore.UPSTREAM_FORMAT, service, mergedInfo.toPath())).and() .commit(); } } } catch (Exception e) { Throwables.propagate(e); } } }
@Timed public QueuedRequestId enqueueRequest(BaragonRequest request, InternalRequestStates state) throws NodeExistsException { final long start = System.currentTimeMillis(); final String queuedRequestPath = String.format(REQUEST_ENQUEUE_FORMAT, request.getLoadBalancerService().getServiceId(), request.getLoadBalancerRequestId()); final String requestPath = String.format(REQUEST_FORMAT, request.getLoadBalancerRequestId()); final String requestStatePath = String.format(REQUEST_STATE_FORMAT, request.getLoadBalancerRequestId()); try { if (!nodeExists(REQUESTS_FORMAT)) { createNode(REQUESTS_FORMAT); } if (!nodeExists(REQUEST_QUEUE_FORMAT)) { createNode(REQUEST_QUEUE_FORMAT); } byte[] requestBytes = objectMapper.writeValueAsBytes(request); byte[] stateBytes = objectMapper.writeValueAsBytes(state); Collection<CuratorTransactionResult> results = curatorFramework.inTransaction() .create().forPath(requestPath, requestBytes).and() .create().forPath(requestStatePath, stateBytes).and() .create().withMode(CreateMode.PERSISTENT_SEQUENTIAL).forPath(queuedRequestPath) .and().commit(); log(OperationType.WRITE, Optional.of(3), Optional.of(requestBytes.length + stateBytes.length), start, String.format("Transaction Paths [%s + %s + %s]", requestPath, requestStatePath, queuedRequestPath)); return QueuedRequestId.fromString(ZKPaths.getNodeFromPath(Iterables.find(results, CuratorTransactionResult.ofTypeAndPath(org.apache.curator.framework.api.transaction.OperationType.CREATE, queuedRequestPath)) .getResultPath())); } catch (NodeExistsException nee) { throw nee; } catch (Exception e) { throw new RuntimeException(e); } }
.delete().forPath(itemPath) .and() .create().withMode(CreateMode.PERSISTENT_SEQUENTIAL).forPath(makeRequeueItemPath(itemPath), bytes) .and() .commit();
private void createTaskAndDeletePendingTaskPrivate(SingularityTask task) throws Exception { // TODO: Should more of the below be done within a transaction? deletePendingTask(task.getTaskRequest().getPendingTask().getPendingTaskId()); final long now = System.currentTimeMillis(); String msg = String.format("Task launched because of %s", task.getTaskRequest().getPendingTask().getPendingTaskId().getPendingType().name()); if (task.getTaskRequest().getPendingTask().getUser().isPresent()) { msg = String.format("%s by %s", msg, task.getTaskRequest().getPendingTask().getUser().get()); } if (task.getTaskRequest().getPendingTask().getMessage().isPresent()) { msg = String.format("%s (%s)", msg, task.getTaskRequest().getPendingTask().getMessage().get()); } saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(task.getTaskId(), now, ExtendedTaskState.TASK_LAUNCHED, Optional.of(msg), Optional.<String>absent())); saveLastActiveTaskStatus(new SingularityTaskStatusHolder(task.getTaskId(), Optional.absent(), now, serverId, Optional.of(task.getAgentId().getValue()))); try { final String path = getTaskPath(task.getTaskId()); CuratorTransactionFinal transaction = curator.inTransaction().create().forPath(path, taskTranscoder.toBytes(task)).and(); transaction.create().forPath(getActivePath(task.getTaskId().getId())).and().commit(); leaderCache.putActiveTask(task); taskCache.set(path, task); } catch (KeeperException.NodeExistsException nee) { LOG.error("Task or active path already existed for {}", task.getTaskId()); } }
@Override public void createGroup(Group group) { ensureConnected(); String groupPath = paths.groupPath(group.getGroupName()); logger.info("Creating group {} for path {}", group.getGroupName(), groupPath); try { zookeeper.inTransaction() .create().forPath(groupPath, mapper.writeValueAsBytes(group)) .and() .create().forPath(paths.topicsPath(group.getGroupName())) .and().commit(); } catch (KeeperException.NodeExistsException ex) { throw new GroupAlreadyExistsException(group.getGroupName(), ex); } catch (Exception ex) { throw new InternalProcessingException(ex); } }
@Override public void createTopic(Topic topic) { groupRepository.ensureGroupExists(topic.getName().getGroupName()); String topicPath = paths.topicPath(topic.getName()); logger.info("Creating topic for path {}", topicPath); try { zookeeper.inTransaction() .create().forPath(topicPath, mapper.writeValueAsBytes(topic)) .and() .create().forPath(paths.subscriptionsPath(topic.getName())) .and().commit(); } catch (KeeperException.NodeExistsException ex) { throw new TopicAlreadyExistsException(topic.getName(), ex); } catch (Exception ex) { throw new InternalProcessingException(ex); } }