@Override public Map<String, Settings> reloadConfiguration(Collection<String> configTypes) { Map<String, Tuple<Long, Settings>> loaded = loadConfigurations(configTypes, false); Map<String, Settings> loaded0 = loaded.entrySet().stream().collect(Collectors.toMap(x -> x.getKey(), x -> x.getValue().v2())); typeToConfig.keySet().removeAll(loaded0.keySet()); typeToConfig.putAll(loaded0); notifyAboutChanges(loaded0); final SearchGuardLicense sgLicense = getLicense(); notifyAboutLicenseChanges(sgLicense); final String license = sgLicense==null?"No license needed because enterprise modules are not enabled" :sgLicense.toString(); LOGGER.info("Search Guard License Info: "+license); if (sgLicense != null) { LOGGER.info("Search Guard License Type: "+sgLicense.getType()+", " + (sgLicense.isValid() ? "valid" : "invalid")); if (sgLicense.getExpiresInDays() <= 30 && sgLicense.isValid()) { LOGGER.warn("Your Search Guard license expires in " + sgLicense.getExpiresInDays() + " days."); System.out.println("Your Search Guard license expires in " + sgLicense.getExpiresInDays() + " days."); } if (!sgLicense.isValid()) { final String reasons = String.join("; ", sgLicense.getMsgs()); LOGGER.error("You are running an unlicensed version of Search Guard. Reason(s): " + reasons); System.out.println("You are running an unlicensed version of Search Guard. Reason(s): " + reasons); System.err.println("You are running an unlicensed version of Search Guard. Reason(s): " + reasons); } } return loaded0; }
throw new ElasticsearchSecurityException(ConfigConstants.SG_MASKED_FIELD_HEADER + " does not match (SG 901D)"); } else { if (log.isDebugEnabled()) { log.debug(ConfigConstants.SG_MASKED_FIELD_HEADER + " already set"); if (log.isDebugEnabled()) { log.debug("attach masked fields info: {}", maskedFieldsMap); final Map<String, Set<String>> dlsQueries = dlsFls.v1(); final Map<String, Set<String>> flsFields = dlsFls.v2();
final long version = singleGetResponse.getVersion(); if (ref == null || ref.length() == 0) { log.error("Empty or null byte reference for {}", type); return null; parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, SearchGuardDeprecationHandler.INSTANCE, ref, XContentType.JSON); parser.nextToken(); parser.nextToken(); if(!type.equals((parser.currentName()))) { log.error("Cannot parse config for type {} because {}!={}", type, type, parser.currentName()); return null; parser.nextToken(); return new Tuple<Long, Settings>(version, Settings.builder().loadFromStream("dummy.json", new ByteArrayInputStream(parser.binaryValue()), true).build()); } catch (final IOException e) { throw ExceptionsHelper.convertToElastic(e);
@Override public String[] provide(String[] original, Object localRequest, boolean supportsReplace) { //CCS if((localRequest instanceof FieldCapabilitiesRequest || localRequest instanceof SearchRequest) && (request instanceof FieldCapabilitiesRequest || request instanceof SearchRequest)) { assert supportsReplace: localRequest.getClass().getName()+" does not support replace"; final Tuple<Boolean, String[]> ccsResult = handleCcs((Replaceable) localRequest); if(ccsResult.v1() == Boolean.TRUE) { if(ccsResult.v2() == null || ccsResult.v2().length == 0) { returnEmpty.set(true); } original = ccsResult.v2(); } } if(returnEmpty.get()) { if(log.isTraceEnabled()) { log.trace("CCS return empty indices for local node"); } } else { final Resolved iResolved = resolveIndexPatterns(original); if(log.isTraceEnabled()) { log.trace("Resolved patterns {} for {} ({}) to {}", original, localRequest.getClass().getSimpleName(), request.getClass().getSimpleName(), iResolved); } resolvedBuilder.add(iResolved); resolvedBuilder.addTypes(resolveTypes(localRequest)); } return IndicesProvider.NOOP; } }, false);
functionStr = new Tuple<>(nameIF,"if(("+caseString+")){"+paramers.get(1).value+"} else {"+paramers.get(2).value+"}"); break; case "split": String generatedFieldName = functionStr.v1(); String returnCommand = ";return " + generatedFieldName +";" ; String newScript = functionStr.v2() + returnCommand; functionStr = new Tuple<>(generatedFieldName, newScript);
private Map<String, Tuple<Long, Settings>> validate(Map<String, Tuple<Long, Settings>> conf, int expectedSize) throws InvalidConfigException { if(conf == null || conf.size() != expectedSize) { throw new InvalidConfigException("Retrieved only partial configuration"); } final Tuple<Long, Settings> roles = conf.get("roles"); final String rolesDelimited; if (roles != null && roles.v2() != null && (rolesDelimited = roles.v2().toDelimitedString('#')) != null) { //<role>.indices.<indice>._dls_= OK //<role>.indices.<indice>._fls_.<num>= OK final String[] rolesString = rolesDelimited.split("#"); for (String role : rolesString) { if (role.contains("_fls_.") && !FLS_PATTERN.matcher(role).matches()) { LOGGER.error("Invalid FLS configuration detected, FLS/DLS will not work correctly: {}", role); } if (role.contains("_dls_=") && !DLS_PATTERN.matcher(role).matches()) { LOGGER.error("Invalid DLS configuration detected, FLS/DLS will not work correctly: {}", role); } } } return conf; }
private void processList(List<Tuple<Item, Consumer<Exception>>> candidates) { Exception exception = null; if (candidates.isEmpty() == false) { try { write(candidates); } catch (Exception ex) { // if this fails we are in deep shit - fail the request logger.debug("failed to write candidates", ex); // this exception is passed to all listeners - we don't retry. if this doesn't work we are in deep shit exception = ex; } } for (Tuple<Item, Consumer<Exception>> tuple : candidates) { Consumer<Exception> consumer = tuple.v2(); try { consumer.accept(exception); } catch (Exception ex) { logger.warn("failed to notify callback", ex); } } }
Decision decision = null; if (index != null) { if (logger.isTraceEnabled()) { logger.trace("Try relocating shard for index index [{}] from node [{}] to node [{}]", idx, maxNode.getNodeId(), minNode.getNodeId()); final AllocationDeciders deciders = allocation.deciders(); for (ShardRouting shard : index) { if (shard.started()) { this, minNode, idx) - weight.weightShardRemoved(this, maxNode, idx); if (delta < minCost || (candidate != null && Float.compare(delta, minCost) == 0 && candidate.id() > shard.id())) { logger.debug("Relocate shard [{}] from node [{}] to node [{}]", candidate, maxNode.getNodeId(), minNode.getNodeId()); minNode.addShard(routingNodes.relocateShard(candidate, minNode.getNodeId(), shardSize, allocation.changes()).v1()); return true; } else {
if (log.isDebugEnabled()) { log.debug("CCS case, original indices: " + Arrays.toString(localIndices)); log.debug("remoteClusterIndices ({}): {}", remoteClusterIndices.size(), remoteClusterIndices); return new Tuple<Boolean, String[]>(modified, localIndices);
Decision allocateDecision = result.v1(); if (allocateDecision.type() != Decision.Type.YES && (explain == false || hasInitiatedFetching(unassignedShard) == false)) { logger.trace("{}: ignoring allocation, can't be allocated on any node", unassignedShard); return AllocateUnassignedDecision.no(UnassignedInfo.AllocationStatus.fromDecision(allocateDecision.type()), result.v2() != null ? new ArrayList<>(result.v2().values()) : null); logger.trace("{}: ignoring allocation, still fetching shard stores", unassignedShard); allocation.setHasPendingAsyncFetch(); List<NodeAllocationResult> nodeDecisions = null; ShardRouting primaryShard = routingNodes.activePrimary(unassignedShard.shardId()); if (primaryShard == null) { assert explain : "primary should only be null here if we are in explain mode, so we didn't " + "exit early when canBeAllocatedToAtLeastOneNode didn't return a YES decision"; return AllocateUnassignedDecision.no(UnassignedInfo.AllocationStatus.fromDecision(allocateDecision.type()), new ArrayList<>(result.v2().values())); logger.trace("{}: no primary shard store found or allocated, letting actual allocation figure it out", unassignedShard); return AllocateUnassignedDecision.NOT_TAKEN; assert explain == false || matchingNodes.nodeDecisions != null : "in explain mode, we must have individual node decisions"; List<NodeAllocationResult> nodeDecisions = augmentExplanationsWithStoreInfo(result.v2(), matchingNodes.nodeDecisions); if (allocateDecision.type() != Decision.Type.YES) { return AllocateUnassignedDecision.no(UnassignedInfo.AllocationStatus.fromDecision(allocateDecision.type()), nodeDecisions);
if(clusterService.state().metaData().index(this.searchguardIndex).mapping("config") != null) { LOGGER.debug("sg index exists and was created before ES 6 (legacy layout)"); retVal.putAll(validate(legacycl.loadLegacy(configTypes.toArray(new String[0]), 5, TimeUnit.SECONDS), configTypes.size())); } else { LOGGER.debug("sg index exists and was created with ES 6 (new layout)"); retVal.putAll(validate(cl.load(configTypes.toArray(new String[0]), 5, TimeUnit.SECONDS), configTypes.size())); LOGGER.debug("sg index not exists (yet)"); retVal.putAll(validate(cl.load(configTypes.toArray(new String[0]), 30, TimeUnit.SECONDS), configTypes.size())); String configurationType = configTypes.iterator().next(); Map<String, String> fields = new HashMap<String, String>(); fields.put(configurationType, Strings.toString(retVal.get(configurationType).v2())); auditLog.logDocumentRead(this.searchguardIndex, configurationType, null, fields, complianceConfig);
private void deleteShardIfExistElseWhere(ClusterState state, IndexShardRoutingTable indexShardRoutingTable) { List<Tuple<DiscoveryNode, ShardActiveRequest>> requests = new ArrayList<>(indexShardRoutingTable.size()); String indexUUID = indexShardRoutingTable.shardId().getIndex().getUUID(); ClusterName clusterName = state.getClusterName(); for (ShardRouting shardRouting : indexShardRoutingTable) { assert shardRouting.started() : "expected started shard but was " + shardRouting; DiscoveryNode currentNode = state.nodes().get(shardRouting.currentNodeId()); requests.add(new Tuple<>(currentNode, new ShardActiveRequest(clusterName, indexUUID, shardRouting.shardId(), deleteShardTimeout))); } ShardActiveResponseHandler responseHandler = new ShardActiveResponseHandler(indexShardRoutingTable.shardId(), state.getVersion(), requests.size()); for (Tuple<DiscoveryNode, ShardActiveRequest> request : requests) { logger.trace("{} sending shard active check to {}", request.v2().shardId, request.v1()); transportService.sendRequest(request.v1(), ACTION_SHARD_EXISTS, request.v2(), responseHandler); } }
/** * Delete shard snapshot */ public void delete() { final Map<String, BlobMetaData> blobs; try { blobs = blobContainer.listBlobs(); } catch (IOException e) { throw new IndexShardSnapshotException(shardId, "Failed to list content of gateway", e); } Tuple<BlobStoreIndexShardSnapshots, Integer> tuple = buildBlobStoreIndexShardSnapshots(blobs); BlobStoreIndexShardSnapshots snapshots = tuple.v1(); int fileListGeneration = tuple.v2(); try { indexShardSnapshotFormat.delete(blobContainer, snapshotId.getUUID()); } catch (IOException e) { logger.debug("[{}] [{}] failed to delete shard snapshot file", shardId, snapshotId); } // Build a list of snapshots that should be preserved List<SnapshotFiles> newSnapshotsList = new ArrayList<>(); for (SnapshotFiles point : snapshots) { if (!point.snapshot().equals(snapshotId.getName())) { newSnapshotsList.add(point); } } // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index finalize(newSnapshotsList, fileListGeneration + 1, blobs, "snapshot deletion [" + snapshotId + "]"); }
/** * Fire some listeners. Does nothing if the list of listeners is null. */ private void fireListeners(List<Tuple<Translog.Location, Consumer<Boolean>>> listenersToFire) { if (listenersToFire != null) { listenerExecutor.execute(() -> { for (Tuple<Translog.Location, Consumer<Boolean>> listener : listenersToFire) { try { listener.v2().accept(false); } catch (Exception e) { logger.warn("Error firing refresh listener", e); } } }); } } }
for (int i = 0; i < previous.getIoStats().devicesStats.length; i++) { FsInfo.DeviceStats deviceStats = previous.getIoStats().devicesStats[i]; deviceMap.put(Tuple.tuple(deviceStats.majorDeviceNumber, deviceStats.minorDeviceNumber), deviceStats); final int majorDeviceNumber = Integer.parseInt(fields[0]); final int minorDeviceNumber = Integer.parseInt(fields[1]); if (!devicesNumbers.contains(Tuple.tuple(majorDeviceNumber, minorDeviceNumber))) { continue; writesCompleted, sectorsWritten, deviceMap.get(Tuple.tuple(majorDeviceNumber, minorDeviceNumber))); devicesStats.add(deviceStats); logger.debug(() -> new ParameterizedMessage( "unexpected exception processing /proc/diskstats for devices {}", devicesNumbers), e); return null;
/** * Execute a scripted upsert, where there is an existing upsert document and a script to be executed. The script is executed and a new * Tuple of operation and updated {@code _source} is returned. */ Tuple<UpdateOpType, Map<String, Object>> executeScriptedUpsert(IndexRequest upsert, Script script, LongSupplier nowInMillis) { Map<String, Object> upsertDoc = upsert.sourceAsMap(); Map<String, Object> ctx = new HashMap<>(3); // Tell the script that this is a create and not an update ctx.put(ContextFields.OP, UpdateOpType.CREATE.toString()); ctx.put(ContextFields.SOURCE, upsertDoc); ctx.put(ContextFields.NOW, nowInMillis.getAsLong()); ctx = executeScript(script, ctx); UpdateOpType operation = UpdateOpType.lenientFromString((String) ctx.get(ContextFields.OP), logger, script.getIdOrCode()); Map newSource = (Map) ctx.get(ContextFields.SOURCE); if (operation != UpdateOpType.CREATE && operation != UpdateOpType.NONE) { // Only valid options for an upsert script are "create" (the default) or "none", meaning abort upsert logger.warn("Invalid upsert operation [{}] for script [{}], doing nothing...", operation, script.getIdOrCode()); operation = UpdateOpType.NONE; } return new Tuple<>(operation, newSource); }
final MoveDecision moveDecision = decideMove(shardRouting); if (moveDecision.isDecisionTaken() && moveDecision.forceMove()) { final ModelNode sourceNode = nodes.get(shardRouting.currentNodeId()); final ModelNode targetNode = nodes.get(moveDecision.getTargetNode().getId()); sourceNode.removeShard(shardRouting); allocation.clusterInfo().getShardSize(shardRouting, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE), allocation.changes()); targetNode.addShard(relocatingShards.v2()); if (logger.isTraceEnabled()) { logger.trace("Moved shard [{}] to node [{}]", shardRouting, targetNode.getRoutingNode()); logger.trace("[{}][{}] can't move", shardRouting.index(), shardRouting.id());
} else { if (timeout == null) { listeners.put(listener, Tuple.tuple(waitingForGlobalCheckpoint, null)); } else { listeners.put( listener, Tuple.tuple( waitingForGlobalCheckpoint, scheduler.schedule( removed = listeners.remove(listener).v2() != null; logger.trace("global checkpoint listener timed out", e); executor.execute(() -> notifyListener(listener, UNASSIGNED_SEQ_NO, e));
@Test public void testComplianceLicenseMap() throws Exception { SearchGuardLicense license = new SearchGuardLicense(XContentHelper .convertToMap(new BytesArray(FileHelper.loadFile("license1.json")), false, JsonXContent.jsonXContent.type()).v2(), cs); Assert.assertFalse(license.hasFeature(Feature.COMPLIANCE)); Assert.assertArrayEquals(license.getFeatures(), new Feature[0]); license = new SearchGuardLicense(XContentHelper .convertToMap(new BytesArray(FileHelper.loadFile("license3.json")), false, JsonXContent.jsonXContent.type()).v2(), cs); Assert.assertFalse(license.hasFeature(Feature.COMPLIANCE)); Assert.assertArrayEquals(license.getFeatures(), new Feature[0]); license = new SearchGuardLicense(XContentHelper .convertToMap(new BytesArray(FileHelper.loadFile("license2.json")), false, JsonXContent.jsonXContent.type()).v2(), cs); Assert.assertTrue(license.hasFeature(Feature.COMPLIANCE)); Assert.assertArrayEquals(license.getFeatures(), Feature.values()); }