public static void checkIndexStatus(Settings settings) { if (!settings.getIndexReadAllowRedStatus()) { RestClient bootstrap = new RestClient(settings); Resource readResource = new Resource(settings, true); try { if (bootstrap.indexExists(readResource.index())) { RestClient.Health status = bootstrap.getHealth(readResource.index()); if (status == RestClient.Health.RED) { throw new EsHadoopIllegalStateException("Index specified [" + readResource.index() + "] is either red or " + "includes an index that is red, and thus all requested data cannot be safely and fully loaded. " + "Bailing out..."); } } } finally { bootstrap.close(); } } }
private RestClient mockClientResponses(RestClient.BulkActionResponse... responses) { RestClient mockClient = Mockito.mock(RestClient.class); OngoingStubbing<RestClient.BulkActionResponse> stubb = Mockito.when( mockClient.bulk(Mockito.eq(resource), Mockito.any(TrackingBytesArray.class)) ); for (RestClient.BulkActionResponse response : responses) { stubb = stubb.thenReturn(response); } stubb.thenThrow(new AssertionError("Exhausted all given test responses.")); return mockClient; }
public long countIndexShard(String index, String shardId, QueryBuilder query) { return count(index, null, shardId, query); }
protected Response executeNotFoundAllowed(Request req) { Response res = execute(req, false); switch (res.status()) { case HttpStatus.OK: break; case HttpStatus.NOT_FOUND: break; default: checkResponse(req, res); } return res; }
private long countBeforeES5X(String indexAndType, String shardId, QueryBuilder query) { StringBuilder uri = new StringBuilder(indexAndType); uri.append("/_count"); if (StringUtils.hasLength(shardId)) { uri.append("?preference=_shards:"); uri.append(shardId); } Response response = execute(GET, uri.toString(), searchRequest(query)); Number count = (Number) parseContent(response.body(), "count"); return (count != null ? count.longValue() : -1); }
public static List<NodeInfo> discoverNodesIfNeeded(Settings settings, Log log) { if (settings.getNodesDiscovery()) { RestClient bootstrap = new RestClient(settings); try { List<NodeInfo> discoveredNodes = bootstrap.getHttpNodes(false); if (log.isDebugEnabled()) { log.debug(String.format("Nodes discovery enabled - found %s", discoveredNodes)); } SettingsUtils.addDiscoveredNodes(settings, discoveredNodes); return discoveredNodes; } finally { bootstrap.close(); } } return null; }
@Test public void testPostDocumentSuccess() throws Exception { String index = "index/type"; Settings settings = new TestSettings(); settings.setInternalVersion(EsMajorVersion.V_7_X); settings.setResourceWrite(index); Resource writeResource = new Resource(settings, false); BytesArray document = new BytesArray("{\"field\":\"value\"}"); SimpleRequest request = new SimpleRequest(Request.Method.POST, null, index, null, document); String response = "{\n" + " \"_index\": \"index\",\n" + " \"_type\": \"type\",\n" + " \"_id\": \"AbcDefGhiJklMnoPqrS_\",\n" + " \"_version\": 1,\n" + " \"result\": \"created\",\n" + " \"_shards\": {\n" + " \"total\": 2,\n" + " \"successful\": 1,\n" + " \"failed\": 0\n" + " },\n" + " \"_seq_no\": 0,\n" + " \"_primary_term\": 1\n" + "}"; NetworkClient mock = Mockito.mock(NetworkClient.class); Mockito.when(mock.execute(Mockito.eq(request))).thenReturn(new SimpleResponse(201, new FastByteArrayInputStream(new BytesArray(response)), "localhost:9200")); RestClient client = new RestClient(new TestSettings(), mock); String id = client.postDocument(writeResource, document); assertEquals("AbcDefGhiJklMnoPqrS_", id); }
RestClient client = new RestClient(testSettings, mock); long count = client.count(index, type, MatchAllQueryBuilder.MATCH_ALL);
client.delete(resources.getResourceWrite().index() + "/" + resources.getResourceWrite().type()); client.delete(resources.getResourceWrite().index() + "/" + resources.getResourceWrite().type() + "/_query?q=*"); } else { client.delete(resources.getResourceWrite().index() + "/_query?q=*"); flush(); client.refresh(resources.getResourceWrite());
public static EsMajorVersion discoverEsVersion(Settings settings, Log log) { String version = settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_VERSION); if (StringUtils.hasText(version)) { if (log.isDebugEnabled()) { log.debug(String.format("Elasticsearch version [%s] already present in configuration; skipping discovery", version)); } return EsMajorVersion.parse(version); } RestClient bootstrap = new RestClient(settings); // first get ES version try { EsMajorVersion esVersion = bootstrap.remoteEsVersion(); if (log.isDebugEnabled()) { log.debug(String.format("Discovered Elasticsearch version [%s]", esVersion)); } settings.setInternalVersion(esVersion); return esVersion; } catch (EsHadoopException ex) { throw new EsHadoopIllegalArgumentException(String.format("Cannot detect ES version - " + "typically this happens if the network/Elasticsearch cluster is not accessible or when targeting " + "a WAN/Cloud instance without the proper setting '%s'", ConfigurationOptions.ES_NODES_WAN_ONLY), ex); } finally { bootstrap.close(); } }
public void waitForShards(String index) throws Exception { restClient.execute(GET, "_cluster/health/" + index + "?level=indices&wait_for_status=yellow"); } }
RestClient bootstrap = new RestClient(settings); try { String message = "No data nodes with HTTP-enabled available"; List<NodeInfo> dataNodes = bootstrap.getHttpDataNodes(); if (dataNodes.isEmpty()) { throw new EsHadoopIllegalArgumentException(message); bootstrap.close();
RestClient bootstrap = new RestClient(settings); try { String message = "Client-only routing specified but no client nodes with HTTP-enabled available"; List<NodeInfo> clientNodes = bootstrap.getHttpClientNodes(); if (clientNodes.isEmpty()) { throw new EsHadoopIllegalArgumentException(message); bootstrap.close();
public List<List<Map<String, Object>>> targetShards(String index, String routing) { List<List<Map<String, Object>>> shardsJson = null; // https://github.com/elasticsearch/elasticsearch/issues/2726 String target = index + "/_search_shards"; if (routing != null) { target += "?routing=" + HttpEncodingTools.encode(routing); } if (indexReadMissingAsEmpty) { Request req = new SimpleRequest(GET, null, target); Response res = executeNotFoundAllowed(req); if (res.status() == HttpStatus.OK) { shardsJson = parseContent(res.body(), "shards"); } else { shardsJson = Collections.emptyList(); } } else { shardsJson = get(target, "shards"); } return shardsJson; }
RestClient bootstrap = new RestClient(settings); try { String message = "Ingest-only routing specified but no ingest nodes with HTTP-enabled available"; List<NodeInfo> clientNodes = bootstrap.getHttpIngestNodes(); if (clientNodes.isEmpty()) { throw new EsHadoopIllegalArgumentException(message); bootstrap.close();
public boolean delete(String indexOrType) { Request req = new SimpleRequest(DELETE, null, indexOrType); Response res = executeNotFoundAllowed(req); return (res.status() == HttpStatus.OK ? true : false); } public boolean deleteScroll(String scrollId) {
public void putMapping(String index, String type, byte[] bytes) { // create index first (if needed) - it might return 403/404 touch(index); if (internalVersion.after(EsMajorVersion.V_6_X)) { execute(PUT, index + "/_mapping", new BytesArray(bytes)); } else { execute(PUT, index + "/_mapping/" + type, new BytesArray(bytes)); } }
@SuppressWarnings("unchecked") private <T> T parseContent(InputStream content, String string) { Map<String, Object> map = Collections.emptyMap(); try { // create parser manually to lower Jackson requirements JsonParser jsonParser = mapper.getJsonFactory().createJsonParser(content); try { map = mapper.readValue(jsonParser, Map.class); } finally { countStreamStats(content); } } catch (IOException ex) { throw new EsHadoopParsingException(ex); } return (T) (string != null ? map.get(string) : map); }
@Override public void close() { if (!closed) { closed = true; finished = true; batch = Collections.emptyList(); reader.close(); // typically the scroll is closed after it is consumed so this will trigger a 404 // however we're closing it either way if (StringUtils.hasText(scrollId)) { repository.getRestClient().deleteScroll(scrollId); } } }
public static EsMajorVersion getEsVersion() { return new RestClient(new TestSettings()).remoteEsVersion(); }