From 246546f485b70a1276a0debf26e3cd585a7ecfc1 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Mon, 5 Oct 2026 17:54:42 +0000 Subject: [PATCH 1/5] SOLR-15823: Add 'nodes' broadcast to the V2 logging set-level endpoint PUT /api/node/logging/levels now accepts a 'nodes' query parameter. When present, NodeLogging fans the request out to the named nodes (or every live node for nodes=all) through V2SolrRequestBasedProxy, the same machinery GetNodeSystemInfo uses, and the response carries one result per node plus the names of any requested nodes that did not respond. Unknown node names are rejected before anything is sent, and using 'nodes' outside SolrCloud mode is a 400 rather than an NPE. The V1 handler now calls the endpoint method with a null nodes value so it does not broadcast twice. --- .../client/api/endpoint/NodeLoggingApis.java | 7 +- .../client/api/model/LoggingResponse.java | 32 ++++ .../solr/handler/admin/LoggingHandler.java | 2 +- .../solr/handler/admin/api/NodeLogging.java | 65 +++++++- .../handler/admin/api/NodeLoggingAPITest.java | 35 +++- .../api/NodeLoggingNodesSolrCloudTest.java | 153 ++++++++++++++++++ 6 files changed, 288 insertions(+), 6 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingNodesSolrCloudTest.java diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeLoggingApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeLoggingApis.java index c5b5cd958070..06d28017a5f0 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeLoggingApis.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeLoggingApis.java @@ -42,8 +42,13 @@ public interface NodeLoggingApis { @Path("/levels") @Operation( summary = "Set one or more logger levels on the target node.", + description = + "If the 'nodes' parameter is provided, the level changes are instead applied to " + + "each of the named nodes (or to every live node, if 'nodes' is 'all'), and the " + + "response reports the per-node results.", tags = {"logging"}) - LoggingResponse modifyLocalLogLevel(List requestBody); + LoggingResponse modifyLocalLogLevel( + @QueryParam("nodes") String nodes, List requestBody); @GET @Path("/messages") diff --git a/solr/api/src/java/org/apache/solr/client/api/model/LoggingResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/LoggingResponse.java index 1ba4be2c0d34..42541b489408 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/LoggingResponse.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/LoggingResponse.java @@ -16,10 +16,42 @@ */ package org.apache.solr.client.api.model; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; /** Generic logging response that includes the name of the log watcher (e.g. "Log4j2") */ public class LoggingResponse extends SolrJerseyResponse { @JsonProperty("watcher") public String watcherName; + + /** + * Per-node results of a request broadcast with the 'nodes' parameter, keyed by node name in "live + * node" format (e.g. "someHost:8983_solr"). Serialized inline, as top-level fields named by node, + * mirroring {@link NodeSystemResponse#remoteNodeData}. Empty for requests that were not + * broadcast. + */ + // Object, not LoggingResponse, since @JsonAnySetter below also feeds this map raw values. + public Map remoteNodeData = new LinkedHashMap<>(); + + @JsonAnyGetter + public Map remoteNodeData() { + return remoteNodeData; + } + + @JsonAnySetter + public void setRemoteNodeResponse(String field, Object value) { + remoteNodeData.put(field, value); + } + + /** + * Nodes that were asked to apply a broadcast request (via the 'nodes' parameter) but did not + * return a response, e.g. because they timed out or errored. Null for requests that were not + * broadcast, and empty when every requested node responded. + */ + @JsonProperty("failedNodes") + public List failedNodes; } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java index 77e65ede7150..deaa364def91 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java @@ -75,7 +75,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw String[] set = params.getParams("set"); if (set != null) { final List changes = NodeLogging.parseLogLevelChanges(set); - squashV2Response(rsp, loggingApi.modifyLocalLogLevel(changes)); + squashV2Response(rsp, loggingApi.modifyLocalLogLevel(null, changes)); } String since = req.getParams().get("since"); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/NodeLogging.java b/solr/core/src/java/org/apache/solr/handler/admin/api/NodeLogging.java index 5594d5f084ef..47085fdf36dc 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/NodeLogging.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/NodeLogging.java @@ -24,7 +24,10 @@ import jakarta.inject.Inject; import java.lang.invoke.MethodHandles; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.apache.solr.api.JerseyResource; @@ -36,17 +39,19 @@ import org.apache.solr.client.api.model.LogMessagesResponse; import org.apache.solr.client.api.model.LoggingResponse; import org.apache.solr.client.api.model.SetThresholdRequestBody; +import org.apache.solr.client.solrj.request.LoggingApi; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.admin.proxy.V2SolrRequestBasedProxy; import org.apache.solr.jersey.PermissionName; import org.apache.solr.logging.LogWatcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -// TODO Add support for 'nodes' param once SOLR-16738 is completed. /** - * V2 APIs for getting or setting log levels on an individual node. + * V2 APIs for getting or setting log levels on an individual node, or for broadcasting log level + * changes across nodes. * *

These APIs ('/api/node/logging' and descendants) are analogous to the v1 /admin/info/logging. */ @@ -84,7 +89,7 @@ public ListLevelsResponse listAllLoggersAndLevels() { @Override @PermissionName(CONFIG_EDIT_PERM) - public LoggingResponse modifyLocalLogLevel(List requestBody) { + public LoggingResponse modifyLocalLogLevel(String nodes, List requestBody) { ensureLogWatcherEnabled(); final LoggingResponse response = instantiateLoggingResponse(LoggingResponse.class); @@ -92,12 +97,66 @@ public LoggingResponse modifyLocalLogLevel(List requestBody) { throw new SolrException(BAD_REQUEST, "Missing request body"); } + if (nodes != null && !nodes.isEmpty()) { + proxyLevelChangesToNodes(response, nodes, requestBody); + return response; + } + for (LogLevelChange change : requestBody) { watcher.setLogLevel(change.logger, change.level); } return response; } + /** + * Applies the given log level changes on other nodes, mirroring how {@link GetNodeSystemInfo} + * fans its request out. The receiving node does not apply the changes locally; it is covered only + * if the resolved node set includes it, in which case it calls itself over HTTP. + */ + private void proxyLevelChangesToNodes( + LoggingResponse response, String nodes, List requestBody) { + if (coreContainer == null || coreContainer.getZkController() == null) { + throw new SolrException( + BAD_REQUEST, "The 'nodes' parameter is only supported in SolrCloud mode"); + } + try { + final var req = new LoggingApi.ModifyLocalLogLevel(); + requestBody.forEach(req::addLogLevelChange); + req.setNodes(nodes); + final var reqProxy = + new V2SolrRequestBasedProxy(coreContainer, req) { + @Override + public void processTypedProxiedResponse( + String nodeName, LoggingResponse proxiedResponse) { + response.remoteNodeData.put(nodeName, proxiedResponse); + } + }; + final Collection destinationNodes = reqProxy.getDestinationNodes(); + // Fail before sending anything if a named node is not part of the cluster; otherwise the + // broadcast would fail partway through, after earlier nodes already applied the change. + final Set liveNodes = + coreContainer.getZkController().zkStateReader.getClusterState().getLiveNodes(); + final List unknownNodes = + destinationNodes.stream().filter(node -> !liveNodes.contains(node)).sorted().toList(); + if (!unknownNodes.isEmpty()) { + throw new SolrException( + BAD_REQUEST, "Requested nodes are not part of the cluster: " + unknownNodes); + } + reqProxy.proxyRequest(); + // The proxy logs and skips nodes that error or time out; surface those nodes here so a + // partial broadcast is visible in the response instead of silent. + final var failedNodes = new ArrayList<>(destinationNodes); + failedNodes.removeAll(response.remoteNodeData.keySet()); + Collections.sort(failedNodes); + response.failedNodes = failedNodes; + } catch (SolrException e) { + throw e; + } catch (Exception e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "Error occurred while proxying to other nodes", e); + } + } + @Override @PermissionName(CONFIG_READ_PERM) public LogMessagesResponse fetchLocalLogMessages(Long boundingTimeMillis) { diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingAPITest.java index 8838ac9e4835..c0b9f0aa94ce 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingAPITest.java @@ -23,6 +23,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -32,6 +33,7 @@ import org.apache.solr.client.api.model.SetThresholdRequestBody; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; import org.apache.solr.core.CoreContainer; import org.apache.solr.logging.LogWatcher; import org.apache.solr.logging.LoggerInfo; @@ -84,13 +86,44 @@ public void testReliesOnLogWatcherToListLogLevels() { public void testReliesOnLogWatcherToModifyLogLevels() { final var responseBody = new NodeLogging(mockCoreContainer) - .modifyLocalLogLevel(List.of(new LogLevelChange("o.a.s.Foo", "WARN"))); + .modifyLocalLogLevel(null, List.of(new LogLevelChange("o.a.s.Foo", "WARN"))); assertNotNull(responseBody); assertNull("Expected error to be null but was " + responseBody.error, responseBody.error); verify(mockLogWatcher).setLogLevel("o.a.s.Foo", "WARN"); } + @Test + public void testModifyLogLevelsWithoutNodesReportsNoBroadcastResults() { + final var responseBody = + new NodeLogging(mockCoreContainer) + .modifyLocalLogLevel(null, List.of(new LogLevelChange("o.a.s.Foo", "WARN"))); + + assertTrue( + "Expected no per-node results for a local request, but was " + responseBody.remoteNodeData, + responseBody.remoteNodeData.isEmpty()); + assertNull( + "Expected failedNodes to be unset for a local request, but was " + responseBody.failedNodes, + responseBody.failedNodes); + } + + @Test + public void testModifyLogLevelsWithNodesFailsOutsideSolrCloud() { + // The mock CoreContainer has no ZkController stubbed, as on a standalone (user-managed) + // node, so NodeLogging sees a null ZkController here. + try { + new NodeLogging(mockCoreContainer) + .modifyLocalLogLevel("all", List.of(new LogLevelChange("o.a.s.Foo", "WARN"))); + fail("Expected a SolrException when 'nodes' is used outside SolrCloud mode"); + } catch (SolrException e) { + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code()); + assertTrue( + "Expected the error to name the 'nodes' parameter, but was: " + e.getMessage(), + e.getMessage().contains("'nodes'")); + } + verify(mockLogWatcher, never()).setLogLevel(any(), any()); + } + private SolrDocumentList logMessageDocList(String... logMessages) { final var docList = new SolrDocumentList(); for (String logMessage : logMessages) { diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingNodesSolrCloudTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingNodesSolrCloudTest.java new file mode 100644 index 000000000000..997337af5fdb --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingNodesSolrCloudTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.charset.StandardCharsets; +import org.apache.solr.client.solrj.RemoteSolrException; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; +import org.apache.solr.client.solrj.request.GenericV2SolrRequest; +import org.apache.solr.client.solrj.response.InputStreamResponseParser; +import org.apache.solr.client.solrj.response.SimpleSolrResponse; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.embedded.JettySolrRunner; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Tests the {@code nodes} broadcast on the V2 {@code PUT /node/logging/levels} endpoint against a + * two-node cluster. + * + *

The requests are sent with a generic V2 request, and the assertions run against the raw JSON + * response body. Log levels themselves cannot be observed per node here: every node of the test + * cluster shares one JVM, and log levels are JVM-wide. What is observable per node is the response: + * a broadcast request is expected to carry one result entry per requested node, keyed by node name, + * plus the names of any requested nodes that did not respond. + */ +public class NodeLoggingNodesSolrCloudTest extends SolrCloudTestCase { + + private static final String LEVEL_CHANGES_JSON = + "[{\"logger\": \"org.apache.solr.handler.admin.api.NodeLoggingNodesSolrCloudTest\"," + + " \"level\": \"WARN\"}]"; + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(2).addConfig("cloud-minimal", configset("cloud-minimal")).configure(); + } + + @Test + public void testBroadcastToAllNodesReportsEveryNode() throws Exception { + final String receivingNode = jettyName(0); + final String otherNode = jettyName(1); + + final JsonNode rsp = putLogLevels(cluster.getJettySolrRunners().get(0), "all"); + + final JsonNode failedNodes = rsp.get("failedNodes"); + assertNotNull("Broadcast response should report failed nodes", failedNodes); + assertTrue("No node should be reported failed: " + failedNodes, failedNodes.isEmpty()); + for (String nodeName : new String[] {receivingNode, otherNode}) { + final JsonNode perNode = rsp.get(nodeName); + assertNotNull("Expected a per-node result for " + nodeName + " in " + rsp, perNode); + // The per-node entry is that node's own logging response. Its presence proves the request + // body (the level changes) reached the node and a response came back. + assertNotNull(perNode.get("watcher")); + } + } + + @Test + public void testBroadcastToSingleNamedNode() throws Exception { + final String receivingNode = jettyName(0); + final String otherNode = jettyName(1); + + final JsonNode rsp = putLogLevels(cluster.getJettySolrRunners().get(0), otherNode); + + assertNotNull(rsp.get("failedNodes")); + assertTrue(rsp.get("failedNodes").isEmpty()); + assertNotNull("Expected a per-node result for " + otherNode, rsp.get(otherNode)); + assertNull("The receiving node was not a target and must not appear", rsp.get(receivingNode)); + } + + @Test + public void testNoNodesKeepsLocalResponseShape() throws Exception { + final JsonNode rsp = putLogLevels(cluster.getJettySolrRunners().get(0), null); + + assertNotNull(rsp.get("watcher")); + assertNull(rsp.get("failedNodes")); + rsp.fieldNames() + .forEachRemaining( + field -> + assertFalse("No per-node entries expected: " + field, field.endsWith("_solr"))); + } + + @Test + public void testUnknownNodeNameFailsFast() { + final GenericV2SolrRequest req = + new GenericV2SolrRequest( + SolrRequest.METHOD.PUT, + "/node/logging/levels", + SolrRequest.SolrRequestType.ADMIN, + new ModifiableSolrParams().set("nodes", "no-such-host.invalid:9999_solr")); + req.withContent(LEVEL_CHANGES_JSON.getBytes(StandardCharsets.UTF_8), "application/json"); + final RemoteSolrException e = + expectThrows(RemoteSolrException.class, () -> sendWithDefaultParser(jetty(0), req)); + assertEquals(400, e.code()); + assertTrue( + "Error should name the problem: " + e.getMessage(), + e.getMessage().contains("not part of the cluster")); + } + + private static String jettyName(int idx) { + return cluster.getJettySolrRunners().get(idx).getNodeName(); + } + + private static JettySolrRunner jetty(int idx) { + return cluster.getJettySolrRunners().get(idx); + } + + private static JsonNode putLogLevels(JettySolrRunner target, String nodes) throws Exception { + final ModifiableSolrParams params = new ModifiableSolrParams(); + if (nodes != null) { + params.set("nodes", nodes); + } + final GenericV2SolrRequest req = + new GenericV2SolrRequest( + SolrRequest.METHOD.PUT, + "/node/logging/levels", + SolrRequest.SolrRequestType.ADMIN, + params); + req.withContent(LEVEL_CHANGES_JSON.getBytes(StandardCharsets.UTF_8), "application/json"); + req.setResponseParser(new InputStreamResponseParser("json")); + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(target.getBaseUrl().toString()).build()) { + final String body = + InputStreamResponseParser.consumeResponseToString(req.process(client).getResponse()); + return MAPPER.readTree(body); + } + } + + private static SimpleSolrResponse sendWithDefaultParser( + JettySolrRunner target, GenericV2SolrRequest req) throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(target.getBaseUrl().toString()).build()) { + return req.process(client); + } + } +} From a76e30e48db11b95592ce0d2cded98553097232d Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Mon, 5 Oct 2026 17:54:42 +0000 Subject: [PATCH 2/5] SOLR-15823: Move the Admin UI logging screen to the V2 set-level endpoint LoggingLevelController.setLevel now calls LoggingV2.modifyLocalLogLevel unconditionally, passing nodes=all only in SolrCloud mode, and the V1 Logging factory is deleted from services.js. --- .../web/js/angular/controllers/logging.js | 19 ++++++++----------- solr/webapp/web/js/angular/services.js | 11 ----------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/solr/webapp/web/js/angular/controllers/logging.js b/solr/webapp/web/js/angular/controllers/logging.js index a3529c3d420f..c2f6d46eb6db 100644 --- a/solr/webapp/web/js/angular/controllers/logging.js +++ b/solr/webapp/web/js/angular/controllers/logging.js @@ -101,7 +101,7 @@ solrAdminApp.controller('LoggingController', ) .controller('LoggingLevelController', - function($scope, $timeout, Logging, LoggingV2, ApiErrorHandler) { + function($scope, $timeout, LoggingV2, ApiErrorHandler) { $scope.resetMenu("logging-levels"); var packageOf = function(logger) { @@ -159,20 +159,17 @@ solrAdminApp.controller('LoggingController', if (!$scope.isPermitted(permissions.CONFIG_EDIT_PERM)) { return; } - var setString = logger.name + ":" + newLevel; logger.showOptions = false; - // Intentionally still v1 (Logging, not LoggingV2): this relies on the "nodes=all" param to - // broadcast the level change to every live node. The v2 NodeLoggingApis endpoint is - // single-node only until SOLR-16738 wires it up to the new V2SolrRequestBasedProxy (see the - // TODO in NodeLogging.java). Move this to LoggingV2.modifyLocalLogLevel once that lands. var doSetLevel = function() { - var params = {set: setString}; - // "nodes=all" only makes sense in SolrCloud mode; in standalone (user-managed) mode the - // request is handled locally, so the param is omitted entirely. + var opts = {logLevelChange: [{logger: logger.name, level: newLevel}]}; + // "nodes=all" only makes sense in SolrCloud mode, where it broadcasts the level + // change to every live node; in standalone (user-managed) mode the request is + // handled locally, so the param is omitted entirely. if ($scope.isCloudEnabled) { - params.nodes = "all"; + opts.nodes = "all"; } - Logging.setLevel(params, function(data) { + LoggingV2.modifyLocalLogLevel(opts, function(error, data, response) { + if (error) { ApiErrorHandler.handle(response); return; } $scope.refresh(); }); }; diff --git a/solr/webapp/web/js/angular/services.js b/solr/webapp/web/js/angular/services.js index c0b525b1698c..3b18921e2aed 100644 --- a/solr/webapp/web/js/angular/services.js +++ b/solr/webapp/web/js/angular/services.js @@ -193,17 +193,6 @@ solrAdminServices.factory('Metrics', } }; }]) -.factory('Logging', - ['$resource', function($resource) { - // This v1 factory only covers "setLevel", which in SolrCloud mode needs the "nodes=all" - // broadcast-to-every-node behavior that the v2 NodeLoggingApis endpoint doesn't support yet - // (see SOLR-16738). The caller (LoggingLevelController) passes nodes:'all' only when - // SolrCloud is enabled; in standalone mode the param is omitted entirely. Retire this - // factory once setLevel moves to LoggingV2. - return $resource('admin/info/logging', {'wt':'json', '_':Date.now()}, { - "setLevel": {} - }); - }]) .factory('Zookeeper', ['$resource', function($resource) { // Tree browsing (formerly "simple"/"detail") moved to ZookeeperReadV2; this factory now only From 100ad2e0df69ff800c269d1db15d26b6d027d1a8 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Mon, 5 Oct 2026 17:54:42 +0000 Subject: [PATCH 3/5] SOLR-15823: Changelog entry and reference guide note for the nodes parameter --- changelog/unreleased/SOLR-15823.yml | 7 +++++++ .../deployment-guide/pages/configuring-logging.adoc | 13 +++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 changelog/unreleased/SOLR-15823.yml diff --git a/changelog/unreleased/SOLR-15823.yml b/changelog/unreleased/SOLR-15823.yml new file mode 100644 index 000000000000..97701cd06bf4 --- /dev/null +++ b/changelog/unreleased/SOLR-15823.yml @@ -0,0 +1,7 @@ +title: V2 logging API can broadcast log level changes to multiple nodes, and the Admin UI logging screen now uses it. +type: added +authors: + - name: Nick Shanin +links: + - name: SOLR-15823 + url: https://issues.apache.org/jira/browse/SOLR-15823 diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/configuring-logging.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/configuring-logging.adoc index 5adfc0bc9f9e..413038d9399d 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/configuring-logging.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/configuring-logging.adoc @@ -102,6 +102,19 @@ curl -X PUT http://localhost:8983/api/node/logging/levels -H 'Content-Type: appl ==== ====== +In SolrCloud mode, the V2 request above changes only the node that receives it. +Add the `nodes` query parameter to apply the same changes on other nodes: `nodes=all` targets every live node, or provide a comma-separated list of node names (in `host:port_solr` form). +The response then reports the result from each node, and names any requested nodes that did not respond under `failedNodes`. + +[source,bash] +---- +curl -X PUT "http://localhost:8983/api/node/logging/levels?nodes=all" -H 'Content-Type: application/json' -d ' + [ + {"logger": "root", "level": "WARN"} + ] +' +---- + === Choosing Log Level at Startup You can temporarily choose a different logging level as you start Solr. From 85d8df82502c4314b2946e413d63e0b4696c4a6a Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Mon, 5 Oct 2026 22:57:46 +0000 Subject: [PATCH 4/5] SOLR-15823: Test the failedNodes report and the applied level change NodeLoggingAPITest gains a broadcast case where one node's proxied request fails: the response must carry a result for the node that answered and name the other in failedNodes. The cloud broadcast test now also checks the JVM-wide level afterwards, showing the payload was applied rather than only delivered, and restores the logger to unset when it finishes. --- .../handler/admin/api/NodeLoggingAPITest.java | 72 +++++++++++++++++ .../api/NodeLoggingNodesSolrCloudTest.java | 77 +++++++++++++++---- 2 files changed, 134 insertions(+), 15 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingAPITest.java index c0b9f0aa94ce..4ec5b7ee38d0 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingAPITest.java @@ -21,19 +21,31 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.lang.reflect.Field; import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; import org.apache.solr.SolrTestCase; import org.apache.solr.client.api.model.LogLevelChange; +import org.apache.solr.client.api.model.LoggingResponse; import org.apache.solr.client.api.model.SetThresholdRequestBody; +import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; +import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.core.CoreContainer; import org.apache.solr.logging.LogWatcher; import org.apache.solr.logging.LoggerInfo; @@ -124,6 +136,66 @@ public void testModifyLogLevelsWithNodesFailsOutsideSolrCloud() { verify(mockLogWatcher, never()).setLogLevel(any(), any()); } + @Test + @SuppressForbidden( + reason = + "places a stubbed ZkStateReader on a ZkController mock; production reads the public" + + " final field directly, so there is no setter or getter to stub") + public void testModifyLogLevelsWithNodesReportsNodesThatDidNotRespond() throws Exception { + final String respondingNode = "host1:8983_solr"; + final String failingNode = "host2:8983_solr"; + + final ZkController mockZkController = mock(ZkController.class); + final ZkStateReader mockZkStateReader = mock(ZkStateReader.class); + final ClusterState mockClusterState = mock(ClusterState.class); + // NodeLogging and the request proxy read ZkController.zkStateReader directly (a public + // final field) rather than calling a getter, so the stubbed reader has to be placed on + // the mock reflectively. + final Field zkStateReaderField = ZkController.class.getField("zkStateReader"); + zkStateReaderField.setAccessible(true); + zkStateReaderField.set(mockZkController, mockZkStateReader); + + when(mockCoreContainer.getZkController()).thenReturn(mockZkController); + when(mockZkController.getCoreContainer()).thenReturn(mockCoreContainer); + when(mockZkStateReader.getClusterState()).thenReturn(mockClusterState); + when(mockClusterState.getLiveNodes()).thenReturn(Set.of(respondingNode, failingNode)); + when(mockZkStateReader.getBaseUrlForNodeName(respondingNode)) + .thenReturn("http://host1:8983/solr"); + when(mockZkStateReader.getBaseUrlForNodeName(failingNode)).thenReturn("http://host2:8983/solr"); + + // One node answers with a logging response; the other's request fails, the way the proxy + // sees a node that errors or times out. + final HttpJettySolrClient mockClient = mock(HttpJettySolrClient.class); + when(mockCoreContainer.getDefaultHttpSolrClient()).thenReturn(mockClient); + final NamedList proxiedResponse = new NamedList<>(); + proxiedResponse.add("response", new LoggingResponse()); + doAnswer( + invocation -> { + if (invocation.getArgument(0).equals("http://host1:8983/solr")) { + return CompletableFuture.completedFuture(proxiedResponse); + } + return CompletableFuture.failedFuture(new RuntimeException("simulated node failure")); + }) + .when(mockClient) + .requestWithBaseUrl(anyString(), any()); + + final var responseBody = + new NodeLogging(mockCoreContainer) + .modifyLocalLogLevel( + respondingNode + "," + failingNode, + List.of(new LogLevelChange("o.a.s.Foo", "WARN"))); + + assertEquals( + "Expected only the responding node to report a result", + Set.of(respondingNode), + responseBody.remoteNodeData.keySet()); + assertEquals( + "Expected the node whose request failed to be reported", + List.of(failingNode), + responseBody.failedNodes); + verify(mockLogWatcher, never()).setLogLevel(any(), any()); + } + private SolrDocumentList logMessageDocList(String... logMessages) { final var docList = new SolrDocumentList(); for (String logMessage : logMessages) { diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingNodesSolrCloudTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingNodesSolrCloudTest.java index 997337af5fdb..8f69aa2e51c0 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingNodesSolrCloudTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingNodesSolrCloudTest.java @@ -39,13 +39,17 @@ * response body. Log levels themselves cannot be observed per node here: every node of the test * cluster shares one JVM, and log levels are JVM-wide. What is observable per node is the response: * a broadcast request is expected to carry one result entry per requested node, keyed by node name, - * plus the names of any requested nodes that did not respond. + * plus the names of any requested nodes that did not respond. The JVM-wide level after a broadcast + * is observable, and one test uses it to show the level change was applied. */ public class NodeLoggingNodesSolrCloudTest extends SolrCloudTestCase { + private static final String TEST_LOGGER = + "org.apache.solr.handler.admin.api.NodeLoggingNodesSolrCloudTest"; private static final String LEVEL_CHANGES_JSON = - "[{\"logger\": \"org.apache.solr.handler.admin.api.NodeLoggingNodesSolrCloudTest\"," - + " \"level\": \"WARN\"}]"; + "[{\"logger\": \"" + TEST_LOGGER + "\", \"level\": \"WARN\"}]"; + private static final String UNSET_LEVEL_JSON = + "[{\"logger\": \"" + TEST_LOGGER + "\", \"level\": \"unset\"}]"; private static final ObjectMapper MAPPER = new ObjectMapper(); @BeforeClass @@ -58,17 +62,29 @@ public void testBroadcastToAllNodesReportsEveryNode() throws Exception { final String receivingNode = jettyName(0); final String otherNode = jettyName(1); - final JsonNode rsp = putLogLevels(cluster.getJettySolrRunners().get(0), "all"); - - final JsonNode failedNodes = rsp.get("failedNodes"); - assertNotNull("Broadcast response should report failed nodes", failedNodes); - assertTrue("No node should be reported failed: " + failedNodes, failedNodes.isEmpty()); - for (String nodeName : new String[] {receivingNode, otherNode}) { - final JsonNode perNode = rsp.get(nodeName); - assertNotNull("Expected a per-node result for " + nodeName + " in " + rsp, perNode); - // The per-node entry is that node's own logging response. Its presence proves the request - // body (the level changes) reached the node and a response came back. - assertNotNull(perNode.get("watcher")); + try { + final JsonNode rsp = putLogLevels(cluster.getJettySolrRunners().get(0), "all"); + + final JsonNode failedNodes = rsp.get("failedNodes"); + assertNotNull("Broadcast response should report failed nodes", failedNodes); + assertTrue("No node should be reported failed: " + failedNodes, failedNodes.isEmpty()); + for (String nodeName : new String[] {receivingNode, otherNode}) { + final JsonNode perNode = rsp.get(nodeName); + assertNotNull("Expected a per-node result for " + nodeName + " in " + rsp, perNode); + // The per-node entry is that node's own logging response. Its presence proves the + // request reached the node and a response came back. + assertNotNull(perNode.get("watcher")); + } + // Log levels are JVM-wide in this cluster, so the receiving node's own listing shows + // whether the level change in the broadcast payload was actually applied, not just + // delivered. + assertEquals( + "The broadcast level change should be in effect", + "WARN", + loggerLevel(getLogLevels(cluster.getJettySolrRunners().get(0)))); + } finally { + // Leave the logger as the test found it (no level set) for the rest of the JVM. + putLogLevels(cluster.getJettySolrRunners().get(0), null, UNSET_LEVEL_JSON); } } @@ -123,6 +139,11 @@ private static JettySolrRunner jetty(int idx) { } private static JsonNode putLogLevels(JettySolrRunner target, String nodes) throws Exception { + return putLogLevels(target, nodes, LEVEL_CHANGES_JSON); + } + + private static JsonNode putLogLevels(JettySolrRunner target, String nodes, String json) + throws Exception { final ModifiableSolrParams params = new ModifiableSolrParams(); if (nodes != null) { params.set("nodes", nodes); @@ -133,7 +154,7 @@ private static JsonNode putLogLevels(JettySolrRunner target, String nodes) throw "/node/logging/levels", SolrRequest.SolrRequestType.ADMIN, params); - req.withContent(LEVEL_CHANGES_JSON.getBytes(StandardCharsets.UTF_8), "application/json"); + req.withContent(json.getBytes(StandardCharsets.UTF_8), "application/json"); req.setResponseParser(new InputStreamResponseParser("json")); try (HttpJettySolrClient client = new HttpJettySolrClient.Builder(target.getBaseUrl().toString()).build()) { @@ -143,6 +164,32 @@ private static JsonNode putLogLevels(JettySolrRunner target, String nodes) throw } } + private static JsonNode getLogLevels(JettySolrRunner target) throws Exception { + final GenericV2SolrRequest req = + new GenericV2SolrRequest( + SolrRequest.METHOD.GET, + "/node/logging/levels", + SolrRequest.SolrRequestType.ADMIN, + new ModifiableSolrParams()); + req.setResponseParser(new InputStreamResponseParser("json")); + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(target.getBaseUrl().toString()).build()) { + final String body = + InputStreamResponseParser.consumeResponseToString(req.process(client).getResponse()); + return MAPPER.readTree(body); + } + } + + private static String loggerLevel(JsonNode listing) { + for (JsonNode logger : listing.get("loggers")) { + if (TEST_LOGGER.equals(logger.get("name").asText())) { + final JsonNode level = logger.get("level"); + return level == null || level.isNull() ? null : level.asText(); + } + } + return null; + } + private static SimpleSolrResponse sendWithDefaultParser( JettySolrRunner target, GenericV2SolrRequest req) throws Exception { try (HttpJettySolrClient client = From 1f60cd39baae5dddbefcf6127b25955dc3518578 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Wed, 7 Oct 2026 04:44:23 +0000 Subject: [PATCH 5/5] SOLR-15823: Note in the ref guide that a failedNodes entry may already have applied the change The response can time out after a node accepted the set-level request, so a node named in failedNodes is not guaranteed to be unchanged. The configuring-logging page now says so next to the failedNodes description. --- .../modules/deployment-guide/pages/configuring-logging.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/configuring-logging.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/configuring-logging.adoc index 413038d9399d..dfdf83c97b2b 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/configuring-logging.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/configuring-logging.adoc @@ -105,6 +105,7 @@ curl -X PUT http://localhost:8983/api/node/logging/levels -H 'Content-Type: appl In SolrCloud mode, the V2 request above changes only the node that receives it. Add the `nodes` query parameter to apply the same changes on other nodes: `nodes=all` targets every live node, or provide a comma-separated list of node names (in `host:port_solr` form). The response then reports the result from each node, and names any requested nodes that did not respond under `failedNodes`. +A node named in `failedNodes` is not guaranteed to be unchanged: if the node accepted the request but its response was lost or timed out, it may already have applied the change. [source,bash] ----