From dad48eb7b2159c3450d40ea31f28a737df538b9b Mon Sep 17 00:00:00 2001 From: along Date: Thu, 17 Sep 2026 10:43:42 -0700 Subject: [PATCH 01/12] feat(cloud-functions): accept routing expressions behind a flag Validate llmConfig.routingMethod as a profile of an RFC 8941 item with parameters, format only, and persist the value exactly as received. The algorithm allow-list is removed; the router is the semantic authority. A new property, nvcf.llm.routing-expressions.enabled (default false), is checked before the grammar: while off, values with tuning parameters are rejected with a clear message and method-only values behave as before. Relates to NVIDIA/nvcf#536 Signed-off-by: along --- .../llm/LlmRoutingExpressionsProperties.java | 29 ++++ .../management/dto/CreateFunctionRequest.java | 2 - .../management/dto/LlmConfigValidator.java | 99 +++++++++---- .../service/function/FunctionLlmService.java | 13 +- .../function/LlmRoutingMethodValidator.java | 38 +++++ .../FunctionsWithLlmModelsTest.java | 48 +++++- .../dto/LlmConfigValidatorTest.java | 79 +++++++++- .../FunctionLlmServicePriorityTest.java | 5 + .../FunctionLlmServiceRoutingMethodTest.java | 137 ++++++++++++++++++ .../LlmRoutingMethodValidatorTest.java | 76 ++++++++++ .../src/test/resources/application-test.yaml | 3 + .../src/main/resources/application.yaml | 3 + 12 files changed, 491 insertions(+), 41 deletions(-) create mode 100644 src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/configuration/llm/LlmRoutingExpressionsProperties.java create mode 100644 src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidator.java create mode 100644 src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServiceRoutingMethodTest.java create mode 100644 src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidatorTest.java diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/configuration/llm/LlmRoutingExpressionsProperties.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/configuration/llm/LlmRoutingExpressionsProperties.java new file mode 100644 index 0000000000..a136b45057 --- /dev/null +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/configuration/llm/LlmRoutingExpressionsProperties.java @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.nvcf.configuration.llm; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Data +@Configuration +@ConfigurationProperties(prefix = "nvcf.llm.routing-expressions") +public class LlmRoutingExpressionsProperties { + + private boolean enabled = false; +} diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/CreateFunctionRequest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/CreateFunctionRequest.java index 6b1d6b22b0..97974949cc 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/CreateFunctionRequest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/CreateFunctionRequest.java @@ -341,8 +341,6 @@ static void validateModelFields(List models, throw new BadRequestException(MESG_MISSING_LLM_MODEL_URIS); } if (isLlmFunction) { - LlmConfigValidator.validateRoutingMethod( - model.getName(), model.getLlmConfig().getRoutingMethod()); LlmConfigValidator.validateTokenRateLimit( model.getName(), model.getLlmConfig().getTokenRateLimit()); } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java index a77fb9a9ae..a4595ca06c 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java @@ -6,57 +6,77 @@ import com.nvidia.boot.exceptions.BadRequestException; import jakarta.annotation.Nullable; -import java.util.Locale; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; import java.util.Set; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; /** - * Rejects invalid {@code llmConfig} routingMethod/tokenRateLimit at create/update, so callers - * get a 400 up front instead of a late failure at invocation. + * Validates routing expression syntax and token rate limits. + * Routing semantics belong to the router. */ @Slf4j public final class LlmConfigValidator { - private LlmConfigValidator() {} - - // Stargate LoadBalancerAlgorithm values; keep in sync. Blank = router default. - private static final Set VALID_ROUTING_METHODS = Set.of( - "power-of-two", - "wait-and-widen", - "round-robin", - "random", - "pulsar", - "pulsar-wait-and-widen", - // Deprecated Stargate aliases retained for existing deployments. - "groq-multiregion", - "pulsar-multiregion"); + private static final int MAX_EXPRESSION_BYTES = 1024; + private static final int MAX_PARAMETERS = 32; + private static final Pattern METHOD_PATTERN = Pattern.compile("[A-Za-z][A-Za-z0-9_-]*"); + private static final Pattern PARAMETER_PATTERN = + Pattern.compile(" *([a-z][a-z0-9_]*)=(\\S(?:.*\\S)?)"); + private static final Pattern INTEGER_PATTERN = Pattern.compile("-?[0-9]{1,15}"); + private static final Pattern DECIMAL_PATTERN = Pattern.compile("-?[0-9]{1,12}\\.[0-9]{1,3}"); + private static final Pattern TOKEN_PATTERN = + Pattern.compile("[A-Za-z*][A-Za-z0-9!#$%&'*+.^_`|~:/-]*"); + private static final Pattern STRING_PATTERN = Pattern.compile( + "\"(?:[\\x20\\x21\\x23-\\x2b\\x2d-\\x3a\\x3c-\\x5b\\x5d-\\x7e]|\\\\[\"\\\\])*\""); // Comma-separated '-' entries, no unit repeated. private static final Pattern TOKEN_RATE_LIMIT_PATTERN = Pattern.compile( "^(?!.*-([SMHDW]).*-\\1)[1-9]\\d*-[SMHDW](,\\s*[1-9]\\d*-[SMHDW])*$"); private static final String MESG_INVALID_ROUTING_METHOD = - "Invalid request: 'llmConfig.routingMethod' for model '%s' is invalid; supported " - + "values are [power-of-two, wait-and-widen, round-robin, random, pulsar, " - + "pulsar-wait-and-widen, groq-multiregion, pulsar-multiregion]"; + "Invalid request: 'llmConfig.routingMethod' for model '%s' is invalid: %s"; + private static final String MESG_EXPRESSION_TOO_LONG = "expression exceeds 1024 bytes"; + private static final String MESG_COMMAS_NOT_ALLOWED = "commas are not allowed"; + private static final String MESG_INVALID_METHOD_NAME = + "method name must match [A-Za-z][A-Za-z0-9_-]*"; + private static final String MESG_TOO_MANY_PARAMETERS = "at most 32 parameters are allowed"; + private static final String MESG_INVALID_PARAMETER = + "parameter '%s' must be key=value with key matching [a-z][a-z0-9_]*"; + private static final String MESG_INVALID_VALUE = + "value for '%s' must be an integer, decimal, token, or quoted string"; + private static final String MESG_DUPLICATE_PARAMETER = "duplicate parameter '%s'"; private static final String MESG_INVALID_TOKEN_RATE_LIMIT = "Invalid request: 'llmConfig.tokenRateLimit' for model '%s' is invalid; expected " + "comma-separated '-' entries with unit in [S, M, H, D, W] " + "(for example '100000-S' or '10-M,5-S')"; - /** Rejects a routingMethod that is not one of the supported router algorithms. */ + private LlmConfigValidator() {} + + /** Validates the routing expression profile without interpreting methods or parameters. */ public static void validateRoutingMethod(String modelName, @Nullable String routingMethod) { if (StringUtils.isBlank(routingMethod)) { return; } - // Match the router: lowercase, '_' -> '-'. - var normalized = routingMethod.trim().toLowerCase(Locale.ROOT).replace('_', '-'); - if (!VALID_ROUTING_METHODS.contains(normalized)) { - var mesg = MESG_INVALID_ROUTING_METHOD.formatted(modelName); - log.error(mesg); - throw new BadRequestException(mesg); + var value = routingMethod.trim(); + if (value.getBytes(StandardCharsets.UTF_8).length > MAX_EXPRESSION_BYTES) { + rejectRoutingMethod(modelName, MESG_EXPRESSION_TOO_LONG); + } + if (value.contains(",")) { + rejectRoutingMethod(modelName, MESG_COMMAS_NOT_ALLOWED); + } + var segments = value.split(";", -1); + if (!METHOD_PATTERN.matcher(segments[0]).matches()) { + rejectRoutingMethod(modelName, MESG_INVALID_METHOD_NAME); + } + if (segments.length - 1 > MAX_PARAMETERS) { + rejectRoutingMethod(modelName, MESG_TOO_MANY_PARAMETERS); + } + var keys = new HashSet(); + for (var index = 1; index < segments.length; index++) { + validateParameter(modelName, segments[index], keys); } } @@ -71,4 +91,31 @@ public static void validateTokenRateLimit(String modelName, @Nullable String tok throw new BadRequestException(mesg); } } + + private static void validateParameter(String modelName, String segment, Set keys) { + var parameter = PARAMETER_PATTERN.matcher(segment); + if (!parameter.matches()) { + rejectRoutingMethod(modelName, MESG_INVALID_PARAMETER.formatted(segment)); + } + var key = parameter.group(1); + if (!isValidBareValue(parameter.group(2))) { + rejectRoutingMethod(modelName, MESG_INVALID_VALUE.formatted(key)); + } + if (!keys.add(key)) { + rejectRoutingMethod(modelName, MESG_DUPLICATE_PARAMETER.formatted(key)); + } + } + + private static boolean isValidBareValue(String value) { + return INTEGER_PATTERN.matcher(value).matches() + || DECIMAL_PATTERN.matcher(value).matches() + || TOKEN_PATTERN.matcher(value).matches() + || STRING_PATTERN.matcher(value).matches(); + } + + private static void rejectRoutingMethod(String modelName, String rule) { + var mesg = MESG_INVALID_ROUTING_METHOD.formatted(modelName, rule); + log.error(mesg); + throw new BadRequestException(mesg); + } } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/FunctionLlmService.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/FunctionLlmService.java index bea23531fc..3d5118e956 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/FunctionLlmService.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/FunctionLlmService.java @@ -73,6 +73,7 @@ public class FunctionLlmService { private final FunctionLookupService functionLookupService; private final FunctionMapperService functionMapperService; + private final LlmRoutingMethodValidator llmRoutingMethodValidator; /** * Resolves the caller's invocation priority: the per-account override when the caller has one, @@ -90,7 +91,7 @@ public Optional resolveInvocationPriority( } /** - * Reject a create request that sets llmInvocationConfig on a non-LLM function. + * Restrict llmInvocationConfig to LLM functions and validate their routing expressions. */ public void validateCreateFunctionRequestWithLlmConfig(CreateFunctionRequest request) { if (request.getLlmInvocationConfig() != null @@ -98,6 +99,14 @@ public void validateCreateFunctionRequestWithLlmConfig(CreateFunctionRequest req log.error(MESG_FUNCTION_NOT_LLM_INVOCATION_CONFIG); throw new BadRequestException(MESG_FUNCTION_NOT_LLM_INVOCATION_CONFIG); } + if (request.getFunctionType() == FunctionTypeEnum.LLM && request.getModels() != null) { + for (var model : request.getModels()) { + if (model.getLlmConfig() != null) { + llmRoutingMethodValidator.validate( + model.getName(), model.getLlmConfig().getRoutingMethod()); + } + } + } } /** @@ -403,7 +412,7 @@ private void updateModels( llmConfig.setTokenRateLimit(llmConfigUpdate.tokenRateLimit()); } if (llmConfigUpdate.routingMethod() != null) { - LlmConfigValidator.validateRoutingMethod( + llmRoutingMethodValidator.validate( modelUpdate.modelName(), llmConfigUpdate.routingMethod()); llmConfig.setRoutingMethod(llmConfigUpdate.routingMethod()); } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidator.java new file mode 100644 index 0000000000..acafdb6472 --- /dev/null +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidator.java @@ -0,0 +1,38 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.nvcf.service.function; + +import com.nvidia.boot.exceptions.BadRequestException; +import com.nvidia.nvcf.configuration.llm.LlmRoutingExpressionsProperties; +import com.nvidia.nvcf.rest.function.management.dto.LlmConfigValidator; +import jakarta.annotation.Nullable; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +@RequiredArgsConstructor +public class LlmRoutingMethodValidator { + + private static final String MESG_ROUTING_PARAMETERS_DISABLED = + "Invalid request: 'llmConfig.routingMethod' for model '%s' carries tuning parameters, " + + "which are not enabled on this deployment; specify the algorithm name only"; + + private final LlmRoutingExpressionsProperties properties; + + public void validate(String modelName, @Nullable String routingMethod) { + if (StringUtils.isBlank(routingMethod)) { + return; + } + if (!properties.isEnabled() && routingMethod.contains(";")) { + var mesg = MESG_ROUTING_PARAMETERS_DISABLED.formatted(modelName); + log.error(mesg); + throw new BadRequestException(mesg); + } + LlmConfigValidator.validateRoutingMethod(modelName, routingMethod); + } +} diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/FunctionsWithLlmModelsTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/FunctionsWithLlmModelsTest.java index af00e91836..6c2a394c4d 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/FunctionsWithLlmModelsTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/FunctionsWithLlmModelsTest.java @@ -595,6 +595,44 @@ private FunctionDto createAdditionalLlmFunctionVersion( return response.getBody().function(); } + @Test + void shouldPersistRoutingExpressionsAsReceivedOnCreateAndUpdate() { + var initialRoutingMethod = "Pulsar_Wait_And_Widen; seed=stable-a;n=2"; + var function = createInitialLlmFunction( + TEST_FUNCTION_NAME + "-" + Instant.now().toEpochMilli(), + "1-M", initialRoutingMethod); + + assertThat(function.models().getFirst().getLlmConfig().getRoutingMethod()) + .isEqualTo(initialRoutingMethod); + assertLlmConfigPersisted(function.versionId(), "1-M", initialRoutingMethod); + + var updatedRoutingMethod = "pulsar;seed=stable-b"; + var updateToken = MOCK_OAUTH2_TOKEN_SERVER.getJwt( + TEST_CLIENT_SUBJECT, List.of(SCOPE_UPDATE_FUNCTION), 100); + var updateRequest = UpdateFunctionRequest.builder() + .modelUpdates(List.of(UpdateFunctionRequest.ModelUpdateDto.builder() + .modelName(TEST_LLM_MODEL_NAME) + .llmConfig(UpdateFunctionRequest.LlmConfigUpdateDto.builder() + .routingMethod(updatedRoutingMethod) + .build()) + .build())) + .build(); + var updateEntity = RequestEntity.put(URI.create("/v2/nvcf/functions/" + function.id() + + "/versions/" + function.versionId())) + .contentType(MediaType.APPLICATION_JSON) + .header("Authorization", "Bearer " + updateToken) + .body(updateRequest); + + var response = testRestTemplate.exchange(updateEntity, FunctionResponse.class); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK); + assertThat(response.getBody()).isNotNull(); + var updatedModel = response.getBody().function().models().getFirst(); + assertThat(updatedModel.getLlmConfig().getRoutingMethod()) + .isEqualTo(updatedRoutingMethod); + assertLlmConfigPersisted(function.versionId(), "1-M", updatedRoutingMethod); + } + @Test void shouldRejectCreateWithInvalidRoutingMethod() { var createToken = MOCK_OAUTH2_TOKEN_SERVER.getJwt(TEST_CLIENT_SUBJECT, @@ -605,7 +643,7 @@ void shouldRejectCreateWithInvalidRoutingMethod() { .inferenceUrl(TEST_INFERENCE_URL) .inferencePort(TEST_INFERENCE_PORT) .functionType(FunctionTypeEnum.LLM) - .models(List.of(llmModel("1-M", "not-a-method"))) + .models(List.of(llmModel("1-M", "pulsar,seed=x"))) .build(); var createEntity = RequestEntity.post(URI.create("/v2/nvcf/functions")) .contentType(MediaType.APPLICATION_JSON) @@ -615,7 +653,8 @@ void shouldRejectCreateWithInvalidRoutingMethod() { var response = testRestTemplate.exchange(createEntity, String.class); assertThat(response.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST); - assertThat(response.getBody()).contains("llmConfig.routingMethod"); + assertThat(response.getBody()).contains( + "llmConfig.routingMethod", TEST_LLM_MODEL_NAME, "commas are not allowed"); } @Test @@ -652,7 +691,7 @@ void shouldRejectUpdateWithInvalidRoutingMethod() { .modelUpdates(List.of(UpdateFunctionRequest.ModelUpdateDto.builder() .modelName(TEST_LLM_MODEL_NAME) .llmConfig(UpdateFunctionRequest.LlmConfigUpdateDto.builder() - .routingMethod("not-a-method") + .routingMethod("pulsar;n=?1") .build()) .build())) .build(); @@ -665,7 +704,8 @@ void shouldRejectUpdateWithInvalidRoutingMethod() { var response = testRestTemplate.exchange(updateEntity, String.class); assertThat(response.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST); - assertThat(response.getBody()).contains("llmConfig.routingMethod"); + assertThat(response.getBody()).contains( + "llmConfig.routingMethod", TEST_LLM_MODEL_NAME, "value for 'n'"); } @Test diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java index 62afbb9a91..208370a665 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java @@ -8,8 +8,12 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.nvidia.boot.exceptions.BadRequestException; -import org.junit.jupiter.api.Test; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.NullAndEmptySource; import org.junit.jupiter.params.provider.ValueSource; @@ -21,10 +25,23 @@ class LlmConfigValidatorTest { @ValueSource(strings = { "power-of-two", "wait-and-widen", "round-robin", "random", "pulsar", "pulsar-wait-and-widen", "groq-multiregion", "pulsar-multiregion", - // Router normalizes case and '_' to '-', so these are accepted too. "Power-Of-Two", "power_of_two", "wait_and_widen", "pulsar_wait_and_widen", - "groq_multiregion", "pulsar_multiregion", " pulsar " + "groq_multiregion", "pulsar_multiregion", " pulsar ", + "pulsar;seed=stable-a", + "pulsar; seed=stable-a; consider_kv_free_tokens=true", + "pulsar-wait-and-widen;seed=stable-a;n=2;max_queue_time_floor_ms=100;" + + "max_queue_time_ceil_ms=500", + "wait-and-widen;next_bucket_unlock_factor=\"0.0625\"", + "power-of-n;sample_count=4;comparator=queue-time", + "wait-and-widen;max_input_work_seconds=1.5", + "wait-and-widen;n=-1", "fastest;widen=2", + "pulsar; seed=x", " pulsar;seed=x ", + "pulsar;n=999999999999999", "pulsar;n=-999999999999999", + "pulsar;n=999999999999.999", "pulsar;n=-999999999999.999", + "pulsar;seed=*", "pulsar;seed=a!#$%&'*+.^_`|~:/-", + "pulsar;seed=\"\"", "pulsar;seed=\"a b\"", "pulsar;seed=\"a\\\"b\\\\c\"" }) + @MethodSource("routingMethodsAtLimits") void validRoutingMethodsAccepted(String routingMethod) { assertThatCode(() -> LlmConfigValidator.validateRoutingMethod(MODEL, routingMethod)) .doesNotThrowAnyException(); @@ -38,13 +55,61 @@ void blankRoutingMethodAccepted(String routingMethod) { .doesNotThrowAnyException(); } - @ParameterizedTest - @ValueSource(strings = {"weighted", "sticky", "not-a-method", "round robin", "power-of-3"}) - void invalidRoutingMethodsRejected(String routingMethod) { + @ParameterizedTest(name = "{index}: {0} violates {1}") + @MethodSource("invalidRoutingMethods") + void invalidRoutingMethodsRejected(String routingMethod, String rule) { assertThatThrownBy(() -> LlmConfigValidator.validateRoutingMethod(MODEL, routingMethod)) .isInstanceOf(BadRequestException.class) .hasMessageContaining("routingMethod") - .hasMessageContaining(MODEL); + .hasMessageContaining(MODEL) + .hasMessageContaining(rule); + } + + private static Stream routingMethodsAtLimits() { + return Stream.of("a".repeat(1024), "pulsar" + parameters(32)); + } + + private static Stream invalidRoutingMethods() { + return Stream.of( + Arguments.of("round robin", "method name must match"), + Arguments.of("power-of-3!", "method name must match"), + Arguments.of(";seed=x", "method name must match"), + Arguments.of("pulsar,seed=x", "commas are not allowed"), + Arguments.of("pulsar;seed=\"a,b\"", "commas are not allowed"), + Arguments.of("pulsar;seed=\"a;b\"", "value for 'seed'"), + Arguments.of("pulsar ;seed=x", "method name must match"), + Arguments.of("pulsar;seed = x", "must be key=value"), + Arguments.of("pulsar;seed= x", "must be key=value"), + Arguments.of("pulsar;seed=x ;n=1", "must be key=value"), + Arguments.of("pulsar;\tseed=x", "must be key=value"), + Arguments.of("pulsar;seed", "must be key=value"), + Arguments.of("pulsar;seed=", "must be key=value"), + Arguments.of("pulsar;=x", "must be key=value"), + Arguments.of("pulsar;Seed=x", "must be key=value"), + Arguments.of("pulsar;2n=1", "must be key=value"), + Arguments.of("pulsar;n=?1", "value for 'n'"), + Arguments.of("pulsar;n=?0", "value for 'n'"), + Arguments.of("pulsar;seed=:YQ==:", "value for 'seed'"), + Arguments.of("pulsar;n=1.2345", "value for 'n'"), + Arguments.of("pulsar;n=1000000000000000", "value for 'n'"), + Arguments.of("pulsar;n=1000000000000.1", "value for 'n'"), + Arguments.of("pulsar;n=1.", "value for 'n'"), + Arguments.of("pulsar;n=1;n=2", "duplicate parameter 'n'"), + Arguments.of("pulsar" + parameters(33), "at most 32 parameters"), + Arguments.of("a".repeat(1025), "expression exceeds 1024 bytes"), + Arguments.of("pulsar;seed=\"" + "\u00e9".repeat(506) + "\"", + "expression exceeds 1024 bytes"), + Arguments.of("pulsar;seed=x;", "must be key=value"), + Arguments.of("pulsar;seed=\"unterminated", "value for 'seed'"), + Arguments.of("pulsar;seed=\"a\\nb\"", "value for 'seed'"), + Arguments.of("pulsar;seed=\"\u00e9\"", "value for 'seed'"), + Arguments.of("pulsar;seed=\"a\tb\"", "value for 'seed'")); + } + + private static String parameters(int count) { + return IntStream.range(0, count) + .mapToObj(index -> ";p" + index + "=x") + .collect(Collectors.joining()); } @ParameterizedTest diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServicePriorityTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServicePriorityTest.java index b8ce20dcb1..b9d9dbf3ef 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServicePriorityTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServicePriorityTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; +import com.nvidia.nvcf.configuration.llm.LlmRoutingExpressionsProperties; import com.nvidia.nvcf.rest.function.management.dto.LlmInvocationConfigDto; import com.nvidia.nvcf.rest.function.management.dto.PriorityDto; import java.util.HashMap; @@ -26,6 +27,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -38,6 +40,9 @@ class FunctionLlmServicePriorityTest { private FunctionLookupService functionLookupService; @Mock private FunctionMapperService functionMapperService; + @Spy + private LlmRoutingMethodValidator llmRoutingMethodValidator = + new LlmRoutingMethodValidator(new LlmRoutingExpressionsProperties()); @InjectMocks private FunctionLlmService service; diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServiceRoutingMethodTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServiceRoutingMethodTest.java new file mode 100644 index 0000000000..2233892a3e --- /dev/null +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServiceRoutingMethodTest.java @@ -0,0 +1,137 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.nvcf.service.function; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.nvidia.boot.exceptions.BadRequestException; +import com.nvidia.nvcf.configuration.llm.LlmRoutingExpressionsProperties; +import com.nvidia.nvcf.persistence.function.entity.ApiBodyFormat; +import com.nvidia.nvcf.persistence.function.entity.FunctionEntity; +import com.nvidia.nvcf.persistence.function.entity.FunctionStatus; +import com.nvidia.nvcf.persistence.function.entity.FunctionType; +import com.nvidia.nvcf.rest.function.management.dto.CreateFunctionRequest; +import com.nvidia.nvcf.rest.function.management.dto.FunctionModelDto; +import com.nvidia.nvcf.rest.function.management.dto.FunctionTypeEnum; +import com.nvidia.nvcf.rest.function.management.dto.UpdateFunctionRequest; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class FunctionLlmServiceRoutingMethodTest { + + @Mock + private FunctionLookupService functionLookupService; + @Mock + private FunctionMapperService functionMapperService; + @Spy + private LlmRoutingMethodValidator llmRoutingMethodValidator = + new LlmRoutingMethodValidator(new LlmRoutingExpressionsProperties()); + @InjectMocks + private FunctionLlmService service; + + @Test + void validatesEveryLlmModelOnce() { + var request = CreateFunctionRequest.builder() + .name("test-function") + .inferenceUrl(URI.create("/v1/chat/completions")) + .functionType(FunctionTypeEnum.LLM) + .models(List.of(model("first", "pulsar"), model("second", "Power_Of_Two"))) + .build(); + + service.validateCreateFunctionRequestWithLlmConfig(request); + + verify(llmRoutingMethodValidator).validate("first", "pulsar"); + verify(llmRoutingMethodValidator).validate("second", "Power_Of_Two"); + verifyNoMoreInteractions(llmRoutingMethodValidator); + } + + @ParameterizedTest + @EnumSource(value = FunctionTypeEnum.class, names = {"DEFAULT", "STREAMING"}) + void skipsRoutingValidationForNonLlmFunctions(FunctionTypeEnum functionType) { + var request = CreateFunctionRequest.builder() + .name("test-function") + .inferenceUrl(URI.create("/v1/chat/completions")) + .functionType(functionType) + .models(List.of(model("first", "pulsar;seed=x"))) + .build(); + + service.validateCreateFunctionRequestWithLlmConfig(request); + + verifyNoInteractions(llmRoutingMethodValidator); + } + + @Test + void createUsesGateBeforeGrammar() { + var request = CreateFunctionRequest.builder() + .name("test-function") + .inferenceUrl(URI.create("/v1/chat/completions")) + .functionType(FunctionTypeEnum.LLM) + .models(List.of(model("first", "round robin;seed=?1"))) + .build(); + + assertThatThrownBy(() -> service.validateCreateFunctionRequestWithLlmConfig(request)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("first") + .hasMessageContaining("not enabled"); + } + + @Test + void updateUsesGateBeforeGrammarAndPreservesStoredValueOnRejection() { + var function = FunctionEntity.builder() + .functionId(UUID.randomUUID()) + .functionVersionId(UUID.randomUUID()) + .ncaId("test-account") + .functionName("test-function") + .functionStatus(FunctionStatus.INACTIVE) + .inferenceUrl("/v1/chat/completions") + .apiBodyFormat(ApiBodyFormat.CUSTOM) + .functionType(FunctionType.LLM) + .modelSpecs(Map.of()) + .build(); + var storedModel = model("first", "pulsar"); + when(functionMapperService.toFunctionModels(function.getModelSpecs())) + .thenReturn(List.of(storedModel)); + var request = UpdateFunctionRequest.builder() + .modelUpdates(List.of(UpdateFunctionRequest.ModelUpdateDto.builder() + .modelName("first") + .llmConfig(UpdateFunctionRequest.LlmConfigUpdateDto.builder() + .routingMethod("round robin;seed=?1") + .build()) + .build())) + .build(); + + assertThatThrownBy(() -> service.applyLlmUpdates(function, request)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("first") + .hasMessageContaining("not enabled"); + assertThat(storedModel.getLlmConfig().getRoutingMethod()).isEqualTo("pulsar"); + } + + private static FunctionModelDto model(String name, String routingMethod) { + return FunctionModelDto.builder() + .name(name) + .llmConfig(FunctionModelDto.LlmConfigDto.builder() + .uris(List.of("/v1/chat/completions")) + .routingMethod(routingMethod) + .build()) + .build(); + } +} diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidatorTest.java new file mode 100644 index 0000000000..aedc148f5d --- /dev/null +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidatorTest.java @@ -0,0 +1,76 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.nvcf.service.function; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.nvidia.boot.exceptions.BadRequestException; +import com.nvidia.nvcf.configuration.llm.LlmRoutingExpressionsProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +class LlmRoutingMethodValidatorTest { + + private static final String MODEL = "test-model"; + + private final LlmRoutingExpressionsProperties properties = + new LlmRoutingExpressionsProperties(); + private final LlmRoutingMethodValidator validator = new LlmRoutingMethodValidator(properties); + + @ParameterizedTest + @ValueSource(strings = {"pulsar;seed=x", "round robin;seed=?1", "pulsar,seed=x;n=1"}) + void disabledRejectsParametersBeforeGrammar(String routingMethod) { + assertThatThrownBy(() -> validator.validate(MODEL, routingMethod)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("not enabled") + .hasMessageContaining(MODEL); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {"pulsar", "Power_Of_Two", "groq-multiregion", " pulsar ", " "}) + void disabledAcceptsMethodOnlyAndBlankValues(String routingMethod) { + assertThatCode(() -> validator.validate(MODEL, routingMethod)).doesNotThrowAnyException(); + } + + @Test + void disabledRejectsMalformedMethodOnlyValue() { + assertThatThrownBy(() -> validator.validate(MODEL, "round robin")) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining(MODEL) + .hasMessageContaining("method name must match"); + } + + @Test + void enabledAcceptsParameters() { + properties.setEnabled(true); + + assertThatCode(() -> validator.validate(MODEL, "pulsar;seed=x")) + .doesNotThrowAnyException(); + } + + @Test + void enabledRejectsMalformedExpression() { + properties.setEnabled(true); + + assertThatThrownBy(() -> validator.validate(MODEL, "pulsar,seed=x")) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining(MODEL) + .hasMessageContaining("commas are not allowed"); + } + + @Test + void readsPropertyOnEachValidation() { + assertThatThrownBy(() -> validator.validate(MODEL, "pulsar;seed=x")) + .isInstanceOf(BadRequestException.class); + properties.setEnabled(true); + + assertThatCode(() -> validator.validate(MODEL, "pulsar;seed=x")) + .doesNotThrowAnyException(); + } +} diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/resources/application-test.yaml b/src/control-plane-services/cloud-functions/nvcf-core/src/test/resources/application-test.yaml index 0a4fa12f8b..dd68659bc8 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/resources/application-test.yaml +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/resources/application-test.yaml @@ -182,6 +182,9 @@ logging: com.github.tomakehurst.wiremock: DEBUG nvcf: + llm: + routing-expressions: + enabled: true request: timeout: 1s # only whole seconds fqdn: http://localhost:${server.port} diff --git a/src/control-plane-services/cloud-functions/nvcf-service/src/main/resources/application.yaml b/src/control-plane-services/cloud-functions/nvcf-service/src/main/resources/application.yaml index e18fc0d68e..4ed248dac4 100644 --- a/src/control-plane-services/cloud-functions/nvcf-service/src/main/resources/application.yaml +++ b/src/control-plane-services/cloud-functions/nvcf-service/src/main/resources/application.yaml @@ -296,6 +296,9 @@ nvcf: oauth2: base-url: https://stg.authn.nvidia.com group-scope: ngc-stg + llm: + routing-expressions: + enabled: false account: max-functions-allowed: 50 max-tasks-allowed: 50 From 25d9a20f9fb0e8ab679603d06d02da1006295beb Mon Sep 17 00:00:00 2001 From: along Date: Thu, 17 Sep 2026 10:53:27 -0700 Subject: [PATCH 02/12] fix(cloud-functions): keep control characters out of routing rejections A rejected parameter segment is echoed in the 400 message and in the error log. The segment is raw request text, so a value containing a newline or carriage return could forge a line on a plain-text log appender. Replace control and line-separator characters with '?' before the segment is formatted; the response still names the offending parameter. Relates to NVIDIA/nvcf#536 Signed-off-by: along --- .../function/management/dto/LlmConfigValidator.java | 11 ++++++++++- .../management/dto/LlmConfigValidatorTest.java | 12 ++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java index a4595ca06c..314a0825f5 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java @@ -31,6 +31,8 @@ public final class LlmConfigValidator { Pattern.compile("[A-Za-z*][A-Za-z0-9!#$%&'*+.^_`|~:/-]*"); private static final Pattern STRING_PATTERN = Pattern.compile( "\"(?:[\\x20\\x21\\x23-\\x2b\\x2d-\\x3a\\x3c-\\x5b\\x5d-\\x7e]|\\\\[\"\\\\])*\""); + private static final Pattern CONTROL_CHARACTERS = + Pattern.compile("[\\p{Cntrl}\\p{Zl}\\p{Zp}]"); // Comma-separated '-' entries, no unit repeated. private static final Pattern TOKEN_RATE_LIMIT_PATTERN = Pattern.compile( @@ -95,7 +97,8 @@ public static void validateTokenRateLimit(String modelName, @Nullable String tok private static void validateParameter(String modelName, String segment, Set keys) { var parameter = PARAMETER_PATTERN.matcher(segment); if (!parameter.matches()) { - rejectRoutingMethod(modelName, MESG_INVALID_PARAMETER.formatted(segment)); + rejectRoutingMethod(modelName, + MESG_INVALID_PARAMETER.formatted(withoutControlCharacters(segment))); } var key = parameter.group(1); if (!isValidBareValue(parameter.group(2))) { @@ -106,6 +109,12 @@ private static void validateParameter(String modelName, String segment, Set LlmConfigValidator.validateRoutingMethod(MODEL, routingMethod)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("parameter 'seed=a?b?c' must be key=value") + .hasMessageNotContaining("\n") + .hasMessageNotContaining("\r"); + } + private static Stream routingMethodsAtLimits() { return Stream.of("a".repeat(1024), "pulsar" + parameters(32)); } From a348948ca2bfffef71c203198e57e1621fee09c3 Mon Sep 17 00:00:00 2001 From: along Date: Thu, 17 Sep 2026 10:54:30 -0700 Subject: [PATCH 03/12] style(cloud-functions): use full license header in new routing classes LlmRoutingMethodValidator and its two tests carried the short SPDX header copied from LlmConfigValidator. Every other file in service/function uses the full Apache-2.0 header; align the three new files with their neighbors. Relates to NVIDIA/nvcf#536 Signed-off-by: along --- .../function/LlmRoutingMethodValidator.java | 14 +++++++++++++- .../FunctionLlmServiceRoutingMethodTest.java | 14 +++++++++++++- .../function/LlmRoutingMethodValidatorTest.java | 14 +++++++++++++- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidator.java index acafdb6472..f16c6b016f 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidator.java @@ -1,6 +1,18 @@ /* - * SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.nvidia.nvcf.service.function; diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServiceRoutingMethodTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServiceRoutingMethodTest.java index 2233892a3e..d052febb5c 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServiceRoutingMethodTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServiceRoutingMethodTest.java @@ -1,6 +1,18 @@ /* - * SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.nvidia.nvcf.service.function; diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidatorTest.java index aedc148f5d..44457e5ea6 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidatorTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidatorTest.java @@ -1,6 +1,18 @@ /* - * SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.nvidia.nvcf.service.function; From a886f3ff20b892a7f16730e10899f68993771644 Mon Sep 17 00:00:00 2001 From: along Date: Thu, 17 Sep 2026 22:37:02 -0700 Subject: [PATCH 04/12] refactor(cloud-functions): validate routing expressions without a flag Drop the nvcf.llm.routing-expressions.enabled property, its LlmRoutingExpressionsProperties class, and the service-level LlmRoutingMethodValidator component. Routing expressions are accepted unconditionally; the rollout needs no feature flag. Move the routing expression grammar out of LlmConfigValidator into a new static LlmRoutingMethodValidator next to it in the request DTO package. CreateFunctionRequest validates routingMethod during deserialization again, as before, and FunctionLlmService.updateModels keeps validating it on model updates; only the callee changed there. LlmConfigValidator keeps the tokenRateLimit check. Tests move with the code: LlmRoutingMethodValidatorTest holds the accept and reject tables, the wiring tests for the removed component are gone, and FunctionsWithLlmModelsTest still covers persist-as-received on create and update plus the 400s on both paths. Relates to NVIDIA/nvcf#536 Signed-off-by: along --- .../llm/LlmRoutingExpressionsProperties.java | 29 ---- .../management/dto/CreateFunctionRequest.java | 2 + .../management/dto/LlmConfigValidator.java | 94 +---------- .../dto/LlmRoutingMethodValidator.java | 122 ++++++++++++++ .../service/function/FunctionLlmService.java | 14 +- .../function/LlmRoutingMethodValidator.java | 50 ------ .../dto/LlmConfigValidatorTest.java | 108 ------------- .../dto/LlmRoutingMethodValidatorTest.java | 138 ++++++++++++++++ .../FunctionLlmServicePriorityTest.java | 5 - .../FunctionLlmServiceRoutingMethodTest.java | 149 ------------------ .../LlmRoutingMethodValidatorTest.java | 88 ----------- .../src/test/resources/application-test.yaml | 3 - .../src/main/resources/application.yaml | 3 - 13 files changed, 268 insertions(+), 537 deletions(-) delete mode 100644 src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/configuration/llm/LlmRoutingExpressionsProperties.java create mode 100644 src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java delete mode 100644 src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidator.java create mode 100644 src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java delete mode 100644 src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServiceRoutingMethodTest.java delete mode 100644 src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidatorTest.java diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/configuration/llm/LlmRoutingExpressionsProperties.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/configuration/llm/LlmRoutingExpressionsProperties.java deleted file mode 100644 index a136b45057..0000000000 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/configuration/llm/LlmRoutingExpressionsProperties.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.nvidia.nvcf.configuration.llm; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; - -@Data -@Configuration -@ConfigurationProperties(prefix = "nvcf.llm.routing-expressions") -public class LlmRoutingExpressionsProperties { - - private boolean enabled = false; -} diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/CreateFunctionRequest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/CreateFunctionRequest.java index 97974949cc..344a0d8bca 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/CreateFunctionRequest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/CreateFunctionRequest.java @@ -341,6 +341,8 @@ static void validateModelFields(List models, throw new BadRequestException(MESG_MISSING_LLM_MODEL_URIS); } if (isLlmFunction) { + LlmRoutingMethodValidator.validate( + model.getName(), model.getLlmConfig().getRoutingMethod()); LlmConfigValidator.validateTokenRateLimit( model.getName(), model.getLlmConfig().getTokenRateLimit()); } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java index 314a0825f5..ee1cdd0394 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java @@ -6,82 +6,28 @@ import com.nvidia.boot.exceptions.BadRequestException; import jakarta.annotation.Nullable; -import java.nio.charset.StandardCharsets; -import java.util.HashSet; -import java.util.Set; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; /** - * Validates routing expression syntax and token rate limits. - * Routing semantics belong to the router. + * Rejects an invalid {@code llmConfig} tokenRateLimit at create/update, so callers get a 400 up + * front instead of a late failure at invocation. */ @Slf4j public final class LlmConfigValidator { - private static final int MAX_EXPRESSION_BYTES = 1024; - private static final int MAX_PARAMETERS = 32; - private static final Pattern METHOD_PATTERN = Pattern.compile("[A-Za-z][A-Za-z0-9_-]*"); - private static final Pattern PARAMETER_PATTERN = - Pattern.compile(" *([a-z][a-z0-9_]*)=(\\S(?:.*\\S)?)"); - private static final Pattern INTEGER_PATTERN = Pattern.compile("-?[0-9]{1,15}"); - private static final Pattern DECIMAL_PATTERN = Pattern.compile("-?[0-9]{1,12}\\.[0-9]{1,3}"); - private static final Pattern TOKEN_PATTERN = - Pattern.compile("[A-Za-z*][A-Za-z0-9!#$%&'*+.^_`|~:/-]*"); - private static final Pattern STRING_PATTERN = Pattern.compile( - "\"(?:[\\x20\\x21\\x23-\\x2b\\x2d-\\x3a\\x3c-\\x5b\\x5d-\\x7e]|\\\\[\"\\\\])*\""); - private static final Pattern CONTROL_CHARACTERS = - Pattern.compile("[\\p{Cntrl}\\p{Zl}\\p{Zp}]"); + private LlmConfigValidator() {} // Comma-separated '-' entries, no unit repeated. private static final Pattern TOKEN_RATE_LIMIT_PATTERN = Pattern.compile( "^(?!.*-([SMHDW]).*-\\1)[1-9]\\d*-[SMHDW](,\\s*[1-9]\\d*-[SMHDW])*$"); - private static final String MESG_INVALID_ROUTING_METHOD = - "Invalid request: 'llmConfig.routingMethod' for model '%s' is invalid: %s"; - private static final String MESG_EXPRESSION_TOO_LONG = "expression exceeds 1024 bytes"; - private static final String MESG_COMMAS_NOT_ALLOWED = "commas are not allowed"; - private static final String MESG_INVALID_METHOD_NAME = - "method name must match [A-Za-z][A-Za-z0-9_-]*"; - private static final String MESG_TOO_MANY_PARAMETERS = "at most 32 parameters are allowed"; - private static final String MESG_INVALID_PARAMETER = - "parameter '%s' must be key=value with key matching [a-z][a-z0-9_]*"; - private static final String MESG_INVALID_VALUE = - "value for '%s' must be an integer, decimal, token, or quoted string"; - private static final String MESG_DUPLICATE_PARAMETER = "duplicate parameter '%s'"; private static final String MESG_INVALID_TOKEN_RATE_LIMIT = "Invalid request: 'llmConfig.tokenRateLimit' for model '%s' is invalid; expected " + "comma-separated '-' entries with unit in [S, M, H, D, W] " + "(for example '100000-S' or '10-M,5-S')"; - private LlmConfigValidator() {} - - /** Validates the routing expression profile without interpreting methods or parameters. */ - public static void validateRoutingMethod(String modelName, @Nullable String routingMethod) { - if (StringUtils.isBlank(routingMethod)) { - return; - } - var value = routingMethod.trim(); - if (value.getBytes(StandardCharsets.UTF_8).length > MAX_EXPRESSION_BYTES) { - rejectRoutingMethod(modelName, MESG_EXPRESSION_TOO_LONG); - } - if (value.contains(",")) { - rejectRoutingMethod(modelName, MESG_COMMAS_NOT_ALLOWED); - } - var segments = value.split(";", -1); - if (!METHOD_PATTERN.matcher(segments[0]).matches()) { - rejectRoutingMethod(modelName, MESG_INVALID_METHOD_NAME); - } - if (segments.length - 1 > MAX_PARAMETERS) { - rejectRoutingMethod(modelName, MESG_TOO_MANY_PARAMETERS); - } - var keys = new HashSet(); - for (var index = 1; index < segments.length; index++) { - validateParameter(modelName, segments[index], keys); - } - } - /** Rejects a tokenRateLimit that is not '-' fragments. */ public static void validateTokenRateLimit(String modelName, @Nullable String tokenRateLimit) { if (StringUtils.isBlank(tokenRateLimit)) { @@ -93,38 +39,4 @@ public static void validateTokenRateLimit(String modelName, @Nullable String tok throw new BadRequestException(mesg); } } - - private static void validateParameter(String modelName, String segment, Set keys) { - var parameter = PARAMETER_PATTERN.matcher(segment); - if (!parameter.matches()) { - rejectRoutingMethod(modelName, - MESG_INVALID_PARAMETER.formatted(withoutControlCharacters(segment))); - } - var key = parameter.group(1); - if (!isValidBareValue(parameter.group(2))) { - rejectRoutingMethod(modelName, MESG_INVALID_VALUE.formatted(key)); - } - if (!keys.add(key)) { - rejectRoutingMethod(modelName, MESG_DUPLICATE_PARAMETER.formatted(key)); - } - } - - // The segment is raw request text echoed in the log and the 400 body; a line break in it - // could forge a log line. - private static String withoutControlCharacters(String segment) { - return CONTROL_CHARACTERS.matcher(segment).replaceAll("?"); - } - - private static boolean isValidBareValue(String value) { - return INTEGER_PATTERN.matcher(value).matches() - || DECIMAL_PATTERN.matcher(value).matches() - || TOKEN_PATTERN.matcher(value).matches() - || STRING_PATTERN.matcher(value).matches(); - } - - private static void rejectRoutingMethod(String modelName, String rule) { - var mesg = MESG_INVALID_ROUTING_METHOD.formatted(modelName, rule); - log.error(mesg); - throw new BadRequestException(mesg); - } } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java new file mode 100644 index 0000000000..0bd8ff5505 --- /dev/null +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java @@ -0,0 +1,122 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.nvcf.rest.function.management.dto; + +import com.nvidia.boot.exceptions.BadRequestException; +import jakarta.annotation.Nullable; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Pattern; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +/** + * Validates the syntax of {@code llmConfig.routingMethod} at create/update: a method name + * optionally followed by {@code ;key=value} parameters. Methods and parameters are not + * interpreted here; the router owns their semantics and the value is stored as received. + */ +@Slf4j +public final class LlmRoutingMethodValidator { + + private static final int MAX_EXPRESSION_BYTES = 1024; + private static final int MAX_PARAMETERS = 32; + private static final Pattern METHOD_PATTERN = Pattern.compile("[A-Za-z][A-Za-z0-9_-]*"); + private static final Pattern PARAMETER_PATTERN = + Pattern.compile(" *([a-z][a-z0-9_]*)=(\\S(?:.*\\S)?)"); + private static final Pattern INTEGER_PATTERN = Pattern.compile("-?[0-9]{1,15}"); + private static final Pattern DECIMAL_PATTERN = Pattern.compile("-?[0-9]{1,12}\\.[0-9]{1,3}"); + private static final Pattern TOKEN_PATTERN = + Pattern.compile("[A-Za-z*][A-Za-z0-9!#$%&'*+.^_`|~:/-]*"); + private static final Pattern STRING_PATTERN = Pattern.compile( + "\"(?:[\\x20\\x21\\x23-\\x2b\\x2d-\\x3a\\x3c-\\x5b\\x5d-\\x7e]|\\\\[\"\\\\])*\""); + private static final Pattern CONTROL_CHARACTERS = + Pattern.compile("[\\p{Cntrl}\\p{Zl}\\p{Zp}]"); + + private static final String MESG_INVALID_ROUTING_METHOD = + "Invalid request: 'llmConfig.routingMethod' for model '%s' is invalid: %s"; + private static final String MESG_EXPRESSION_TOO_LONG = "expression exceeds 1024 bytes"; + private static final String MESG_COMMAS_NOT_ALLOWED = "commas are not allowed"; + private static final String MESG_INVALID_METHOD_NAME = + "method name must match [A-Za-z][A-Za-z0-9_-]*"; + private static final String MESG_TOO_MANY_PARAMETERS = "at most 32 parameters are allowed"; + private static final String MESG_INVALID_PARAMETER = + "parameter '%s' must be key=value with key matching [a-z][a-z0-9_]*"; + private static final String MESG_INVALID_VALUE = + "value for '%s' must be an integer, decimal, token, or quoted string"; + private static final String MESG_DUPLICATE_PARAMETER = "duplicate parameter '%s'"; + + private LlmRoutingMethodValidator() {} + + /** Rejects a routingMethod whose syntax the router could not parse. Blank is allowed. */ + public static void validate(String modelName, @Nullable String routingMethod) { + if (StringUtils.isBlank(routingMethod)) { + return; + } + var value = routingMethod.trim(); + if (value.getBytes(StandardCharsets.UTF_8).length > MAX_EXPRESSION_BYTES) { + reject(modelName, MESG_EXPRESSION_TOO_LONG); + } + if (value.contains(",")) { + reject(modelName, MESG_COMMAS_NOT_ALLOWED); + } + var segments = value.split(";", -1); + if (!METHOD_PATTERN.matcher(segments[0]).matches()) { + reject(modelName, MESG_INVALID_METHOD_NAME); + } + if (segments.length - 1 > MAX_PARAMETERS) { + reject(modelName, MESG_TOO_MANY_PARAMETERS); + } + var keys = new HashSet(); + for (var index = 1; index < segments.length; index++) { + validateParameter(modelName, segments[index], keys); + } + } + + private static void validateParameter(String modelName, String segment, Set keys) { + var parameter = PARAMETER_PATTERN.matcher(segment); + if (!parameter.matches()) { + reject(modelName, MESG_INVALID_PARAMETER.formatted(withoutControlCharacters(segment))); + } + var key = parameter.group(1); + if (!isValidBareValue(parameter.group(2))) { + reject(modelName, MESG_INVALID_VALUE.formatted(key)); + } + if (!keys.add(key)) { + reject(modelName, MESG_DUPLICATE_PARAMETER.formatted(key)); + } + } + + // The segment is raw request text echoed in the log and the 400 body; a line break in it + // could forge a log line. + private static String withoutControlCharacters(String segment) { + return CONTROL_CHARACTERS.matcher(segment).replaceAll("?"); + } + + private static boolean isValidBareValue(String value) { + return INTEGER_PATTERN.matcher(value).matches() + || DECIMAL_PATTERN.matcher(value).matches() + || TOKEN_PATTERN.matcher(value).matches() + || STRING_PATTERN.matcher(value).matches(); + } + + private static void reject(String modelName, String rule) { + var mesg = MESG_INVALID_ROUTING_METHOD.formatted(modelName, rule); + log.error(mesg); + throw new BadRequestException(mesg); + } +} diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/FunctionLlmService.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/FunctionLlmService.java index 3d5118e956..59854dc0e5 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/FunctionLlmService.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/FunctionLlmService.java @@ -26,6 +26,7 @@ import com.nvidia.nvcf.rest.function.management.dto.FunctionTypeEnum; import com.nvidia.nvcf.rest.function.management.dto.LlmConfigValidator; import com.nvidia.nvcf.rest.function.management.dto.LlmInvocationConfigDto; +import com.nvidia.nvcf.rest.function.management.dto.LlmRoutingMethodValidator; import com.nvidia.nvcf.rest.function.management.dto.UpdateFunctionRequest; import jakarta.annotation.Nullable; import java.util.Comparator; @@ -73,7 +74,6 @@ public class FunctionLlmService { private final FunctionLookupService functionLookupService; private final FunctionMapperService functionMapperService; - private final LlmRoutingMethodValidator llmRoutingMethodValidator; /** * Resolves the caller's invocation priority: the per-account override when the caller has one, @@ -91,7 +91,7 @@ public Optional resolveInvocationPriority( } /** - * Restrict llmInvocationConfig to LLM functions and validate their routing expressions. + * Reject a create request that sets llmInvocationConfig on a non-LLM function. */ public void validateCreateFunctionRequestWithLlmConfig(CreateFunctionRequest request) { if (request.getLlmInvocationConfig() != null @@ -99,14 +99,6 @@ public void validateCreateFunctionRequestWithLlmConfig(CreateFunctionRequest req log.error(MESG_FUNCTION_NOT_LLM_INVOCATION_CONFIG); throw new BadRequestException(MESG_FUNCTION_NOT_LLM_INVOCATION_CONFIG); } - if (request.getFunctionType() == FunctionTypeEnum.LLM && request.getModels() != null) { - for (var model : request.getModels()) { - if (model.getLlmConfig() != null) { - llmRoutingMethodValidator.validate( - model.getName(), model.getLlmConfig().getRoutingMethod()); - } - } - } } /** @@ -412,7 +404,7 @@ private void updateModels( llmConfig.setTokenRateLimit(llmConfigUpdate.tokenRateLimit()); } if (llmConfigUpdate.routingMethod() != null) { - llmRoutingMethodValidator.validate( + LlmRoutingMethodValidator.validate( modelUpdate.modelName(), llmConfigUpdate.routingMethod()); llmConfig.setRoutingMethod(llmConfigUpdate.routingMethod()); } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidator.java deleted file mode 100644 index f16c6b016f..0000000000 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidator.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.nvidia.nvcf.service.function; - -import com.nvidia.boot.exceptions.BadRequestException; -import com.nvidia.nvcf.configuration.llm.LlmRoutingExpressionsProperties; -import com.nvidia.nvcf.rest.function.management.dto.LlmConfigValidator; -import jakarta.annotation.Nullable; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; - -@Slf4j -@Component -@RequiredArgsConstructor -public class LlmRoutingMethodValidator { - - private static final String MESG_ROUTING_PARAMETERS_DISABLED = - "Invalid request: 'llmConfig.routingMethod' for model '%s' carries tuning parameters, " - + "which are not enabled on this deployment; specify the algorithm name only"; - - private final LlmRoutingExpressionsProperties properties; - - public void validate(String modelName, @Nullable String routingMethod) { - if (StringUtils.isBlank(routingMethod)) { - return; - } - if (!properties.isEnabled() && routingMethod.contains(";")) { - var mesg = MESG_ROUTING_PARAMETERS_DISABLED.formatted(modelName); - log.error(mesg); - throw new BadRequestException(mesg); - } - LlmConfigValidator.validateRoutingMethod(modelName, routingMethod); - } -} diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java index 266f920d8a..83f7ff5d1f 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java @@ -8,13 +8,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.nvidia.boot.exceptions.BadRequestException; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.NullAndEmptySource; import org.junit.jupiter.params.provider.ValueSource; @@ -22,108 +16,6 @@ class LlmConfigValidatorTest { private static final String MODEL = "meta/llama-3.1-8b-instruct"; - @ParameterizedTest - @ValueSource(strings = { - "power-of-two", "wait-and-widen", "round-robin", "random", "pulsar", - "pulsar-wait-and-widen", "groq-multiregion", "pulsar-multiregion", - "Power-Of-Two", "power_of_two", "wait_and_widen", "pulsar_wait_and_widen", - "groq_multiregion", "pulsar_multiregion", " pulsar ", - "pulsar;seed=stable-a", - "pulsar; seed=stable-a; consider_kv_free_tokens=true", - "pulsar-wait-and-widen;seed=stable-a;n=2;max_queue_time_floor_ms=100;" - + "max_queue_time_ceil_ms=500", - "wait-and-widen;next_bucket_unlock_factor=\"0.0625\"", - "power-of-n;sample_count=4;comparator=queue-time", - "wait-and-widen;max_input_work_seconds=1.5", - "wait-and-widen;n=-1", "fastest;widen=2", - "pulsar; seed=x", " pulsar;seed=x ", - "pulsar;n=999999999999999", "pulsar;n=-999999999999999", - "pulsar;n=999999999999.999", "pulsar;n=-999999999999.999", - "pulsar;seed=*", "pulsar;seed=a!#$%&'*+.^_`|~:/-", - "pulsar;seed=\"\"", "pulsar;seed=\"a b\"", "pulsar;seed=\"a\\\"b\\\\c\"" - }) - @MethodSource("routingMethodsAtLimits") - void validRoutingMethodsAccepted(String routingMethod) { - assertThatCode(() -> LlmConfigValidator.validateRoutingMethod(MODEL, routingMethod)) - .doesNotThrowAnyException(); - } - - @ParameterizedTest - @NullAndEmptySource - @ValueSource(strings = {" "}) - void blankRoutingMethodAccepted(String routingMethod) { - assertThatCode(() -> LlmConfigValidator.validateRoutingMethod(MODEL, routingMethod)) - .doesNotThrowAnyException(); - } - - @ParameterizedTest(name = "{index}: {0} violates {1}") - @MethodSource("invalidRoutingMethods") - void invalidRoutingMethodsRejected(String routingMethod, String rule) { - assertThatThrownBy(() -> LlmConfigValidator.validateRoutingMethod(MODEL, routingMethod)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("routingMethod") - .hasMessageContaining(MODEL) - .hasMessageContaining(rule); - } - - @Test - void controlCharactersInRejectedSegmentReplaced() { - var routingMethod = "pulsar;seed=a\nb\rc"; - - assertThatThrownBy(() -> LlmConfigValidator.validateRoutingMethod(MODEL, routingMethod)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("parameter 'seed=a?b?c' must be key=value") - .hasMessageNotContaining("\n") - .hasMessageNotContaining("\r"); - } - - private static Stream routingMethodsAtLimits() { - return Stream.of("a".repeat(1024), "pulsar" + parameters(32)); - } - - private static Stream invalidRoutingMethods() { - return Stream.of( - Arguments.of("round robin", "method name must match"), - Arguments.of("power-of-3!", "method name must match"), - Arguments.of(";seed=x", "method name must match"), - Arguments.of("pulsar,seed=x", "commas are not allowed"), - Arguments.of("pulsar;seed=\"a,b\"", "commas are not allowed"), - Arguments.of("pulsar;seed=\"a;b\"", "value for 'seed'"), - Arguments.of("pulsar ;seed=x", "method name must match"), - Arguments.of("pulsar;seed = x", "must be key=value"), - Arguments.of("pulsar;seed= x", "must be key=value"), - Arguments.of("pulsar;seed=x ;n=1", "must be key=value"), - Arguments.of("pulsar;\tseed=x", "must be key=value"), - Arguments.of("pulsar;seed", "must be key=value"), - Arguments.of("pulsar;seed=", "must be key=value"), - Arguments.of("pulsar;=x", "must be key=value"), - Arguments.of("pulsar;Seed=x", "must be key=value"), - Arguments.of("pulsar;2n=1", "must be key=value"), - Arguments.of("pulsar;n=?1", "value for 'n'"), - Arguments.of("pulsar;n=?0", "value for 'n'"), - Arguments.of("pulsar;seed=:YQ==:", "value for 'seed'"), - Arguments.of("pulsar;n=1.2345", "value for 'n'"), - Arguments.of("pulsar;n=1000000000000000", "value for 'n'"), - Arguments.of("pulsar;n=1000000000000.1", "value for 'n'"), - Arguments.of("pulsar;n=1.", "value for 'n'"), - Arguments.of("pulsar;n=1;n=2", "duplicate parameter 'n'"), - Arguments.of("pulsar" + parameters(33), "at most 32 parameters"), - Arguments.of("a".repeat(1025), "expression exceeds 1024 bytes"), - Arguments.of("pulsar;seed=\"" + "\u00e9".repeat(506) + "\"", - "expression exceeds 1024 bytes"), - Arguments.of("pulsar;seed=x;", "must be key=value"), - Arguments.of("pulsar;seed=\"unterminated", "value for 'seed'"), - Arguments.of("pulsar;seed=\"a\\nb\"", "value for 'seed'"), - Arguments.of("pulsar;seed=\"\u00e9\"", "value for 'seed'"), - Arguments.of("pulsar;seed=\"a\tb\"", "value for 'seed'")); - } - - private static String parameters(int count) { - return IntStream.range(0, count) - .mapToObj(index -> ";p" + index + "=x") - .collect(Collectors.joining()); - } - @ParameterizedTest @ValueSource(strings = {"100000-S", "10-M", "5-H", "1-D", "2-W", "10-M,5-S", "10-M, 5-S"}) void validTokenRateLimitsAccepted(String tokenRateLimit) { diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java new file mode 100644 index 0000000000..d2e0b4090e --- /dev/null +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.nvcf.rest.function.management.dto; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.nvidia.boot.exceptions.BadRequestException; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +class LlmRoutingMethodValidatorTest { + + private static final String MODEL = "meta/llama-3.1-8b-instruct"; + + @ParameterizedTest + @ValueSource(strings = { + "power-of-two", "wait-and-widen", "round-robin", "random", "pulsar", + "pulsar-wait-and-widen", "groq-multiregion", "pulsar-multiregion", + "Power-Of-Two", "power_of_two", "wait_and_widen", "pulsar_wait_and_widen", + "groq_multiregion", "pulsar_multiregion", " pulsar ", + "pulsar;seed=stable-a", + "pulsar; seed=stable-a; consider_kv_free_tokens=true", + "pulsar-wait-and-widen;seed=stable-a;n=2;max_queue_time_floor_ms=100;" + + "max_queue_time_ceil_ms=500", + "wait-and-widen;next_bucket_unlock_factor=\"0.0625\"", + "power-of-n;sample_count=4;comparator=queue-time", + "wait-and-widen;max_input_work_seconds=1.5", + "wait-and-widen;n=-1", "fastest;widen=2", + "pulsar; seed=x", " pulsar;seed=x ", + "pulsar;n=999999999999999", "pulsar;n=-999999999999999", + "pulsar;n=999999999999.999", "pulsar;n=-999999999999.999", + "pulsar;seed=*", "pulsar;seed=a!#$%&'*+.^_`|~:/-", + "pulsar;seed=\"\"", "pulsar;seed=\"a b\"", "pulsar;seed=\"a\\\"b\\\\c\"" + }) + @MethodSource("routingMethodsAtLimits") + void validRoutingMethodsAccepted(String routingMethod) { + assertThatCode(() -> LlmRoutingMethodValidator.validate(MODEL, routingMethod)) + .doesNotThrowAnyException(); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {" "}) + void blankRoutingMethodAccepted(String routingMethod) { + assertThatCode(() -> LlmRoutingMethodValidator.validate(MODEL, routingMethod)) + .doesNotThrowAnyException(); + } + + @ParameterizedTest(name = "{index}: {0} violates {1}") + @MethodSource("invalidRoutingMethods") + void invalidRoutingMethodsRejected(String routingMethod, String rule) { + assertThatThrownBy(() -> LlmRoutingMethodValidator.validate(MODEL, routingMethod)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("routingMethod") + .hasMessageContaining(MODEL) + .hasMessageContaining(rule); + } + + @Test + void controlCharactersInRejectedSegmentReplaced() { + var routingMethod = "pulsar;seed=a\nb\rc"; + + assertThatThrownBy(() -> LlmRoutingMethodValidator.validate(MODEL, routingMethod)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("parameter 'seed=a?b?c' must be key=value") + .hasMessageNotContaining("\n") + .hasMessageNotContaining("\r"); + } + + private static Stream routingMethodsAtLimits() { + return Stream.of("a".repeat(1024), "pulsar" + parameters(32)); + } + + private static Stream invalidRoutingMethods() { + return Stream.of( + Arguments.of("round robin", "method name must match"), + Arguments.of("power-of-3!", "method name must match"), + Arguments.of(";seed=x", "method name must match"), + Arguments.of("pulsar,seed=x", "commas are not allowed"), + Arguments.of("pulsar;seed=\"a,b\"", "commas are not allowed"), + Arguments.of("pulsar;seed=\"a;b\"", "value for 'seed'"), + Arguments.of("pulsar ;seed=x", "method name must match"), + Arguments.of("pulsar;seed = x", "must be key=value"), + Arguments.of("pulsar;seed= x", "must be key=value"), + Arguments.of("pulsar;seed=x ;n=1", "must be key=value"), + Arguments.of("pulsar;\tseed=x", "must be key=value"), + Arguments.of("pulsar;seed", "must be key=value"), + Arguments.of("pulsar;seed=", "must be key=value"), + Arguments.of("pulsar;=x", "must be key=value"), + Arguments.of("pulsar;Seed=x", "must be key=value"), + Arguments.of("pulsar;2n=1", "must be key=value"), + Arguments.of("pulsar;n=?1", "value for 'n'"), + Arguments.of("pulsar;n=?0", "value for 'n'"), + Arguments.of("pulsar;seed=:YQ==:", "value for 'seed'"), + Arguments.of("pulsar;n=1.2345", "value for 'n'"), + Arguments.of("pulsar;n=1000000000000000", "value for 'n'"), + Arguments.of("pulsar;n=1000000000000.1", "value for 'n'"), + Arguments.of("pulsar;n=1.", "value for 'n'"), + Arguments.of("pulsar;n=1;n=2", "duplicate parameter 'n'"), + Arguments.of("pulsar" + parameters(33), "at most 32 parameters"), + Arguments.of("a".repeat(1025), "expression exceeds 1024 bytes"), + Arguments.of("pulsar;seed=\"" + "\u00e9".repeat(506) + "\"", + "expression exceeds 1024 bytes"), + Arguments.of("pulsar;seed=x;", "must be key=value"), + Arguments.of("pulsar;seed=\"unterminated", "value for 'seed'"), + Arguments.of("pulsar;seed=\"a\\nb\"", "value for 'seed'"), + Arguments.of("pulsar;seed=\"\u00e9\"", "value for 'seed'"), + Arguments.of("pulsar;seed=\"a\tb\"", "value for 'seed'")); + } + + private static String parameters(int count) { + return IntStream.range(0, count) + .mapToObj(index -> ";p" + index + "=x") + .collect(Collectors.joining()); + } +} diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServicePriorityTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServicePriorityTest.java index b9d9dbf3ef..b8ce20dcb1 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServicePriorityTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServicePriorityTest.java @@ -18,7 +18,6 @@ import static org.assertj.core.api.Assertions.assertThat; -import com.nvidia.nvcf.configuration.llm.LlmRoutingExpressionsProperties; import com.nvidia.nvcf.rest.function.management.dto.LlmInvocationConfigDto; import com.nvidia.nvcf.rest.function.management.dto.PriorityDto; import java.util.HashMap; @@ -27,7 +26,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -40,9 +38,6 @@ class FunctionLlmServicePriorityTest { private FunctionLookupService functionLookupService; @Mock private FunctionMapperService functionMapperService; - @Spy - private LlmRoutingMethodValidator llmRoutingMethodValidator = - new LlmRoutingMethodValidator(new LlmRoutingExpressionsProperties()); @InjectMocks private FunctionLlmService service; diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServiceRoutingMethodTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServiceRoutingMethodTest.java deleted file mode 100644 index d052febb5c..0000000000 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/FunctionLlmServiceRoutingMethodTest.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.nvidia.nvcf.service.function; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -import com.nvidia.boot.exceptions.BadRequestException; -import com.nvidia.nvcf.configuration.llm.LlmRoutingExpressionsProperties; -import com.nvidia.nvcf.persistence.function.entity.ApiBodyFormat; -import com.nvidia.nvcf.persistence.function.entity.FunctionEntity; -import com.nvidia.nvcf.persistence.function.entity.FunctionStatus; -import com.nvidia.nvcf.persistence.function.entity.FunctionType; -import com.nvidia.nvcf.rest.function.management.dto.CreateFunctionRequest; -import com.nvidia.nvcf.rest.function.management.dto.FunctionModelDto; -import com.nvidia.nvcf.rest.function.management.dto.FunctionTypeEnum; -import com.nvidia.nvcf.rest.function.management.dto.UpdateFunctionRequest; -import java.net.URI; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Spy; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class FunctionLlmServiceRoutingMethodTest { - - @Mock - private FunctionLookupService functionLookupService; - @Mock - private FunctionMapperService functionMapperService; - @Spy - private LlmRoutingMethodValidator llmRoutingMethodValidator = - new LlmRoutingMethodValidator(new LlmRoutingExpressionsProperties()); - @InjectMocks - private FunctionLlmService service; - - @Test - void validatesEveryLlmModelOnce() { - var request = CreateFunctionRequest.builder() - .name("test-function") - .inferenceUrl(URI.create("/v1/chat/completions")) - .functionType(FunctionTypeEnum.LLM) - .models(List.of(model("first", "pulsar"), model("second", "Power_Of_Two"))) - .build(); - - service.validateCreateFunctionRequestWithLlmConfig(request); - - verify(llmRoutingMethodValidator).validate("first", "pulsar"); - verify(llmRoutingMethodValidator).validate("second", "Power_Of_Two"); - verifyNoMoreInteractions(llmRoutingMethodValidator); - } - - @ParameterizedTest - @EnumSource(value = FunctionTypeEnum.class, names = {"DEFAULT", "STREAMING"}) - void skipsRoutingValidationForNonLlmFunctions(FunctionTypeEnum functionType) { - var request = CreateFunctionRequest.builder() - .name("test-function") - .inferenceUrl(URI.create("/v1/chat/completions")) - .functionType(functionType) - .models(List.of(model("first", "pulsar;seed=x"))) - .build(); - - service.validateCreateFunctionRequestWithLlmConfig(request); - - verifyNoInteractions(llmRoutingMethodValidator); - } - - @Test - void createUsesGateBeforeGrammar() { - var request = CreateFunctionRequest.builder() - .name("test-function") - .inferenceUrl(URI.create("/v1/chat/completions")) - .functionType(FunctionTypeEnum.LLM) - .models(List.of(model("first", "round robin;seed=?1"))) - .build(); - - assertThatThrownBy(() -> service.validateCreateFunctionRequestWithLlmConfig(request)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("first") - .hasMessageContaining("not enabled"); - } - - @Test - void updateUsesGateBeforeGrammarAndPreservesStoredValueOnRejection() { - var function = FunctionEntity.builder() - .functionId(UUID.randomUUID()) - .functionVersionId(UUID.randomUUID()) - .ncaId("test-account") - .functionName("test-function") - .functionStatus(FunctionStatus.INACTIVE) - .inferenceUrl("/v1/chat/completions") - .apiBodyFormat(ApiBodyFormat.CUSTOM) - .functionType(FunctionType.LLM) - .modelSpecs(Map.of()) - .build(); - var storedModel = model("first", "pulsar"); - when(functionMapperService.toFunctionModels(function.getModelSpecs())) - .thenReturn(List.of(storedModel)); - var request = UpdateFunctionRequest.builder() - .modelUpdates(List.of(UpdateFunctionRequest.ModelUpdateDto.builder() - .modelName("first") - .llmConfig(UpdateFunctionRequest.LlmConfigUpdateDto.builder() - .routingMethod("round robin;seed=?1") - .build()) - .build())) - .build(); - - assertThatThrownBy(() -> service.applyLlmUpdates(function, request)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("first") - .hasMessageContaining("not enabled"); - assertThat(storedModel.getLlmConfig().getRoutingMethod()).isEqualTo("pulsar"); - } - - private static FunctionModelDto model(String name, String routingMethod) { - return FunctionModelDto.builder() - .name(name) - .llmConfig(FunctionModelDto.LlmConfigDto.builder() - .uris(List.of("/v1/chat/completions")) - .routingMethod(routingMethod) - .build()) - .build(); - } -} diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidatorTest.java deleted file mode 100644 index 44457e5ea6..0000000000 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/service/function/LlmRoutingMethodValidatorTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.nvidia.nvcf.service.function; - -import static org.assertj.core.api.Assertions.assertThatCode; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import com.nvidia.boot.exceptions.BadRequestException; -import com.nvidia.nvcf.configuration.llm.LlmRoutingExpressionsProperties; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.NullAndEmptySource; -import org.junit.jupiter.params.provider.ValueSource; - -class LlmRoutingMethodValidatorTest { - - private static final String MODEL = "test-model"; - - private final LlmRoutingExpressionsProperties properties = - new LlmRoutingExpressionsProperties(); - private final LlmRoutingMethodValidator validator = new LlmRoutingMethodValidator(properties); - - @ParameterizedTest - @ValueSource(strings = {"pulsar;seed=x", "round robin;seed=?1", "pulsar,seed=x;n=1"}) - void disabledRejectsParametersBeforeGrammar(String routingMethod) { - assertThatThrownBy(() -> validator.validate(MODEL, routingMethod)) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("not enabled") - .hasMessageContaining(MODEL); - } - - @ParameterizedTest - @NullAndEmptySource - @ValueSource(strings = {"pulsar", "Power_Of_Two", "groq-multiregion", " pulsar ", " "}) - void disabledAcceptsMethodOnlyAndBlankValues(String routingMethod) { - assertThatCode(() -> validator.validate(MODEL, routingMethod)).doesNotThrowAnyException(); - } - - @Test - void disabledRejectsMalformedMethodOnlyValue() { - assertThatThrownBy(() -> validator.validate(MODEL, "round robin")) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining(MODEL) - .hasMessageContaining("method name must match"); - } - - @Test - void enabledAcceptsParameters() { - properties.setEnabled(true); - - assertThatCode(() -> validator.validate(MODEL, "pulsar;seed=x")) - .doesNotThrowAnyException(); - } - - @Test - void enabledRejectsMalformedExpression() { - properties.setEnabled(true); - - assertThatThrownBy(() -> validator.validate(MODEL, "pulsar,seed=x")) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining(MODEL) - .hasMessageContaining("commas are not allowed"); - } - - @Test - void readsPropertyOnEachValidation() { - assertThatThrownBy(() -> validator.validate(MODEL, "pulsar;seed=x")) - .isInstanceOf(BadRequestException.class); - properties.setEnabled(true); - - assertThatCode(() -> validator.validate(MODEL, "pulsar;seed=x")) - .doesNotThrowAnyException(); - } -} diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/resources/application-test.yaml b/src/control-plane-services/cloud-functions/nvcf-core/src/test/resources/application-test.yaml index dd68659bc8..0a4fa12f8b 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/resources/application-test.yaml +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/resources/application-test.yaml @@ -182,9 +182,6 @@ logging: com.github.tomakehurst.wiremock: DEBUG nvcf: - llm: - routing-expressions: - enabled: true request: timeout: 1s # only whole seconds fqdn: http://localhost:${server.port} diff --git a/src/control-plane-services/cloud-functions/nvcf-service/src/main/resources/application.yaml b/src/control-plane-services/cloud-functions/nvcf-service/src/main/resources/application.yaml index 4ed248dac4..e18fc0d68e 100644 --- a/src/control-plane-services/cloud-functions/nvcf-service/src/main/resources/application.yaml +++ b/src/control-plane-services/cloud-functions/nvcf-service/src/main/resources/application.yaml @@ -296,9 +296,6 @@ nvcf: oauth2: base-url: https://stg.authn.nvidia.com group-scope: ngc-stg - llm: - routing-expressions: - enabled: false account: max-functions-allowed: 50 max-tasks-allowed: 50 From a76e656aee3a32b994e589935446140ea61c13c2 Mon Sep 17 00:00:00 2001 From: along Date: Fri, 18 Sep 2026 08:53:52 -0700 Subject: [PATCH 05/12] fix(cloud-functions): validate routingMethod exactly as persisted Count the 1024-byte limit on the raw routingMethod and strip only outer spaces before the grammar check. String.trim() also removed outer control characters, so "pulsar\n" validated as "pulsar" while callers persisted and forwarded the newline, and outer whitespace did not count toward the limit. A value made only of control characters is now rejected instead of passing as blank. Addresses review feedback on NVIDIA/nvcf#1955. Relates to NVIDIA/nvcf#536 Signed-off-by: along --- .../dto/LlmRoutingMethodValidator.java | 16 ++++++++++++---- .../dto/LlmRoutingMethodValidatorTest.java | 7 ++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java index 0bd8ff5505..e963d2ba80 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java @@ -62,15 +62,23 @@ public final class LlmRoutingMethodValidator { private LlmRoutingMethodValidator() {} - /** Rejects a routingMethod whose syntax the router could not parse. Blank is allowed. */ + /** + * Rejects a routingMethod whose syntax the router could not parse. Empty or space-only values + * are allowed. + */ public static void validate(String modelName, @Nullable String routingMethod) { - if (StringUtils.isBlank(routingMethod)) { + if (routingMethod == null) { return; } - var value = routingMethod.trim(); - if (value.getBytes(StandardCharsets.UTF_8).length > MAX_EXPRESSION_BYTES) { + // Callers persist and forward the value as received, so the limit counts the raw bytes and + // only outer spaces are insignificant; other outer control characters must fail below. + if (routingMethod.getBytes(StandardCharsets.UTF_8).length > MAX_EXPRESSION_BYTES) { reject(modelName, MESG_EXPRESSION_TOO_LONG); } + var value = StringUtils.strip(routingMethod, " "); + if (value.isEmpty()) { + return; + } if (value.contains(",")) { reject(modelName, MESG_COMMAS_NOT_ALLOWED); } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java index d2e0b4090e..6f79d6703a 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java @@ -90,7 +90,7 @@ void controlCharactersInRejectedSegmentReplaced() { } private static Stream routingMethodsAtLimits() { - return Stream.of("a".repeat(1024), "pulsar" + parameters(32)); + return Stream.of("a".repeat(1024), " " + "a".repeat(1023), "pulsar" + parameters(32)); } private static Stream invalidRoutingMethods() { @@ -98,6 +98,11 @@ private static Stream invalidRoutingMethods() { Arguments.of("round robin", "method name must match"), Arguments.of("power-of-3!", "method name must match"), Arguments.of(";seed=x", "method name must match"), + Arguments.of("pulsar\n", "method name must match"), + Arguments.of("\tpulsar", "method name must match"), + Arguments.of("\n", "method name must match"), + Arguments.of("pulsar;seed=x\n", "must be key=value"), + Arguments.of(" " + "a".repeat(1024), "expression exceeds 1024 bytes"), Arguments.of("pulsar,seed=x", "commas are not allowed"), Arguments.of("pulsar;seed=\"a,b\"", "commas are not allowed"), Arguments.of("pulsar;seed=\"a;b\"", "value for 'seed'"), From a8d18e7abee449f392006ffedea5b0eb364fa9cf Mon Sep 17 00:00:00 2001 From: along Date: Fri, 18 Sep 2026 09:23:48 -0700 Subject: [PATCH 06/12] fix(cloud-functions): store routingMethod without outer spaces Trim leading and trailing spaces from llmConfig.routingMethod before validation and store the trimmed value on create, version create, and update, including the copies propagated to sibling versions. Validated and stored bytes are now identical, the 1024-byte limit applies to the stored value, and whitespace variants no longer register as different values in sibling reconciliation. Only the space character is trimmed; tabs, line breaks, and other control characters are still rejected. Relates to NVIDIA/nvcf#536 Signed-off-by: along --- .../management/dto/CreateFunctionRequest.java | 4 ++-- .../dto/LlmRoutingMethodValidator.java | 21 +++++++++--------- .../service/function/FunctionLlmService.java | 8 +++---- .../FunctionsWithLlmModelsTest.java | 20 +++++++++++------ .../dto/LlmRoutingMethodValidatorTest.java | 22 +++++++++++++++++-- 5 files changed, 50 insertions(+), 25 deletions(-) diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/CreateFunctionRequest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/CreateFunctionRequest.java index 344a0d8bca..6ef3a3cde6 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/CreateFunctionRequest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/CreateFunctionRequest.java @@ -341,8 +341,8 @@ static void validateModelFields(List models, throw new BadRequestException(MESG_MISSING_LLM_MODEL_URIS); } if (isLlmFunction) { - LlmRoutingMethodValidator.validate( - model.getName(), model.getLlmConfig().getRoutingMethod()); + model.getLlmConfig().setRoutingMethod(LlmRoutingMethodValidator.validate( + model.getName(), model.getLlmConfig().getRoutingMethod())); LlmConfigValidator.validateTokenRateLimit( model.getName(), model.getLlmConfig().getTokenRateLimit()); } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java index e963d2ba80..97e0a79b73 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java @@ -63,21 +63,21 @@ public final class LlmRoutingMethodValidator { private LlmRoutingMethodValidator() {} /** - * Rejects a routingMethod whose syntax the router could not parse. Empty or space-only values - * are allowed. + * Rejects a routingMethod whose syntax the router could not parse and returns the value to + * store: the input without outer spaces, so validated and stored bytes are identical. Any + * other outer character, including tabs and line breaks, fails the grammar. */ - public static void validate(String modelName, @Nullable String routingMethod) { + @Nullable + public static String validate(String modelName, @Nullable String routingMethod) { if (routingMethod == null) { - return; - } - // Callers persist and forward the value as received, so the limit counts the raw bytes and - // only outer spaces are insignificant; other outer control characters must fail below. - if (routingMethod.getBytes(StandardCharsets.UTF_8).length > MAX_EXPRESSION_BYTES) { - reject(modelName, MESG_EXPRESSION_TOO_LONG); + return null; } var value = StringUtils.strip(routingMethod, " "); if (value.isEmpty()) { - return; + return value; + } + if (value.getBytes(StandardCharsets.UTF_8).length > MAX_EXPRESSION_BYTES) { + reject(modelName, MESG_EXPRESSION_TOO_LONG); } if (value.contains(",")) { reject(modelName, MESG_COMMAS_NOT_ALLOWED); @@ -93,6 +93,7 @@ public static void validate(String modelName, @Nullable String routingMethod) { for (var index = 1; index < segments.length; index++) { validateParameter(modelName, segments[index], keys); } + return value; } private static void validateParameter(String modelName, String segment, Set keys) { diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/FunctionLlmService.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/FunctionLlmService.java index 59854dc0e5..c5a3da7fe8 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/FunctionLlmService.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/service/function/FunctionLlmService.java @@ -265,7 +265,8 @@ private Map propagateModelUpdatesToSiblings( UpdateFunctionRequest.ModelUpdateDto::modelName, u -> FunctionModelDto.LlmConfigDto.builder() .tokenRateLimit(u.llmConfig().tokenRateLimit()) - .routingMethod(u.llmConfig().routingMethod()) + .routingMethod(LlmRoutingMethodValidator.validate( + u.modelName(), u.llmConfig().routingMethod())) .build())); if (overrides.isEmpty()) { return Map.of(); @@ -404,9 +405,8 @@ private void updateModels( llmConfig.setTokenRateLimit(llmConfigUpdate.tokenRateLimit()); } if (llmConfigUpdate.routingMethod() != null) { - LlmRoutingMethodValidator.validate( - modelUpdate.modelName(), llmConfigUpdate.routingMethod()); - llmConfig.setRoutingMethod(llmConfigUpdate.routingMethod()); + llmConfig.setRoutingMethod(LlmRoutingMethodValidator.validate( + modelUpdate.modelName(), llmConfigUpdate.routingMethod())); } updated = true; break; diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/FunctionsWithLlmModelsTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/FunctionsWithLlmModelsTest.java index 6c2a394c4d..526f5024ad 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/FunctionsWithLlmModelsTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/FunctionsWithLlmModelsTest.java @@ -596,15 +596,20 @@ private FunctionDto createAdditionalLlmFunctionVersion( } @Test - void shouldPersistRoutingExpressionsAsReceivedOnCreateAndUpdate() { - var initialRoutingMethod = "Pulsar_Wait_And_Widen; seed=stable-a;n=2"; + void shouldStoreRoutingExpressionsWithoutOuterSpacesOnCreateAndUpdate() { + var functionName = TEST_FUNCTION_NAME + "-" + Instant.now().toEpochMilli(); + var storedRoutingMethod = "Pulsar_Wait_And_Widen; seed=stable-a;n=2"; var function = createInitialLlmFunction( - TEST_FUNCTION_NAME + "-" + Instant.now().toEpochMilli(), - "1-M", initialRoutingMethod); + functionName, "1-M", " " + storedRoutingMethod + " "); assertThat(function.models().getFirst().getLlmConfig().getRoutingMethod()) - .isEqualTo(initialRoutingMethod); - assertLlmConfigPersisted(function.versionId(), "1-M", initialRoutingMethod); + .isEqualTo(storedRoutingMethod); + assertLlmConfigPersisted(function.versionId(), "1-M", storedRoutingMethod); + + var secondVersion = createAdditionalLlmFunctionVersion( + function.id(), functionName, "1-M", " " + storedRoutingMethod); + assertThat(secondVersion.models().getFirst().getLlmConfig().getRoutingMethod()) + .isEqualTo(storedRoutingMethod); var updatedRoutingMethod = "pulsar;seed=stable-b"; var updateToken = MOCK_OAUTH2_TOKEN_SERVER.getJwt( @@ -613,7 +618,7 @@ void shouldPersistRoutingExpressionsAsReceivedOnCreateAndUpdate() { .modelUpdates(List.of(UpdateFunctionRequest.ModelUpdateDto.builder() .modelName(TEST_LLM_MODEL_NAME) .llmConfig(UpdateFunctionRequest.LlmConfigUpdateDto.builder() - .routingMethod(updatedRoutingMethod) + .routingMethod(" " + updatedRoutingMethod + " ") .build()) .build())) .build(); @@ -631,6 +636,7 @@ void shouldPersistRoutingExpressionsAsReceivedOnCreateAndUpdate() { assertThat(updatedModel.getLlmConfig().getRoutingMethod()) .isEqualTo(updatedRoutingMethod); assertLlmConfigPersisted(function.versionId(), "1-M", updatedRoutingMethod); + assertLlmConfigPersisted(secondVersion.versionId(), "1-M", updatedRoutingMethod); } @Test diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java index 6f79d6703a..2b62412892 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java @@ -16,6 +16,7 @@ */ package com.nvidia.nvcf.rest.function.management.dto; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -89,8 +90,26 @@ void controlCharactersInRejectedSegmentReplaced() { .hasMessageNotContaining("\r"); } + @ParameterizedTest(name = "{index}: [{0}] is stored as [{1}]") + @MethodSource("storedValues") + void returnsValueWithoutOuterSpaces(String routingMethod, String stored) { + assertThat(LlmRoutingMethodValidator.validate(MODEL, routingMethod)).isEqualTo(stored); + } + + private static Stream storedValues() { + return Stream.of( + Arguments.of(null, null), + Arguments.of("", ""), + Arguments.of(" ", ""), + Arguments.of("pulsar", "pulsar"), + Arguments.of(" pulsar ", "pulsar"), + Arguments.of(" Power_Of_Two;seed=x ", "Power_Of_Two;seed=x"), + Arguments.of("pulsar; seed=x", "pulsar; seed=x")); + } + private static Stream routingMethodsAtLimits() { - return Stream.of("a".repeat(1024), " " + "a".repeat(1023), "pulsar" + parameters(32)); + return Stream.of( + "a".repeat(1024), " " + "a".repeat(1024) + " ", "pulsar" + parameters(32)); } private static Stream invalidRoutingMethods() { @@ -102,7 +121,6 @@ private static Stream invalidRoutingMethods() { Arguments.of("\tpulsar", "method name must match"), Arguments.of("\n", "method name must match"), Arguments.of("pulsar;seed=x\n", "must be key=value"), - Arguments.of(" " + "a".repeat(1024), "expression exceeds 1024 bytes"), Arguments.of("pulsar,seed=x", "commas are not allowed"), Arguments.of("pulsar;seed=\"a,b\"", "commas are not allowed"), Arguments.of("pulsar;seed=\"a;b\"", "value for 'seed'"), From 079d4455ff85bbe981d3a6f5ed59f8de99c9bb32 Mon Sep 17 00:00:00 2001 From: along Date: Fri, 18 Sep 2026 10:25:09 -0700 Subject: [PATCH 07/12] test(cloud-functions): cover version create and more grammar rows Add an API test that rejects a malformed routingMethod on version create, persist an unknown method and parameter through the update path to show that only syntax is checked, and extend the reject table with hyphen and underscore-leading keys, digit- and minus-leading tokens, and control characters inside an unquoted value. Relates to NVIDIA/nvcf#536 Signed-off-by: along --- .../FunctionsWithLlmModelsTest.java | 31 ++++++++++++++++++- .../dto/LlmRoutingMethodValidatorTest.java | 6 ++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/FunctionsWithLlmModelsTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/FunctionsWithLlmModelsTest.java index 526f5024ad..424ebf58bd 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/FunctionsWithLlmModelsTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/FunctionsWithLlmModelsTest.java @@ -611,7 +611,8 @@ void shouldStoreRoutingExpressionsWithoutOuterSpacesOnCreateAndUpdate() { assertThat(secondVersion.models().getFirst().getLlmConfig().getRoutingMethod()) .isEqualTo(storedRoutingMethod); - var updatedRoutingMethod = "pulsar;seed=stable-b"; + // Unknown method and parameter: well formed, so it persists; the router owns semantics. + var updatedRoutingMethod = "fastest;widen=2"; var updateToken = MOCK_OAUTH2_TOKEN_SERVER.getJwt( TEST_CLIENT_SUBJECT, List.of(SCOPE_UPDATE_FUNCTION), 100); var updateRequest = UpdateFunctionRequest.builder() @@ -663,6 +664,34 @@ void shouldRejectCreateWithInvalidRoutingMethod() { "llmConfig.routingMethod", TEST_LLM_MODEL_NAME, "commas are not allowed"); } + @Test + void shouldRejectVersionCreateWithInvalidRoutingMethod() { + var functionName = TEST_FUNCTION_NAME + "-" + Instant.now().toEpochMilli(); + var function = createInitialLlmFunction(functionName, "1-M", "round-robin"); + + var createToken = MOCK_OAUTH2_TOKEN_SERVER.getJwt(TEST_CLIENT_SUBJECT, + List.of(SCOPE_REGISTER_FUNCTION), 100); + var createRequest = CreateFunctionRequest.builder() + .name(functionName) + .containerImage(TEST_NGC_CONTAINER_IMAGE) + .inferenceUrl(TEST_INFERENCE_URL) + .inferencePort(TEST_INFERENCE_PORT) + .functionType(FunctionTypeEnum.LLM) + .models(List.of(llmModel("1-M", "pulsar;seed="))) + .build(); + var createEntity = RequestEntity.post(URI.create( + "/v2/nvcf/functions/" + function.id() + "/versions")) + .contentType(MediaType.APPLICATION_JSON) + .header("Authorization", "Bearer " + createToken) + .body(createRequest); + + var response = testRestTemplate.exchange(createEntity, String.class); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST); + assertThat(response.getBody()).contains( + "llmConfig.routingMethod", TEST_LLM_MODEL_NAME, "must be key=value"); + } + @Test void shouldRejectCreateWithInvalidTokenRateLimit() { var createToken = MOCK_OAUTH2_TOKEN_SERVER.getJwt(TEST_CLIENT_SUBJECT, diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java index 2b62412892..a949b147f9 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java @@ -134,6 +134,12 @@ private static Stream invalidRoutingMethods() { Arguments.of("pulsar;=x", "must be key=value"), Arguments.of("pulsar;Seed=x", "must be key=value"), Arguments.of("pulsar;2n=1", "must be key=value"), + Arguments.of("pulsar;max-queued=1", "must be key=value"), + Arguments.of("pulsar;_n=1", "must be key=value"), + Arguments.of("pulsar;seed=1abc", "value for 'seed'"), + Arguments.of("pulsar;seed=-abc", "value for 'seed'"), + Arguments.of("pulsar;seed=a\tb", "value for 'seed'"), + Arguments.of("pulsar;seed=a" + (char) 0x7f + "b", "value for 'seed'"), Arguments.of("pulsar;n=?1", "value for 'n'"), Arguments.of("pulsar;n=?0", "value for 'n'"), Arguments.of("pulsar;seed=:YQ==:", "value for 'seed'"), From 16699aed619b5acf51cfa9d1bf2865c945c43ec8 Mon Sep 17 00:00:00 2001 From: along Date: Fri, 18 Sep 2026 13:41:43 -0700 Subject: [PATCH 08/12] docs(user): describe routing expressions for llmConfig.routingMethod The load-balancing guide said the function API accepts only the fixed algorithm names. Describe the expression profile it accepts now: the algorithm name, key=value parameters, value forms, the 32-parameter and 1024-byte limits, comma and control-character rules, and outer-space trimming. Point the CLI and gateway guides at that section and state that nvcf-cli still accepts only the algorithm names. Relates to NVIDIA/nvcf#536 Signed-off-by: along --- docs/user/cli.md | 2 +- docs/user/llm-gateway.md | 9 ++++--- .../user/llm-request-router-load-balancing.md | 24 ++++++++++++++++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/docs/user/cli.md b/docs/user/cli.md index 9ece976581..c81053c1ab 100644 --- a/docs/user/cli.md +++ b/docs/user/cli.md @@ -799,7 +799,7 @@ LLM functions use `functionType: "LLM"` and define model routing metadata under } ``` -For LLM models, `llmConfig.routingMethod` accepts `round_robin`, `power_of_two`, `groq_multiregion`, `pulsar`, or `random`. +For LLM models, `nvcf-cli` accepts `round_robin`, `power_of_two`, `groq_multiregion`, `pulsar`, or `random` for `llmConfig.routingMethod`. The function API also accepts routing expressions with `;key=value` parameters, described in [LLM Request Router Load Balancing](./llm-request-router-load-balancing.md); the CLI does not pass them through yet. Supported LLM paths are `/v1/chat/completions`, `/v1/responses`, and `/v1/embeddings`. `llmConfig.tokenRateLimit` accepts one or more comma-separated positive integer token limits in `-` format. Supported units are `S` (seconds), `M` (minutes), `H` (hours), `D` (days), and `W` (weeks). Use `1000-S` for a single limit, or `1000-S,5000-M,100000-H,500000-D,1000000-W` for a combined limit with distinct units. Use JSON input for combined limits because inline CLI model specs use commas as field separators. diff --git a/docs/user/llm-gateway.md b/docs/user/llm-gateway.md index 4e07be33d3..0f50d9e81a 100644 --- a/docs/user/llm-gateway.md +++ b/docs/user/llm-gateway.md @@ -91,10 +91,13 @@ gateway does not wrap it in a second NVCF envelope. | `/v1/responses` | Supports native Responses API requests. Streaming clients receive server-sent events (SSE). Non-streaming clients receive the terminal Responses JSON object. | | `/v1/embeddings` | Supports embeddings requests with string or string array input. | -`nvcf-cli` accepts `round_robin`, `power_of_two`, `groq_multiregion`, -`pulsar`, or `random` for `llmConfig.routingMethod`. +The function API accepts a routing expression for `llmConfig.routingMethod`: +an algorithm name, optionally followed by `;key=value` parameters. `nvcf-cli` +currently accepts only the algorithm names `round_robin`, `power_of_two`, +`groq_multiregion`, `pulsar`, or `random`. -For the mapping to Stargate algorithms and the request-router allowlist, see +For the expression syntax, the mapping to Stargate algorithms, and the +request-router allowlist, see [LLM Request Router Load Balancing](./llm-request-router-load-balancing.md). `llmConfig.tokenRateLimit` applies a per-model token limit. Use one or more comma-separated limits in `-` format, where `` is a positive integer and `` is one of `S` (seconds), `M` (minutes), `H` (hours), `D` (days), or `W` (weeks). A single limit is one token budget over one time window, such as `1000-S`. A combined limit is multiple token budgets over distinct time windows, such as `1000-S,5000-M,100000-H,500000-D,1000000-W`; do not repeat a unit in the same value. diff --git a/docs/user/llm-request-router-load-balancing.md b/docs/user/llm-request-router-load-balancing.md index c4a12a3dec..3c4acab6a6 100644 --- a/docs/user/llm-request-router-load-balancing.md +++ b/docs/user/llm-request-router-load-balancing.md @@ -94,7 +94,7 @@ Algorithm availability is enforced at separate layers: | Layer | Input contract | | --- | --- | | `lb-config.json` | Canonical Stargate algorithm names: `power-of-two`, `wait-and-widen`, `round-robin`, `random`, `pulsar`, and `pulsar-wait-and-widen`. Legacy `groq-multiregion` and `pulsar-multiregion` aliases remain accepted for existing deployments. | -| Function model `llmConfig.routingMethod` | The same algorithm names, with underscores accepted in place of hyphens. Legacy aliases remain accepted for existing functions. | +| Function model `llmConfig.routingMethod` | A routing expression: an algorithm name, optionally followed by `;key=value` parameters. The control plane checks the syntax described below and stores the value as received apart from outer spaces. Stargate decides at request time whether the algorithm and parameters apply. | | LLM API Gateway | Nonblank routing method from authenticated model metadata, trimmed and forwarded as `x-routing-method` without algorithm validation. | | Stargate `x-routing-method` | Case-insensitive algorithm name with hyphens or underscores. It must match the effective algorithm or a model or top-level `request_algorithms` entry. Otherwise, Stargate returns HTTP `400`. | @@ -108,6 +108,28 @@ Use `wait-and-widen` and `pulsar-wait-and-widen` in new function metadata, Existing `groq-multiregion` and `pulsar-multiregion` values continue to work through the Stargate and control-plane compatibility aliases. +### Routing expression syntax + +`llmConfig.routingMethod` follows a profile of RFC 8941 structured field +parameters: + +- An algorithm name matching `[A-Za-z][A-Za-z0-9_-]*`, optionally followed by + parameters separated by `;`, for example `pulsar; seed=stable-a; n=2`. +- Each parameter is `key=value`. Keys match `[a-z][a-z0-9_]*`. Spaces are + allowed before a key but not around `=`. +- A value is an integer of up to 15 digits, a decimal with up to 12 integer + digits and 1 to 3 fraction digits, a token that starts with a letter or `*`, + or a double-quoted string of printable ASCII in which only `\"` and `\\` + are escaped. +- At most 32 parameters and 1024 bytes. Commas are not allowed anywhere, + including inside quoted strings, and a quoted string cannot contain `;`. +- Leading and trailing spaces are removed before the value is stored. Tabs, + line breaks, and other control characters are rejected. + +The control plane does not check algorithm names or parameter names. A +well-formed expression that Stargate does not understand is stored and is +rejected at request time with HTTP `400`. + ## Keep router headers trusted The gateway can send the following headers to Stargate. Derive or validate From b7d4a7f327c94f160315a293e679cdb4cc3e283a Mon Sep 17 00:00:00 2001 From: along Date: Fri, 18 Sep 2026 14:25:31 -0700 Subject: [PATCH 09/12] fix(cloud-functions): scrub C1 controls and model names in rejections Match control characters with \p{Cc} instead of \p{Cntrl}, which only covers ASCII: U+0085 (NEL) is a line terminator that the parameter pattern refuses, so it reached the 400 body and the log line unchanged. Scrub the model name the same way in both llmConfig validators, since it is raw request text in the same message. Log the rejections at warn, as a malformed client value is not an operator problem, and derive the limit messages from the limit constants so they cannot drift. The end-to-end test now creates the second version with a different routing method so the sibling write really happens and both stored rows are asserted. Addresses review feedback on NVIDIA/nvcf#1955. Relates to NVIDIA/nvcf#536 Signed-off-by: along --- .../management/dto/LlmConfigValidator.java | 5 +++-- .../dto/LlmRoutingMethodValidator.java | 22 +++++++++++-------- .../FunctionsWithLlmModelsTest.java | 8 +++++-- .../dto/LlmConfigValidatorTest.java | 9 ++++++++ .../dto/LlmRoutingMethodValidatorTest.java | 19 ++++++++++++++++ 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java index ee1cdd0394..7f34a746cf 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java @@ -34,8 +34,9 @@ public static void validateTokenRateLimit(String modelName, @Nullable String tok return; } if (!TOKEN_RATE_LIMIT_PATTERN.matcher(tokenRateLimit).matches()) { - var mesg = MESG_INVALID_TOKEN_RATE_LIMIT.formatted(modelName); - log.error(mesg); + var mesg = MESG_INVALID_TOKEN_RATE_LIMIT.formatted( + LlmRoutingMethodValidator.withoutControlCharacters(modelName)); + log.warn(mesg); throw new BadRequestException(mesg); } } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java index 97e0a79b73..6a85f88664 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java @@ -44,16 +44,19 @@ public final class LlmRoutingMethodValidator { Pattern.compile("[A-Za-z*][A-Za-z0-9!#$%&'*+.^_`|~:/-]*"); private static final Pattern STRING_PATTERN = Pattern.compile( "\"(?:[\\x20\\x21\\x23-\\x2b\\x2d-\\x3a\\x3c-\\x5b\\x5d-\\x7e]|\\\\[\"\\\\])*\""); + // \p{Cc} covers the C1 controls such as U+0085 (NEL), which \p{Cntrl} does not. private static final Pattern CONTROL_CHARACTERS = - Pattern.compile("[\\p{Cntrl}\\p{Zl}\\p{Zp}]"); + Pattern.compile("[\\p{Cc}\\p{Zl}\\p{Zp}]"); private static final String MESG_INVALID_ROUTING_METHOD = "Invalid request: 'llmConfig.routingMethod' for model '%s' is invalid: %s"; - private static final String MESG_EXPRESSION_TOO_LONG = "expression exceeds 1024 bytes"; + private static final String MESG_EXPRESSION_TOO_LONG = + "expression exceeds %d bytes".formatted(MAX_EXPRESSION_BYTES); private static final String MESG_COMMAS_NOT_ALLOWED = "commas are not allowed"; private static final String MESG_INVALID_METHOD_NAME = "method name must match [A-Za-z][A-Za-z0-9_-]*"; - private static final String MESG_TOO_MANY_PARAMETERS = "at most 32 parameters are allowed"; + private static final String MESG_TOO_MANY_PARAMETERS = + "at most %d parameters are allowed".formatted(MAX_PARAMETERS); private static final String MESG_INVALID_PARAMETER = "parameter '%s' must be key=value with key matching [a-z][a-z0-9_]*"; private static final String MESG_INVALID_VALUE = @@ -110,10 +113,10 @@ private static void validateParameter(String modelName, String segment, Set LlmConfigValidator.validateTokenRateLimit("m\nx", "20-X")) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("for model 'm?x'") + .hasMessageNotContaining("\n"); + } } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java index a949b147f9..e5ff225f75 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java @@ -90,6 +90,25 @@ void controlCharactersInRejectedSegmentReplaced() { .hasMessageNotContaining("\r"); } + @Test + void nextLineControlCharacterInRejectedSegmentReplaced() { + var nextLine = String.valueOf((char) 0x85); + var routingMethod = "pulsar;seed=a" + nextLine + "b"; + + assertThatThrownBy(() -> LlmRoutingMethodValidator.validate(MODEL, routingMethod)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("parameter 'seed=a?b' must be key=value") + .hasMessageNotContaining(nextLine); + } + + @Test + void controlCharactersInModelNameReplaced() { + assertThatThrownBy(() -> LlmRoutingMethodValidator.validate("m\nx", "pulsar,seed=x")) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("for model 'm?x'") + .hasMessageNotContaining("\n"); + } + @ParameterizedTest(name = "{index}: [{0}] is stored as [{1}]") @MethodSource("storedValues") void returnsValueWithoutOuterSpaces(String routingMethod, String stored) { From 6243ee7c34a1731114c87dc94e0c9fea6c26d428 Mon Sep 17 00:00:00 2001 From: along Date: Fri, 18 Sep 2026 14:36:19 -0700 Subject: [PATCH 10/12] revert(docs): describe routing expressions for llmConfig.routingMethod This reverts commit 16699aed619b5acf51cfa9d1bf2865c945c43ec8. The request router does not parse routing expressions yet and nvcf-cli cannot send them, so documenting the grammar now would advertise a configuration that fails at request time. The user docs follow in a separate change once the router accepts expressions. Relates to NVIDIA/nvcf#536 Signed-off-by: along --- docs/user/cli.md | 2 +- docs/user/llm-gateway.md | 9 +++---- .../user/llm-request-router-load-balancing.md | 24 +------------------ 3 files changed, 5 insertions(+), 30 deletions(-) diff --git a/docs/user/cli.md b/docs/user/cli.md index c81053c1ab..9ece976581 100644 --- a/docs/user/cli.md +++ b/docs/user/cli.md @@ -799,7 +799,7 @@ LLM functions use `functionType: "LLM"` and define model routing metadata under } ``` -For LLM models, `nvcf-cli` accepts `round_robin`, `power_of_two`, `groq_multiregion`, `pulsar`, or `random` for `llmConfig.routingMethod`. The function API also accepts routing expressions with `;key=value` parameters, described in [LLM Request Router Load Balancing](./llm-request-router-load-balancing.md); the CLI does not pass them through yet. +For LLM models, `llmConfig.routingMethod` accepts `round_robin`, `power_of_two`, `groq_multiregion`, `pulsar`, or `random`. Supported LLM paths are `/v1/chat/completions`, `/v1/responses`, and `/v1/embeddings`. `llmConfig.tokenRateLimit` accepts one or more comma-separated positive integer token limits in `-` format. Supported units are `S` (seconds), `M` (minutes), `H` (hours), `D` (days), and `W` (weeks). Use `1000-S` for a single limit, or `1000-S,5000-M,100000-H,500000-D,1000000-W` for a combined limit with distinct units. Use JSON input for combined limits because inline CLI model specs use commas as field separators. diff --git a/docs/user/llm-gateway.md b/docs/user/llm-gateway.md index 0f50d9e81a..4e07be33d3 100644 --- a/docs/user/llm-gateway.md +++ b/docs/user/llm-gateway.md @@ -91,13 +91,10 @@ gateway does not wrap it in a second NVCF envelope. | `/v1/responses` | Supports native Responses API requests. Streaming clients receive server-sent events (SSE). Non-streaming clients receive the terminal Responses JSON object. | | `/v1/embeddings` | Supports embeddings requests with string or string array input. | -The function API accepts a routing expression for `llmConfig.routingMethod`: -an algorithm name, optionally followed by `;key=value` parameters. `nvcf-cli` -currently accepts only the algorithm names `round_robin`, `power_of_two`, -`groq_multiregion`, `pulsar`, or `random`. +`nvcf-cli` accepts `round_robin`, `power_of_two`, `groq_multiregion`, +`pulsar`, or `random` for `llmConfig.routingMethod`. -For the expression syntax, the mapping to Stargate algorithms, and the -request-router allowlist, see +For the mapping to Stargate algorithms and the request-router allowlist, see [LLM Request Router Load Balancing](./llm-request-router-load-balancing.md). `llmConfig.tokenRateLimit` applies a per-model token limit. Use one or more comma-separated limits in `-` format, where `` is a positive integer and `` is one of `S` (seconds), `M` (minutes), `H` (hours), `D` (days), or `W` (weeks). A single limit is one token budget over one time window, such as `1000-S`. A combined limit is multiple token budgets over distinct time windows, such as `1000-S,5000-M,100000-H,500000-D,1000000-W`; do not repeat a unit in the same value. diff --git a/docs/user/llm-request-router-load-balancing.md b/docs/user/llm-request-router-load-balancing.md index 3c4acab6a6..c4a12a3dec 100644 --- a/docs/user/llm-request-router-load-balancing.md +++ b/docs/user/llm-request-router-load-balancing.md @@ -94,7 +94,7 @@ Algorithm availability is enforced at separate layers: | Layer | Input contract | | --- | --- | | `lb-config.json` | Canonical Stargate algorithm names: `power-of-two`, `wait-and-widen`, `round-robin`, `random`, `pulsar`, and `pulsar-wait-and-widen`. Legacy `groq-multiregion` and `pulsar-multiregion` aliases remain accepted for existing deployments. | -| Function model `llmConfig.routingMethod` | A routing expression: an algorithm name, optionally followed by `;key=value` parameters. The control plane checks the syntax described below and stores the value as received apart from outer spaces. Stargate decides at request time whether the algorithm and parameters apply. | +| Function model `llmConfig.routingMethod` | The same algorithm names, with underscores accepted in place of hyphens. Legacy aliases remain accepted for existing functions. | | LLM API Gateway | Nonblank routing method from authenticated model metadata, trimmed and forwarded as `x-routing-method` without algorithm validation. | | Stargate `x-routing-method` | Case-insensitive algorithm name with hyphens or underscores. It must match the effective algorithm or a model or top-level `request_algorithms` entry. Otherwise, Stargate returns HTTP `400`. | @@ -108,28 +108,6 @@ Use `wait-and-widen` and `pulsar-wait-and-widen` in new function metadata, Existing `groq-multiregion` and `pulsar-multiregion` values continue to work through the Stargate and control-plane compatibility aliases. -### Routing expression syntax - -`llmConfig.routingMethod` follows a profile of RFC 8941 structured field -parameters: - -- An algorithm name matching `[A-Za-z][A-Za-z0-9_-]*`, optionally followed by - parameters separated by `;`, for example `pulsar; seed=stable-a; n=2`. -- Each parameter is `key=value`. Keys match `[a-z][a-z0-9_]*`. Spaces are - allowed before a key but not around `=`. -- A value is an integer of up to 15 digits, a decimal with up to 12 integer - digits and 1 to 3 fraction digits, a token that starts with a letter or `*`, - or a double-quoted string of printable ASCII in which only `\"` and `\\` - are escaped. -- At most 32 parameters and 1024 bytes. Commas are not allowed anywhere, - including inside quoted strings, and a quoted string cannot contain `;`. -- Leading and trailing spaces are removed before the value is stored. Tabs, - line breaks, and other control characters are rejected. - -The control plane does not check algorithm names or parameter names. A -well-formed expression that Stargate does not understand is stored and is -rejected at request time with HTTP `400`. - ## Keep router headers trusted The gateway can send the following headers to Stargate. Derive or validate From b4a75bc47e4f633ebe3ebb94b0a73a247c286c14 Mon Sep 17 00:00:00 2001 From: along Date: Fri, 18 Sep 2026 15:02:52 -0700 Subject: [PATCH 11/12] refactor(cloud-functions): drop model name scrub from llmConfig errors The routing method validator owns the routing value, not the model name, and scrubbing the name in its one message left every other log line that carries a model name untouched. Format the name as received again in both llmConfig validators; control characters in model names belong to a name constraint or the log layer. Relates to NVIDIA/nvcf#536 Signed-off-by: along --- .../function/management/dto/LlmConfigValidator.java | 3 +-- .../management/dto/LlmRoutingMethodValidator.java | 10 +++++----- .../management/dto/LlmConfigValidatorTest.java | 9 --------- .../management/dto/LlmRoutingMethodValidatorTest.java | 8 -------- 4 files changed, 6 insertions(+), 24 deletions(-) diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java index 7f34a746cf..4dd0d7811a 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java @@ -34,8 +34,7 @@ public static void validateTokenRateLimit(String modelName, @Nullable String tok return; } if (!TOKEN_RATE_LIMIT_PATTERN.matcher(tokenRateLimit).matches()) { - var mesg = MESG_INVALID_TOKEN_RATE_LIMIT.formatted( - LlmRoutingMethodValidator.withoutControlCharacters(modelName)); + var mesg = MESG_INVALID_TOKEN_RATE_LIMIT.formatted(modelName); log.warn(mesg); throw new BadRequestException(mesg); } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java index 6a85f88664..316558336d 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java @@ -113,10 +113,10 @@ private static void validateParameter(String modelName, String segment, Set LlmConfigValidator.validateTokenRateLimit("m\nx", "20-X")) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("for model 'm?x'") - .hasMessageNotContaining("\n"); - } } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java index e5ff225f75..466c0eefad 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java @@ -101,14 +101,6 @@ void nextLineControlCharacterInRejectedSegmentReplaced() { .hasMessageNotContaining(nextLine); } - @Test - void controlCharactersInModelNameReplaced() { - assertThatThrownBy(() -> LlmRoutingMethodValidator.validate("m\nx", "pulsar,seed=x")) - .isInstanceOf(BadRequestException.class) - .hasMessageContaining("for model 'm?x'") - .hasMessageNotContaining("\n"); - } - @ParameterizedTest(name = "{index}: [{0}] is stored as [{1}]") @MethodSource("storedValues") void returnsValueWithoutOuterSpaces(String routingMethod, String stored) { From d4bc6ddb3dcec6d17963b800f1a137cd2e4a1afc Mon Sep 17 00:00:00 2001 From: along Date: Fri, 18 Sep 2026 15:31:55 -0700 Subject: [PATCH 12/12] fix(cloud-functions): scrub model names in routing rejections The model name is echoed into the log line and the 400 body of a routing method rejection. Replace control characters in it with '?' the same way as in the echoed parameter segment, so a name containing a line break cannot forge a log line. The name itself is not validated. Relates to NVIDIA/nvcf#536 Signed-off-by: along --- .../management/dto/LlmRoutingMethodValidator.java | 2 +- .../management/dto/LlmRoutingMethodValidatorTest.java | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java index 316558336d..c07f273cb3 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidator.java @@ -128,7 +128,7 @@ private static boolean isValidBareValue(String value) { // A malformed client value is not an operator problem, so it is logged below error level. private static void reject(String modelName, String rule) { - var mesg = MESG_INVALID_ROUTING_METHOD.formatted(modelName, rule); + var mesg = MESG_INVALID_ROUTING_METHOD.formatted(withoutControlCharacters(modelName), rule); log.warn(mesg); throw new BadRequestException(mesg); } diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java index 466c0eefad..e5ff225f75 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmRoutingMethodValidatorTest.java @@ -101,6 +101,14 @@ void nextLineControlCharacterInRejectedSegmentReplaced() { .hasMessageNotContaining(nextLine); } + @Test + void controlCharactersInModelNameReplaced() { + assertThatThrownBy(() -> LlmRoutingMethodValidator.validate("m\nx", "pulsar,seed=x")) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("for model 'm?x'") + .hasMessageNotContaining("\n"); + } + @ParameterizedTest(name = "{index}: [{0}] is stored as [{1}]") @MethodSource("storedValues") void returnsValueWithoutOuterSpaces(String routingMethod, String stored) {