Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/control-plane-services/cloud-functions/NOTICE
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@

Lists of 288 third-party dependencies.
Lists of 291 third-party dependencies.
(Apache License, Version 2.0) LZ4 Java Compression (at.yawk.lz4:lz4-java:1.11.2 - https://github.com/yawkat/lz4-java)
(EPL-2.0) (LGPL-2.1-only) Logback Classic Module (ch.qos.logback:logback-classic:1.5.38 - http://logback.qos.ch)
(EPL-2.0) (LGPL-2.1-only) Logback Core Module (ch.qos.logback:logback-core:1.5.38 - http://logback.qos.ch)
Expand Down Expand Up @@ -48,6 +48,9 @@ Lists of 288 third-party dependencies.
(Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.21.0 - https://commons.apache.org/proper/commons-codec/)
(Apache-2.0) Apache Commons IO (commons-io:commons-io:2.20.0 - https://commons.apache.org/proper/commons-io/)
(Apache-2.0) Apache Commons Logging (commons-logging:commons-logging:1.4.0 - https://commons.apache.org/proper/commons-logging/)
(The Apache Software License, Version 2.0) CloudEvents - API (io.cloudevents:cloudevents-api:4.1.1 - https://cloudevents.github.io/sdk-java/)
(The Apache Software License, Version 2.0) CloudEvents - Core (io.cloudevents:cloudevents-core:4.1.1 - https://cloudevents.github.io/sdk-java/)
(The Apache Software License, Version 2.0) CloudEvents - JSON Jackson (io.cloudevents:cloudevents-json-jackson:4.1.1 - https://cloudevents.github.io/sdk-java/)
(Apache 2.0) io.grpc:grpc-api (io.grpc:grpc-api:1.83.1 - https://github.com/grpc/grpc-java)
(Apache 2.0) io.grpc:grpc-context (io.grpc:grpc-context:1.83.1 - https://github.com/grpc/grpc-java)
(Apache 2.0) io.grpc:grpc-core (io.grpc:grpc-core:1.83.1 - https://github.com/grpc/grpc-java)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,9 @@ nvcf_java_library(
"@nv_third_party_deps//:io_grpc_grpc_api",
"@nv_third_party_deps//:io_grpc_grpc_netty_shaded",
"@nv_third_party_deps//:io_grpc_grpc_stub",
"@nv_third_party_deps//:io_cloudevents_cloudevents_api",
"@nv_third_party_deps//:io_cloudevents_cloudevents_core",
"@nv_third_party_deps//:io_cloudevents_cloudevents_json_jackson",
"@nv_third_party_deps//:net_devh_grpc_common_spring_boot",
"@nv_third_party_deps//:io_micrometer_micrometer_core",
"@nv_third_party_deps//:io_micrometer_micrometer_observation",
Expand Down Expand Up @@ -190,6 +193,8 @@ NVCF_CORE_TEST_DEPS = [
"@nv_third_party_deps//:io_grpc_grpc_api",
"@nv_third_party_deps//:io_grpc_grpc_services",
"@nv_third_party_deps//:io_grpc_grpc_stub",
"@nv_third_party_deps//:io_cloudevents_cloudevents_core",
"@nv_third_party_deps//:io_cloudevents_cloudevents_json_jackson",
Comment thread
coderabbitai[bot] marked this conversation as resolved.
"@nv_third_party_deps//:io_micrometer_micrometer_core",
"@nv_third_party_deps//:io_micrometer_micrometer_observation",
"@nv_third_party_deps//:io_micrometer_micrometer_tracing",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.nvidia.nvcf.service.eventledger;

import static io.cloudevents.jackson.JsonFormat.CONTENT_TYPE;

import com.nvidia.nvcf.persistence.function.entity.FunctionStatus;
import com.nvidia.nvcf.util.NvcfOAuth2ClientUtils;
import io.cloudevents.CloudEvent;
import io.cloudevents.core.builder.CloudEventBuilder;
import io.cloudevents.core.format.EventFormat;
import io.cloudevents.core.provider.EventFormatProvider;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.util.Map;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.WebClient;
import tools.jackson.databind.json.JsonMapper;

@Slf4j
@Service
@RefreshScope
public class EventLedgerClient {

static final String CLIENT_REGISTRATION_ID = "event-ledger";
static final String CLOUD_EVENTS_PATH = "/v3/ledger/cloudevents";
static final String CLOUD_EVENTS_CONTENT_TYPE = CONTENT_TYPE;
static final String CLOUD_EVENT_SOURCE = "nvidia-cloud-functions";

private static final String MESG_UNKNOWN_FUNCTION_STATUS =
"Event Ledger unknown function status: {}, skip publishing.";
private static final String MESG_FAILED_TO_PUBLISH_FUNCTION_STATUS =
"Failed to publish function status to Event Ledger: accountId={}, functionId={}, "
+ "functionVersionId={}, deploymentId={}, status={}";
private static final Map<FunctionStatus, String> EVENT_NAMES = Map.of(
FunctionStatus.DEPLOYING, "Function.Deploying",
FunctionStatus.ACTIVE, "Function.Ready",
FunctionStatus.DEGRADING, "Function.Degrading",
FunctionStatus.DEGRADED, "Function.Degraded",
FunctionStatus.ERROR, "Function.Error",
FunctionStatus.INACTIVE, "Function.Inactive");
private final Duration timeout;
private final boolean enabled;
private final WebClient webClient;
private final JsonMapper jsonMapper;
private final EventFormat eventFormat;

private record FunctionStatusTransition(
UUID functionId,
UUID functionVersionId,
UUID deploymentId,
FunctionStatus previousStatus,
FunctionStatus currentStatus,
Instant persistedAt) {
}

@Autowired
public EventLedgerClient(
@Value("${nvcf.event-ledger.enabled:false}") boolean enabled,
@Value("${nvcf.event-ledger.base-url:http://event-ledger.nvcf.svc.cluster.local:8080}")
String baseUrl,
Comment thread
dmikhaylovnv marked this conversation as resolved.
@Value("${nvcf.event-ledger.timeout:2s}") Duration timeout,
@Value("${spring.security.oauth2.client.registration.event-ledger.client-id:}")
String clientId,
@Value("${spring.security.oauth2.client.registration.event-ledger.client-secret:}")
String clientSecret,
@Value("${spring.security.oauth2.client.registration.event-ledger.scope:}") String scope,
@Value("${spring.security.oauth2.client.provider.event-ledger.token-uri:}") String tokenUri,
WebClient.Builder webClientBuilder,
JsonMapper jsonMapper) {
this(enabled, timeout, enabled
? authenticatedWebClient(
baseUrl, clientId, clientSecret, scope, tokenUri, webClientBuilder)
: webClientBuilder.baseUrl(baseUrl).build(), jsonMapper);
}

EventLedgerClient(
boolean enabled,
Duration timeout,
WebClient webClient,
JsonMapper jsonMapper) {
this.enabled = enabled;
this.timeout = timeout;
this.webClient = webClient;
this.jsonMapper = jsonMapper;
this.eventFormat = EventFormatProvider.getInstance().resolveFormat(CONTENT_TYPE);
}

public void publish(
String ncaId,
UUID functionId,
UUID functionVersionId,
UUID deploymentId,
FunctionStatus previousStatus,
FunctionStatus currentStatus,
Instant persistedAt) {
if (!enabled) {
return;
}

var transition = new FunctionStatusTransition(
functionId,
functionVersionId,
deploymentId,
previousStatus,
currentStatus,
persistedAt);
var eventName = EVENT_NAMES.get(transition.currentStatus());
if (eventName == null) {
log.warn(MESG_UNKNOWN_FUNCTION_STATUS, transition.currentStatus());
return;
}

send(ncaId, transition, eventName);
}

private void send(
String ncaId, FunctionStatusTransition transition, String eventName) {
try {
webClient.post()
.uri(CLOUD_EVENTS_PATH)
.contentType(MediaType.parseMediaType(CLOUD_EVENTS_CONTENT_TYPE))
.bodyValue(eventFormat.serialize(buildCloudEvent(ncaId, transition, eventName)))
.retrieve()
.toBodilessEntity()
.block(timeout);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
} catch (Exception ex) {
log.warn(MESG_FAILED_TO_PUBLISH_FUNCTION_STATUS,
ncaId, transition.functionId(), transition.functionVersionId(),
transition.deploymentId(), transition.currentStatus(), ex);
}
}

private CloudEvent buildCloudEvent(
String ncaId, FunctionStatusTransition transition, String eventName) throws Exception {
var details = Map.of(
"previousStatus", transition.previousStatus().toString(),
"currentStatus", transition.currentStatus().toString());
return CloudEventBuilder.v1()
.withId(UUID.randomUUID().toString())
.withSource(URI.create(CLOUD_EVENT_SOURCE))
.withTime(transition.persistedAt().atOffset(ZoneOffset.UTC))
.withType(eventName)
.withExtension("namespace", transition.functionVersionId().toString())
Comment thread
shelleyshen-0 marked this conversation as resolved.
.withExtension("functionid", transition.functionId().toString())
.withExtension("functionversionid", transition.functionVersionId().toString())
.withExtension("deploymentid", transition.deploymentId().toString())
.withExtension("ncaid", ncaId)
.withData(jsonMapper.writeValueAsString(details).getBytes(StandardCharsets.UTF_8))
.build();
}

private static WebClient authenticatedWebClient(
String baseUrl,
String clientId,
String clientSecret,
String scope,
String tokenUri,
WebClient.Builder builder) {
return builder.baseUrl(baseUrl)
.filter(NvcfOAuth2ClientUtils.getOAuth2ExchangeFilter(
builder, CLIENT_REGISTRATION_ID, tokenUri, clientId, clientSecret, scope))
.build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ public class FunctionDeploymentService {
private final JsonMapper jsonMapper;
private final WorkerNatsService workerNatsService;
private final RegistryArtifactService artifactService;
private final FunctionStatusTransitionService functionStatusTransitionService;

// Function deployment is created by the admin.
@SneakyThrows
Expand Down Expand Up @@ -228,13 +229,12 @@ public FunctionDeploymentDto createFunctionDeployment(
// functions_deployment_v2 and gpu_specifications in one batch.
deploymentBatchWriter.createDeployment(
new FunctionDeploymentContext(deployment, gpuSpecEntities));
var newStatus = DEPLOYING;
if (isZeroScaling) {
log.info(MESG_ZERO_SCALE, functionId, functionVersionId);
function.setFunctionStatus(ACTIVE);
} else {
function.setFunctionStatus(DEPLOYING);
newStatus = ACTIVE;
}
functionsRepository.insert(function);
functionStatusTransitionService.persist(function, deploymentId, newStatus);
} catch (Exception ex) {
var mesg = format(MESG_FAILED_CREATE_DEPLOYMENT,
functionId, functionVersionId, status, ex.getMessage());
Expand Down Expand Up @@ -321,8 +321,8 @@ private FunctionDto deleteFunctionDeployment(

try {
// Mark function as INACTIVE to stop accepting new jobs/tasks for this version.
function.setFunctionStatus(INACTIVE);
functionsRepository.insert(function);
functionStatusTransitionService.persist(
function, deployment.getDeploymentId(), INACTIVE);

// The current default behavior for deleting a deployment is ungraceful as we whack the
// queue and the workers. In the future, when we get the opportunity to break backward
Expand Down Expand Up @@ -482,8 +482,7 @@ public FunctionEntity transitionFunctionToActive(
});
var jsonBefore = jsonMapper.valueToTree(function);

function.setFunctionStatus(ACTIVE);
functionsRepository.insert(function);
functionStatusTransitionService.persist(function, deploymentId, ACTIVE);

var summary = SUMMARY_ACTIVATE_FUNCTION.formatted(functionId, functionVersionId);
functionAuditService.auditFunctionUpdate(summary, STATE_ACTIVATED, jsonBefore, function);
Expand All @@ -502,8 +501,7 @@ public FunctionEntity transitionDeployingFunctionToError(
// Once we set the function status to ERROR we'll no longer attempt to
// clean its deployment. Org Admin should delete the deployment to reset
// the function's status to ACTIVE.
function.setFunctionStatus(FunctionStatus.ERROR);
functionsRepository.insert(function);
functionStatusTransitionService.persist(function, deploymentId, FunctionStatus.ERROR);

var summary = SUMMARY_ERROR_FUNCTION.formatted(functionId, functionVersionId);
functionAuditService.auditFunctionUpdate(summary, STATE_ERROR, jsonBefore, function);
Expand All @@ -529,8 +527,7 @@ public FunctionEntity transitionFunctionToDegrading(
});
var jsonBefore = jsonMapper.valueToTree(function);

function.setFunctionStatus(DEGRADING);
functionsRepository.insert(function);
functionStatusTransitionService.persist(function, deploymentId, DEGRADING);

var summary = SUMMARY_DEGRADING_FUNCTION.formatted(functionId, functionVersionId);
functionAuditService.auditFunctionUpdate(summary, STATE_DEGRADING, jsonBefore, function);
Expand All @@ -556,8 +553,7 @@ public FunctionEntity transitionFunctionToDegraded(
});
var jsonBefore = jsonMapper.valueToTree(function);

function.setFunctionStatus(DEGRADED);
functionsRepository.insert(function);
functionStatusTransitionService.persist(function, deploymentId, DEGRADED);

var summary = SUMMARY_DEGRADED_FUNCTION.formatted(functionId, functionVersionId);
functionAuditService.auditFunctionUpdate(summary, STATE_DEGRADED, jsonBefore, function);
Expand All @@ -582,8 +578,7 @@ private FunctionEntity transitionFunctionToInactive(
});
var jsonBefore = jsonMapper.valueToTree(function);

function.setFunctionStatus(INACTIVE);
functionsRepository.insert(function);
functionStatusTransitionService.persist(function, deploymentId, INACTIVE);

var summary = SUMMARY_INACTIVATE_FUNCTION.formatted(functionId, functionVersionId);
functionAuditService.auditFunctionUpdate(summary, STATE_INACTIVE, jsonBefore, function);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.nvidia.nvcf.service.function;

import com.nvidia.nvcf.persistence.function.FunctionsRepository;
import com.nvidia.nvcf.persistence.function.entity.FunctionEntity;
import com.nvidia.nvcf.persistence.function.entity.FunctionStatus;
import com.nvidia.nvcf.service.eventledger.EventLedgerClient;
import java.time.Clock;
import java.time.Instant;
import java.util.UUID;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;

@Service
@RequiredArgsConstructor
public class FunctionStatusTransitionService {

private final FunctionsRepository functionsRepository;
private final EventLedgerClient eventLedgerClient;
// Inject the time source so transition timestamps can be controlled in tests.
private final Clock clock;

public void persist(
FunctionEntity function, UUID deploymentId, FunctionStatus newStatus) {
var previousStatus = function.getFunctionStatus();
if (previousStatus == newStatus) {
return;
}
function.setFunctionStatus(newStatus);
functionsRepository.insert(function);
eventLedgerClient.publish(
function.getNcaId(),
function.getFunctionId(),
function.getFunctionVersionId(),
deploymentId,
previousStatus,
newStatus,
Instant.now(clock));
}
}
Loading
Loading