diff --git a/.github/workflows/validate-branch-into-main.yaml b/.github/workflows/validate-branch-into-main.yaml index 77b2498b..336c5860 100644 --- a/.github/workflows/validate-branch-into-main.yaml +++ b/.github/workflows/validate-branch-into-main.yaml @@ -10,8 +10,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Check source branch + env: + SOURCE_BRANCH: ${{ github.head_ref }} run: | - SOURCE_BRANCH="${{ github.head_ref }}" if [[ "$SOURCE_BRANCH" != "develop" ]]; then echo "Error: Only pull requests from develop branch are allowed into main" echo "Current source branch ($SOURCE_BRANCH)." diff --git a/internal/lambda-managed-instances/aws-lambda-rie/internal/app_test.go b/internal/lambda-managed-instances/aws-lambda-rie/internal/app_test.go index 31144c88..ed1d7420 100644 --- a/internal/lambda-managed-instances/aws-lambda-rie/internal/app_test.go +++ b/internal/lambda-managed-instances/aws-lambda-rie/internal/app_test.go @@ -79,7 +79,7 @@ func TestApp_ServeHTTP(t *testing.T) { defer mockApp.AssertExpectations(t) mockApp.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(tt.initResponse) if tt.initResponse == nil { - mockApp.On("Invoke", mock.Anything, mock.Anything, mock.Anything).Return(tt.invokeErr, tt.responseSent) + mockApp.On("Invoke", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tt.invokeErr, tt.responseSent, false) } initMsg := intmodel.InitRequestMessage{ diff --git a/internal/lambda-managed-instances/aws-lambda-rie/internal/run.go b/internal/lambda-managed-instances/aws-lambda-rie/internal/run.go index 138b1038..2010ebb5 100644 --- a/internal/lambda-managed-instances/aws-lambda-rie/internal/run.go +++ b/internal/lambda-managed-instances/aws-lambda-rie/internal/run.go @@ -75,7 +75,7 @@ func Run(supv supvmodel.ProcessSupervisor, args []string, fileUtil utils.FileUti } rieApp := NewHTTPHandler(raptorApp, initMsg) - s, err := raptor.StartServer(raptorApp, rieApp, &raptor.TCPAddress{AddrPort: rieAddr}) + s, err := raptor.StartServer(raptorApp, rieApp, &raptor.TCPAddress{AddrPort: rieAddr}, false) if err != nil { return nil, nil, nil, fmt.Errorf("could not start RIE server: %w", err) } diff --git a/internal/lambda-managed-instances/aws-lambda-rie/internal/telemetry/README.md b/internal/lambda-managed-instances/aws-lambda-rie/internal/telemetry/README.md deleted file mode 100644 index 49f45e51..00000000 --- a/internal/lambda-managed-instances/aws-lambda-rie/internal/telemetry/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# RIE Telemetry Package - -The RIE (Runtime Interface Emulator) telemetry package provides Telemetry API. - -## Architecture Overview - -``` -┌─────────────────┐ ┌─────────────────┐ ┌──────────────────┐ -│ EventsAPI │ │ LogsEgress │ │ SubscriptionAPI │ -│ │ │ │ │ │ -│ • Platform │ │ • Runtime logs │ │ • Subscription │ -│ events │ │ • Extension │ │ management │ -│ • Lifecycle │ │ logs │ │ • Schema │ -│ events │ │ • Log capture │ │ validation │ -└─────────┬───────┘ └─────────┬───────┘ └──────────┬───────┘ - │ │ │ - └──────────────┬───────────────────────────────┘ - │ - ┌────▼────┐ - │ Relay │ - │ │ - │ Event │ - │ Broker │ - └────┬────┘ - │ - ┌──────────────┼──────────────┐ - │ │ │ - ┌─────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐ - │Subscriber │ │Subscriber │ │Subscriber │ - │ A │ │ B │ │ C │ - └─────┬─────┘ └─────┬─────┘ └─────┬─────┘ - │ │ │ - ┌─────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐ - │TCP Client │ │HTTP Client│ │TCP Client │ - └───────────┘ └───────────┘ └───────────┘ -``` - -## Core Components - -### 1. EventsAPI (`events_api.go`) -**Responsibility**: Platform event generation and distribution - -The EventsAPI serves as the primary interface for generating and broadcasting AWS Lambda platform events. It implements the `EventsAPI` interface and handles various lifecycle events including initialization, invocation, and error reporting. - -### 2. LogsEgress (`logs_egress.go`) -**Responsibility**: Log capture and forwarding - -The LogsEgress component implements the `StdLogsEgressAPI` interface to capture stdout/stderr from both runtime and extensions, forwarding them to telemetry subscribers while maintaining original console output. - -### 3. Relay (`relay.go`) -**Responsibility**: Event broadcasting and subscriber management - -The Relay acts as a central event broker, managing subscribers and broadcasting events to all registered telemetry consumers. - -### 4. SubscriptionAPI (`subscription_api.go`) -**Responsibility**: Subscription management and validation - -The SubscriptionAPI handles telemetry subscription requests, validates them against JSON schemas, and manages the subscription lifecycle. - -## Internal Components - -### 1. Subscriber (`internal/subscriber.go`) -**Responsibility**: Event batching and delivery - -Each subscriber represents a telemetry consumer and manages efficient event delivery through batching and asynchronous processing. - -### 2. Client (`internal/client.go`) -**Responsibility**: Protocol-specific event delivery - -The client abstraction provides protocol-specific implementations for delivering events to telemetry consumers. - -### 3. Batch (`internal/batch.go`) -**Responsibility**: Event collection and timing - -The batch component manages collections of events with size and time-based flushing logic. - -### 4. Types (`internal/types.go`) -**Responsibility**: Type definitions and constants - -Centralized type definitions for protocols, event categories, and configuration structures. - -## Event Flow - -### 1. Subscription Flow -``` -Extension/Agent → SubscriptionAPI → Schema Validation → Subscriber Creation → Relay Registration -``` - -### 2. Event Flow -``` -Event Source → EventsAPI → Relay → Subscribers → Batching → Client -``` - -### 3. Log Flow -``` -Runtime/Extension → LogsEgress → Console Output + Relay → Subscribers → Batching → Client -``` diff --git a/internal/lambda-managed-instances/aws-lambda-rie/main.go b/internal/lambda-managed-instances/aws-lambda-rie/main.go deleted file mode 100644 index 7f23d80e..00000000 --- a/internal/lambda-managed-instances/aws-lambda-rie/main.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda-managed-instances/aws-lambda-rie/run" -) - -func main() { - run.Run() -} diff --git a/internal/lambda-managed-instances/interop/sandbox_model.go b/internal/lambda-managed-instances/interop/sandbox_model.go index 2bb04530..d88e441c 100644 --- a/internal/lambda-managed-instances/interop/sandbox_model.go +++ b/internal/lambda-managed-instances/interop/sandbox_model.go @@ -253,6 +253,7 @@ type EEStaticData struct { XRayTracingMode intmodel.XrayTracingMode ArtefactType intmodel.ArtefactType RuntimeVersion string + RuntimeRelease string AmiId string AvailabilityZoneId string } diff --git a/internal/lambda-managed-instances/interop/service_log_values.go b/internal/lambda-managed-instances/interop/service_log_values.go index cb6d1a5c..1ac339b6 100644 --- a/internal/lambda-managed-instances/interop/service_log_values.go +++ b/internal/lambda-managed-instances/interop/service_log_values.go @@ -27,6 +27,10 @@ const ( ShutdownWaitAllProcessesDuration = "WaitCustomerProcessesExitDuration" ShutdownRuntimeServerDuration = "StopRuntimeServerDuration" + RuntimeNextCountMetric = "RuntimeNextCount" + + RuntimeWorkerCountMetric = "RuntimeWorkerCount" + ClientErrorMetric = "ClientError" ClientErrorReasonTemplate = "ClientErrorReason-%s" CustomerErrorMetric = "CustomerError" diff --git a/internal/lambda-managed-instances/invoke/invoke_router.go b/internal/lambda-managed-instances/invoke/invoke_router.go index 86a97d13..6625bd2f 100644 --- a/internal/lambda-managed-instances/invoke/invoke_router.go +++ b/internal/lambda-managed-instances/invoke/invoke_router.go @@ -195,6 +195,12 @@ func (ir *InvokeRouter) GetRuntimePoolCounts() RuntimePoolCounts { func (ir *InvokeRouter) ReserveIdleRuntime(ctx context.Context, invokeID interop.InvokeID, timeout time.Duration) (interop.ReserveIdleRuntimeResponse, model.AppError) { logging.Debug(ctx, "InvokeRouter: reserving idle runtime") + if _, exists := ir.runningInvokes.Get(invokeID); exists { + logging.Warn(ctx, "InvokeRouter: reservation collides with in-flight invoke") + return interop.ReserveIdleRuntimeFailureResponse{ErrorType: model.ErrorDuplicatedInvokeId}, + model.NewClientError(ErrInvokeIdAlreadyExists, model.ErrorSeverityError, model.ErrorDuplicatedInvokeId) + } + err := ir.runtimePool.Reserve(invokeID, timeout, func() { if ir.runtimePool.ExpireReservation(invokeID) { logging.Info(ctx, "InvokeRouter: reservation expired") diff --git a/internal/lambda-managed-instances/invoke/invoke_router_test.go b/internal/lambda-managed-instances/invoke/invoke_router_test.go index c5353e8e..86685947 100644 --- a/internal/lambda-managed-instances/invoke/invoke_router_test.go +++ b/internal/lambda-managed-instances/invoke/invoke_router_test.go @@ -399,6 +399,29 @@ func TestReserveIdleRuntime_DuplicateInvokeID(t *testing.T) { assert.Equal(t, model.ErrorDuplicatedInvokeId, appErr.ErrorType()) } +func TestReserveIdleRuntime_DuplicateAgainstRunningInvoke(t *testing.T) { + t.Parallel() + + mocks, router := createMocksAndInitRouter() + + _, err := router.RuntimeNext(mocks.ctx, mocks.runtimeNextRequest) + require.NoError(t, err) + + const inFlightID = "reserve-vs-running" + router.runningInvokes.Set(inFlightID, &mocks.runnningInvoke) + + resp, appErr := router.ReserveIdleRuntime(mocks.ctx, inFlightID, 100*time.Millisecond) + + require.NotNil(t, appErr) + failResp, ok := resp.(interop.ReserveIdleRuntimeFailureResponse) + assert.True(t, ok, "expected ReserveIdleRuntimeFailureResponse") + assert.Equal(t, model.ErrorDuplicatedInvokeId, failResp.ErrorType) + assert.Equal(t, model.ErrorDuplicatedInvokeId, appErr.ErrorType()) + + assert.Equal(t, 0, router.runtimePool.ReservedCount(), "no reservation should have been recorded") + assert.Equal(t, 1, router.GetRuntimePoolCounts().Idle, "idle runtime should still be available") +} + func TestReserveIdleRuntime_Expiration(t *testing.T) { t.Parallel() diff --git a/internal/lambda-managed-instances/invoke/reservation_metrics.go b/internal/lambda-managed-instances/invoke/reservation_metrics.go index cd7446d4..57ef4963 100644 --- a/internal/lambda-managed-instances/invoke/reservation_metrics.go +++ b/internal/lambda-managed-instances/invoke/reservation_metrics.go @@ -13,11 +13,13 @@ import ( ) const ( - ReserveSuccessMetric = "ReserveSuccess" - ReserveFailedMetric = "ReserveFailed" + ReserveSuccessMetric = "ReserveSuccess" + ReserveFailedMetric = "ReserveFailed" + ReserveParseDurationMetric = "ReserveParseDuration" + ReserveLogicDurationMetric = "ReserveLogicDuration" ) -func ReserveServiceLog(logger servicelogs.Logger, opStart time.Time, invokeID string, appErr model.AppError) { +func ReserveServiceLog(logger servicelogs.Logger, opStart time.Time, invokeID string, appErr model.AppError, parseDuration, reserveDuration time.Duration) { props := []servicelogs.Property{ {Name: "invoke_id", Value: invokeID}, } @@ -43,6 +45,11 @@ func ReserveServiceLog(logger servicelogs.Logger, opStart time.Time, invokeID st servicelogs.Counter(ReserveFailedMetric, failed), servicelogs.Counter(interop.ClientErrorMetric, clientErrCnt), servicelogs.Counter(interop.NonCustomerErrorMetric, nonCustomerErrCnt), + servicelogs.Timer(ReserveParseDurationMetric, parseDuration), + } + + if reserveDuration > 0 { + metrics = append(metrics, servicelogs.Timer(ReserveLogicDurationMetric, reserveDuration)) } if appErr != nil { diff --git a/internal/lambda-managed-instances/invoke/runtime_pool.go b/internal/lambda-managed-instances/invoke/runtime_pool.go index df96162c..981d6972 100644 --- a/internal/lambda-managed-instances/invoke/runtime_pool.go +++ b/internal/lambda-managed-instances/invoke/runtime_pool.go @@ -5,6 +5,7 @@ package invoke import ( "errors" + "log/slog" "sync" "time" @@ -44,9 +45,14 @@ func (p *RuntimePool) Add(runtime runningInvoke) error { } func (p *RuntimePool) Reserve(invokeID interop.InvokeID, timeout time.Duration, onExpire func()) error { + lockStart := time.Now() p.mu.Lock() defer p.mu.Unlock() + if lockWait := time.Since(lockStart); lockWait > time.Millisecond { + slog.Warn("RuntimePool.Reserve lock contention", "wait_us", lockWait.Microseconds()) + } + if _, exists := p.reserved[invokeID]; exists { return ErrInvokeIdAlreadyExists } diff --git a/internal/lambda-managed-instances/invoke/runtime_response_request.go b/internal/lambda-managed-instances/invoke/runtime_response_request.go index b764af6c..3d3c2ba8 100644 --- a/internal/lambda-managed-instances/invoke/runtime_response_request.go +++ b/internal/lambda-managed-instances/invoke/runtime_response_request.go @@ -9,6 +9,7 @@ import ( "io" "log/slog" "net/http" + "time" "github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda-managed-instances/interop" "github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda-managed-instances/logging" @@ -23,6 +24,7 @@ const ( type runtimeResponse struct { request *http.Request + rc *http.ResponseController parsingErr model.AppError @@ -31,7 +33,7 @@ type runtimeResponse struct { responseMode string } -func NewRuntimeResponse(ctx context.Context, request *http.Request, invokeID interop.InvokeID) runtimeResponse { +func NewRuntimeResponse(ctx context.Context, request *http.Request, writer http.ResponseWriter, invokeID interop.InvokeID) runtimeResponse { contentType := request.Header.Get(RuntimeContentTypeHeader) if contentType == "" { @@ -39,6 +41,7 @@ func NewRuntimeResponse(ctx context.Context, request *http.Request, invokeID int } resp := runtimeResponse{ request: request, + rc: http.NewResponseController(writer), contentType: contentType, invokeID: invokeID, } @@ -74,6 +77,12 @@ func (r *runtimeResponse) BodyReader() io.Reader { return r.request.Body } +func (r *runtimeResponse) Cancel() { + if err := r.rc.SetReadDeadline(time.Unix(0, 0)); err != nil { + slog.Warn("Cancel: SetReadDeadline failed", "err", err) + } +} + func (r *runtimeResponse) ResponseMode() string { return r.responseMode } diff --git a/internal/lambda-managed-instances/invoke/runtime_response_request_test.go b/internal/lambda-managed-instances/invoke/runtime_response_request_test.go index cb11bf6f..a0c6e968 100644 --- a/internal/lambda-managed-instances/invoke/runtime_response_request_test.go +++ b/internal/lambda-managed-instances/invoke/runtime_response_request_test.go @@ -5,6 +5,7 @@ package invoke import ( "net/http" + "net/http/httptest" "testing" "github.com/stretchr/testify/assert" @@ -66,7 +67,7 @@ func TestRuntimeResponse_TrailerError(t *testing.T) { req.Trailer = make(http.Header) req.Trailer.Set(FunctionErrorTypeTrailer, tc.errorTypeTrailer) req.Trailer.Set(FunctionErrorBodyTrailer, tc.errorBodyTrailer) - resp := NewRuntimeResponse(req.Context(), req, "test-invoke-id") + resp := NewRuntimeResponse(req.Context(), req, httptest.NewRecorder(), "test-invoke-id") actualTrailerError := resp.TrailerError() diff --git a/internal/lambda-managed-instances/invoke/runtime_response_sender.go b/internal/lambda-managed-instances/invoke/runtime_response_sender.go index 9604a8dd..813282df 100644 --- a/internal/lambda-managed-instances/invoke/runtime_response_sender.go +++ b/internal/lambda-managed-instances/invoke/runtime_response_sender.go @@ -33,9 +33,15 @@ func sendInvokeToRuntime(ctx context.Context, initData interop.InitStaticDataPro runtimeReq.Header().Set(RuntimeRequestIdHeader, invokeReq.InvokeID()) runtimeReq.Header().Set(RuntimeDeadlineHeader, strconv.FormatInt(invokeReq.Deadline().UnixMilli(), 10)) runtimeReq.Header().Set(RuntimeFunctionArnHeader, initData.FunctionARN()) - runtimeReq.Header().Set(RuntimeTraceIdHeader, traceId) - runtimeReq.Header().Set(RuntimeClientContextHeader, invokeReq.ClientContext()) - runtimeReq.Header().Set(RuntimeCognitoIdentifyHeader, buildCognitoIdentifyHeader(invokeReq)) + if traceId != "" { + runtimeReq.Header().Set(RuntimeTraceIdHeader, traceId) + } + if cc := invokeReq.ClientContext(); cc != "" { + runtimeReq.Header().Set(RuntimeClientContextHeader, cc) + } + if cogId := buildCognitoIdentifyHeader(invokeReq); cogId != "" { + runtimeReq.Header().Set(RuntimeCognitoIdentifyHeader, cogId) + } runtimeReq.WriteHeader(http.StatusOK) timedReader := &utils.TimedReader{ diff --git a/internal/lambda-managed-instances/invoke/runtime_response_sender_test.go b/internal/lambda-managed-instances/invoke/runtime_response_sender_test.go index 9727e5bc..3d527b42 100644 --- a/internal/lambda-managed-instances/invoke/runtime_response_sender_test.go +++ b/internal/lambda-managed-instances/invoke/runtime_response_sender_test.go @@ -134,3 +134,36 @@ func TestSendResponseFailure_CtxCancelled(t *testing.T) { checkResponseSenderExpectations(t, mocks) } + +func TestSendResponse_OmitsEmptyOptionalHeaders(t *testing.T) { + t.Parallel() + + mocks := createMocksAndRuntimeResponder() + buildInitDataMocks(&mocks.initData) + + mocks.invokeReq.On("ContentType").Return("application/json") + mocks.invokeReq.On("InvokeID").Return("123456") + mocks.invokeReq.On("Deadline").Return(time.Now().Add(time.Second)) + mocks.invokeReq.On("ClientContext").Return("") + mocks.invokeReq.On("CognitoId").Return("") + mocks.invokeReq.On("CognitoPoolId").Return("") + mocks.invokeReq.On("BodyReader").Return(mocks.reader) + + recorder := httptest.NewRecorder() + mocks.runtimeReq = recorder + + _, _, _, err := sendInvokeToRuntime(mocks.ctx, &mocks.initData, &mocks.invokeReq, mocks.runtimeReq, "") + assert.NoError(t, err) + + headers := recorder.Header() + assert.Empty(t, headers.Get(RuntimeTraceIdHeader)) + assert.Empty(t, headers.Get(RuntimeClientContextHeader)) + assert.Empty(t, headers.Get(RuntimeCognitoIdentifyHeader)) + + assert.NotEmpty(t, headers.Get(RuntimeRequestIdHeader)) + assert.NotEmpty(t, headers.Get(RuntimeDeadlineHeader)) + assert.NotEmpty(t, headers.Get(RuntimeFunctionArnHeader)) + assert.NotEmpty(t, headers.Get(RuntimeContentTypeHeader)) + + checkResponseSenderExpectations(t, mocks) +} diff --git a/internal/lambda-managed-instances/rapi/handler/invocationresponse.go b/internal/lambda-managed-instances/rapi/handler/invocationresponse.go index 8fb6ce28..661d0d45 100644 --- a/internal/lambda-managed-instances/rapi/handler/invocationresponse.go +++ b/internal/lambda-managed-instances/rapi/handler/invocationresponse.go @@ -28,7 +28,7 @@ func (h *invocationResponseHandler) ServeHTTP(writer http.ResponseWriter, reques ctx := logging.WithInvokeID(request.Context(), invokeID) logging.Debug(ctx, "Received Runtime Response") - resp := invoke.NewRuntimeResponse(ctx, request, invokeID) + resp := invoke.NewRuntimeResponse(ctx, request, writer, invokeID) err := h.runtimeRespHandler.RuntimeResponse(ctx, &resp) if err == nil { diff --git a/internal/lambda-managed-instances/rapid/handlers.go b/internal/lambda-managed-instances/rapid/handlers.go index 5da1a59c..44fb6b94 100644 --- a/internal/lambda-managed-instances/rapid/handlers.go +++ b/internal/lambda-managed-instances/rapid/handlers.go @@ -346,6 +346,7 @@ func doInitRuntime( return customerErr } + execCtx.initExecutionData.StaticData.RuntimeRelease = appctx.GetRuntimeRelease(execCtx.appCtx) telemetry.SendInitRuntimeDoneLogEvent(execCtx.eventsAPI, execCtx.appCtx, phase, nil) return nil diff --git a/internal/lambda-managed-instances/rapid/model/client_error.go b/internal/lambda-managed-instances/rapid/model/client_error.go index 57802779..f2c8e5e2 100644 --- a/internal/lambda-managed-instances/rapid/model/client_error.go +++ b/internal/lambda-managed-instances/rapid/model/client_error.go @@ -17,6 +17,11 @@ const ( ErrorInvalidResponseBandwidthRate ErrorType = "ErrInvalidResponseBandwidthRate" ErrorInvalidResponseBandwidthBurstSize ErrorType = "ErrInvalidResponseBandwidthBurstSize" ErrorExecutionEnvironmentShutdown ErrorType = "Client.ExecutionEnvironmentShutDown" + + ErrorInvalidInvokeId ErrorType = "Client.InvalidInvokeId" + + ErrorInvalidConnectionHoldTimeout ErrorType = "ErrInvalidConnectionHoldTimeout" + ErrorInvalidResponseHoldTimeout ErrorType = "ErrInvalidResponseHoldTimeout" ) type ClientError struct { diff --git a/internal/lambda-managed-instances/rapid/model/function_metadata.go b/internal/lambda-managed-instances/rapid/model/function_metadata.go index 8349b7fe..f0147d02 100644 --- a/internal/lambda-managed-instances/rapid/model/function_metadata.go +++ b/internal/lambda-managed-instances/rapid/model/function_metadata.go @@ -4,12 +4,13 @@ package model type FunctionMetadata struct { - AccountID string - FunctionName string - FunctionVersion string - MemorySizeBytes uint64 - Handler string - RuntimeInfo RuntimeInfo + AccountID string + FunctionName string + FunctionVersion string + MemorySizeBytes uint64 + Handler string + RuntimeInfo RuntimeInfo + RuntimeWorkerCount int } type RuntimeInfo struct { diff --git a/internal/lambda-managed-instances/rapid/model/platform_error.go b/internal/lambda-managed-instances/rapid/model/platform_error.go index 3f67bc6e..2a787910 100644 --- a/internal/lambda-managed-instances/rapid/model/platform_error.go +++ b/internal/lambda-managed-instances/rapid/model/platform_error.go @@ -20,7 +20,8 @@ const ( ErrSandboxLogSocketsUnavailable ErrorType = "Sandbox.LogSocketsUnavailable" ErrSandboxEventSetupFailure ErrorType = "Sandbox.EventSetupFailure" - ErrSandboxShutdownFailed ErrorType = "Sandbox.ShutdownFailed" + ErrSandboxShutdownFailed ErrorType = "Sandbox.ShutdownFailed" + ErrorResponseReplayFailed ErrorType = "Sandbox.ResponseReplayFailed" ) type PlatformError struct { diff --git a/internal/lambda-managed-instances/raptor/raptor_utils.go b/internal/lambda-managed-instances/raptor/raptor_utils.go index dfdb4f40..3ea9b6ad 100644 --- a/internal/lambda-managed-instances/raptor/raptor_utils.go +++ b/internal/lambda-managed-instances/raptor/raptor_utils.go @@ -43,11 +43,12 @@ func getInitExecutionData(initRequest *internalModel.InitRequestMessage, runtime LogGroupName: initRequest.LogGroupName, LogStreamName: initRequest.LogStreamName, FunctionMetadata: model.FunctionMetadata{ - AccountID: initRequest.AccountID, - FunctionName: initRequest.TaskName, - FunctionVersion: initRequest.FunctionVersion, - MemorySizeBytes: uint64(initRequest.MemorySizeBytes), - Handler: initRequest.Handler, + AccountID: initRequest.AccountID, + FunctionName: initRequest.TaskName, + FunctionVersion: initRequest.FunctionVersion, + MemorySizeBytes: uint64(initRequest.MemorySizeBytes), + Handler: initRequest.Handler, + RuntimeWorkerCount: initRequest.RuntimeWorkerCount, RuntimeInfo: model.RuntimeInfo{ Arn: initRequest.RuntimeArn, Version: initRequest.RuntimeVersion, diff --git a/internal/lambda-managed-instances/raptor/server.go b/internal/lambda-managed-instances/raptor/server.go index 23374f88..f2b46376 100644 --- a/internal/lambda-managed-instances/raptor/server.go +++ b/internal/lambda-managed-instances/raptor/server.go @@ -4,6 +4,7 @@ package raptor import ( + "context" "log/slog" "net" "net/http" @@ -18,7 +19,7 @@ import ( "github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda-managed-instances/rapid/model" ) -func StartServer(shutdownHandler shutdownHandler, handler http.Handler, addr Address) (*Server, error) { +func StartServer(shutdownHandler shutdownHandler, handler http.Handler, addr Address, h2Only bool) (*Server, error) { listener, err := net.Listen(addr.Protocol(), addr.String()) if err != nil { return nil, err @@ -27,11 +28,26 @@ func StartServer(shutdownHandler shutdownHandler, handler http.Handler, addr Add addr.UpdateFromListener(listener) s := &Server{ - httpServer: &http.Server{Handler: handler, ReadHeaderTimeout: 15 * time.Second}, + httpServer: &http.Server{ + Handler: handler, + ReadHeaderTimeout: 15 * time.Second, + Protocols: &http.Protocols{}, + }, doneCh: make(chan struct{}), shutdownHandler: shutdownHandler, Addr: addr, } + if h2Only { + + s.httpServer.Protocols.SetUnencryptedHTTP2(true) + + s.httpServer.HTTP2 = &http.HTTP2Config{ + MaxConcurrentStreams: 2048, + } + } else { + + s.httpServer.Protocols.SetHTTP1(true) + } go func() { if err := s.httpServer.Serve(listener); err != nil { @@ -46,9 +62,17 @@ func (s *Server) Shutdown(err error) { s.shutdownOnce.Do(func() { s.shutdownHandler.Shutdown(model.NewClientError(err, model.ErrorSeverityFatal, model.ErrorExecutionEnvironmentShutdown)) slog.Info("Shutting down HTTP server...") - if err := s.httpServer.Close(); err != nil { - slog.Warn("error shutdown EA http server", "err", err) + + start := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + if err := s.httpServer.Shutdown(ctx); err != nil { + slog.Warn("EA server graceful shutdown timed out, forcing close", "err", err) + if closeErr := s.httpServer.Close(); closeErr != nil { + slog.Warn("EA server force close failed", "err", closeErr) + } } + slog.Info("EA HTTP server closed", "duration", time.Since(start)) if err != nil { s.err.Store(err) diff --git a/internal/lambda-managed-instances/raptor/server_test.go b/internal/lambda-managed-instances/raptor/server_test.go index eabae715..a574f85c 100644 --- a/internal/lambda-managed-instances/raptor/server_test.go +++ b/internal/lambda-managed-instances/raptor/server_test.go @@ -30,7 +30,7 @@ func TestStartNewServer_UDS(t *testing.T) { server, err := StartServer(mockShutdownHandler, handler, &UnixAddress{ Path: socketPath, - }) + }, true) require.NoError(t, err) assert.Equal(t, socketPath, server.Addr.String()) @@ -53,7 +53,7 @@ func TestStartNewServer_TCP(t *testing.T) { server, err := StartServer(mockShutdownHandler, handler, &TCPAddress{ eaAPIAddrPort, - }) + }, false) require.NoError(t, err) assert.Equal(t, eaAPIAddrPort, server.Addr.(*TCPAddress).AddrPort) @@ -69,7 +69,7 @@ func TestStartNewServer_UDS_ListenError(t *testing.T) { _, err := StartServer(mockShutdownHandler, handler, &UnixAddress{ Path: invalidSocketPath, - }) + }, true) assert.Error(t, err) assert.Contains(t, err.Error(), invalidSocketPath) } @@ -80,7 +80,7 @@ func TestStartNewServe_TCP_ListenError(t *testing.T) { mockShutdownHandler := newMockShutdownHandler(t) handler := mocks.NewNoOpHandler() - _, err := StartServer(mockShutdownHandler, handler, &TCPAddress{eaAPIAddrPort}) + _, err := StartServer(mockShutdownHandler, handler, &TCPAddress{eaAPIAddrPort}, false) assert.Error(t, err) assert.Contains(t, err.Error(), "1.1.1.1:49275") } diff --git a/internal/lambda-managed-instances/supervisor/local/process.go b/internal/lambda-managed-instances/supervisor/local/process.go index ec0bc0c3..9e64a501 100644 --- a/internal/lambda-managed-instances/supervisor/local/process.go +++ b/internal/lambda-managed-instances/supervisor/local/process.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "strconv" + "strings" "sync" "syscall" "time" @@ -159,7 +160,11 @@ func (s *ProcessSupervisor) Exec(ctx context.Context, req *model.ExecRequest) er cell = int32(status.Signal()) signo = &cell - cause = model.Signaled + if status.Signal() == syscall.SIGKILL && wasOomKill() { + cause = model.OomKilled + } else { + cause = model.Signaled + } } } } @@ -320,5 +325,43 @@ func (s *ProcessSupervisor) setProcessGroupPriorities(pid int) error { return fmt.Errorf("failed to set nice score for %d: %w", pgid, err) } + oomScorePath := fmt.Sprintf("/proc/%d/oom_score_adj", pid) + + f, err := os.OpenFile(oomScorePath, os.O_WRONLY, 0) + if err != nil { + return fmt.Errorf("could not open file %s: %w", oomScorePath, err) + } + defer func() { + if err := f.Close(); err != nil { + slog.Error("could not close file", "file", oomScorePath, "err", err) + } + }() + if _, err := f.WriteString("1000"); err != nil { + return fmt.Errorf("could not write to file %s: %w", oomScorePath, err) + } + return nil } + +func wasOomKill() bool { + return checkOomKill("/sys/fs/cgroup/memory.events") +} + +func checkOomKill(eventsPath string) bool { + data, err := os.ReadFile(eventsPath) + if err != nil { + slog.Warn("could not read memory.events", "path", eventsPath, "err", err) + return false + } + for _, line := range strings.Split(string(data), "\n") { + if after, found := strings.CutPrefix(line, "oom_kill "); found { + count, err := strconv.Atoi(after) + if err != nil { + slog.Warn("could not parse oom_kill count", "line", line, "err", err) + return false + } + return count > 0 + } + } + return false +} diff --git a/internal/lambda-managed-instances/supervisor/local/process_test.go b/internal/lambda-managed-instances/supervisor/local/process_test.go index bd1f10fb..fe5f83dc 100644 --- a/internal/lambda-managed-instances/supervisor/local/process_test.go +++ b/internal/lambda-managed-instances/supervisor/local/process_test.go @@ -6,6 +6,8 @@ package local import ( "context" "errors" + "os" + "path/filepath" "syscall" "testing" "time" @@ -226,3 +228,37 @@ func TestTerminateCheckStatus(t *testing.T) { require.NotNil(t, term.Signaled()) require.EqualValues(t, syscall.SIGTERM, *term.Signo) } + +func TestCheckOomKill_OomKilled(t *testing.T) { + path := filepath.Join(t.TempDir(), "memory.events") + os.WriteFile(path, []byte("low 0\nhigh 0\nmax 96\noom 1\noom_kill 1\noom_group_kill 0\n"), 0644) + assert.True(t, checkOomKill(path)) +} + +func TestCheckOomKill_NoOom(t *testing.T) { + path := filepath.Join(t.TempDir(), "memory.events") + os.WriteFile(path, []byte("low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n"), 0644) + assert.False(t, checkOomKill(path)) +} + +func TestCheckOomKill_FileNotFound(t *testing.T) { + assert.False(t, checkOomKill("/nonexistent/memory.events")) +} + +func TestCheckOomKill_MultipleOomKills(t *testing.T) { + path := filepath.Join(t.TempDir(), "memory.events") + os.WriteFile(path, []byte("low 0\nhigh 0\nmax 50\noom 3\noom_kill 3\noom_group_kill 0\n"), 0644) + assert.True(t, checkOomKill(path)) +} + +func TestCheckOomKill_MalformedCount(t *testing.T) { + path := filepath.Join(t.TempDir(), "memory.events") + os.WriteFile(path, []byte("low 0\nhigh 0\noom_kill abc\n"), 0644) + assert.False(t, checkOomKill(path)) +} + +func TestCheckOomKill_NoOomKillLine(t *testing.T) { + path := filepath.Join(t.TempDir(), "memory.events") + os.WriteFile(path, []byte("low 0\nhigh 0\nmax 0\n"), 0644) + assert.False(t, checkOomKill(path)) +} diff --git a/internal/lambda-managed-instances/testutils/blocking_read_closer.go b/internal/lambda-managed-instances/testutils/blocking_read_closer.go new file mode 100644 index 00000000..a9f5d5dc --- /dev/null +++ b/internal/lambda-managed-instances/testutils/blocking_read_closer.go @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package testutils + +import "io" + +type BlockingReadCloser struct { + done chan struct{} +} + +func NewBlockingReadCloser() *BlockingReadCloser { + return &BlockingReadCloser{done: make(chan struct{})} +} + +func (r *BlockingReadCloser) Read(p []byte) (int, error) { + <-r.done + return 0, io.EOF +} + +func (r *BlockingReadCloser) Close() error { + select { + case <-r.done: + default: + close(r.done) + } + return nil +} diff --git a/internal/lambda-managed-instances/testutils/functional/runtime_actions.go b/internal/lambda-managed-instances/testutils/functional/runtime_actions.go index ea1559ae..bea7a1f6 100644 --- a/internal/lambda-managed-instances/testutils/functional/runtime_actions.go +++ b/internal/lambda-managed-instances/testutils/functional/runtime_actions.go @@ -215,6 +215,7 @@ func (a InvocationResponseErrorAction) String() string { type InvocationStreamingResponseAction struct { Chunks []string + Body io.Reader ContentType string InvokeID interop.InvokeID ResponseModeHeader string @@ -223,10 +224,14 @@ type InvocationStreamingResponseAction struct { } func (a InvocationStreamingResponseAction) Execute(t *testing.T, client *Client) (*http.Response, error) { + var body io.Reader + if a.Body != nil { + body = a.Body + } else { + body = NewChunkedReader(a.Chunks, a.ChunkDelay) + } - chunkedReader := NewChunkedReader(a.Chunks, a.ChunkDelay) - - return client.Response(a.InvokeID, chunkedReader, a.ContentType, a.ResponseModeHeader, a.Trailers) + return client.Response(a.InvokeID, body, a.ContentType, a.ResponseModeHeader, a.Trailers) } func (a InvocationStreamingResponseAction) ValidateStatus(t *testing.T, resp *http.Response) {} diff --git a/internal/lambda-managed-instances/testutils/socket_utils.go b/internal/lambda-managed-instances/testutils/socket_utils.go index b5fa7be9..3e3b5588 100644 --- a/internal/lambda-managed-instances/testutils/socket_utils.go +++ b/internal/lambda-managed-instances/testutils/socket_utils.go @@ -11,7 +11,6 @@ import ( "os" "path/filepath" "testing" - "time" "github.com/google/uuid" ) @@ -38,10 +37,22 @@ func NewUnixSocketClient(socketPath string) *http.Client { } transport := &http.Transport{ - DialContext: dialer, - DisableCompression: true, - ResponseHeaderTimeout: 5 * time.Second, + DialContext: dialer, + DisableCompression: true, + Protocols: &http.Protocols{}, } + transport.Protocols.SetUnencryptedHTTP2(true) + + return &http.Client{ + Transport: transport, + } +} + +func NewH2CClient() *http.Client { + transport := &http.Transport{ + Protocols: &http.Protocols{}, + } + transport.Protocols.SetUnencryptedHTTP2(true) return &http.Client{ Transport: transport,