From 0726174986d24254258f2cdedbe8fb7bab59b07d Mon Sep 17 00:00:00 2001 From: Ryo Kajiwara Date: Tue, 23 Jun 2026 16:05:06 +0900 Subject: [PATCH 1/3] Add DHKEM support and test cases On OpenSSL 3.2-3.4, DHKEM on EC/ECX keys need the operation (DHKEM) to be explicitly selected, so this adds a helper to the implementation to fill the operation parameter --- ext/openssl/ossl_pkey.c | 35 +++++++++++++++++++++++++++++++++-- test/openssl/test_pkey.rb | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c index 0763cfb8b..4242a1868 100644 --- a/ext/openssl/ossl_pkey.c +++ b/ext/openssl/ossl_pkey.c @@ -13,6 +13,11 @@ # include #endif +#ifdef HAVE_EVP_PKEY_ENCAPSULATE_INIT +# include +# include +#endif + /* * Classes */ @@ -1515,6 +1520,30 @@ ossl_pkey_derive(int argc, VALUE *argv, VALUE self) } #ifdef HAVE_EVP_PKEY_ENCAPSULATE_INIT +/* + * X25519, X448 and EC keys only support the RFC 9180 DH-Based KEM (DHKEM). + * OpenSSL 3.2-3.4 require the operation to be selected explicitly via + * OSSL_KEM_PARAM_OPERATION before encapsulate/decapsulate; without it the + * operation fails with "invalid mode". OpenSSL 3.5 defaults to DHKEM. Build + * the parameter for these key types so the operation works across versions. + * Returns NULL (use the implementation default) for other key types. + */ +static const OSSL_PARAM * +ossl_pkey_kem_params(EVP_PKEY *pkey, OSSL_PARAM *params) +{ +#ifdef OSSL_KEM_PARAM_OPERATION_DHKEM + if (EVP_PKEY_is_a(pkey, "X25519") || EVP_PKEY_is_a(pkey, "X448") || + EVP_PKEY_is_a(pkey, "EC")) { + params[0] = OSSL_PARAM_construct_utf8_string( + OSSL_KEM_PARAM_OPERATION, + (char *)OSSL_KEM_PARAM_OPERATION_DHKEM, 0); + params[1] = OSSL_PARAM_construct_end(); + return params; + } +#endif + return NULL; +} + /* * call-seq: * pkey.encapsulate -> [ciphertext, shared_secret] @@ -1531,13 +1560,14 @@ ossl_pkey_encapsulate(VALUE self) EVP_PKEY_CTX *ctx; VALUE ciphertext, shared_secret; size_t ciphertextlen, shared_secretlen; + OSSL_PARAM params[2]; int state; GetPKey(self, pkey); ctx = EVP_PKEY_CTX_new(pkey, /* engine */NULL); if (!ctx) ossl_raise(ePKeyError, "EVP_PKEY_CTX_new"); - if (EVP_PKEY_encapsulate_init(ctx, NULL) <= 0) { + if (EVP_PKEY_encapsulate_init(ctx, ossl_pkey_kem_params(pkey, params)) <= 0) { EVP_PKEY_CTX_free(ctx); ossl_raise(ePKeyError, "EVP_PKEY_encapsulate_init"); } @@ -1589,6 +1619,7 @@ ossl_pkey_decapsulate(VALUE self, VALUE ciphertext) EVP_PKEY_CTX *ctx; VALUE shared_secret; size_t shared_secretlen; + OSSL_PARAM params[2]; int state; GetPKey(self, pkey); @@ -1597,7 +1628,7 @@ ossl_pkey_decapsulate(VALUE self, VALUE ciphertext) ctx = EVP_PKEY_CTX_new(pkey, /* engine */NULL); if (!ctx) ossl_raise(ePKeyError, "EVP_PKEY_CTX_new"); - if (EVP_PKEY_decapsulate_init(ctx, NULL) <= 0) { + if (EVP_PKEY_decapsulate_init(ctx, ossl_pkey_kem_params(pkey, params)) <= 0) { EVP_PKEY_CTX_free(ctx); ossl_raise(ePKeyError, "EVP_PKEY_decapsulate_init"); } diff --git a/test/openssl/test_pkey.rb b/test/openssl/test_pkey.rb index c9b5f56f7..189730c96 100644 --- a/test/openssl/test_pkey.rb +++ b/test/openssl/test_pkey.rb @@ -307,6 +307,42 @@ def test_ml_kem assert_equal(shared_secret, privkey.decapsulate(ciphertext)) end + def test_dhkem_x25519 + # EVP_KEM-X25519 / EVP_KEM-X448 were added in OpenSSL 3.2. + omit "DHKEM is not supported" unless openssl?(3, 2, 0) + + pkey = OpenSSL::PKey.generate_key("X25519") + raw_public_key = pkey.raw_public_key + raw_private_key = pkey.raw_private_key + + assert_match(/type_name=X25519/, pkey.inspect) + assert_equal(32, raw_public_key.bytesize) # Npk + assert_equal(32, raw_private_key.bytesize) # Nsk + + pubkey = OpenSSL::PKey.new_raw_public_key("X25519", raw_public_key) + ciphertext, shared_secret = pubkey.encapsulate + assert_equal(32, ciphertext.bytesize) # Nenc + assert_equal(32, shared_secret.bytesize) # Nsecret + assert_equal(shared_secret, pkey.decapsulate(ciphertext)) + + privkey = OpenSSL::PKey.new_raw_private_key("X25519", raw_private_key) + assert_equal(shared_secret, privkey.decapsulate(ciphertext)) + end + + def test_dhkem_ec + # EVP_KEM-EC (RFC 9180 DHKEM over NIST curves) was added in OpenSSL 3.2. + omit "DHKEM is not supported" unless openssl?(3, 2, 0) + + pkey = OpenSSL::PKey::EC.generate("prime256v1") + assert_match(/type_name=EC/, pkey.inspect) + + pubkey = OpenSSL::PKey.read(pkey.public_to_der) + ciphertext, shared_secret = pubkey.encapsulate + assert_equal(65, ciphertext.bytesize) # Nenc + assert_equal(32, shared_secret.bytesize) # Nsecret + assert_equal(shared_secret, pkey.decapsulate(ciphertext)) + end + def test_raw_initialize_errors assert_raise(OpenSSL::PKey::PKeyError) { OpenSSL::PKey.new_raw_private_key("foo123", "xxx") } assert_raise(OpenSSL::PKey::PKeyError) { OpenSSL::PKey.new_raw_private_key("ED25519", "xxx") } From 61e5235042e1348360df405a5067df53ed11670c Mon Sep 17 00:00:00 2001 From: Ryo Kajiwara Date: Tue, 23 Jun 2026 16:22:08 +0900 Subject: [PATCH 2/3] omit DHKEM tests on FIPS mode --- test/openssl/test_pkey.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/openssl/test_pkey.rb b/test/openssl/test_pkey.rb index 189730c96..abb23c562 100644 --- a/test/openssl/test_pkey.rb +++ b/test/openssl/test_pkey.rb @@ -308,6 +308,9 @@ def test_ml_kem end def test_dhkem_x25519 + # DHKEM (RFC 9180) and the X25519/X448 curves are not FIPS-approved; the + # KEM is built into the default provider only, not the FIPS module. + omit_on_fips # EVP_KEM-X25519 / EVP_KEM-X448 were added in OpenSSL 3.2. omit "DHKEM is not supported" unless openssl?(3, 2, 0) @@ -330,6 +333,9 @@ def test_dhkem_x25519 end def test_dhkem_ec + # DHKEM (RFC 9180) is built into the default provider only, not the FIPS + # module. + omit_on_fips # EVP_KEM-EC (RFC 9180 DHKEM over NIST curves) was added in OpenSSL 3.2. omit "DHKEM is not supported" unless openssl?(3, 2, 0) From b7d163e3fd645a201a0e4035afa9e7dbee7c1d9e Mon Sep 17 00:00:00 2001 From: Ryo Kajiwara Date: Tue, 23 Jun 2026 18:30:59 +0900 Subject: [PATCH 3/3] Revert adding parameter helper As OpenSSL <3.4 is approaching EOL, I opted to remove the helper altogether and changed the version gating to 3.5+ --- ext/openssl/ossl_pkey.c | 35 ++--------------------------------- test/openssl/test_pkey.rb | 12 ++++++++---- 2 files changed, 10 insertions(+), 37 deletions(-) diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c index 4242a1868..0763cfb8b 100644 --- a/ext/openssl/ossl_pkey.c +++ b/ext/openssl/ossl_pkey.c @@ -13,11 +13,6 @@ # include #endif -#ifdef HAVE_EVP_PKEY_ENCAPSULATE_INIT -# include -# include -#endif - /* * Classes */ @@ -1520,30 +1515,6 @@ ossl_pkey_derive(int argc, VALUE *argv, VALUE self) } #ifdef HAVE_EVP_PKEY_ENCAPSULATE_INIT -/* - * X25519, X448 and EC keys only support the RFC 9180 DH-Based KEM (DHKEM). - * OpenSSL 3.2-3.4 require the operation to be selected explicitly via - * OSSL_KEM_PARAM_OPERATION before encapsulate/decapsulate; without it the - * operation fails with "invalid mode". OpenSSL 3.5 defaults to DHKEM. Build - * the parameter for these key types so the operation works across versions. - * Returns NULL (use the implementation default) for other key types. - */ -static const OSSL_PARAM * -ossl_pkey_kem_params(EVP_PKEY *pkey, OSSL_PARAM *params) -{ -#ifdef OSSL_KEM_PARAM_OPERATION_DHKEM - if (EVP_PKEY_is_a(pkey, "X25519") || EVP_PKEY_is_a(pkey, "X448") || - EVP_PKEY_is_a(pkey, "EC")) { - params[0] = OSSL_PARAM_construct_utf8_string( - OSSL_KEM_PARAM_OPERATION, - (char *)OSSL_KEM_PARAM_OPERATION_DHKEM, 0); - params[1] = OSSL_PARAM_construct_end(); - return params; - } -#endif - return NULL; -} - /* * call-seq: * pkey.encapsulate -> [ciphertext, shared_secret] @@ -1560,14 +1531,13 @@ ossl_pkey_encapsulate(VALUE self) EVP_PKEY_CTX *ctx; VALUE ciphertext, shared_secret; size_t ciphertextlen, shared_secretlen; - OSSL_PARAM params[2]; int state; GetPKey(self, pkey); ctx = EVP_PKEY_CTX_new(pkey, /* engine */NULL); if (!ctx) ossl_raise(ePKeyError, "EVP_PKEY_CTX_new"); - if (EVP_PKEY_encapsulate_init(ctx, ossl_pkey_kem_params(pkey, params)) <= 0) { + if (EVP_PKEY_encapsulate_init(ctx, NULL) <= 0) { EVP_PKEY_CTX_free(ctx); ossl_raise(ePKeyError, "EVP_PKEY_encapsulate_init"); } @@ -1619,7 +1589,6 @@ ossl_pkey_decapsulate(VALUE self, VALUE ciphertext) EVP_PKEY_CTX *ctx; VALUE shared_secret; size_t shared_secretlen; - OSSL_PARAM params[2]; int state; GetPKey(self, pkey); @@ -1628,7 +1597,7 @@ ossl_pkey_decapsulate(VALUE self, VALUE ciphertext) ctx = EVP_PKEY_CTX_new(pkey, /* engine */NULL); if (!ctx) ossl_raise(ePKeyError, "EVP_PKEY_CTX_new"); - if (EVP_PKEY_decapsulate_init(ctx, ossl_pkey_kem_params(pkey, params)) <= 0) { + if (EVP_PKEY_decapsulate_init(ctx, NULL) <= 0) { EVP_PKEY_CTX_free(ctx); ossl_raise(ePKeyError, "EVP_PKEY_decapsulate_init"); } diff --git a/test/openssl/test_pkey.rb b/test/openssl/test_pkey.rb index abb23c562..544d9e0b3 100644 --- a/test/openssl/test_pkey.rb +++ b/test/openssl/test_pkey.rb @@ -311,8 +311,10 @@ def test_dhkem_x25519 # DHKEM (RFC 9180) and the X25519/X448 curves are not FIPS-approved; the # KEM is built into the default provider only, not the FIPS module. omit_on_fips - # EVP_KEM-X25519 / EVP_KEM-X448 were added in OpenSSL 3.2. - omit "DHKEM is not supported" unless openssl?(3, 2, 0) + # EVP_KEM-X25519 / EVP_KEM-X448 were added in OpenSSL 3.2, but the DHKEM + # operation defaults correctly (without an explicit OSSL_KEM_PARAM_OPERATION) + # only since OpenSSL 3.5. + omit "DHKEM is not supported" unless openssl?(3, 5, 0) pkey = OpenSSL::PKey.generate_key("X25519") raw_public_key = pkey.raw_public_key @@ -336,8 +338,10 @@ def test_dhkem_ec # DHKEM (RFC 9180) is built into the default provider only, not the FIPS # module. omit_on_fips - # EVP_KEM-EC (RFC 9180 DHKEM over NIST curves) was added in OpenSSL 3.2. - omit "DHKEM is not supported" unless openssl?(3, 2, 0) + # EVP_KEM-EC (RFC 9180 DHKEM over NIST curves) was added in OpenSSL 3.2, but + # the DHKEM operation defaults correctly (without an explicit + # OSSL_KEM_PARAM_OPERATION) only since OpenSSL 3.5. + omit "DHKEM is not supported" unless openssl?(3, 5, 0) pkey = OpenSSL::PKey::EC.generate("prime256v1") assert_match(/type_name=EC/, pkey.inspect)