Skip to content

Improve RSA PKCS#11 interoperability and provider support #655

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
olszomal wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Improve RSA PKCS#11 interoperability and provider support #655

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c5c5140
tests: improve RSA interoperability and coverage
olszomal May 18, 2026
bc06606
provider: improve RSA signing and decrypt handling
olszomal May 18, 2026
caef7e4
provider: add DigestInfo encoding for RSA PKCS#1 v1.5 signatures
olszomal May 18, 2026
470b9aa
provider: add RSA verify-recover support
olszomal May 18, 2026
98e23c9
provider: add X9.31 digest encoding for RSA signatures
olszomal May 18, 2026
96ded1c
Refactor PKCS#11 RSA decrypt handling and logging
olszomal May 19, 2026
46d2397
Restore RSA PKCS#1 v1.5 private key decryption support
olszomal May 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions NEWS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ NEWS for Libp11 -- History of user visible changes

New in 0.4.19; unreleased
* Define LIBP11_VERSION_NUMBER (Michał Trojnara)
* Added PKCS#11 provider support for OpenSSL 4.x. (Małgorzata Olszówka)

New in 0.4.18; 2026-02-16; Michał Trojnara
* Support for RSA-PSS and RSA-OAEP using keys retrieved using the
Expand Down
6 changes: 3 additions & 3 deletions src/libp11-int.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,8 +383,8 @@ extern int pkcs11_sign(int type,

/* Sign input data using RSA private key via PKCS#11 mechanism */
extern int pkcs11_evp_pkey_rsa_sign(PKCS11_OBJECT_private *key, EVP_PKEY *pkey,
const char *mdname, const int pad_mode, const int salt_len,
const char *mgf1_mdname, unsigned char *oaep_label, const int oaep_labellen,
const char *mdname, const int pad_mode,
const int salt_len, const char *mgf1_mdname,
unsigned char *sig, size_t *siglen,
const unsigned char *tbs, size_t tbslen);

Expand All @@ -409,7 +409,7 @@ extern int pkcs11_evp_pkey_rsa_decrypt(PKCS11_OBJECT_private *key, EVP_PKEY *pke
const char *mdname, const int pad_mode,
const char *mgf1_mdname, unsigned char *oaep_label, const int oaep_labellen,
unsigned char *out, size_t *outlen,
size_t *outsize, const unsigned char *in, size_t inlen);
const unsigned char *in, size_t inlen);

/* This function has never been implemented */
extern int pkcs11_verify(int type,
Expand Down
3 changes: 1 addition & 2 deletions src/libp11.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -537,7 +537,6 @@ extern int PKCS11_verify(int type,
/* Perform a private-key operation using a PKCS#11-backed EVP_PKEY */
extern int PKCS11_evp_pkey_sign(EVP_PKEY *pkey, int type, const char *mdname,
const int pad_mode, const int salt_len, const char *mgf1_mdname,
unsigned char *oaep_label, const int oaep_labellen,
unsigned char *sig, size_t *siglen,
const unsigned char *tbs, size_t tbslen);

Expand All @@ -546,7 +545,7 @@ extern int PKCS11_evp_pkey_decrypt(EVP_PKEY *pk, int type, const char *mdname,
const int pad_mode, const char *mgf1_mdname,
unsigned char *oaep_label, const int oaep_labellen,
unsigned char *sig, size_t *siglen,
size_t *outsize, const unsigned char *in, size_t inlen);
const unsigned char *in, size_t inlen);
#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */

/* Encrypts data using the private key */
Expand Down
6 changes: 2 additions & 4 deletions src/p11_front.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -551,7 +551,6 @@ int PKCS11_sign(int type, const unsigned char *m, unsigned int m_len,
#if OPENSSL_VERSION_NUMBER >= 0x30000000L
int PKCS11_evp_pkey_sign(EVP_PKEY *pk, int type, const char *mdname,
const int pad_mode, const int pss_saltlen, const char *mgf1_mdname,
unsigned char *oaep_label, const int oaep_labellen,
unsigned char *sig, size_t *siglen,
const unsigned char *tbs, size_t tbslen)
{
Expand All @@ -569,7 +568,6 @@ int PKCS11_evp_pkey_sign(EVP_PKEY *pk, int type, const char *mdname,
case EVP_PKEY_RSA:
return pkcs11_evp_pkey_rsa_sign(key, pk, mdname,
pad_mode, pss_saltlen, mgf1_mdname,
oaep_label, oaep_labellen,
sig, siglen, tbs, tbslen);
#ifndef OPENSSL_NO_EC
case EVP_PKEY_EC:
Expand All @@ -589,7 +587,7 @@ int PKCS11_evp_pkey_decrypt(EVP_PKEY *pk, int type, const char *mdname,
const int pad_mode, const char *mgf1_mdname,
unsigned char *oaep_label, const int oaep_labellen,
unsigned char *out, size_t *outlen,
size_t *outsize, const unsigned char *in, size_t inlen)
const unsigned char *in, size_t inlen)
{
PKCS11_OBJECT_private *key;
PKCS11_KEY *pkey = pkcs11_get_pkcs11_key(pk);
Expand All @@ -607,7 +605,7 @@ int PKCS11_evp_pkey_decrypt(EVP_PKEY *pk, int type, const char *mdname,
return pkcs11_evp_pkey_rsa_decrypt(key, pk, mdname,
pad_mode, mgf1_mdname,
oaep_label, oaep_labellen,
out, outlen, outsize, in, inlen);
out, outlen, in, inlen);
default:
return -2; /* type not supported */
}
Expand Down
56 changes: 32 additions & 24 deletions src/p11_key.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1257,8 +1257,7 @@ static int pkcs11_try_pkey_rsa_sign(EVP_PKEY_CTX *evp_pkey_ctx,
mgf1_mdname = EVP_MD_name(mgf1_md);

return pkcs11_evp_pkey_rsa_sign(key, pkey, mdname, padding,
salt_len, mgf1_mdname, NULL, 0,
sig, siglen, tbs, tbslen);
salt_len, mgf1_mdname, sig, siglen, tbs, tbslen);
}

/* Attempt to decrypt using the PKCS#11-backed RSA implementation */
Expand All @@ -1269,14 +1268,13 @@ static int pkcs11_try_pkey_rsa_decrypt(EVP_PKEY_CTX *evp_pkey_ctx,
EVP_PKEY *pkey;
RSA *rsa;
int padding;
CK_ULONG outsize = (CK_ULONG)*outlen;
PKCS11_OBJECT_private *key;
PKCS11_SLOT_private *slot;
CK_SESSION_HANDLE session;
const EVP_MD *md, *mgf1_md;
const char *mdname, *mgf1_mdname;
const char *mdname = NULL, *mgf1_mdname = NULL;
unsigned char *oaep_label = NULL;
int oaep_labellen;
int oaep_labellen = 0;

/* RSA method has EVP_PKEY_FLAG_AUTOARGLEN set. OpenSSL core will handle
* the size inquiry internally. */
Expand All @@ -1302,34 +1300,44 @@ static int pkcs11_try_pkey_rsa_decrypt(EVP_PKEY_CTX *evp_pkey_ctx,
if (EVP_PKEY_CTX_get_rsa_padding(evp_pkey_ctx, &padding) <= 0)
return -1;

if (padding != RSA_PKCS1_OAEP_PADDING)
return -1; /* unsupported */

/* retrieve OAEP parameters */
if (EVP_PKEY_CTX_get_rsa_oaep_md(evp_pkey_ctx, &md) <= 0)
return -1;

if (EVP_PKEY_CTX_get_rsa_mgf1_md(evp_pkey_ctx, &mgf1_md) <= 0)
return -1;

oaep_labellen = EVP_PKEY_CTX_get0_rsa_oaep_label(evp_pkey_ctx, &oaep_label);
if (oaep_labellen < 0) {
oaep_labellen = 0;
oaep_label = NULL;
}

slot = key->slot;
if (!slot)
return -1;

if (pkcs11_get_session(slot, 0, &session))
return -1;

mdname = EVP_MD_name(md);
mgf1_mdname = EVP_MD_name(mgf1_md);
switch (padding) {
case RSA_PKCS1_PADDING:
break;

case RSA_PKCS1_OAEP_PADDING:
/* retrieve OAEP parameters */
if (EVP_PKEY_CTX_get_rsa_oaep_md(evp_pkey_ctx, &md) <= 0 ||
md == NULL)
return -1;

if (EVP_PKEY_CTX_get_rsa_mgf1_md(evp_pkey_ctx, &mgf1_md) <= 0 ||
mgf1_md == NULL)
return -1;

mdname = EVP_MD_name(md);
mgf1_mdname = EVP_MD_name(mgf1_md);

oaep_labellen = EVP_PKEY_CTX_get0_rsa_oaep_label(evp_pkey_ctx,
&oaep_label);
if (oaep_labellen < 0) {
oaep_labellen = 0;
oaep_label = NULL;
}
break;

default:
return -1;
}

return pkcs11_evp_pkey_rsa_decrypt(key, pkey, mdname, padding, mgf1_mdname,
oaep_label, oaep_labellen, out, outlen, &outsize, in, inlen);
oaep_label, oaep_labellen, out, outlen, in, inlen);
}

static int pkcs11_pkey_rsa_sign(EVP_PKEY_CTX *evp_pkey_ctx,
Expand Down
Loading
Loading