summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAdam Langley <agl@chromium.org>2014-07-25 10:06:44 -0700
committerAdam Langley <agl@google.com>2014-07-29 21:47:51 +0000
commit5129e2d69582c0c54a335eb7e0bc794a02418403 (patch)
tree83756c13dae7277532dadb94415d1e4bae7393dc
parentd82d79dffe59b9282d6eb00c4b2ee93f9fcd2e75 (diff)
downloadsrc-5129e2d69582c0c54a335eb7e0bc794a02418403.tar.gz
Align EVP return values with BoringSSL convention.
Where possible, functions should return one for success and zero for error. The use of additional negative values to indicate an error is, itself, error prone. This change fixes many EVP functions to remove the possibility of negative return values. Existing code that is testing for <= 0 will continue to function, although there is the possibility that some code was differentiating between negative values (error) and zero (invalid signature) for the verify functions and will now show the wrong error message. Change-Id: I982512596bb18a82df65861394dbd7487783bd3d Reviewed-on: https://boringssl-review.googlesource.com/1333 Reviewed-by: Adam Langley <agl@google.com>
-rw-r--r--crypto/ecdsa/ecdsa.c10
-rw-r--r--crypto/evp/evp_ctx.c99
-rw-r--r--crypto/evp/p_ec.c10
-rw-r--r--crypto/evp/p_rsa.c195
-rw-r--r--include/openssl/ecdsa.h13
-rw-r--r--include/openssl/evp.h45
-rw-r--r--include/openssl/rsa.h4
7 files changed, 164 insertions, 212 deletions
diff --git a/crypto/ecdsa/ecdsa.c b/crypto/ecdsa/ecdsa.c
index 0e8e42f..067fd6c 100644
--- a/crypto/ecdsa/ecdsa.c
+++ b/crypto/ecdsa/ecdsa.c
@@ -73,7 +73,7 @@ int ECDSA_sign(int type, const uint8_t *digest, size_t digest_len, uint8_t *sig,
int ECDSA_verify(int type, const uint8_t *digest, size_t digest_len,
const uint8_t *sig, size_t sig_len, EC_KEY *eckey) {
ECDSA_SIG *s;
- int ret = -1;
+ int ret = 0;
if (eckey->ecdsa_meth && eckey->ecdsa_meth->verify) {
return eckey->ecdsa_meth->verify(digest, digest_len, sig, sig_len, eckey);
@@ -127,7 +127,7 @@ ECDSA_SIG *ECDSA_do_sign(const uint8_t *digest, size_t digest_len,
int ECDSA_do_verify(const uint8_t *digest, size_t digest_len,
const ECDSA_SIG *sig, EC_KEY *eckey) {
- int ret = -1;
+ int ret = 0;
BN_CTX *ctx;
BIGNUM *order, *u1, *u2, *m, *X;
EC_POINT *point = NULL;
@@ -136,20 +136,20 @@ int ECDSA_do_verify(const uint8_t *digest, size_t digest_len,
if (eckey->ecdsa_meth && eckey->ecdsa_meth->verify) {
OPENSSL_PUT_ERROR(ECDSA, ECDSA_do_verify, ECDSA_R_NOT_IMPLEMENTED);
- return -1;
+ return 0;
}
/* check input values */
if (eckey == NULL || (group = EC_KEY_get0_group(eckey)) == NULL ||
(pub_key = EC_KEY_get0_public_key(eckey)) == NULL || sig == NULL) {
OPENSSL_PUT_ERROR(ECDSA, ECDSA_do_verify, ECDSA_R_MISSING_PARAMETERS);
- return -1;
+ return 0;
}
ctx = BN_CTX_new();
if (!ctx) {
OPENSSL_PUT_ERROR(ECDSA, ECDSA_do_verify, ERR_R_MALLOC_FAILURE);
- return -1;
+ return 0;
}
BN_CTX_start(ctx);
order = BN_CTX_get(ctx);
diff --git a/crypto/evp/evp_ctx.c b/crypto/evp/evp_ctx.c
index 1365dae..1416d3a 100644
--- a/crypto/evp/evp_ctx.c
+++ b/crypto/evp/evp_ctx.c
@@ -232,7 +232,6 @@ int EVP_PKEY_CTX_ctrl(EVP_PKEY_CTX *ctx, int keytype, int optype, int cmd,
}
int EVP_PKEY_sign_init(EVP_PKEY_CTX *ctx) {
- int ret;
if (!ctx || !ctx->pmeth || !ctx->pmeth->sign) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_sign_init,
EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
@@ -244,14 +243,12 @@ int EVP_PKEY_sign_init(EVP_PKEY_CTX *ctx) {
return 1;
}
- ret = ctx->pmeth->sign_init(ctx);
- /* TODO(fork): normalise the return value convention. */
- if (ret <= 0) {
+ if (!ctx->pmeth->sign_init(ctx)) {
ctx->operation = EVP_PKEY_OP_UNDEFINED;
return 0;
}
- return ret;
+ return 1;
}
int EVP_PKEY_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *sig_len,
@@ -269,22 +266,21 @@ int EVP_PKEY_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *sig_len,
}
int EVP_PKEY_verify_init(EVP_PKEY_CTX *ctx) {
- int ret;
-
if (!ctx || !ctx->pmeth || !ctx->pmeth->verify) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_verify_init,
EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
- return -2;
+ return 0;
}
ctx->operation = EVP_PKEY_OP_VERIFY;
if (!ctx->pmeth->verify_init) {
return 1;
}
- ret = ctx->pmeth->verify_init(ctx);
- if (ret <= 0) {
+ if (!ctx->pmeth->verify_init(ctx)) {
ctx->operation = EVP_PKEY_OP_UNDEFINED;
+ return 0;
}
- return ret;
+
+ return 1;
}
int EVP_PKEY_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, size_t sig_len,
@@ -292,31 +288,30 @@ int EVP_PKEY_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, size_t sig_len,
if (!ctx || !ctx->pmeth || !ctx->pmeth->verify) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_verify,
EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
- return -2;
+ return 0;
}
if (ctx->operation != EVP_PKEY_OP_VERIFY) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_verify, EVP_R_OPERATON_NOT_INITIALIZED);
- return -1;
+ return 0;
}
return ctx->pmeth->verify(ctx, sig, sig_len, data, data_len);
}
int EVP_PKEY_encrypt_init(EVP_PKEY_CTX *ctx) {
- int ret;
if (!ctx || !ctx->pmeth || !ctx->pmeth->encrypt) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_encrypt_init,
EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
- return -2;
+ return 0;
}
ctx->operation = EVP_PKEY_OP_ENCRYPT;
if (!ctx->pmeth->encrypt_init) {
return 1;
}
- ret = ctx->pmeth->encrypt_init(ctx);
- if (ret <= 0) {
+ if (!ctx->pmeth->encrypt_init(ctx)) {
ctx->operation = EVP_PKEY_OP_UNDEFINED;
+ return 0;
}
- return ret;
+ return 1;
}
int EVP_PKEY_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
@@ -324,32 +319,30 @@ int EVP_PKEY_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
if (!ctx || !ctx->pmeth || !ctx->pmeth->encrypt) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_encrypt,
EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
- return -2;
+ return 0;
}
if (ctx->operation != EVP_PKEY_OP_ENCRYPT) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_encrypt, EVP_R_OPERATON_NOT_INITIALIZED);
- return -1;
+ return 0;
}
return ctx->pmeth->encrypt(ctx, out, outlen, in, inlen);
}
int EVP_PKEY_decrypt_init(EVP_PKEY_CTX *ctx) {
- int ret;
-
if (!ctx || !ctx->pmeth || !ctx->pmeth->decrypt) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_decrypt_init,
EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
- return -2;
+ return 0;
}
ctx->operation = EVP_PKEY_OP_DECRYPT;
if (!ctx->pmeth->decrypt_init) {
return 1;
}
- ret = ctx->pmeth->decrypt_init(ctx);
- if (ret <= 0) {
+ if (!ctx->pmeth->decrypt_init(ctx)) {
ctx->operation = EVP_PKEY_OP_UNDEFINED;
+ return 0;
}
- return ret;
+ return 1;
}
int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
@@ -357,31 +350,30 @@ int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
if (!ctx || !ctx->pmeth || !ctx->pmeth->decrypt) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_decrypt,
EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
- return -2;
+ return 0;
}
if (ctx->operation != EVP_PKEY_OP_DECRYPT) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_decrypt, EVP_R_OPERATON_NOT_INITIALIZED);
- return -1;
+ return 0;
}
return ctx->pmeth->decrypt(ctx, out, outlen, in, inlen);
}
int EVP_PKEY_derive_init(EVP_PKEY_CTX *ctx) {
- int ret;
if (!ctx || !ctx->pmeth || !ctx->pmeth->derive) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_init,
EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
- return -2;
+ return 0;
}
ctx->operation = EVP_PKEY_OP_DERIVE;
if (!ctx->pmeth->derive_init) {
return 1;
}
- ret = ctx->pmeth->derive_init(ctx);
- if (ret <= 0) {
+ if (!ctx->pmeth->derive_init(ctx)) {
ctx->operation = EVP_PKEY_OP_UNDEFINED;
+ return 0;
}
- return ret;
+ return 1;
}
int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) {
@@ -391,20 +383,20 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) {
!ctx->pmeth->ctrl) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer,
EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
- return -2;
+ return 0;
}
if (ctx->operation != EVP_PKEY_OP_DERIVE &&
ctx->operation != EVP_PKEY_OP_ENCRYPT &&
ctx->operation != EVP_PKEY_OP_DECRYPT) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer,
EVP_R_OPERATON_NOT_INITIALIZED);
- return -1;
+ return 0;
}
ret = ctx->pmeth->ctrl(ctx, EVP_PKEY_CTRL_PEER_KEY, 0, peer);
if (ret <= 0) {
- return ret;
+ return 0;
}
if (ret == 2) {
@@ -413,12 +405,12 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) {
if (!ctx->pkey) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer, EVP_R_NO_KEY_SET);
- return -1;
+ return 0;
}
if (ctx->pkey->type != peer->type) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer, EVP_R_DIFFERENT_KEY_TYPES);
- return -1;
+ return 0;
}
/* ran@cryptocom.ru: For clarity. The error is if parameters in peer are
@@ -430,7 +422,7 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) {
!EVP_PKEY_cmp_parameters(ctx->pkey, peer)) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer,
EVP_R_DIFFERENT_PARAMETERS);
- return -1;
+ return 0;
}
if (ctx->peerkey) {
@@ -442,7 +434,7 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) {
if (ret <= 0) {
ctx->peerkey = NULL;
- return ret;
+ return 0;
}
CRYPTO_add(&peer->references, 1, CRYPTO_LOCK_EVP_PKEY);
@@ -453,58 +445,55 @@ int EVP_PKEY_derive(EVP_PKEY_CTX *ctx, uint8_t *key, size_t *out_key_len) {
if (!ctx || !ctx->pmeth || !ctx->pmeth->derive) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive,
EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
- return -2;
+ return 0;
}
if (ctx->operation != EVP_PKEY_OP_DERIVE) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive, EVP_R_OPERATON_NOT_INITIALIZED);
- return -1;
+ return 0;
}
return ctx->pmeth->derive(ctx, key, out_key_len);
}
int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx) {
- int ret;
if (!ctx || !ctx->pmeth || !ctx->pmeth->keygen) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_keygen_init,
EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
- return -2;
+ return 0;
}
ctx->operation = EVP_PKEY_OP_KEYGEN;
if (!ctx->pmeth->keygen_init) {
return 1;
}
- ret = ctx->pmeth->keygen_init(ctx);
- if (ret <= 0) {
+ if (!ctx->pmeth->keygen_init(ctx)) {
ctx->operation = EVP_PKEY_OP_UNDEFINED;
+ return 0;
}
- return ret;
+ return 1;
}
int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey) {
- int ret;
-
if (!ctx || !ctx->pmeth || !ctx->pmeth->keygen) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_keygen,
EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
- return -2;
+ return 0;
}
if (ctx->operation != EVP_PKEY_OP_KEYGEN) {
OPENSSL_PUT_ERROR(EVP, EVP_PKEY_keygen, EVP_R_OPERATON_NOT_INITIALIZED);
- return -1;
+ return 0;
}
if (!ppkey) {
- return -1;
+ return 0;
}
if (!*ppkey) {
*ppkey = EVP_PKEY_new();
}
- ret = ctx->pmeth->keygen(ctx, *ppkey);
- if (ret <= 0) {
+ if (!ctx->pmeth->keygen(ctx, *ppkey)) {
EVP_PKEY_free(*ppkey);
*ppkey = NULL;
+ return 0;
}
- return ret;
+ return 1;
}
diff --git a/crypto/evp/p_ec.c b/crypto/evp/p_ec.c
index 2305e6e..e429f77 100644
--- a/crypto/evp/p_ec.c
+++ b/crypto/evp/p_ec.c
@@ -164,7 +164,7 @@ static void pkey_ec_cleanup(EVP_PKEY_CTX *ctx) {
static int pkey_ec_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *siglen,
const uint8_t *tbs, size_t tbslen) {
- int ret, type;
+ int type;
unsigned int sltmp;
EC_PKEY_CTX *dctx = ctx->data;
EC_KEY *ec = ctx->pkey->pkey.ec;
@@ -182,10 +182,8 @@ static int pkey_ec_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *siglen,
type = EVP_MD_type(dctx->md);
}
- ret = ECDSA_sign(type, tbs, tbslen, sig, &sltmp, ec);
-
- if (ret <= 0) {
- return ret;
+ if (!ECDSA_sign(type, tbs, tbslen, sig, &sltmp, ec)) {
+ return 0;
}
*siglen = (size_t)sltmp;
return 1;
@@ -235,7 +233,7 @@ static int pkey_ec_derive(EVP_PKEY_CTX *ctx, uint8_t *key,
ret = ECDH_compute_key(key, outlen, pubkey, eckey, 0);
if (ret < 0) {
- return ret;
+ return 0;
}
*keylen = ret;
return 1;
diff --git a/crypto/evp/p_rsa.c b/crypto/evp/p_rsa.c
index 458b606..bcaca4b 100644
--- a/crypto/evp/p_rsa.c
+++ b/crypto/evp/p_rsa.c
@@ -168,58 +168,57 @@ static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk) {
static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *siglen,
const uint8_t *tbs, size_t tbslen) {
- int ret;
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
+ const size_t key_len = EVP_PKEY_size(ctx->pkey);
if (!sig) {
- *siglen = RSA_size(rsa);
+ *siglen = key_len;
return 1;
- } else if (*siglen < (size_t)RSA_size(rsa)) {
+ }
+
+ if (*siglen < key_len) {
OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_BUFFER_TOO_SMALL);
return 0;
}
if (rctx->md) {
+ unsigned int out_len;
+
if (tbslen != EVP_MD_size(rctx->md)) {
OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_INVALID_DIGEST_LENGTH);
- return -1;
+ return 0;
}
if (EVP_MD_type(rctx->md) == NID_mdc2) {
OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_NO_MDC2_SUPPORT);
- ret = -1;
- } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
- unsigned int sltmp;
- ret = RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig, &sltmp, rsa);
- if (ret <= 0) {
- return ret;
- }
- ret = sltmp;
- } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
- if (!setup_tbuf(rctx, ctx) ||
- !RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf, tbs, rctx->md,
- rctx->mgf1md, rctx->saltlen)) {
- return -1;
- }
- /* TODO(fork): don't use private_encrypt. */
- ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf, sig, rsa,
- RSA_NO_PADDING);
- } else {
- return -1;
+ return 0;
}
- } else {
- /* TODO(fork): don't use private_encrypt. */
- ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
- rctx->pad_mode);
- }
- if (ret < 0) {
- return ret;
+ switch (rctx->pad_mode) {
+ case RSA_PKCS1_PADDING:
+ if (!RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig, &out_len, rsa)) {
+ return 0;
+ }
+ *siglen = out_len;
+ return 1;
+
+ case RSA_PKCS1_PSS_PADDING:
+ if (!setup_tbuf(rctx, ctx) ||
+ !RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf, tbs, rctx->md,
+ rctx->mgf1md, rctx->saltlen) ||
+ !RSA_encrypt(rsa, siglen, sig, *siglen, rctx->tbuf, key_len,
+ RSA_NO_PADDING)) {
+ return 0;
+ }
+ return 1;
+
+ default:
+ return 0;
+ }
}
- *siglen = ret;
- return 1;
+ return RSA_sign_raw(rsa, siglen, sig, *siglen, tbs, tbslen, rctx->pad_mode);
}
static int pkey_rsa_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig,
@@ -228,42 +227,33 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig,
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
size_t rslen;
+ const size_t key_len = EVP_PKEY_size(ctx->pkey);
if (rctx->md) {
- if (rctx->pad_mode == RSA_PKCS1_PADDING) {
- return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen, sig, siglen, rsa);
- }
+ switch (rctx->pad_mode) {
+ case RSA_PKCS1_PADDING:
+ return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen, sig, siglen, rsa);
+
+ case RSA_PKCS1_PSS_PADDING:
+ if (!setup_tbuf(rctx, ctx) ||
+ !RSA_verify_raw(rsa, &rslen, rctx->tbuf, key_len, sig, siglen,
+ RSA_NO_PADDING) ||
+ !RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md, rctx->mgf1md,
+ rctx->tbuf, rctx->saltlen)) {
+ return 0;
+ }
+ return 1;
- if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
- int ret;
- if (!setup_tbuf(rctx, ctx)) {
- return -1;
- }
- /* TODO(fork): don't use public_decrypt. */
- ret = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa, RSA_NO_PADDING);
- if (ret <= 0) {
- return 0;
- }
- ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md, rctx->mgf1md,
- rctx->tbuf, rctx->saltlen);
- if (ret <= 0) {
+ default:
return 0;
- }
- return 1;
- } else {
- return -1;
- }
- } else {
- if (!setup_tbuf(rctx, ctx)) {
- return -1;
- }
- rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa, rctx->pad_mode);
- if (rslen == 0) {
- return 0;
}
}
- if (rslen != tbslen || CRYPTO_memcmp(tbs, rctx->tbuf, rslen) != 0) {
+ if (!setup_tbuf(rctx, ctx) ||
+ !RSA_verify_raw(rsa, &rslen, rctx->tbuf, key_len, sig, siglen,
+ rctx->pad_mode) ||
+ rslen != tbslen ||
+ CRYPTO_memcmp(tbs, rctx->tbuf, rslen) != 0) {
return 0;
}
@@ -272,77 +262,73 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig,
static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
const uint8_t *in, size_t inlen) {
- int ret;
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
+ const size_t key_len = EVP_PKEY_size(ctx->pkey);
if (!out) {
- *outlen = RSA_size(rsa);
+ *outlen = key_len;
return 1;
- } else if (*outlen < (size_t)RSA_size(rsa)) {
+ }
+
+ if (*outlen < key_len) {
OPENSSL_PUT_ERROR(EVP, pkey_rsa_encrypt, EVP_R_BUFFER_TOO_SMALL);
return 0;
}
if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
- int klen = RSA_size(rsa);
- if (!setup_tbuf(rctx, ctx)) {
- return -1;
- }
- if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen, in, inlen,
+ if (!setup_tbuf(rctx, ctx) ||
+ !RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, key_len, in, inlen,
rctx->oaep_label, rctx->oaep_labellen,
- rctx->md, rctx->mgf1md)) {
- return -1;
+ rctx->md, rctx->mgf1md) ||
+ !RSA_encrypt(rsa, outlen, out, *outlen, rctx->tbuf, key_len,
+ RSA_NO_PADDING)) {
+ return 0;
}
- ret = RSA_public_encrypt(klen, rctx->tbuf, out, rsa, RSA_NO_PADDING);
- } else {
- ret = RSA_public_encrypt(inlen, in, out, rsa, rctx->pad_mode);
- }
-
- if (ret < 0) {
- return ret;
+ return 1;
}
- *outlen = ret;
- return 1;
+ return RSA_encrypt(rsa, outlen, out, *outlen, in, inlen, rctx->pad_mode);
}
static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out,
size_t *outlen, const uint8_t *in,
size_t inlen) {
- int ret;
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
+ const size_t key_len = EVP_PKEY_size(ctx->pkey);
if (!out) {
- *outlen = RSA_size(rsa);
+ *outlen = key_len;
return 1;
- } else if (*outlen < (size_t)RSA_size(rsa)) {
+ }
+
+ if (*outlen < key_len) {
OPENSSL_PUT_ERROR(EVP, pkey_rsa_decrypt, EVP_R_BUFFER_TOO_SMALL);
return 0;
}
if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
- if (!setup_tbuf(rctx, ctx)) {
- return -1;
- }
- ret = RSA_private_decrypt(inlen, in, rctx->tbuf, rsa, RSA_NO_PADDING);
- if (ret <= 0) {
- return ret;
+ size_t plaintext_len;
+ int message_len;
+
+ if (!setup_tbuf(rctx, ctx) ||
+ !RSA_decrypt(rsa, &plaintext_len, rctx->tbuf, key_len, in, inlen,
+ RSA_NO_PADDING)) {
+ return 0;
}
- ret = RSA_padding_check_PKCS1_OAEP_mgf1(
- out, ret, rctx->tbuf, ret, rctx->oaep_label,
- rctx->oaep_labellen, rctx->md, rctx->mgf1md);
- } else {
- ret = RSA_private_decrypt(inlen, in, out, rsa, rctx->pad_mode);
- }
- if (ret < 0) {
- return ret;
+ message_len = RSA_padding_check_PKCS1_OAEP_mgf1(
+ out, key_len, rctx->tbuf, plaintext_len, rctx->oaep_label,
+ rctx->oaep_labellen, rctx->md, rctx->mgf1md);
+ if (message_len < 0) {
+ return 0;
+ }
+ *outlen = message_len;
+ return 1;
}
- *outlen = ret;
- return 1;
+ return RSA_decrypt(rsa, outlen, out, key_len, in, inlen, rctx->pad_mode);
}
static int check_padding_md(const EVP_MD *md, int padding) {
@@ -507,7 +493,6 @@ static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) {
RSA *rsa = NULL;
RSA_PKEY_CTX *rctx = ctx->data;
- int ret;
if (!rctx->pub_exp) {
rctx->pub_exp = BN_new();
if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
@@ -518,13 +503,13 @@ static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) {
return 0;
}
- ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, NULL);
- if (ret > 0) {
- EVP_PKEY_assign_RSA(pkey, rsa);
- } else {
+ if (!RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, NULL)) {
RSA_free(rsa);
+ return 0;
}
- return ret;
+
+ EVP_PKEY_assign_RSA(pkey, rsa);
+ return 1;
}
const EVP_PKEY_METHOD rsa_pkey_meth = {
diff --git a/include/openssl/ecdsa.h b/include/openssl/ecdsa.h
index 8a74ecb..a2436f1 100644
--- a/include/openssl/ecdsa.h
+++ b/include/openssl/ecdsa.h
@@ -78,12 +78,8 @@ int ECDSA_sign(int type, const uint8_t *digest, size_t digest_len, uint8_t *sig,
/* ECDSA_verify verifies that |sig_len| bytes from |sig| constitute a valid
* signature by |key| of |digest|. (The |type| argument should be zero.) It
- * returns one on success, zero if the signature is invalid and -1 on error.
- *
- * DANGER: this function differs from the usual OpenSSL return value
- * convention.
- *
- * TODO(fork): change this to return the usual 0/1. */
+ * returns one on success or zero if the signature is invalid or an error
+ * occured. */
int ECDSA_verify(int type, const uint8_t *digest, size_t digest_len,
const uint8_t *sig, size_t sig_len, EC_KEY *key);
@@ -116,12 +112,9 @@ ECDSA_SIG *ECDSA_do_sign(const uint8_t *digest, size_t digest_len,
EC_KEY *key);
/* ECDSA_verify verifies that |sig| constitutes a valid signature by |key| of
- * |digest|. It returns one on success, zero if the signature is invalid and -1
+ * |digest|. It returns one on success or zero if the signature is invalid or
* on error.
*
- * DANGER: this function differs from the usual OpenSSL return value
- * convention.
- *
* TODO(fork): remove this function. */
int ECDSA_do_verify(const uint8_t *digest, size_t digest_len,
const ECDSA_SIG *sig, EC_KEY *key);
diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index b522e8f..d47d51f 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -222,8 +222,7 @@ int i2d_PublicKey(EVP_PKEY *key, uint8_t **outp);
* operation will be written to |*pctx|; this can be used to set alternative
* signing options.
*
- * It returns one on success, or <= 0 on error. WARNING: this differs from the
- * usual OpenSSL return convention. */
+ * It returns one on success, or zero on error. */
int EVP_DigestSignInit(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, const EVP_MD *type,
ENGINE *e, EVP_PKEY *pkey);
@@ -239,8 +238,7 @@ int EVP_DigestSignUpdate(EVP_MD_CTX *ctx, const void *data, size_t len);
* is successful, the signature is written to |out_sig| and |*out_sig_len| is
* set to its length.
*
- * It returns one on success and <= 0 on error. WARNING: this differs from the
- * usual, OpenSSL return value convention. */
+ * It returns one on success, or zero on error. */
int EVP_DigestSignFinal(EVP_MD_CTX *ctx, uint8_t *out_sig, size_t *out_sig_len);
@@ -252,8 +250,7 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, uint8_t *out_sig, size_t *out_sig_len);
* operation will be written to |*pctx|; this can be used to set alternative
* signing options.
*
- * It returns one on success, or <= 0 on error. WARNING: this differs from the
- * usual OpenSSL return convention. */
+ * It returns one on success, or zero on error. */
int EVP_DigestVerifyInit(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx,
const EVP_MD *type, ENGINE *e, EVP_PKEY *pkey);
@@ -422,8 +419,7 @@ int EVP_PKEY_CTX_ctrl(EVP_PKEY_CTX *ctx, int keytype, int optype, int cmd,
/* EVP_PKEY_sign_init initialises an |EVP_PKEY_CTX| for a signing operation. It
* should be called before |EVP_PKEY_sign|.
*
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_sign_init(EVP_PKEY_CTX *ctx);
/* EVP_PKEY_sign signs |data_len| bytes from |data| using |ctx|. If |sig| is
@@ -443,23 +439,20 @@ int EVP_PKEY_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *sig_len,
/* EVP_PKEY_verify_init initialises an |EVP_PKEY_CTX| for a signature
* verification operation. It should be called before |EVP_PKEY_verify|.
*
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_verify_init(EVP_PKEY_CTX *ctx);
/* EVP_PKEY_verify verifies that |sig_len| bytes from |sig| are a valid signature
* for |data|.
*
- * It returns one on success or zero on error. (Note: this differs from
- * OpenSSL, which can also return negative values to indicate an error. ) */
+ * It returns one on success or zero on error. */
int EVP_PKEY_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, size_t sig_len,
const uint8_t *data, size_t data_len);
/* EVP_PKEY_encrypt_init initialises an |EVP_PKEY_CTX| for an encryption
* operation. It should be called before |EVP_PKEY_encrypt|.
*
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_encrypt_init(EVP_PKEY_CTX *ctx);
/* EVP_PKEY_encrypt encrypts |in_len| bytes from |in|. If |out| is NULL, the
@@ -471,16 +464,14 @@ int EVP_PKEY_encrypt_init(EVP_PKEY_CTX *ctx);
* WARNING: Setting |out| to NULL only gives the maximum size of the
* ciphertext. The actual ciphertext may be smaller.
*
- * It returns one on success or <= 0 on error. (Note: this differs from
- * OpenSSL, which can also return negative values to indicate an error. ) */
+ * It returns one on success or zero on error. */
int EVP_PKEY_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *out_len,
const uint8_t *in, size_t in_len);
/* EVP_PKEY_decrypt_init initialises an |EVP_PKEY_CTX| for a decryption
* operation. It should be called before |EVP_PKEY_decrypt|.
*
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_decrypt_init(EVP_PKEY_CTX *ctx);
/* EVP_PKEY_decrypt decrypts |in_len| bytes from |in|. If |out| is NULL, the
@@ -492,8 +483,7 @@ int EVP_PKEY_decrypt_init(EVP_PKEY_CTX *ctx);
* WARNING: Setting |out| to NULL only gives the maximum size of the
* plaintext. The actual plaintext may be smaller.
*
- * It returns one on success or <= 0 on error. (Note: this differs from
- * OpenSSL, which can also return negative values to indicate an error. ) */
+ * It returns one on success or zero on error. */
int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *out_len,
const uint8_t *in, size_t in_len);
@@ -501,15 +491,13 @@ int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *out_len,
* operation. It should be called before |EVP_PKEY_derive_set_peer| and
* |EVP_PKEY_derive|.
*
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_derive_init(EVP_PKEY_CTX *ctx);
/* EVP_PKEY_derive_set_peer sets the peer's key to be used for key derivation
* by |ctx| to |peer|. It should be called after |EVP_PKEY_derive_init|. (For
* example, this is used to set the peer's key in (EC)DH.) It returns one on
- * success and <= 0 on error. WARNING: this differs from the usual return value
- * convention. */
+ * success and zero on error. */
int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer);
/* EVP_PKEY_derive derives a shared key between the two keys configured in
@@ -521,21 +509,18 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer);
* WARNING: Setting |out| to NULL only gives the maximum size of the key. The
* actual key may be smaller.
*
- * It returns one on success and <= 0 on error. WARNING: this differs from the
- * usual return convention. */
+ * It returns one on success and zero on error. */
int EVP_PKEY_derive(EVP_PKEY_CTX *ctx, uint8_t *key, size_t *out_key_len);
/* EVP_PKEY_keygen_init initialises an |EVP_PKEY_CTX| for a key generation
* operation. It should be called before |EVP_PKEY_keygen|.
*
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx);
/* EVP_PKEY_keygen performs a key generation operation using the values from
* |ctx| and sets |*ppkey| to a fresh |EVP_PKEY| containing the resulting key.
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * normal return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey);
diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index 821bb1b..9cc465d 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -93,7 +93,9 @@ int RSA_up_ref(RSA *rsa);
* for |e|. If |cb| is not NULL then it is called during the key generation
* process. In addition to the calls documented for |BN_generate_prime_ex|, it
* is called with event=2 when the n'th prime is rejected as unsuitable and
- * with event=3 when a suitable value for |p| is found. */
+ * with event=3 when a suitable value for |p| is found.
+ *
+ * It returns one on success or zero on error. */
int RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);