Make EVP_DigestVerifyFinal return only zero or one.

It was already almost there. Just a malloc failure away. now all the
EVP_Digest{Sign,Verify}* functions may be used without worrying about -1 return
values.

Change-Id: I96a9750b300010615979bd5f1522b1d241764665
Reviewed-on: https://boringssl-review.googlesource.com/2064
Reviewed-by: Adam Langley <agl@google.com>

3 files changed, 30 insertions, 40 deletions

diff --git a/crypto/evp/digestsign.c b/crypto/evp/digestsign.c
index 08968ed..c86b805 100644
--- a/crypto/evp/digestsign.c
+++ b/crypto/evp/digestsign.c
@@ -168,22 +168,15 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, uint8_t *out_sig,
     if (has_signctx || !r) {
       return r;
     }
-    if (EVP_PKEY_sign(ctx->pctx, out_sig, out_sig_len, md, mdlen) <= 0) {
-      return 0;
-    }
+    return EVP_PKEY_sign(ctx->pctx, out_sig, out_sig_len, md, mdlen);
   } else {
     if (has_signctx) {
-      if (ctx->pctx->pmeth->signctx(ctx->pctx, out_sig, out_sig_len, ctx) <= 0) {
-        return 0;
-      }
+      return ctx->pctx->pmeth->signctx(ctx->pctx, out_sig, out_sig_len, ctx);
     } else {
       size_t s = EVP_MD_size(ctx->digest);
-      if (EVP_PKEY_sign(ctx->pctx, out_sig, out_sig_len, NULL, s) <= 0) {
-        return 0;
-      }
+      return EVP_PKEY_sign(ctx->pctx, out_sig, out_sig_len, NULL, s);
     }
   }
-  return 1;
 }
 
 int EVP_DigestVerifyFinal(EVP_MD_CTX *ctx, const uint8_t *sig,
@@ -196,7 +189,7 @@ int EVP_DigestVerifyFinal(EVP_MD_CTX *ctx, const uint8_t *sig,
 
   EVP_MD_CTX_init(&tmp_ctx);
   if (!EVP_MD_CTX_copy_ex(&tmp_ctx, ctx)) {
-    return -1;
+    return 0;
   }
   if (has_verifyctx) {
     r = tmp_ctx.pctx->pmeth->verifyctx(tmp_ctx.pctx, sig, sig_len, &tmp_ctx);
diff --git a/crypto/evp/example_sign.c b/crypto/evp/example_sign.c
index 42a19ec..2d4c071 100644
--- a/crypto/evp/example_sign.c
+++ b/crypto/evp/example_sign.c
@@ -196,12 +196,12 @@ static int example_EVP_DigestSignInit(void) {
 
   pkey = load_example_rsa_key();
   if (pkey == NULL ||
-      EVP_DigestSignInit(&md_ctx, NULL, EVP_sha256(), NULL, pkey) != 1 ||
-      EVP_DigestSignUpdate(&md_ctx, kMsg, sizeof(kMsg)) != 1) {
+      !EVP_DigestSignInit(&md_ctx, NULL, EVP_sha256(), NULL, pkey) ||
+      !EVP_DigestSignUpdate(&md_ctx, kMsg, sizeof(kMsg))) {
     goto out;
   }
   /* Determine the size of the signature. */
-  if (EVP_DigestSignFinal(&md_ctx, NULL, &sig_len) != 1) {
+  if (!EVP_DigestSignFinal(&md_ctx, NULL, &sig_len)) {
     goto out;
   }
   /* Sanity check for testing. */
@@ -211,14 +211,14 @@ static int example_EVP_DigestSignInit(void) {
   }
 
   sig = malloc(sig_len);
-  if (sig == NULL || EVP_DigestSignFinal(&md_ctx, sig, &sig_len) != 1) {
+  if (sig == NULL || !EVP_DigestSignFinal(&md_ctx, sig, &sig_len)) {
     goto out;
   }
 
   /* Ensure that the signature round-trips. */
-  if (EVP_DigestVerifyInit(&md_ctx_verify, NULL, EVP_sha256(), NULL, pkey) != 1 ||
-      EVP_DigestVerifyUpdate(&md_ctx_verify, kMsg, sizeof(kMsg)) != 1 ||
-      EVP_DigestVerifyFinal(&md_ctx_verify, sig, sig_len) != 1) {
+  if (!EVP_DigestVerifyInit(&md_ctx_verify, NULL, EVP_sha256(), NULL, pkey) ||
+      !EVP_DigestVerifyUpdate(&md_ctx_verify, kMsg, sizeof(kMsg)) ||
+      !EVP_DigestVerifyFinal(&md_ctx_verify, sig, sig_len)) {
     goto out;
   }
 
@@ -250,9 +250,9 @@ static int example_EVP_DigestVerifyInit(void) {
 
   pkey = load_example_rsa_key();
   if (pkey == NULL ||
-      EVP_DigestVerifyInit(&md_ctx, NULL, EVP_sha256(), NULL, pkey) != 1 ||
-      EVP_DigestVerifyUpdate(&md_ctx, kMsg, sizeof(kMsg)) != 1 ||
-      EVP_DigestVerifyFinal(&md_ctx, kSignature, sizeof(kSignature)) != 1) {
+      !EVP_DigestVerifyInit(&md_ctx, NULL, EVP_sha256(), NULL, pkey) ||
+      !EVP_DigestVerifyUpdate(&md_ctx, kMsg, sizeof(kMsg)) ||
+      !EVP_DigestVerifyFinal(&md_ctx, kSignature, sizeof(kSignature))) {
     goto out;
   }
   ret = 1;
@@ -282,7 +282,7 @@ static int test_algorithm_roundtrip(EVP_MD_CTX *md_ctx, EVP_PKEY *pkey) {
 
   EVP_MD_CTX_init(&md_ctx_verify);
 
-  if (EVP_DigestSignUpdate(md_ctx, kMsg, sizeof(kMsg)) != 1) {
+  if (!EVP_DigestSignUpdate(md_ctx, kMsg, sizeof(kMsg))) {
     goto out;
   }
 
@@ -293,7 +293,7 @@ static int test_algorithm_roundtrip(EVP_MD_CTX *md_ctx, EVP_PKEY *pkey) {
   }
 
   /* Determine the size of the signature. */
-  if (EVP_DigestSignFinal(md_ctx, NULL, &sig_len) != 1) {
+  if (!EVP_DigestSignFinal(md_ctx, NULL, &sig_len)) {
     goto out;
   }
   /* Sanity check for testing. */
@@ -303,14 +303,14 @@ static int test_algorithm_roundtrip(EVP_MD_CTX *md_ctx, EVP_PKEY *pkey) {
   }
 
   sig = malloc(sig_len);
-  if (sig == NULL || EVP_DigestSignFinal(md_ctx, sig, &sig_len) != 1) {
+  if (sig == NULL || !EVP_DigestSignFinal(md_ctx, sig, &sig_len)) {
     goto out;
   }
 
   /* Ensure that the signature round-trips. */
-  if (EVP_DigestVerifyInitFromAlgorithm(&md_ctx_verify, algor, pkey) != 1 ||
-      EVP_DigestVerifyUpdate(&md_ctx_verify, kMsg, sizeof(kMsg)) != 1 ||
-      EVP_DigestVerifyFinal(&md_ctx_verify, sig, sig_len) != 1) {
+  if (!EVP_DigestVerifyInitFromAlgorithm(&md_ctx_verify, algor, pkey) ||
+      !EVP_DigestVerifyUpdate(&md_ctx_verify, kMsg, sizeof(kMsg)) ||
+      !EVP_DigestVerifyFinal(&md_ctx_verify, sig, sig_len)) {
     goto out;
   }
 
@@ -342,7 +342,7 @@ static int test_EVP_DigestSignAlgorithm(void) {
   }
 
   /* Test a simple AlgorithmIdentifier. */
-  if (EVP_DigestSignInit(&md_ctx, &pkey_ctx, EVP_sha256(), NULL, pkey) != 1 ||
+  if (!EVP_DigestSignInit(&md_ctx, &pkey_ctx, EVP_sha256(), NULL, pkey) ||
       !test_algorithm_roundtrip(&md_ctx, pkey)) {
     fprintf(stderr, "RSA with SHA-256 failed\n");
     goto out;
@@ -352,7 +352,7 @@ static int test_EVP_DigestSignAlgorithm(void) {
   EVP_MD_CTX_init(&md_ctx);
 
   /* Test RSA-PSS with custom parameters. */
-  if (EVP_DigestSignInit(&md_ctx, &pkey_ctx, EVP_sha256(), NULL, pkey) != 1 ||
+  if (!EVP_DigestSignInit(&md_ctx, &pkey_ctx, EVP_sha256(), NULL, pkey) ||
       EVP_PKEY_CTX_set_rsa_padding(pkey_ctx, RSA_PKCS1_PSS_PADDING) != 1 ||
       EVP_PKEY_CTX_set_rsa_mgf1_md(pkey_ctx, EVP_sha512()) != 1 ||
       !test_algorithm_roundtrip(&md_ctx, pkey)) {
@@ -412,11 +412,11 @@ static int example_EVP_DigestVerifyInitFromAlgorithm(void) {
 
   pkey = load_example_rsa_key();
   if (pkey == NULL ||
-      EVP_DigestVerifyInitFromAlgorithm(&md_ctx, algor, pkey) != 1||
-      EVP_DigestVerifyUpdate(&md_ctx, CBS_data(&tbs_cert),
-                             CBS_len(&tbs_cert)) != 1 ||
-      EVP_DigestVerifyFinal(&md_ctx, CBS_data(&signature),
-                            CBS_len(&signature)) != 1) {
+      !EVP_DigestVerifyInitFromAlgorithm(&md_ctx, algor, pkey) ||
+      !EVP_DigestVerifyUpdate(&md_ctx, CBS_data(&tbs_cert),
+                              CBS_len(&tbs_cert)) ||
+      !EVP_DigestVerifyFinal(&md_ctx, CBS_data(&signature),
+                             CBS_len(&signature))) {
     goto out;
   }
   ret = 1;
diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index e3922a3..1f60145 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -290,10 +290,7 @@ OPENSSL_EXPORT int EVP_DigestVerifyUpdate(EVP_MD_CTX *ctx, const void *data,
 
 /* EVP_DigestVerifyFinal verifies that |sig_len| bytes of |sig| are a valid
  * signature for the data that has been included by one or more calls to
- * |EVP_DigestVerifyUpdate|.
- *
- * It returns one on success and <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * |EVP_DigestVerifyUpdate|. It returns one on success and zero otherwise. */
 OPENSSL_EXPORT int EVP_DigestVerifyFinal(EVP_MD_CTX *ctx, const uint8_t *sig,
                                          size_t sig_len);
 
@@ -462,8 +459,8 @@ OPENSSL_EXPORT int EVP_PKEY_sign_init(EVP_PKEY_CTX *ctx);
  * space available at |sig|. If sufficient, the signature will be written to
  * |sig| and |*sig_len| updated with the true length.
  *
- * WARNING: Setting |out| to NULL only gives the maximum size of the
- * plaintext. The actual plaintext may be smaller.
+ * WARNING: Setting |sig| to NULL only gives the maximum size of the
+ * signature. The actual signature may be smaller.
  *
  * It returns one on success or zero on error. (Note: this differs from
  * OpenSSL, which can also return negative values to indicate an error. ) */