Fix all warnings when compiling with -Wall -W -pedantic.

[tinc] / src / openssl / rsa.c

diff --git a/src/openssl/rsa.c b/src/openssl/rsa.c
index 20bfb65..0e81172 100644 (file)
--- a/src/openssl/rsa.c
+++ b/src/openssl/rsa.c
@@ -22,7 +22,7 @@
 #include <openssl/pem.h>
 #include <openssl/err.h>
 
-#define __TINC_RSA_INTERNAL__
+#define TINC_RSA_INTERNAL
 typedef RSA rsa_t;
 
 #include "../logger.h"
@@ -30,29 +30,59 @@ typedef RSA rsa_t;
 
 // Set RSA keys
 
+#ifndef HAVE_RSA_SET0_KEY
+int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d) {
+       BN_free(r->n);
+       r->n = n;
+       BN_free(r->e);
+       r->e = e;
+       BN_free(r->d);
+       r->d = d;
+       return 1;
+}
+#endif
+
 rsa_t *rsa_set_hex_public_key(char *n, char *e) {
-       rsa_t *rsa = RSA_new();
-       if(!rsa)
-               return NULL;
+       BIGNUM *bn_n = NULL;
+       BIGNUM *bn_e = NULL;
 
-       if(BN_hex2bn(&rsa->n, n) != strlen(n) || BN_hex2bn(&rsa->e, e) != strlen(e)) {
-               RSA_free(rsa);
+       if((size_t)BN_hex2bn(&bn_n, n) != strlen(n) || (size_t)BN_hex2bn(&bn_e, e) != strlen(e)) {
+               BN_free(bn_e);
+               BN_free(bn_n);
                return false;
        }
 
+       rsa_t *rsa = RSA_new();
+
+       if(!rsa) {
+               return NULL;
+       }
+
+       RSA_set0_key(rsa, bn_n, bn_e, NULL);
+
        return rsa;
 }
 
 rsa_t *rsa_set_hex_private_key(char *n, char *e, char *d) {
+       BIGNUM *bn_n = NULL;
+       BIGNUM *bn_e = NULL;
+       BIGNUM *bn_d = NULL;
+
+       if((size_t)BN_hex2bn(&bn_n, n) != strlen(n) || (size_t)BN_hex2bn(&bn_e, e) != strlen(e) || (size_t)BN_hex2bn(&bn_d, d) != strlen(d)) {
+               BN_free(bn_d);
+               BN_free(bn_e);
+               BN_free(bn_n);
+               return false;
+       }
+
        rsa_t *rsa = RSA_new();
-       if(!rsa)
-               return NULL;
 
-       if(BN_hex2bn(&rsa->n, n) != strlen(n) || BN_hex2bn(&rsa->e, e) != strlen(e) || BN_hex2bn(&rsa->d, d) != strlen(d)) {
-               RSA_free(rsa);
-               return false;
+       if(!rsa) {
+               return NULL;
        }
 
+       RSA_set0_key(rsa, bn_n, bn_e, bn_d);
+
        return rsa;
 }
 
@@ -61,11 +91,14 @@ rsa_t *rsa_set_hex_private_key(char *n, char *e, char *d) {
 rsa_t *rsa_read_pem_public_key(FILE *fp) {
        rsa_t *rsa = PEM_read_RSAPublicKey(fp, NULL, NULL, NULL);
 
-       if(!rsa)
+       if(!rsa) {
+               rewind(fp);
                rsa = PEM_read_RSA_PUBKEY(fp, NULL, NULL, NULL);
+       }
 
-       if(!rsa)
+       if(!rsa) {
                logger(DEBUG_ALWAYS, LOG_ERR, "Unable to read RSA public key: %s", ERR_error_string(ERR_get_error(), NULL));
+       }
 
        return rsa;
 }
@@ -73,8 +106,9 @@ rsa_t *rsa_read_pem_public_key(FILE *fp) {
 rsa_t *rsa_read_pem_private_key(FILE *fp) {
        rsa_t *rsa = PEM_read_RSAPrivateKey(fp, NULL, NULL, NULL);
 
-       if(!rsa)
+       if(!rsa) {
                logger(DEBUG_ALWAYS, LOG_ERR, "Unable to read RSA private key: %s", ERR_error_string(ERR_get_error(), NULL));
+       }
 
        return rsa;
 }
@@ -84,16 +118,18 @@ size_t rsa_size(rsa_t *rsa) {
 }
 
 bool rsa_public_encrypt(rsa_t *rsa, void *in, size_t len, void *out) {
-       if(RSA_public_encrypt(len, in, out, rsa, RSA_NO_PADDING) == len)
+       if((size_t)RSA_public_encrypt(len, in, out, rsa, RSA_NO_PADDING) == len) {
                return true;
+       }
 
        logger(DEBUG_ALWAYS, LOG_ERR, "Unable to perform RSA encryption: %s", ERR_error_string(ERR_get_error(), NULL));
        return false;
 }
 
 bool rsa_private_decrypt(rsa_t *rsa, void *in, size_t len, void *out) {
-       if(RSA_private_decrypt(len, in, out, rsa, RSA_NO_PADDING) == len)
+       if((size_t)RSA_private_decrypt(len, in, out, rsa, RSA_NO_PADDING) == len) {
                return true;
+       }
 
        logger(DEBUG_ALWAYS, LOG_ERR, "Unable to perform RSA decryption: %s", ERR_error_string(ERR_get_error(), NULL));
        return false;
@@ -104,6 +140,7 @@ bool rsa_active(rsa_t *rsa) {
 }
 
 void rsa_free(rsa_t *rsa) {
-       if(rsa)
+       if(rsa) {
                RSA_free(rsa);
+       }
 }