How to correctly generate a RSA key pair in C?

Question

I'm trying to generate a RSA key pair in C with the following function:

int generate_key(const int bits, char* public_key_name, char* private_key_name){
    int ret = 0;
    RSA *r = NULL;
    BIGNUM *bne = NULL;
    FILE *bp_public = NULL;
    FILE* bp_private = NULL;
    
    unsigned long   e = RSA_F4;

    // 1. generate rsa key
    bne = BN_secure_new();
    ret = BN_set_word(bne,e);
    if(ret != 1){
        goto free_all;
    }

    r = RSA_new();
    ret = RSA_generate_key_ex(r, bits, bne, NULL);
    if(ret != 1){
        goto free_all;
    }

    // 2. save public key
    bp_public = fopen(public_key_name, "w+");
    ret = PEM_write_RSAPublicKey(bp_public, r);
    fclose(bp_public);
    if(ret != 1){
        goto free_all;
    }

    // 3. save private key
    bp_private = fopen(private_key_name, "w+");
    ret = PEM_write_RSAPrivateKey(bp_private, r, NULL, NULL, 0, NULL, NULL);
    fclose(bp_private);

    // 4. free
    free_all:
    RSA_free(r);
    BN_clear_free(bne);

    return ret;
}

It works fine except for the fact that when I used the public key to verify a signed message an error pops out (error:0909006C:PEM routines:get_name:no) as result of EVP_VerifyFinal. Notice that, weird thing, the returning value of EVP_VerifyFinal is not 0 but -1: if it was 0 then It would be unrefutably a signature verification failure but it is not, it happens some other thing that I could not figure out. In the following I past the code used to sign and verify.

void sign(const unsigned char* const restrict clear_buf, const unsigned long clear_size, unsigned char* restrict* const restrict sgnt_buf, unsigned long* const restrict sgnt_size, const char* const restrict prvkey_file_name){
   const EVP_MD* md;
   EVP_MD_CTX* md_ctx;
   int ret;
   FILE* prvkey_file;
   EVP_PKEY* prvkey;
   unsigned expected_sgn_size;
   unsigned tmp;
   
   if(clear_size > INT_MAX){
       fprintf(stderr, "Buffer to sign too big\n");
       exit(1);
   }
   
   if((prvkey_file = fopen(prvkey_file_name, "r")) == NULL){
       fprintf(stderr, "Error: cannot open file '%s' (missing?)\n", prvkey_file_name);
       exit(1);
   }
   prvkey = PEM_read_PrivateKey(prvkey_file, NULL, NULL, NULL);
   fclose(prvkey_file);
   expected_sgn_size = (unsigned) EVP_PKEY_size(prvkey);

   if((*sgnt_buf = (unsigned char*)malloc((size_t)expected_sgn_size)) == NULL){
       fprintf(stderr, "Error in allocating memory for signature. Error: %s\n", strerror(errno));
       exit(1);
   }
   // create the signature context:
   md = EVP_sha256();
   md_ctx = EVP_MD_CTX_new();
   if(!md_ctx){
       fprintf(stderr, "Error: EVP_MD_CTX_new returned NULL\n");
       exit(1);
   }

   if(EVP_SignInit(md_ctx, md) == 0){
       fprintf(stderr, "Error: EVP_SignInit returned %d\n", ret);
       exit(1);
   }
   if(EVP_SignUpdate(md_ctx, clear_buf, (unsigned)clear_size) == 0){
       fprintf(stderr, "Error: EVP_SignUpdate returned %d\n", ret);
       exit(1);
   }
   if(EVP_SignFinal(md_ctx, *sgnt_buf, &tmp, prvkey) == 0){
       fprintf(stderr, "Error: EVP_SignFinal returned %d\n", ret);
       exit(1);
   }
   if(tmp < expected_sgn_size){
       fprintf(stderr, "Error in signing, signature size does not match expected size\n");
       exit(1);
   }
   
   *sgnt_size = (unsigned long)tmp;
   
   // delete the digest and the private key from memory:
   EVP_MD_CTX_free(md_ctx);
   EVP_PKEY_free(prvkey);
}

void verify(const unsigned char* const restrict file_buf, unsigned long* const restrict file_size, const char* const restrict pubkey_file_name){
   // declare some useful variables:
   int ret;
   FILE* pubkey_file;
   EVP_PKEY* pubkey;
   unsigned char* sgnt_buf;
   unsigned sgnt_size;
   
   pubkey_file = fopen(pubkey_file_name, "r");
   if(!pubkey_file){
       fprintf(stderr, "Error: cannot open file '%s' (missing?)\n", pubkey_file_name);
       exit(1);
   }
   pubkey = PEM_read_PUBKEY(pubkey_file, NULL, NULL, NULL);
   sgnt_size = (unsigned) EVP_PKEY_size(pubkey);
   fclose(pubkey_file);
   
   const EVP_MD* md = EVP_sha256();
   EVP_MD_CTX* md_ctx;
   
   *file_size -= (unsigned long)sgnt_size;
   if((sgnt_buf = (unsigned char*)malloc((size_t)sgnt_size)) == NULL){
       fprintf(stderr, "Error in allocating memory for signature. Error: %s\n", strerror(errno));
       exit(1);
   }
   memcpy((void*)sgnt_buf, (void*)(file_buf + *file_size), (size_t)sgnt_size);
   
   // create the signature context:
   md_ctx = EVP_MD_CTX_new();
   if(!md_ctx){
       fprintf(stderr, "Error: EVP_MD_CTX_new returned NULL\n");
       exit(1);
   }

   if(EVP_VerifyInit(md_ctx, md) == 0){
       fprintf(stderr, "Error in EVP_VerifyInit\n");
       exit(1);
   }
   if(EVP_VerifyUpdate(md_ctx, file_buf, *file_size) == 0){
       fprintf(stderr, "Error in EVP_VerifyUpdate\n");
       exit(1); 
   }
   ret = EVP_VerifyFinal(md_ctx, sgnt_buf, sgnt_size, pubkey);
   if(ret == 0){ // it is 0 if invalid signature, -1 if some other error, 1 if success.
       fprintf(stderr, "Error: EVP_VerifyFinal failed: invalid signature\n");
       exit(1);
   } else if(ret == -1){
       fprintf(stderr, "Some error occured during signature verification\n");
       exit(1);
   }else if (ret == 1){
       // fprintf(stdout, "Signature verified\n");
   }else{
       fprintf(stderr, "I shouldn't be printed. EVP_VerifyFinal returned %d\n", ret);
       exit(1);
   }
   EVP_MD_CTX_free(md_ctx);
   EVP_PKEY_free(pubkey);
   free(sgnt_buf);
}

On the web I found some solution but either they are not in C, or/and they are deprecated don't solve the problem (one solution that I found here generates a private key that it doesn't even sign for some reason...)

You write the pubkey using `PEM_write_RSAPublicKey` which is the 'legacy' algorithm-specific PKCS1 format, and attempt to read it with `PEM_read_PUBKEY` which expects the X.509-based generic format, which doesn't work; your 'key' is actually null, although you don't detect this, and verifying with a null/nonexistent key (and possibly a null signature buffer, since malloc(0) is implementation-dependent) doesn't work. [In general whenever an OpenSSL libcrypto routine returns a failure value you should display the error stack.](https://www.openssl.org/docs/faq.html#PROG8) — dave_thompson_085, Apr 13 '21 at 19:06
Also you should know RSA signatures are always the size of the key, i.e. `EVP_PKEY_size(x)`, but that is not true for other algorithms like DSA and ECDSA, so assuming the signature must be and is this fixed size constrains your usage. — dave_thompson_085, Apr 13 '21 at 19:08
@dave_thompson_085 ok I (think I) understand but what should I modify in the code? I'm pretty new at OpenSSL — Luigi, Apr 13 '21 at 20:27
The simplest solution is to write the pubkey using `PEM_write_RSA_PUBKEY` which will be compatible with your read. — dave_thompson_085, Apr 14 '21 at 22:30

score 0 · Answer 1 · answered Apr 13 '21 at 19:19

Your verify code makes a assumation that is incorrect.

If you read the EVP_PKEY_size documentation is states:

VP_PKEY_size() returns the maximum suitable size for the output buffers for almost all operations that can be done with pkey.

The sign function is correctly returning the reported signature size based on the returned value from EVP_SignFinal.

The verify funciton is assuming that the size of the signature is the returned value from EVP_PKEY_size. This is most likely incorrect.

You need to change how you "store" the signature to also store it's size somewhere as well. You can confirm that the size looks correct in that is must always be <= the returned value from EVP_PKEY_size.

e.g. If you had a storage format like:

[file data][signature][size]EOF

You could change your verify to look like this:

void verify(const unsigned char* const file_buf, unsigned long* const file_size, const char* const pubkey_file_name){
   // declare some useful variables:
   ...
   unsigned sgnt_size;
   unsigned max_sgnt_size;
   
   ...

   max_sgnt_size = (unsigned) EVP_PKEY_size(pubkey);

   ...

   if(*file_size < sizeof(unsigned)) {
       fprintf(stderr, "Error: File size to small\n");
       exit(1);
   }

   *file_size -= sizeof(unsigned);
   sgnt_size = *(const unsigned *)(file_buf + *file_size);
   if(sgnt_size > max_sgnt_size) {
       fprintf(stderr, "Error: Invalid signature size\n");
       exit(1);
   }

   if(sgnt_size <= *file_size) {
       fprintf(stderr, "Error: Invalid file size\n");
       exit(1);
   }

   *file_size -= (unsigned long)sgnt_size;

   ...
}

That would be a problem in general, but not for RSA; RSA signatures (and cryptograms) are always the (same) size of the modulus. See e.g. RFC 8017 section 4. — dave_thompson_085, Apr 14 '21 at 22:31
Note that `sgnt_size = *(const unsigned *)(file_buf + *file_size);` is always a [strict aliasing violation](https://stackoverflow.com/questions/98650/what-is-the-strict-aliasing-rule) and invokes undefined behavior. You can not safely refer to an array of `[[un]signed] char` values as an `unsigned int` value. And if the referred-to `char` value is not properly aligned for an `unsigned int` value, that will *also* invoke undefined behavior. — Andrew Henle, May 24 '23 at 00:07

Ahab Devoid · Answer 2 · 2023-05-23T23:34:52.540

According to the openssl documentation, and compiler warnings, the PEM_write functions are now depreciated and shouldn't be used.

Here's where to start looking for the documentation:

(encoder) OSSL_ENCODER_CTX_new_for_pkey

(Key Management Flags)

With that out of the way, I was able to successfully generate and save into files after a bit of research. Here's the function that I created for the very purpose of generating PEM RSA public and private key files.

Do with the code as you please. The code is implemented in a class. I just snipped the unimportant stuff out.

EVP_PKEY *keypair = NULL;
OSSL_ENCODER_CTX *encoderCtx = NULL;
void generateRSAKeyFree(){
        EVP_PKEY_free(keypair);
        OSSL_ENCODER_CTX_free(encoderCtx);
}
/*
 * NOTES: 
 *  Generic RSA Key Sizes : 1024 | 4096 | 8192
 * */
bool generateRsaKeyPairToFile(int bits, string publicKeyLoc, string privateKeyLoc){
        failed = false;
        keypair = EVP_RSA_gen(bits);
        if(keypair == NULL){
            failed = true;
            return false;
        }
        
        // Write out Public Key
        encoderCtx = OSSL_ENCODER_CTX_new_for_pkey(keypair, OSSL_KEYMGMT_SELECT_PUBLIC_KEY, "PEM", NULL, NULL);
        if(encoderCtx == NULL){
            failed = true;
            generateRSAKeyFree();
            return false;
        }

        FILE *fp = fopen(publicKeyLoc.c_str(), "w+");
        if(fp == NULL){
                failed = true;
                generateRSAKeyFree();
                return false;
        }

        if(!OSSL_ENCODER_to_fp(encoderCtx, fp)){
                fclose(fp);
                generateRSAKeyFree();
                failed = true;
                return false;
        }
        fclose(fp);
        OSSL_ENCODER_CTX_free(encoderCtx);


        //Write out Private Key
        encoderCtx = OSSL_ENCODER_CTX_new_for_pkey(keypair, OSSL_KEYMGMT_SELECT_PRIVATE_KEY, "PEM", NULL, NULL);
        if(encoderCtx == NULL){
                failed = true;
                generateRSAKeyFree();
                return false;
        }

        fp = fopen(privateKeyLoc.c_str(), "w+");
        if(fp == NULL){
                failed = true;
                generateRSAKeyFree();
                return false;
        }

        if(!OSSL_ENCODER_to_fp(encoderCtx, fp)){
                fclose(fp);
                generateRSAKeyFree();
                failed = true;
                return false;
        }
        fclose(fp);
        generateRSAKeyFree();
        return true;
}

I use vim, so there's large 8 spaces in my tabs, not gonna go through it all. But with that said, if you for some reason decided that you want to use DER encoding, you can just change the strings PEM to DER.

This sample code should be openssl 3.0+ compatible.

How to correctly generate a RSA key pair in C?

2 Answers2