Can't encrypt long string with XOR C crypter

Question

I wrote a C program to perform an XOR encryption, my problem is that the program is not able to encrypt files with more than 24 characters.

The code:

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#define BUF_SIZE  2

char* xor(char*, char*);
char* gen_key(size_t);

int main(int argc, char **argv) {
    char *buffer = NULL,* encrypted_buffer = NULL;
    size_t file_size;
    char *key = gen_key(6);
    char tmp_buffer[BUF_SIZE];
    FILE *finput = NULL, *foutput = NULL;

    finput = fopen("file.txt", "rb");
    fseek(finput, 0, SEEK_END);
    file_size = ftell(finput);
    rewind(finput);
    printf("File size : %d\n", (int)file_size);
    buffer = (char*)malloc((file_size + 1) * sizeof(char));
    memset(buffer, 0, sizeof(buffer));
    while (!feof(finput)) {
        memset(tmp_buffer, 0, sizeof(tmp_buffer));
        fgets(tmp_buffer, sizeof(tmp_buffer), finput);
        strcat(buffer, tmp_buffer);
    }
    printf("%s", buffer);
    encrypted_buffer = xor(buffer, key);
    free(buffer);
    buffer = xor(encrypted_buffer, key);
    printf("Encrypted : %s\n", encrypted_buffer);
    printf("Decrypted : %s\n", buffer);
    printf("EOF\n");
    free(encrypted_buffer);
    fclose(finput);
    return 0;
}

char *gen_key(size_t length) {
    srand(time(NULL));
    const char charset[] = "AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz012345679";
    const size_t charset_size = (sizeof(charset) - 1);
    unsigned int i;
    char *key = NULL;
    key = (char*)malloc((length + 1) * sizeof(char));
    memset(key, 0, sizeof(key));
    for (i = 0; i < length; i++)
        key[i] = charset[rand() % charset_size];
    return key;
}

char *xor(char *file, char *key) {
    unsigned int i;
    char *xor = NULL;
    xor = (char*)malloc(sizeof(file));
    memset(xor, 0, sizeof(xor));
    for (i = 0; i < strlen(file); i++)
        *(xor + i) = *(file + i) ^ *(key + (i % strlen(key) - 1));
    return xor;
}

And the output is :

File size : 55
ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklimnopqrstuvwxyz

Encrypted : A2#G8-  M   >7S$1!

Decrypted : ABCDEFGHIJKLMNOPQRSTUVWX!:!e!

EOF

Answer 1

There are multiple problems in your code:

• The buffer size is very small: #define BUF_SIZE 2. You should use a reasonable size for the line buffer, such as 80 or 100.

• in memset(buffer, 0, sizeof(buffer));, buffer is a pointer, so sizeof(buffer) is not the size of the array, just the size of the pointer. Use file_size + 1 in this case.

• You make the same mistake in other places in your code: pass the size of the buffers instead of relying on the sizeof() operator.

• while (!feof(f)) is always wrong: you could jut use fread to read the file in one step or read line by line with:

  while (fgets(tmp_buffer, sizeof(tmp_buffer), finput)) {
      ...
  

Note that there is a major problem with your approach: the key is composed of letters and digits and the file is assumed to contain text. If the file contains one of the characters in the key at the appropriate position, xoring this character with the key byte will produce a null byte that will stop the output in printf() and that would also stop the decryption if you were to store it in an output file. You rightfully use binary mode ("rb") for the file stream, but you should also make no assumptions on the file contents and handle null bytes transparently.

Here is a modified version of your program:

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

unsigned char *xor(unsigned char *file, size_t size, const char *key);
char *gen_key(size_t length);

void print_buffer(const char *msg, unsigned char *buf, size_t size) {
    printf("%s: ", msg);
    for (size_t i = 0; i < size; i++) {
        switch (buf[i]) {
        case '\n':
            printf("\\n");
            break;
        case '\\':
            printf("\\\\");
            break;
        default:
            if (buf[i] >= ' ' && buf[i] < 127)
                putchar(buf[i]);
            else
                printf("\\%02x", buf[i]);
            break;
        }
    }
    printf("\n");
}

int main(int argc, char **argv) {
    long file_size, nread, nwritten;
    unsigned char *buffer, *encrypted_buffer, *decrypted_buffer;
    char *key = gen_key(6);
    FILE *finput = fopen("file.txt", "rb");

    if (finput == NULL) {
        fprintf(stderr, "cannot open file.txt: %s\n", strerror(errno));
        return 1;
    }
    fseek(finput, 0, SEEK_END);
    file_size = ftell(finput);
    rewind(finput);
    printf("File size: %ld\n", file_size);

    buffer = calloc(file_size, sizeof(char));
    nread = fread(buffer, 1, file_size, finput);
    if (nread != file_size) {
        fprintf(stderr, "error reading file.txt: read %ld bytes, expected %ld\n",
                nread, file_size);
    }
    fclose(finput);

    FILE *foutput = fopen("output.bin", "wb");
    if (foutput == NULL) {
        fprintf(stderr, "cannot open output.bin: %s\n", strerror(errno));
        return 1;
    }
    encrypted_buffer = xor(buffer, nread, key);
    nwritten = fwrite(encrypted_buffer, 1, nread, foutput);
    if (nwritten != nread) {
        fprintf(stderr, "error writing output.bin: wrote %ld bytes, expected %ld\n",
                nwritten, nread);
    }
    fclose(foutput);

    decrypted_buffer = xor(encrypted_buffer, nread, key);

    printf("Key      : %s\n", key);
    print_buffer("Original ", buffer, nread);
    print_buffer("Encrypted", encrypted_buffer, nread);
    print_buffer("Decrypted", decrypted_buffer, nread);
    if (!memcmp(decrypted_buffer, buffer, nread))
        printf("OK\n");

    free(decrypted_buffer);
    free(encrypted_buffer);
    free(buffer);
    return 0;
}

char *gen_key(size_t length) {
    const char charset[] = "AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz012345679";
    const size_t charset_size = sizeof(charset) - 1;
    char *key = (char*)calloc(length + 1, sizeof(char));
    size_t i;

    srand(time(NULL));
    for (i = 0; i < length; i++)
        key[i] = charset[rand() % charset_size];
    return key;
}

unsigned char *xor(unsigned char *file, size_t size, const char *key) {
    size_t i, keylen = strlen(key);
    unsigned char *xor = calloc(size, sizeof(char));
    for (i = 0; i < size; i++)
        xor[i] = file[i] ^ key[i % keylen];
    return xor;
}