I understand that the OP had at least a vague understanding of the bit-level arithmetic but was lost on converting the strings to numbers and its inverse. here's a working (with minimal testing anyway) example, using froh42's calculation.
jcomeau@aspire:~/rentacoder/jcomeau/freifunk$ cat inet.c; make inet; ./inet 192.168.3.1 255.255.255.0
#include <arpa/inet.h>
#include <stdio.h>
int main(int argc, char **argv) {
    char *host_ip = argc > 1 ? argv[1] : "127.0.0.1";
    char *netmask = argc > 2 ? argv[2] : "255.255.255.255";
    struct in_addr host, mask, broadcast;
    char broadcast_address[INET_ADDRSTRLEN];
    if (inet_pton(AF_INET, host_ip, &host) == 1 &&
        inet_pton(AF_INET, netmask, &mask) == 1)
        broadcast.s_addr = host.s_addr | ~mask.s_addr;
    else {
        fprintf(stderr, "Failed converting strings to numbers\n");
        return 1;
    }
    if (inet_ntop(AF_INET, &broadcast, broadcast_address, INET_ADDRSTRLEN) != NULL)
        printf("Broadcast address of %s with netmask %s is %s\n",
            host_ip, netmask, broadcast_address);
    else {
        fprintf(stderr, "Failed converting number to string\n");
        return 1;
    }
    return 0;
}
cc     inet.c   -o inet
Broadcast address of 192.168.3.1 with netmask 255.255.255.0 is 192.168.3.255