How do I implement this proxy object?
You construct the proxy object with custom semantics (usually overloaded operators) and return it from operator[] of the parent object. Below I present an example, with even two proxy objects - first operator[] returns a Bitset::Byte, then Bitset::Byte::operator[] returns Bitset::Bit. The cool think is, Bitset::Bit has special operator= that allows to "just set" it's value.
#include <iostream>
#include <cstddef>
#include <climits>
#include <array>
#include <cassert>
#include <iomanip>
template<size_t N>
struct Bitset {
    std::array<unsigned char, N> mem{};
    struct Bit {
        unsigned char& byte;
        unsigned mask;
        Bit(unsigned char &byte, unsigned idx) :
            byte(byte), mask(1 << idx) {}
        operator unsigned () {
            return (byte & mask) ? 1 : 0;
        }
        Bit& operator=(unsigned v) {
            if (v) {
                byte |= mask;
            } else {
                byte &= mask;
            }
            return *this;
        }
    };
    struct Byte {
        unsigned char& byte;
        Bit operator[](unsigned idx) {
            return {byte, idx};
        }
        operator unsigned () {
            return byte;
        }
    };
    Byte operator[](unsigned idx) {
        return { mem.at(idx) };
    }
};
int main() {
    Bitset<20> a;
    std::cout << "a[1][3] = " << a[1][3] << "\n";
    // let's set 2nd byte 4th bit
    a[1][3] = 1; // yay!
    std::cout << "a[1] = 0x" << std::hex << std::setw(2) << std::setfill('0') <<  a[1] << "\n";
    std::cout << "a[1][3] = " << a[1][3] << "\n";
}
which outputs:
a[1][3] = 0
a[1] = 0x08
a[1][3] = 1
Doing a[1][3] = something is great and amazing functionality, it's clear and precise. But it's way more writing then just providing a function in parent object:
#include <iostream>
#include <cstddef>
#include <climits>
#include <array>
#include <cassert>
#include <iomanip>
template<size_t N>
struct Bitset {
    std::array<unsigned char, N> mem{};
    unsigned char& operator()(size_t n) {
        return mem.at(n);
    }
    void set(size_t n, size_t idx, unsigned v) {
        if (v) {
            mem.at(n) |= 1 << idx;
        } else {
            mem.at(n) &= 1 << idx;
        }
    }
    unsigned operator()(size_t n, size_t idx) {
        return (mem.at(n) & 1 << idx) ? 1 : 0;
    }
};
int main() {
    Bitset<20> a;
    std::cout << "a[1][3] = " << a(1, 3) << "\n";\
    // let's set 2nd byte 4th bit
    // a(1, 3) = 1; // ugh, not possible
    a.set(1, 3, 1);
    std::cout << "a[1] = 0x" << std::hex << std::setw(2) << std::setfill('0') << (unsigned)a(1) << "\n";
    std::cout << "a[1][3] = " << a(1, 3) << "\n";
}