The only solution that I came up with is to define rows with a unique set of elements. It is somewhat of a hack, but it works good and is a way to have a solution until the Streamlit community comes up with a better way.
In this example I will have a row with 4 columns and that is unique for my sidebar.
col1, col2, col3, col4 = st.sidebar.columns([1, 1, 1, 1])
The buttons:
with col1:
    st.button("", on_click=style_button_row, kwargs={
        'clicked_button_ix': 1, 'n_buttons': 4
    })
with col2:
    st.button("", on_click=style_button_row, kwargs={
        'clicked_button_ix': 2, 'n_buttons': 4
    })
with col3:
    st.button("◀", on_click=style_button_row, kwargs={
       'clicked_button_ix': 3, 'n_buttons': 4
    })
with col4:
    st.button("", on_click=style_button_row, kwargs={
        'clicked_button_ix': 4, 'n_buttons': 4
    })
The styling way inspired from Can CSS detect the number of children an element has?:
div[data-testid*="stHorizontalBlock"] > div:nth-child(%(nth_child)s):nth-last-child(%(nth_last_child)s) button
The styling function:
def style_button_row(clicked_button_ix, n_buttons):
    def get_button_indices(button_ix):
        return {
            'nth_child': button_ix,
            'nth_last_child': n_buttons - button_ix + 1
        }
    clicked_style = """
    div[data-testid*="stHorizontalBlock"] > div:nth-child(%(nth_child)s):nth-last-child(%(nth_last_child)s) button {
        border-color: rgb(255, 75, 75);
        color: rgb(255, 75, 75);
        box-shadow: rgba(255, 75, 75, 0.5) 0px 0px 0px 0.2rem;
        outline: currentcolor none medium;
    }
    """
    unclicked_style = """
    div[data-testid*="stHorizontalBlock"] > div:nth-child(%(nth_child)s):nth-last-child(%(nth_last_child)s) button {
        pointer-events: none;
        cursor: not-allowed;
        opacity: 0.65;
        filter: alpha(opacity=65);
        -webkit-box-shadow: none;
        box-shadow: none;
    }
    """
    style = ""
    for ix in range(n_buttons):
        ix += 1
        if ix == clicked_button_ix:
            style += clicked_style % get_button_indices(ix)
        else:
            style += unclicked_style % get_button_indices(ix)
    st.markdown(f"<style>{style}</style>", unsafe_allow_html=True)
Result:
