Switching between long mode and compatibility mode is done by changing CS. User mode code cannot modify the descriptor table, but it can perform a far jump or far call to a code segment that is already present in the descriptor table. I think that in Linux (for example) the required compatibility mode descriptor is present.
Here is sample code for Linux (Ubuntu). Build with
$ gcc -no-pie switch_mode.c switch_cs.s
switch_mode.c:
#include <stdlib.h>
#include <stdio.h>
#include <stdbool.h>
extern bool switch_cs(int cs, bool (*f)());
extern bool check_mode();
int main(int argc, char **argv)
{
    int cs = 0x23;
    if (argc > 1)
        cs = strtoull(argv[1], 0, 16);
    printf("switch to CS=%02x\n", cs);
    bool r = switch_cs(cs, check_mode);
    if (r)
        printf("cs=%02x: 64-bit mode\n", cs);
    else
        printf("cs=%02x: 32-bit mode\n", cs);
    return 0;
}
switch_cs.s:
        .intel_syntax noprefix
        .code64
        .text
        .globl switch_cs
switch_cs:
        push    rbx
        push    rbp
        mov     rbp, rsp
        sub     rsp, 0x18
        mov     rbx, rsp
        movq    [rbx], offset .L1
        mov     [rbx+4], edi
        // Before the lcall, switch to a stack below 4GB.
        // This assumes that the data segment is below 4GB.
        mov     rsp, offset stack+0xf0
        lcall   [rbx]
        // restore rsp to the original stack
        leave
        pop     rbx
        ret
        .code32
.L1:
        call    esi
        lret
        .code64
        .globl check_mode
// returns false for 32-bit mode; true for 64-bit mode
check_mode:
        xor     eax, eax
        // In 32-bit mode, this instruction is executed as
        // inc eax; test eax, eax
        test    rax, rax
        setz    al
        ret
        .data
        .align  16
stack:  .space 0x100