how to avoid caching when writing to mmio registers?

Question

I'm writing a custom os in virtualbox and having trouble writing and reading successfully from the IOAPIC mmio registers. i.e. It seems to ignore the index register write. After loading R8 with the IOAPIC base address (determined from ACPI enumeration to be 0xFEC00000), I use the following routines to read/write:

; -----------------------------------------------------------------------------
; IN :  RAX = ioapic address, EBX = index register
; OUT:  ECX = return value
ioapic_read:
    mov [r8], ebx
    mov ecx, [r8 + 0x10]
    ret
; -----------------------------------------------------------------------------
; IN :  RAX = ioapic address, EBX = index register, ECX = value
; OUT:  -
ioapic_write:
    mov [r8], ebx
    mov [r8 + 0x10], ecx
    ret        

But an ioapic_read will always return the last value written (by ioapic_write) irrespective of the index used. I have identity paging setup to use 0x9B which I think should disable caching.

I have tried using pause after each of the movs. Didn't help. Tried mfences between the movs. Didn't help.

I have confirmed the 0xFEC00000 address is successfully identity mapped.

It looks like there's still some caching going on. What am I missing?

EDIT

I have discovered it's not a caching issue but something a lot stranger - at least to my ignorant brain. My identity paging, works on demand such that a page fault will generate the correct physical page in the tables.

This seems to be working but in the case of the IOAPIC mmio registers, I need to cause a page fault by doing a dummy read or write to the 0xFEC00000 address prior to attempting to use it. The even odder thing is that I need to do this dummy read enough instructions prior or it doesn't work. e.g.

This WORKS!

 mov eax, [os_IOAPICAddress]
 mov dword[rax], 0
 mov r8, rax
 .
 .
 .
 call ioapic_read

... this DOESN'T!

 mov eax, [os_IOAPICAddress]
 mov r8, rax
 mov dword[rax], 0
 .
 .
 .
 call ioapic_read

I suspect a pipelining/serializing issue but I would really love to learn both why I need to page fault the address into the tables before using it in an MMIO register, and why I need to do it far enough in advance. In the latter case, how to fix it so it is serialized such so I don't need to worry about it.

My identity paging routine:

pageFault_identity_0x0E:
    pop r8
    push rsi rdi rax rcx rdx r9

    test r8, 1
    jnz exception_gate_14
    mov rdx, cr2                                   ; faulting address
    shr rdx, 39
    and rdx, 0x1FF                                 ; get 9 bit index      

    mov rdi, cr3
    lea rsi, [rdi + rdx*8]
    mov rdi, [rsi]
    test rdi, 1
    jnz @f
    call set_new_page_table                                               
@@:
    shr rdi, 12                                     ; get rid of flags
    shl rdi, 12

    mov rdx, cr2
    shr rdx, 30                                     ; get 9 bit index    
    and rdx, 0x1FF

    lea rsi, [rdi + rdx*8]
    mov rdi, [rsi]
    test rdi, 1
    jnz @f
    call set_new_page_table                                               
@@:
    shr rdi, 12                                     ; get rid of flags
    shl rdi, 12

    mov rdx, cr2
    shr rdx, 21
    mov rax, rdx
    and rdx, 0x1FF                                  ; get 9 bit index    
    lea rsi, [rdi + rdx*8]

    shl rax, 21
    or rax, 0x83
    mov [rsi], rax

    shr rax, 21
    shl rax, 21

    pop r9 rdx rcx rax rdi rsi
    iretq
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;
; IN:   rsi = address of blank entry
; OUT:  rdi = base address of new table, changes rax & rcx
;
set_new_page_table:                                ; make table, get it, zero it, insert base into previous table
    movzx rdi, [page_table_count]
    shl rdi, 12
    add rdi, NEW_PAGE_TABLES

    CLEAR_BLOCK rdi, 0x200                     ; clears 4096 bytes in rdi, returns rdi + 4096

    sub rdi, 0x1000
    lea rax, [rdi + 0x3]                              ; table base address
    mov [rsi], rax
    inc [page_table_count]
    ret

Answer 1

Given the original code it looked as if you were setting the page directory entry bits properly to mark the MMIO region uncachable. I was convinced there was some other issue. With your subsequent edit you showed us your page fault handler pageFault_identity_0x0:

pageFault_identity_0x0E:
    pop r8
    push rsi rdi rax rcx rdx r9

When the processor transfers control to to this page fault exception handler it will pass an error code on the top of the stack as a parameter. The problem is that you replace the contents of R8 with the error number without saving and then restoring the register.

You'll have to modify your exception handler to preserve R8, move the contents from the proper stack offset where the error number is into R8. Just remember to ensure that the error number is no longer on the top of the stack prior to the IRETQ.

Likely the odd behaviour you have been getting is directly related to R8 not being properly restored upon return from a page fault.

---

A solution that may work is:

pageFault_identity_0x0E:
    push rsi
    push rdi
    push rax
    push rcx
    push rdx
    push r9
    push r8

    mov r8, [rsp+7*8]    ; Error Code is at offset RSP+7*8 after all the pushes
    ; Do exception handling work here

    pop r8
    pop r9
    pop rdx
    pop rcx
    pop rax
    pop rdi
    pop rsi

    add rsp, 8           ; Remove the error code
    iretq

Answer 2

Michael solved it but in the interests of completeness, I will post my final implementation.

pageFault_identity_0x0E:
    test qword[rsp], 1
    jnz exception_gate_14
    add rsp, 8

    push rsi rdi rax rcx rdx
    mov rdx, cr2                                   ; faulting address
    .
    .
    .
    pop rdx rcx rax rdi rsi
    iretq

EDIT: Have edited to remove the xchg.