Is there an x86 instruction to create a core dump?

Question

General question that I would like answered

I have some x86 assembly code that I'm trying to debug. I'd like to get a core dump so I can inspect what is going on. Is there an x86 instruction (or set of instructions) that will generate a core dump at a given point in a program? Is there a way to assemble the assembly to make it core dump if there is an error?

Specific issue (explained here for context)

I am writing a compiler for a small lambda calculus following An Incremental Approach to Compiler Construction. I'm working now on implementing closures, and I need to issue an indirect jump. I'm trying to compile this code:

(labels ((f (code (n) () (+ n 1)))) (app (closure f) 3))

My compiler generates the following:

     .text
     .p2align 4,,15
     .globl _scheme_entry
 _scheme_entry:
     movq %rdi, %r15
     jmp _definition_end38349
 _func_f38350:
     movq $4, %rax
     movq %rax, -16(%rsp)
     movq -8(%rsp), %rax
     addq -16(%rsp), %rax
     ret
 _definition_end38349:
     movq $12, %rax
     movq %rax, -24(%rsp)
     movq %rdi, -8(%rsp)
     leaq _func_f38350(%rip), %rax
     movq %rax, 0(%r15)
     movq %r15, %rax
     orq $6, %rax
     addq $8, %r15
     xorq $6, %rax
     movq %rax, %rdi
     addq $8, %rsp
     callq *%rdi
     subq $8, %rsp
     movq -8(%rsp), %rdi
     ret

I have an accompanying driver file written in C that handles the formatting and display of the result of the compiled code. For reference, here it is:

#include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>

 #define fixnum_mask  3
 #define fixnum_tag   0
 #define fixnum_shift 2

 #define data_mask    7
 #define cons_tag     1
 #define vector_tag   2
 #define string_tag   3
 #define symb_tag     5
 #define closure_tag  6

 #define empty_list   47

 #define char_tag     15
 #define char_mask    255
 #define char_shift   8

 #define bool_tag     31
 #define bool_mask    127
 #define bool_shift   7

 #define heap_size    8192

 size_t scheme_entry(size_t *heap);
 void format_val(size_t val);

 int main(int argc, char** argv) {
   size_t *heap = malloc(heap_size);
   size_t val = scheme_entry(heap);

   format_val(val);
   return 0;
 }

 void format_val(size_t val) {
   if ((val & bool_mask) == bool_tag) {
     printf((val >> bool_shift) ? "#t" : "#f");
   }
   else if ((val & fixnum_mask) == fixnum_tag) {
     printf("%zu", val >> fixnum_shift);
   }
   else if ((val & data_mask) == closure_tag) {
     printf("#<closure %zx>", val);
   }
   else if ((val & fixnum_mask) == cons_tag) {
     val--;
     size_t car = *((size_t*)val);
     size_t cdr = *((size_t*)val + 1);
     printf("("); format_val(car); printf(" . "); format_val(cdr); printf(")");
   }
   else if (val == empty_list) {
     printf("()");
   }
   /* else if ((val & char_mask) == char_tag) { */
   /*   printf("%c", val >> char_shift); */
   /* } */
   else {
     printf("#<unknown value: %zx>", val);
   }
 }

It compiles without complaint on macOS when I run gcc assembly-file.s driver.c. When I run the resulting a.out file, I get the following error:

[2]    84530 bus error  ./a.out

Is there a way I can get a core dump so I can inspect the values of the registers?

Bonus: if you can see what's wrong with my assembly, I wouldn't mind an answer to that either. ;-) I've tried using the GDB with my code, but it freezes every time I try it out on the a.out file.

I'm running this on macOS; gcc --version gives:

Configured with: --prefix=/Applications/Xcode.app/Contents/Developer/usr --with-gxx-include-dir=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/c++/4.2.1
Apple clang version 11.0.3 (clang-1103.0.32.62)
Target: x86_64-apple-darwin19.4.0
Thread model: posix
InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin

Much appreciated.

Answer 1

You can ask GCC:

$ cat tmp.c
void foo() { __builtin_trap(); }
$ gcc -O2 tmp.c
$ ./a.out 
Illegal instruction (core dumped)
$ gcc -O2 tmp.c -S -o-
        ...
        ud2