The operating system(RHEL) was accidentally crashed suddenly, after reboot it is back to normal, there is no way to reproduce.
And I am new to both C and assembly language.Because of my lack of ability, I can only analyze that it is line 122 of jhash.h that causes mov 0x5c(%rax),%ecx to access a null pointer, but I would like to know what the null pointer is and where it comes from?
OS version: Linux version 3.10.0-1160.15.2.el7.x86_64
gcc version: 4.8.5 20150623
backtrace:
crash> bt
PID: 0 TASK: ffff885f33b4a100 CPU: 6 COMMAND: "swapper/6"
#0 [ffff8865dd783640] machine_kexec at ffffffffbae662c4
#1 [ffff8865dd7836a0] __crash_kexec at ffffffffbaf227a2
#2 [ffff8865dd783770] crash_kexec at ffffffffbaf22890
#3 [ffff8865dd783788] oops_end at ffffffffbb58c798
#4 [ffff8865dd7837b0] no_context at ffffffffbae75d14
#5 [ffff8865dd783800] __bad_area_nosemaphore at ffffffffbae75fe2
#6 [ffff8865dd783850] bad_area_nosemaphore at ffffffffbae76104
#7 [ffff8865dd783860] __do_page_fault at ffffffffbb58f750
#8 [ffff8865dd7838d0] do_page_fault at ffffffffbb58f975
#9 [ffff8865dd783900] page_fault at ffffffffbb58b778
[exception RIP: hash_ip4_test+41]
RIP: ffffffffc08e5909 RSP: ffff8865dd7839b8 RFLAGS: 00010296
RAX: 0000000000000000 RBX: ffff88656f363800 RCX: ffff8865dd783aa0
RDX: ffff8865dd783a10 RSI: ffff8865dd783a0c RDI: ffff88656f363800
RBP: ffff8865dd7839f8 R8: 0000000000000000 R9: ffff8865dd783a10
R10: ffff8865dd783aa0 R11: ffff885e7af65b80 R12: ffff8865dd783a98
R13: ffff8863c8f22f00 R14: ffff88656f363800 R15: ffff8865dd783a0c
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
#10 [ffff8865dd783a00] hash_ip4_kadt at ffffffffc08e4106 [ip_set_hash_ip]
#11 [ffff8865dd783a58] ip_set_test at ffffffffc08ffcc0 [ip_set]
#12 [ffff8865dd783a90] set_match_v4 at ffffffffc09a4dc0 [xt_set]
#13 [ffff8865dd783ae8] ipt_do_table at ffffffffc0545640 [ip_tables]
#14 [ffff8865dd783c38] iptable_mangle_hook at ffffffffc0878043 [iptable_mangle]
#15 [ffff8865dd783c78] nf_iterate at ffffffffbb495a48
#16 [ffff8865dd783cb8] nf_hook_slow at ffffffffbb495b38
#17 [ffff8865dd783cf0] ip_rcv at ffffffffbb4a0559
#18 [ffff8865dd783d60] __netif_receive_skb_core at ffffffffbb455829
#19 [ffff8865dd783dd0] __netif_receive_skb at ffffffffbb455b28
#20 [ffff8865dd783df0] netif_receive_skb_internal at ffffffffbb455bb0
#21 [ffff8865dd783e20] napi_gro_receive at ffffffffbb456838
#22 [ffff8865dd783e48] gro_cell_poll at ffffffffc09ba187 [vxlan]
#23 [ffff8865dd783e78] net_rx_action at ffffffffbb4561cf
#24 [ffff8865dd783ef8] __do_softirq at ffffffffbaea4b35
#25 [ffff8865dd783f68] call_softirq at ffffffffbb5984ec
#26 [ffff8865dd783f80] do_softirq at ffffffffbae2f715
#27 [ffff8865dd783fa0] irq_exit at ffffffffbaea4eb5
#28 [ffff8865dd783fb8] do_IRQ at ffffffffbb599936
--- <IRQ stack> ---
#29 [ffff885f33b5bdf8] ret_from_intr at ffffffffbb58b36a
[exception RIP: native_safe_halt+11]
RIP: ffffffffbb589ebb RSP: ffff885f33b5bea8 RFLAGS: 00000286
RAX: ffffffffbb589c70 RBX: 006cc0c03aa693c0 RCX: 0100000000000000
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000046
RBP: ffff885f33b5bea8 R8: 0000000000000000 R9: 0000000000000001
R10: 0000000000000000 R11: 7fffffffffffffff R12: 006cc0c03aa693c0
R13: 0000000000000006 R14: 006cc0c03a0dfd40 R15: 0dd5743fda151c77
ORIG_RAX: ffffffffffffff73 CS: 0010 SS: 0018
#30 [ffff885f33b5beb0] default_idle at ffffffffbb589c8e
#31 [ffff885f33b5bed0] arch_cpu_idle at ffffffffbae37ca0
#32 [ffff885f33b5bee0] cpu_startup_entry at ffffffffbaf0142a
#33 [ffff885f33b5bf28] start_secondary at ffffffffbae5a827
#34 [ffff885f33b5bf50] start_cpu at ffffffffbae000d5
Disassembly:
crash> dis -rl ffffffffc08e5909
/usr/src/debug/kernel-3.10.0-1160.15.2.el7/linux-3.10.0-1160.15.2.el7.x86_64/net/netfilter/ipset/ip_set_hash_gen.h: 988
0xffffffffc08e58e0 <hash_ip4_test>: nopl 0x0(%rax,%rax,1) [FTRACE NOP]
0xffffffffc08e58e5 <hash_ip4_test+5>: push %rbp
0xffffffffc08e58e6 <hash_ip4_test+6>: mov %rcx,%r10
0xffffffffc08e58e9 <hash_ip4_test+9>: mov %rdx,%r9
0xffffffffc08e58ec <hash_ip4_test+12>: mov %rsp,%rbp
0xffffffffc08e58ef <hash_ip4_test+15>: push %r15
0xffffffffc08e58f1 <hash_ip4_test+17>: mov %rsi,%r15
0xffffffffc08e58f4 <hash_ip4_test+20>: push %r14
0xffffffffc08e58f6 <hash_ip4_test+22>: mov %rdi,%r14
0xffffffffc08e58f9 <hash_ip4_test+25>: push %r13
0xffffffffc08e58fb <hash_ip4_test+27>: push %r12
0xffffffffc08e58fd <hash_ip4_test+29>: push %rbx
0xffffffffc08e58fe <hash_ip4_test+30>: sub $0x18,%rsp
/usr/src/debug/kernel-3.10.0-1160.15.2.el7/linux-3.10.0-1160.15.2.el7.x86_64/net/netfilter/ipset/ip_set_hash_gen.h: 989
0xffffffffc08e5902 <hash_ip4_test+34>: mov 0x80(%rdi),%rax
/usr/src/debug/kernel-3.10.0-1160.15.2.el7/linux-3.10.0-1160.15.2.el7.x86_64/include/linux/jhash.h: 122
0xffffffffc08e5909 <hash_ip4_test+41>: mov 0x5c(%rax),%ecx
Disassembly:
crash> dis -l hash_ip4_test+41
/usr/src/debug/kernel-3.10.0-1160.15.2.el7/linux-3.10.0-1160.15.2.el7.x86_64/include/linux/jhash.h: 122
0xffffffffc08e5909 <hash_ip4_test+41>: mov 0x5c(%rax),%ecx
Lines 57-132 of the /usr/src/debug/kernel-3.10.0-1160.15.2.el7/linux-3.10.0-1160.15.2.el7.x86_64/include/linux/jhash.h file:
...
57 /* An arbitrary initial parameter */
58 #define JHASH_INITVAL 0xdeadbeef
59
60 /* jhash - hash an arbitrary key
61 * @k: sequence of bytes as key
62 * @length: the length of the key
63 * @initval: the previous hash, or an arbitray value
64 *
65 * The generic version, hashes an arbitrary sequence of bytes.
66 * No alignment or length assumptions are made about the input key.
67 *
68 * Returns the hash value of the key. The result depends on endianness.
69 */
70 static inline u32 jhash(const void *key, u32 length, u32 initval)
71 {
72 u32 a, b, c;
73 const u8 *k = key;
74
75 /* Set up the internal state */
76 a = b = c = JHASH_INITVAL + length + initval;
77
78 /* All but the last block: affect some 32 bits of (a,b,c) */
79 while (length > 12) {
80 a += __get_unaligned_cpu32(k);
81 b += __get_unaligned_cpu32(k + 4);
82 c += __get_unaligned_cpu32(k + 8);
83 __jhash_mix(a, b, c);
84 length -= 12;
85 k += 12;
86 }
87 /* Last block: affect all 32 bits of (c) */
88 /* All the case statements fall through */
89 switch (length) {
90 case 12: c += (u32)k[11]<<24;
91 case 11: c += (u32)k[10]<<16;
92 case 10: c += (u32)k[9]<<8;
93 case 9: c += k[8];
94 case 8: b += (u32)k[7]<<24;
95 case 7: b += (u32)k[6]<<16;
96 case 6: b += (u32)k[5]<<8;
97 case 5: b += k[4];
98 case 4: a += (u32)k[3]<<24;
99 case 3: a += (u32)k[2]<<16;
100 case 2: a += (u32)k[1]<<8;
101 case 1: a += k[0];
102 __jhash_final(a, b, c);
103 case 0: /* Nothing left to add */
104 break;
105 }
106
107 return c;
108 }
109
110 /* jhash2 - hash an array of u32's
111 * @k: the key which must be an array of u32's
112 * @length: the number of u32's in the key
113 * @initval: the previous hash, or an arbitray value
114 *
115 * Returns the hash value of the key.
116 */
117 static inline u32 jhash2(const u32 *k, u32 length, u32 initval)
118 {
119 u32 a, b, c;
120
121 /* Set up the internal state */
122 a = b = c = JHASH_INITVAL + (length<<2) + initval;
123
124 /* Handle most of the key */
125 while (length > 3) {
126 a += k[0];
127 b += k[1];
128 c += k[2];
129 __jhash_mix(a, b, c);
130 length -= 3;
131 k += 3;
132 }
...
vmcore-dmesg.txt:
...
[30611156.397794] BUG: unable to handle kernel NULL pointer dereference at 000000000000005c
[30611156.397866] IP: [<ffffffffc08e5909>] hash_ip4_test+0x29/0x120 [ip_set_hash_ip]
[30611156.397894] PGD 53dac5067 PUD 79d27a067 PMD 0
[30611156.397913] Oops: 0000 [#1] SMP
[30611156.397927] Modules linked in: ext4 mbcache jbd2 binfmt_misc veth vxlan ip6_udp_tunnel ipt_REJECT udp_tunnel nf_reject_ipv4 xt_set ip_set_hash_ip ip_set_hash_net ip_set nf_conntrack_netlink xt_addrtype xt_nat xt_statistic ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_mark xt_comment xt_conntrack iptable_filter iptable_nat nf_nat_ipv4 nf_nat iptable_mangle nf_tables nfnetlink nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack overlay(T) vsock_diag tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag vmw_vsock_vmci_transport vsock sunrpc ppdev iosf_mbi crc32_pclmul vmw_balloon ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi
[30611156.398212] vmwgfx drm_kms_helper syscopyarea sd_mod sysfillrect sysimgblt fb_sys_fops crc_t10dif crct10dif_generic ttm ahci drm ata_piix libahci crct10dif_pclmul crct10dif_common crc32c_intel libata nfit libnvdimm serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
[30611156.398321] CPU: 6 PID: 0 Comm: swapper/6 Kdump: loaded Tainted: G ------------ T 3.10.0-1160.15.2.el7.x86_64 #1
[30611156.398351] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
[30611156.398379] task: ffff885f33b4a100 ti: ffff885f33b58000 task.ti: ffff885f33b58000
[30611156.398401] RIP: 0010:[<ffffffffc08e5909>] [<ffffffffc08e5909>] hash_ip4_test+0x29/0x120 [ip_set_hash_ip]
[30611156.398429] RSP: 0018:ffff8865dd7839b8 EFLAGS: 00010296
[30611156.398446] RAX: 0000000000000000 RBX: ffff88656f363800 RCX: ffff8865dd783aa0
[30611156.398467] RDX: ffff8865dd783a10 RSI: ffff8865dd783a0c RDI: ffff88656f363800
[30611156.398487] RBP: ffff8865dd7839f8 R08: 0000000000000000 R09: ffff8865dd783a10
[30611156.398507] R10: ffff8865dd783aa0 R11: ffff885e7af65b80 R12: ffff8865dd783a98
[30611156.398533] R13: ffff8863c8f22f00 R14: ffff88656f363800 R15: ffff8865dd783a0c
[30611156.398554] FS: 0000000000000000(0000) GS:ffff8865dd780000(0000) knlGS:0000000000000000
[30611156.398576] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[30611156.398593] CR2: 000000000000005c CR3: 000000072c816000 CR4: 00000000007607e0
[30611156.398643] PKRU: 00000000
[30611156.398653] Call Trace:
[30611156.398664] <IRQ>
[30611156.398678] [<ffffffffc0942190>] ? hash_net_create+0x410/0x410 [ip_set_hash_net]
[30611156.398702] [<ffffffffc08e4106>] hash_ip4_kadt+0xb6/0xf0 [ip_set_hash_ip]
[30611156.398725] [<ffffffffc08ffcc0>] ip_set_test+0xb0/0x1b0 [ip_set]
[30611156.398744] [<ffffffffc09a4dc0>] set_match_v4+0xa0/0xe0 [xt_set]
[30611156.398769] [<ffffffffc0545640>] ipt_do_table+0x280/0x740 [ip_tables]
[30611156.398819] [<ffffffffc07e7cc8>] ? tcp_packet+0x3b8/0xa60 [nf_conntrack]
[30611156.398851] [<ffffffffbaea3fb7>] ? local_bh_enable+0x17/0x20
[30611156.398871] [<ffffffffc0878043>] iptable_mangle_hook+0x43/0x130 [iptable_mangle]
[30611156.398897] [<ffffffffbb495a48>] nf_iterate+0x98/0xe0
[30611156.398913] [<ffffffffbb495b38>] nf_hook_slow+0xa8/0x110
[30611156.398929] [<ffffffffbb4a0559>] ip_rcv+0x339/0x420
[30611156.398945] [<ffffffffbb49fa80>] ? inet_del_offload+0x40/0x40
[30611156.398964] [<ffffffffbb455829>] __netif_receive_skb_core+0x729/0xa10
[30611156.398985] [<ffffffffbb4ccd00>] ? tcp4_gro_receive+0x40/0x1a0
[30611156.399004] [<ffffffffbaf07b1f>] ? __getnstimeofday64+0x3f/0xd0
[30611156.399022] [<ffffffffbb455b28>] __netif_receive_skb+0x18/0x60
[30611156.399040] [<ffffffffbb455bb0>] netif_receive_skb_internal+0x40/0xc0
[30611156.399826] [<ffffffffbb456838>] napi_gro_receive+0xd8/0x100
[30611156.400612] [<ffffffffc09ba187>] gro_cell_poll+0x57/0x80 [vxlan]
[30611156.401487] [<ffffffffbb4561cf>] net_rx_action+0x26f/0x390
[30611156.402226] [<ffffffffbaea4b35>] __do_softirq+0xf5/0x280
[30611156.402981] [<ffffffffbb5984ec>] call_softirq+0x1c/0x30
[30611156.403710] [<ffffffffbae2f715>] do_softirq+0x65/0xa0
[30611156.404404] [<ffffffffbaea4eb5>] irq_exit+0x105/0x110
[30611156.405260] [<ffffffffbb599936>] do_IRQ+0x56/0xf0
[30611156.405968] [<ffffffffbb58b36a>] common_interrupt+0x16a/0x16a
[30611156.406645] <EOI>
[30611156.406656] [<ffffffffbb589c70>] ? __cpuidle_text_start+0x8/0x8
[30611156.407913] [<ffffffffbb589ebb>] ? native_safe_halt+0xb/0x20
[30611156.408499] [<ffffffffbb589c8e>] default_idle+0x1e/0xc0
[30611156.409068] [<ffffffffbae37ca0>] arch_cpu_idle+0x20/0xc0
[30611156.409604] [<ffffffffbaf0142a>] cpu_startup_entry+0x14a/0x1e0
[30611156.410112] [<ffffffffbae5a827>] start_secondary+0x1f7/0x270
[30611156.410819] [<ffffffffbae000d5>] start_cpu+0x5/0x14
[30611156.411325] Code: ff 90 0f 1f 44 00 00 55 49 89 ca 49 89 d1 48 89 e5 41 57 49 89 f7 41 56 49 89 fe 41 55 41 54 53 48 83 ec 18 48 8b 87 80 00 00 00 <8b> 48 5c 48 8b 30 41 8b 07 81 e9 0d 41 52 21 89 ca 01 c8 c1 ca
[30611156.412403] RIP [<ffffffffc08e5909>] hash_ip4_test+0x29/0x120 [ip_set_hash_ip]
[30611156.412911] RSP <ffff8865dd7839b8>
[30611156.413496] CR2: 000000000000005c
file net/netfilter/ipset/ip_set_hash_gen.h:
...
986 mtype_test(struct ip_set *set, void *value, const struct ip_set_ext *ext,
987 struct ip_set_ext *mext, u32 flags)
988 {
989 struct htype *h = set->data;
990 struct htable *t;
991 struct mtype_elem *d = value;
992 struct hbucket *n;
993 struct mtype_elem *data;
994 int i, ret = 0;
995 u32 key, multi = 0;
996
997 t = rcu_dereference_bh(h->table);
998 #ifdef IP_SET_HASH_WITH_NETS
999 /* If we test an IP address and not a network address,
1000 * try all possible network sizes
1001 */
...
Based on peter's suggestion, I added some information, I wonder if
can be infer that set->data is the null pointer?
ip_set is the first argument of the mtype_test function.
crash> struct -o ip_set
struct ip_set {
[0x0] char name[32];
[0x20] spinlock_t lock;
[0x24] u32 ref;
[0x28] u32 ref_netlink;
[0x30] struct ip_set_type *type;
[0x38] const struct ip_set_type_variant *variant;
[0x40] u8 family;
[0x41] u8 revision;
[0x42] u8 extensions;
[0x43] u8 flags;
[0x44] u32 timeout;
[0x48] u32 elements;
[0x50] size_t ext_size;
[0x58] size_t dsize;
[0x60] size_t offset[4];
[0x80] void *data;
}
SIZE: 0x88
crash> struct ip_set ffff88656f363800
struct ip_set {
name = "\200\060\066oe\210\377\377T-NODE-IP-V4-tmp\000\000\000\000\000\000\000",
lock = {
{
rlock = {
raw_lock = {
val = {
counter = 0x0
}
}
}
}
},
ref = 0x0,
ref_netlink = 0x0,
type = 0xffffffffc08ea380 <hash_ip_type>,
variant = 0xffffffffc08e9480 <hash_ip4_variant>,
family = 0x2,
revision = 0x4,
extensions = 0x1,
flags = 0x0,
timeout = 0x0,
elements = 0x20,
ext_size = 0xba0,
dsize = 0x10,
offset = {0x0, 0x8, 0x0, 0x0},
data = 0x0
}
crash>
