I am running FreeBSD 14.2 kernel on NanoPi NEO (Allwinner H3 ARMv7 instruction set architecture), and the kernel is built from source code using Git Commit ID: 5395ddd7aa130a8d8063f6eccf893937cc8baf0f.
The phenomenon I encountered was that the system randomly got stuck during operation, the debug serial port no longer printed anything, and the network was disconnected. I suspected that there was a panic.
Then I made some adjustments to the kernel (mainly turning on the EARLY_PRINTF option and enabling DDB), and now I can see the panic information reported by the kernel.
I found that panic seems to be related to pmap, and perhaps to the TF card. I changed to another TF card to boot the system, and the probability of the problem became lower.
I will continue to debug in depth, but I would like to ask if you have encountered similar problems?
Can you provide some valuable information? Thank you!
The phenomenon I encountered was that the system randomly got stuck during operation, the debug serial port no longer printed anything, and the network was disconnected. I suspected that there was a panic.
Then I made some adjustments to the kernel (mainly turning on the EARLY_PRINTF option and enabling DDB), and now I can see the panic information reported by the kernel.
Code:
Fatal kernel mode data abort: 'Translation Fault (L1)' on read
trapframe: 0xd6838898
FSR=00000005, FAR=5c000000, spsr=20000013
r0 =5c000000, r1 =d6fe9000, r2 =00000001, r3 =00000001
r4 =00000000, r5 =00000001, r6 =5c000000, r7 =d6838930
r8 =00000000, r9 =00000000, r10=c08a4363, r11=d6838950
r12=d683892c, ssp=d6838928, slr=c065f9cc, pc =c065fa94
panic: Fatal abort
cpuid = 0
time = 1738910008
KDB: stack backtrace:
db_trace_self() at db_trace_self
pc = 0xc06444e0 lr = 0xc007db18 (db_trace_self_wrapper+0x30)
sp = 0xd6838670 fp = 0xd6838788
db_trace_self_wrapper() at db_trace_self_wrapper+0x30
pc = 0xc007db18 lr = 0xc0307424 (vpanic+0x140)
sp = 0xd6838790 fp = 0xd68387b0
r4 = 0x00000100 r5 = 0x00000000
r6 = 0xc06f13a2 r7 = 0xc08df7a4
vpanic() at vpanic+0x140
pc = 0xc0307424 lr = 0xc03072e4 (vpanic)
sp = 0xd68387b8 fp = 0xd68387bc
r4 = 0xd6838898 r5 = 0x00000013
r6 = 0x5c000000 r7 = 0x00000005
r8 = 0x00000005 r9 = 0x00000013
r10 = 0x5c000000
vpanic() at vpanic
pc = 0xc03072e4 lr = 0xc0669cb4 (abort_align)
sp = 0xd68387c4 fp = 0xd68387f0
r4 = 0x00000005 r5 = 0x00000005
r6 = 0x00000013 r7 = 0x5c000000
r8 = 0xd68387bc r9 = 0xc03072e4
r10 = 0xd68387c4
abort_align() at abort_align
pc = 0xc0669cb4 lr = 0xc06697dc (abort_handler+0x288)
sp = 0xd68387f8 fp = 0xd6838890
r4 = 0xd6fe9000 r10 = 0x5c000000
abort_handler() at abort_handler+0x288
pc = 0xc06697dc lr = 0xc0646d48 (exception_exit)
sp = 0xd6838898 fp = 0xd6838950
r4 = 0x00000000 r5 = 0x00000001
r6 = 0x5c000000 r7 = 0xd6838930
r8 = 0x00000000 r9 = 0x00000000
r10 = 0xc08a4363
exception_exit() at exception_exit
pc = 0xc0646d48 lr = 0xc065f9cc (pmap_page_wired_mappings+0xcc)
sp = 0xd6838928 fp = 0xd6838950
r0 = 0x5c000000 r1 = 0xd6fe9000
r2 = 0x00000001 r3 = 0x00000001
r4 = 0x00000000 r5 = 0x00000001
r6 = 0x5c000000 r7 = 0xd6838930
r8 = 0x00000000 r9 = 0x00000000
r10 = 0xc08a4363 r12 = 0xd683892c
pmap_pvh_wired_mappings() at pmap_pvh_wired_mappings+0x48
pc = 0xc065fa94 lr = 0xc065f9cc (pmap_page_wired_mappings+0xcc)
sp = 0xd6838958 fp = 0xd6838970
r4 = 0xc0d70c30 r5 = 0x00000001
r6 = 0xc08a4363 r7 = 0xc08fd8c0
r8 = 0x00000000 r9 = 0x00000000
r10 = 0x00000036
pmap_page_wired_mappings() at pmap_page_wired_mappings+0xcc
pc = 0xc065f9cc lr = 0xc03e235c (brelse+0x460)
sp = 0xd6838978 fp = 0xd68389b0
r4 = 0xc3361c10 r5 = 0xc0d70c30
r6 = 0x00001000 r7 = 0x00007000
brelse() at brelse+0x460
pc = 0xc03e235c lr = 0xc05b89e8 (trunc_dependencies+0xccc)
sp = 0xd68389b8 fp = 0xd6838a18
r4 = 0x00000800 r5 = 0xffffffff
r6 = 0x00000000 r7 = 0xc3361c10
r8 = 0xd68389f8 r9 = 0x00000000
r10 = 0xd6d18d24
trunc_dependencies() at trunc_dependencies+0xccc
pc = 0xc05b89e8 lr = 0xc05ba470 (softdep_setup_freeblocks+0xbdc)
sp = 0xd6838a20 fp = 0xd6838aa8
r4 = 0xd6838a80 r5 = 0xd4c2f180
r6 = 0xd6dd5b00 r7 = 0xd6b1a660
r8 = 0x00000800 r9 = 0xd6b1a640
r10 = 0xcfe6f580
softdep_setup_freeblocks() at softdep_setup_freeblocks+0xbdc
pc = 0xc05ba470 lr = 0xc05a0678 (ffs_truncate+0x9c0)
sp = 0xd6838ab0 fp = 0xd6838c00
r4 = 0xd6b1a640 r5 = 0xd6d18c90
r6 = 0xd4bb5538 r7 = 0xffffffff
r8 = 0xffffffff r9 = 0x00000c00
r10 = 0x00000000
ffs_truncate() at ffs_truncate+0x9c0
pc = 0xc05a0678 lr = 0xc05e254c (ufs_inactive+0x30c)
sp = 0xd6838c08 fp = 0xd6838c30
r4 = 0xd6d18c90 r5 = 0xd6b1a640
r6 = 0x00000000 r7 = 0xc08a4363
r8 = 0xd4b98800 r9 = 0xd6fe9000
r10 = 0x00000000
ufs_inactive() at ufs_inactive+0x30c
pc = 0xc05e254c lr = 0xc041098c (vinactivef+0xb8)
sp = 0xd6838c38 fp = 0xd6838c50
r4 = 0xd6d18c90 r5 = 0xd6d18cfc
r6 = 0xd6838c38 r7 = 0xc08a4363
r8 = 0xd4b98800 r9 = 0xd6fe9000
vinactivef() at vinactivef+0xb8
pc = 0xc041098c lr = 0xc0410378 (vput_final+0x354)
sp = 0xd6838c58 fp = 0xd6838c70
r4 = 0xd6d18c90 r5 = 0x00000000
r6 = 0x00000000 r7 = 0x00000000
vput_final() at vput_final+0x354
pc = 0xc0410378 lr = 0xc042b560 (vn_close1+0x1d4)
sp = 0xd6838c78 fp = 0xd6838cb0
r4 = 0xd6d18c90 r5 = 0x00000000
r6 = 0x00000000 r7 = 0xc08a4365
vn_close1() at vn_close1+0x1d4
pc = 0xc042b560 lr = 0xc04297ec (vn_closefile+0x40)
sp = 0xd6838cb8 fp = 0xd6838cf0
r4 = 0xd4c51a80 r5 = 0x00000003
r6 = 0xd6d18c90 r7 = 0xd4b98800
r8 = 0xc08a4363 r9 = 0xd4c51a80
r10 = 0xd4c51a80
vn_closefile() at vn_closefile+0x40
pc = 0xc04297ec lr = 0xc02982c8 (_fdrop+0x1c)
sp = 0xd6838cf8 fp = 0xd6838d00
r4 = 0xd4c51a80 r5 = 0x00000000
r6 = 0xd4c51a84 r7 = 0x00000000
_fdrop() at _fdrop+0x1c
pc = 0xc02982c8 lr = 0xc029c124 (closef+0x360)
sp = 0xd6838d08 fp = 0xd6838d68
r4 = 0xd6fe9000 r5 = 0x00000000
closef() at closef+0x360
pc = 0xc029c124 lr = 0xc02989a0 (closefp+0x94)
sp = 0xd6838d70 fp = 0xd6838d88
r4 = 0xd6fe9000 r5 = 0x00000001
r6 = 0x00000001 r7 = 0xd6838d74
r8 = 0xc08a4363 r9 = 0xd6fe9000
r10 = 0xd4c51a80
closefp() at closefp+0x94
pc = 0xc02989a0 lr = 0xc0299738 (kern_close+0x160)
sp = 0xd6838d90 fp = 0xd6838db8
r4 = 0xd4c4c720 r5 = 0x0000000f
r6 = 0xd4c4c410 r7 = 0x0000000f
kern_close() at kern_close+0x160
pc = 0xc0299738 lr = 0xc0668df0 (swi_handler+0x15c)
sp = 0xd6838dc0 fp = 0xd6838e38
r4 = 0xd6fe9000 r5 = 0x00000000
r6 = 0xc084d568 r7 = 0x00000001
r8 = 0xbfbfdadc r9 = 0x00000000
r10 = 0xd4bf8000
swi_handler() at swi_handler+0x15c
pc = 0xc0668df0 lr = 0xc0646cdc (swi_exit)
sp = 0xd6838e40 fp = 0xbfbfdac8
r4 = 0x0000000f r5 = 0x203a6000
r6 = 0x0000000f r7 = 0x00000006
r8 = 0xbfbfdadc r9 = 0x00000000
r10 = 0x00000000
swi_exit() at swi_exit
pc = 0xc0646cdc lr = 0xc0646cdc (swi_exit)
sp = 0xd6838e40 fp = 0xbfbfdac8
KDB: enter: panic
[ thread pid 1422 tid 100159 ]
Stopped at kdb_enter+0x54: ldrb r15, [r15, r15, ror r15]!
db>
Code:
# pkg install stressdisk
The package management tool is not yet installed on your system.
Do you want to fetch and install it now? [y/N]: y
Bootstrapping pkg from pkg+https://pkg.FreeBSD.org/FreeBSD:14:armv7/quarterly, please wait...
Fatal kernel mode data abort: 'Translation Fault (L1)' on read
trapframe: 0xcf1d2b38
FSR=00000005, FAR=5c000000, spsr=20000013
r0 =5c000000, r1 =00000001, r2 =00000001, r3 =5c000000
r4 =c0d704b0, r5 =c08fd8c0, r6 =4000f000, r7 =5c000000
r8 =00000000, r9 =c2200000, r10=00000000, r11=cf1d2bf8
r12=c08a429b, ssp=cf1d2bc8, slr=cf1d2bd8, pc =c06602b8
panic: Fatal abort
cpuid = 2
time = 1738915369
KDB: stack backtrace:
db_trace_self() at db_trace_self
pc = 0xc0644438 lr = 0xc007db18 (db_trace_self_wrapper+0x30)
sp = 0xcf1d2910 fp = 0xcf1d2a28
db_trace_self_wrapper() at db_trace_self_wrapper+0x30
pc = 0xc007db18 lr = 0xc0307424 (vpanic+0x140)
sp = 0xcf1d2a30 fp = 0xcf1d2a50
r4 = 0x00000100 r5 = 0x00000000
r6 = 0xc06f1322 r7 = 0xc08df7a4
vpanic() at vpanic+0x140
pc = 0xc0307424 lr = 0xc03072e4 (vpanic)
sp = 0xcf1d2a58 fp = 0xcf1d2a5c
r4 = 0xcf1d2b38 r5 = 0x00000013
r6 = 0x5c000000 r7 = 0x00000005
r8 = 0x00000005 r9 = 0x00000013
r10 = 0x5c000000
vpanic() at vpanic
pc = 0xc03072e4 lr = 0xc0669c34 (abort_align)
sp = 0xcf1d2a64 fp = 0xcf1d2a90
r4 = 0x00000005 r5 = 0x00000005
r6 = 0x00000013 r7 = 0x5c000000
r8 = 0xcf1d2a5c r9 = 0xc03072e4
r10 = 0xcf1d2a64
abort_align() at abort_align
pc = 0xc0669c34 lr = 0xc066975c (abort_handler+0x288)
sp = 0xcf1d2a98 fp = 0xcf1d2b30
r4 = 0xd2faec00 r10 = 0x5c000000
abort_handler() at abort_handler+0x288
pc = 0xc066975c lr = 0xc0646ca0 (exception_exit)
sp = 0xcf1d2b38 fp = 0xcf1d2bf8
r4 = 0xc0d704b0 r5 = 0xc08fd8c0
r6 = 0x4000f000 r7 = 0x5c000000
r8 = 0x00000000 r9 = 0xc2200000
r10 = 0x00000000
exception_exit() at exception_exit
pc = 0xc0646ca0 lr = 0xcf1d2bd8 (0xcf1d2bd8)
sp = 0xcf1d2bc8 fp = 0xcf1d2bf8
r0 = 0x5c000000 r1 = 0x00000001
r2 = 0x00000001 r3 = 0x5c000000
r4 = 0xc0d704b0 r5 = 0xc08fd8c0
r6 = 0x4000f000 r7 = 0x5c000000
r8 = 0x00000000 r9 = 0xc2200000
r10 = 0x00000000 r12 = 0xc08a429b
pmap_ts_referenced() at pmap_ts_referenced+0xe4
pc = 0xc06602b8 lr = 0xc062be10 (vm_pageout_worker+0xf9c)
sp = 0xcf1d2c00 fp = 0xcf1d2dd0
r4 = 0xc0d704b0 r5 = 0xc08f9900
r6 = 0xd674e8f0 r7 = 0x00000000
r8 = 0xd2faec00 r9 = 0x0000ff8d
r10 = 0x00000001
vm_pageout_worker() at vm_pageout_worker+0xf9c
pc = 0xc062be10 lr = 0xc062ae28 (vm_pageout+0x204)
sp = 0xcf1d2dd8 fp = 0xcf1d2e10
r4 = 0xc22f0000 r5 = 0x00000001
r6 = 0xc08f94c0 r7 = 0xc062c428
r8 = 0x00000000 r9 = 0xc06ea3ba
r10 = 0x00000000
vm_pageout() at vm_pageout+0x204
pc = 0xc062ae28 lr = 0xc02b97d0 (fork_exit+0xc0)
sp = 0xcf1d2e18 fp = 0xcf1d2e38
r4 = 0xd2faec00 r5 = 0xc22f0000
r6 = 0xc062ac24 r7 = 0x00000000
r8 = 0xcf1d2e40 r9 = 0xc08a429d
r10 = 0xc08cf200
fork_exit() at fork_exit+0xc0
pc = 0xc02b97d0 lr = 0xc0646c34 (swi_exit)
sp = 0xcf1d2e40 fp = 0x00000000
r4 = 0xc062ac24 r5 = 0x00000000
r6 = 0x7ff6d861 r7 = 0xd21b4c00
r8 = 0xc08ef3fc r10 = 0xc08cf200
swi_exit() at swi_exit
pc = 0xc0646c34 lr = 0xc0646c34 (swi_exit)
sp = 0xcf1d2e40 fp = 0x00000000
KDB: enter: panic
[ thread pid 7 tid 100081 ]
Stopped at kdb_enter+0x54: ldrb r15, [r15, r15, ror r15]!
db>
Code:
Number of packages to be installed: 1
The process will require 2 MiB more space.
819 KiB to be downloaded.
Proceed with this action? [y/N]: y
[1/1] Fetching stressdisk-1.0.13_10.pkg: 100% 819 KiB 167.7kB/s 00:05
Checking integrity... done (0 conflicting)
[1/1] Installing stressdisk-1.0.13_10...
[1/1] Extracting stressdisk-1.0.13_10: 100%
Fatal kernel mode data abort: 'Translation Fault (L1)' on read
trapframe: 0xd4af9808
FSR=00000005, FAR=8c000000, spsr=a0000013
r0 =8c000000, r1 =d4b4ac00, r2 =00000001, r3 =00000001
r4 =00000000, r5 =00000001, r6 =8c000000, r7 =d4af98a0
r8 =00000000, r9 =00000000, r10=c08a429b, r11=d4af98c0
r12=d4af989c, ssp=d4af9898, slr=c065f92c, pc =c065f9f4
panic: Fatal abort
cpuid = 1
time = 1738916584
KDB: stack backtrace:
db_trace_self() at db_trace_self
pc = 0xc0644438 lr = 0xc007db18 (db_trace_self_wrapper+0x30)
sp = 0xd4af95e0 fp = 0xd4af96f8
db_trace_self_wrapper() at db_trace_self_wrapper+0x30
pc = 0xc007db18 lr = 0xc0307424 (vpanic+0x140)
sp = 0xd4af9700 fp = 0xd4af9720
r4 = 0x00000100 r5 = 0x00000000
r6 = 0xc06f1322 r7 = 0xc08df7a4
vpanic() at vpanic+0x140
pc = 0xc0307424 lr = 0xc03072e4 (vpanic)
sp = 0xd4af9728 fp = 0xd4af972c
r4 = 0xd4af9808 r5 = 0x00000013
r6 = 0x8c000000 r7 = 0x00000005
r8 = 0x00000005 r9 = 0x00000013
r10 = 0x8c000000
vpanic() at vpanic
pc = 0xc03072e4 lr = 0xc0669c34 (abort_align)
sp = 0xd4af9734 fp = 0xd4af9760
r4 = 0x00000005 r5 = 0x00000005
r6 = 0x00000013 r7 = 0x8c000000
r8 = 0xd4af972c r9 = 0xc03072e4
r10 = 0xd4af9734
abort_align() at abort_align
pc = 0xc0669c34 lr = 0xc066975c (abort_handler+0x288)
sp = 0xd4af9768 fp = 0xd4af9800
r4 = 0xd4b4ac00 r10 = 0x8c000000
abort_handler() at abort_handler+0x288
pc = 0xc066975c lr = 0xc0646ca0 (exception_exit)
sp = 0xd4af9808 fp = 0xd4af98c0
r4 = 0x00000000 r5 = 0x00000001
r6 = 0x8c000000 r7 = 0xd4af98a0
r8 = 0x00000000 r9 = 0x00000000
r10 = 0xc08a429b
exception_exit() at exception_exit
pc = 0xc0646ca0 lr = 0xc065f92c (pmap_page_wired_mappings+0xcc)
sp = 0xd4af9898 fp = 0xd4af98c0
r0 = 0x8c000000 r1 = 0xd4b4ac00
r2 = 0x00000001 r3 = 0x00000001
r4 = 0x00000000 r5 = 0x00000001
r6 = 0x8c000000 r7 = 0xd4af98a0
r8 = 0x00000000 r9 = 0x00000000
r10 = 0xc08a429b r12 = 0xd4af989c
pmap_pvh_wired_mappings() at pmap_pvh_wired_mappings+0x48
pc = 0xc065f9f4 lr = 0xc065f92c (pmap_page_wired_mappings+0xcc)
sp = 0xd4af98c8 fp = 0xd4af98e0
r4 = 0xc0d74fb0 r5 = 0x00000001
r6 = 0xc08a429b r7 = 0xc08fd8c0
r8 = 0x00000000 r9 = 0x00000000
r10 = 0x0000003a
pmap_page_wired_mappings() at pmap_page_wired_mappings+0xcc
pc = 0xc065f92c lr = 0xc03e22b4 (brelse+0x460)
sp = 0xd4af98e8 fp = 0xd4af9920
r4 = 0xc330b7c0 r5 = 0xc0d74fb0
r6 = 0x00001000 r7 = 0x00003000
brelse() at brelse+0x460
pc = 0xc03e22b4 lr = 0xc05b8940 (trunc_dependencies+0xccc)
sp = 0xd4af9928 fp = 0xd4af9988
r4 = 0x00000800 r5 = 0xffffffff
r6 = 0x00000000 r7 = 0xc330b7c0
r8 = 0xd4af9968 r9 = 0x00000000
r10 = 0xd6f1d094
trunc_dependencies() at trunc_dependencies+0xccc
pc = 0xc05b8940 lr = 0xc05ba3c8 (softdep_setup_freeblocks+0xbdc)
sp = 0xd4af9990 fp = 0xd4af9a18
r4 = 0xd4af99f0 r5 = 0xc2262780
r6 = 0xd6e08300 r7 = 0xd6de5520
r8 = 0x00000800 r9 = 0xd6de5500
r10 = 0xd4b2ec00
softdep_setup_freeblocks() at softdep_setup_freeblocks+0xbdc
pc = 0xc05ba3c8 lr = 0xc05a05d0 (ffs_truncate+0x9c0)
sp = 0xd4af9a20 fp = 0xd4af9b70
r4 = 0xd6de5500 r5 = 0xd6f1d000
r6 = 0xd4bac538 r7 = 0xffffffff
r8 = 0xffffffff r9 = 0x00000c00
r10 = 0x00000000
ffs_truncate() at ffs_truncate+0x9c0
pc = 0xc05a05d0 lr = 0xc05e24a4 (ufs_inactive+0x30c)
sp = 0xd4af9b78 fp = 0xd4af9ba0
r4 = 0xd6f1d000 r5 = 0xd6de5500
r6 = 0x00000000 r7 = 0xc08a429b
r8 = 0x00000000 r9 = 0xd4af9cf8
r10 = 0xd6f1d000
ufs_inactive() at ufs_inactive+0x30c
pc = 0xc05e24a4 lr = 0xc04108e4 (vinactivef+0xb8)
sp = 0xd4af9ba8 fp = 0xd4af9bc0
r4 = 0xd6f1d000 r5 = 0xd6f1d06c
r6 = 0xd4af9ba8 r7 = 0xc08a429b
r8 = 0x00000000 r9 = 0xd4af9cf8
vinactivef() at vinactivef+0xb8
pc = 0xc04108e4 lr = 0xc04102d0 (vput_final+0x354)
sp = 0xd4af9bc8 fp = 0xd4af9be0
r4 = 0xd6f1d000 r5 = 0x00000001
r6 = 0x00000000 r7 = 0xc08a429b
vput_final() at vput_final+0x354
pc = 0xc04102d0 lr = 0xc0421a2c (kern_funlinkat+0x29c)
sp = 0xd4af9be8 fp = 0xd4af9d98
r4 = 0x00000000 r5 = 0x00000000
r6 = 0x20f61ed0 r7 = 0x00000005
kern_funlinkat() at kern_funlinkat+0x29c
pc = 0xc0421a2c lr = 0xc0421ca4 (sys_unlinkat+0x58)
sp = 0xd4af9da0 fp = 0xd4af9db8
r4 = 0xd4b4ac00 r5 = 0x20f61ed0
r6 = 0xc0850340 r7 = 0x00000001
r8 = 0x20f61e60 r9 = 0x00000000
r10 = 0xd4bdd000
sys_unlinkat() at sys_unlinkat+0x58
pc = 0xc0421ca4 lr = 0xc0668d70 (swi_handler+0x15c)
sp = 0xd4af9dc0 fp = 0xd4af9e38
r5 = 0x00000000 r10 = 0xd4bdd000
swi_handler() at swi_handler+0x15c
pc = 0xc0668d70 lr = 0xc0646c34 (swi_exit)
sp = 0xd4af9e40 fp = 0xbfbfd640
r4 = 0x20f61ecf r5 = 0x00000000
r6 = 0x00377504 r7 = 0x000001f7
r8 = 0x20f61e60 r9 = 0x00000000
r10 = 0x20f61c88
swi_exit() at swi_exit
pc = 0xc0646c34 lr = 0xc0646c34 (swi_exit)
sp = 0xd4af9e40 fp = 0xbfbfd640
KDB: enter: panic
[ thread pid 1322 tid 100167 ]
Stopped at kdb_enter+0x54: ldrb r15, [r15, r15, ror r15]!
db>I found that panic seems to be related to pmap, and perhaps to the TF card. I changed to another TF card to boot the system, and the probability of the problem became lower.
I will continue to debug in depth, but I would like to ask if you have encountered similar problems?
Can you provide some valuable information? Thank you!