TrueNAS NFS random crash

TrueNAS General
,
1

Hi, everyone. I got an problem while I uses TrueNAS scale.

The hardware is:

  1. OS Version:TrueNAS-SCALE-24.04.0
  2. Product:RS540-E9-RS36-E
  3. Model:Intel(R) Xeon(R) Gold 6138 CPU @ 2.00GHz
  4. RAM: 128G ECC
  5. 40G Mellanox NIC
  6. 16T HDD disk * 18
  7. HBA card

I setup a NFS server and there is 5 client, all client is ubuntu 22.04 with 10G Mellanox NIC

This is the mount option from the client:

nfs rw,noatime,rsize=1048576,wsize=1048576,nofail,nodev,nosuid,_netdev,soft 0 0

There is no other service on the TrueNAS server, only NFS setup.

And the server just crash randomly.

This is the syndrome while NFS server crashing:
Client side:

  1. Mount nfs:
    • Just stuck and no response
  2. Already mounted nfs:
    • Just no response.
    • No fail message at dmesg, it seems server is still responsive, but not respond any read/write request

Server side:

  1. /usr/sbin/rpc.nfsd stuck at D state. Not able to kill
  2. No error message at dmesg -T
  3. Not able to restart nfs-server service. The restart command will stuck
  4. In the netdata->NFS server->filehandles, We can observe the stale counter grows very rapidly
  5. Not able to reslove this state unless reboot the NFS server.

I just want a stable NFS server which allows 5 client to access it’s data

It’s really annoying that the server just crash every week, I can’t do anything except reboot the TrueNAS scale.

Any suggesstion? Like where I can get additional logs, or how to resolve this issue?

2

I’m seeing issues which may be similar too. Will check the handles thing, but have been seeing the rpc failures too. Even in the nightlies.

FWIW, Cobia (23.10.2) works fine for me.

3

Update at 2024/8/1
It crashed again.

In this creah, there 5 client, and one of the client are not able to connect to the server

What am i tries:

From client:

  1. dmesg -T not showing any error message related to nfs
  2. ls /data/nas07 will cause infifity hang, and the ls process becomes D state

Attemp 1: ls with strace

Result 1: stucks at AT_FDCWD
Logs 1: strace ls at client

# strace ls /data/nas07
execve("/usr/bin/ls", ["ls", "/data/nas07"], 0x7ffdc925e188 /* 32 vars */) = 0
brk(NULL)                               = 0x592527cf9000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffddf02c780) = -1 EINVAL (Invalid argument)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7bee415ff000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=34323, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 34323, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7bee415f6000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=166280, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 177672, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7bee415ca000
mprotect(0x7bee415d0000, 139264, PROT_NONE) = 0
mmap(0x7bee415d0000, 106496, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7bee415d0000
mmap(0x7bee415ea000, 28672, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x20000) = 0x7bee415ea000
mmap(0x7bee415f2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x27000) = 0x7bee415f2000
mmap(0x7bee415f4000, 5640, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7bee415f4000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\237\2\0\0\0\0\0"..., 832) = 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0 \0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0"..., 48, 848) = 48
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0I\17\357\204\3$\f\221\2039x\324\224\323\236S"..., 68, 896) = 68
newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=2220400, ...}, AT_EMPTY_PATH) = 0
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
mmap(NULL, 2264656, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7bee41200000
mprotect(0x7bee41228000, 2023424, PROT_NONE) = 0
mmap(0x7bee41228000, 1658880, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x28000) = 0x7bee41228000
mmap(0x7bee413bd000, 360448, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bd000) = 0x7bee413bd000
mmap(0x7bee41416000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x215000) = 0x7bee41416000
mmap(0x7bee4141c000, 52816, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7bee4141c000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=613064, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 615184, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7bee41533000
mmap(0x7bee41535000, 438272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7bee41535000
mmap(0x7bee415a0000, 163840, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6d000) = 0x7bee415a0000
mmap(0x7bee415c8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x94000) = 0x7bee415c8000
close(3)                                = 0
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7bee41530000
arch_prctl(ARCH_SET_FS, 0x7bee41530800) = 0
set_tid_address(0x7bee41530ad0)         = 135008
set_robust_list(0x7bee41530ae0, 24)     = 0
rseq(0x7bee415311a0, 0x20, 0, 0x53053053) = 0
mprotect(0x7bee41416000, 16384, PROT_READ) = 0
mprotect(0x7bee415c8000, 4096, PROT_READ) = 0
mprotect(0x7bee415f2000, 4096, PROT_READ) = 0
mprotect(0x59252690f000, 4096, PROT_READ) = 0
mprotect(0x7bee41639000, 8192, PROT_READ) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
munmap(0x7bee415f6000, 34323)           = 0
statfs("/sys/fs/selinux", 0x7ffddf02c7c0) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7ffddf02c7c0)      = -1 ENOENT (No such file or directory)
getrandom("\xb6\xba\xe2\x00\xe6\x5e\x1e\xb5", 8, GRND_NONBLOCK) = 8
brk(NULL)                               = 0x592527cf9000
brk(0x592527d1a000)                     = 0x592527d1a000
openat(AT_FDCWD, "/proc/filesystems", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0444, st_size=0, ...}, AT_EMPTY_PATH) = 0
read(3, "nodev\tsysfs\nnodev\ttmpfs\nnodev\tbd"..., 1024) = 466
read(3, "", 1024)                       = 0
close(3)                                = 0
access("/etc/selinux/config", F_OK)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=3048928, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 3048928, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7bee40e00000
close(3)                                = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TIOCGWINSZ, {ws_row=34, ws_col=237, ws_xpixel=0, ws_ypixel=0}) = 0
statx(AT_FDCWD, "/data/nas07", AT_STATX_SYNC_AS_STAT, STATX_MODE,

Logs 1: tcpdump at truenas server:
The client just keep sending nfs packet, no matter what I am doing such as ls or etc.

root@deepserver-03 ~# tcpdump -XX -i any "host 172.22.5.7"
tcpdump: data link type LINUX_SLL2
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on any, link-type LINUX_SLL2 (Linux cooked v2), snapshot length 262144 bytes
06:35:28.318308 enp10s0 Out IP deepserver-03.848 > 172.22.5.7.nfs: Flags [.], ack 3621257462, win 24576, options [nop,nop,TS val 1451083031 ecr 3644682304], length 0
        0x0000:  0800 0000 0000 0004 0001 0406 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 2405 4000 4006 b588  M...E..4$.@.@...
        0x0020:  ac16 0403 ac16 0507 0350 0801 def2 9f5f  .........P....._
        0x0030:  d7d8 00f6 8010 6000 615d 0000 0101 080a  ......`.a]......
        0x0040:  567d c517 d93d 7040                      V}...=p@
06:35:28.318378 enp10s0 In  IP 172.22.5.7.nfs > deepserver-03.848: Flags [.], ack 1, win 14784, options [nop,nop,TS val 3644713024 ecr 1407061674], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0034 beaf 4000 4006 1ade  4...E..4..@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0350 d7d8 00f6  ...........P....
        0x0030:  def2 9f60 8010 39c0 554c 0000 0101 080a  ...`..9.UL......
        0x0040:  d93d e840 53de 0eaa                      .=.@S...
06:35:33.438274 enp10s0 Out ARP, Request who-has 172.22.5.7 tell deepserver-03, length 28
        0x0000:  0806 0000 0000 0004 0001 0406 6c92 bf13  ............l...
        0x0010:  4dca 0000 0001 0800 0604 0001 6c92 bf13  M...........l...
        0x0020:  4dca ac16 0403 0000 0000 0000 ac16 0507  M...............
06:35:33.438320 enp10s0 In  ARP, Reply 172.22.5.7 is-at e4:1d:2d:df:34:80 (oui Unknown), length 46
        0x0000:  0806 0000 0000 0004 0001 0006 e41d 2ddf  ..............-.
        0x0010:  3480 0000 0001 0800 0604 0002 e41d 2ddf  4.............-.
        0x0020:  3480 ac16 0507 6c92 bf13 4dca ac16 0403  4.....l...M.....
        0x0030:  0000 0000 0000 0000 0000 0000 0000 0000  ................
        0x0040:  0000                                     ..

Attemp 2: block connection from client

Result: the client still sending nfs and ignores all my requests such as ls /data/nas07

command at server:

iptables -A INPUT -s 172.22.4.3 -j REJECT

Logs 2-1: from truenas server:

listening on any, link-type LINUX_SLL2 (Linux cooked v2), snapshot length 262144 bytes
14:46:44.155020 enp138s0 In  IP 172.22.4.3.848 > 172.22.5.7.nfs: Flags [.], ack 3621257462, win 24576, options [nop,nop,TS val 1451758871 ecr 3645327424], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 241b 4000 4006 b572  M...E..4$.@.@..r
        0x0020:  ac16 0403 ac16 0507 0350 0801 def2 9f5f  .........P....._
        0x0030:  d7d8 00f6 8010 6000 c5ec 0000 0101 080a  ......`.........
        0x0040:  5688 1517 d947 4840                      V....GH@
14:46:44.155063 enp138s0 Out IP 172.22.5.7 > 172.22.4.3: ICMP 172.22.5.7 tcp port nfs unreachable, length 60
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 45c0 0050 eff8 0000 4001 28be  4...E..P....@.(.
        0x0020:  ac16 0507 ac16 0403 0303 5e5a 0000 0000  ..........^Z....
        0x0030:  4500 0034 241b 4000 4006 b572 ac16 0403  E..4$.@.@..r....
        0x0040:  ac16 0507 0350 0801 def2 9f5f d7d8 00f6  .....P....._....
        0x0050:  8010 6000 c5ec 0000 0101 080a 5688 1517  ..`.........V...
        0x0060:  d947 4840                                .GH@
14:46:49.275007 enp138s0 In  ARP, Request who-has 172.22.5.7 tell 172.22.4.3, length 46
        0x0000:  0806 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 0001 0800 0604 0001 6c92 bf13  M...........l...
        0x0020:  4dca ac16 0403 0000 0000 0000 ac16 0507  M...............
        0x0030:  0000 0000 0000 0000 0000 0000 0000 0000  ................
        0x0040:  0000                                     ..
14:46:49.275027 enp138s0 Out ARP, Reply 172.22.5.7 is-at e4:1d:2d:df:34:80 (oui Unknown), length 28
        0x0000:  0806 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 0001 0800 0604 0002 e41d 2ddf  4.............-.
        0x0020:  3480 ac16 0507 6c92 bf13 4dca ac16 0403  4.....l...M.....
14:47:14.875004 enp138s0 In  IP 172.22.4.3.848 > 172.22.5.7.nfs: Flags [R.], seq 1, ack 1, win 24576, options [nop,nop,TS val 1451789591 ecr 3645327424], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 241c 4000 4006 b571  M...E..4$.@.@..q
        0x0020:  ac16 0403 ac16 0507 0350 0801 def2 9f60  .........P.....`
        0x0030:  d7d8 00f6 8014 6000 4de7 0000 0101 080a  ......`.M.......
        0x0040:  5688 8d17 d947 4840                      V....GH@
14:47:14.875056 enp138s0 Out IP 172.22.5.7 > 172.22.4.3: ICMP 172.22.5.7 tcp port nfs unreachable, length 60
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 45c0 0050 0cac 0000 4001 0c0b  4...E..P....@...
        0x0020:  ac16 0507 ac16 0403 0303 5e5a 0000 0000  ..........^Z....
        0x0030:  4500 0034 241c 4000 4006 b571 ac16 0403  E..4$.@.@..q....
        0x0040:  ac16 0507 0350 0801 def2 9f60 d7d8 00f6  .....P.....`....
        0x0050:  8014 6000 4de7 0000 0101 080a 5688 8d17  ..`.M.......V...
        0x0060:  d947 4840                                .GH@

Logs 2-2 from client with dmesg -T
After 5 minutes, the client finally shows nfs error message

[Thu Aug  1 06:48:47 2024] nfs: server 172.22.5.7 not responding, timed out
[Thu Aug  1 06:48:47 2024] nfs: server 172.22.5.7 not responding, timed out
[Thu Aug  1 06:48:47 2024] nfs: server 172.22.5.7 not responding, timed out
[Thu Aug  1 06:48:47 2024] nfs: server 172.22.5.7 not responding, timed out
[Thu Aug  1 06:48:47 2024] nfs: server 172.22.5.7 not responding, timed out
[Thu Aug  1 06:48:47 2024] nfs: server 172.22.5.7 not responding, timed out
[Thu Aug  1 06:48:47 2024] nfs: server 172.22.5.7 not responding, timed out
[Thu Aug  1 06:48:47 2024] nfs: server 172.22.5.7 not responding, timed out
[Thu Aug  1 06:48:47 2024] nfs: server 172.22.5.7 not responding, timed out

Attemp 3: Allowing connection ,and remount from client

# TrueNAS
iptables -D INPUT -s 172.22.4.3 -j REJECT

Result: remount nfs again from client, still stucked

Logs 3: strace mount from client

# strace mount -t nfs -o rw,noatime,rsize=1048576,wsize=1048576,nofail,nodev,nosuid,_netdev,soft 172.22.5.
7:/mnt/MainPool/DS/PublicData /data/nas07

execve("/usr/bin/mount", ["mount", "-t", "nfs", "-o", "rw,noatime,rsize=1048576,wsize=1"..., "172.22.5.7:/mnt/MainPool/DS/Publ"..., "/data/nas07"], 0x7ffc54bf9ae0 /* 32 vars */) = 0
brk(NULL)                               = 0x651b3f3e4000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffc6eb32d10) = -1 EINVAL (Invalid argument)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x79f1c1427000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=34323, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 34323, PROT_READ, MAP_PRIVATE, 3, 0) = 0x79f1c141e000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libmount.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=273064, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 274984, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x79f1c13da000
mprotect(0x79f1c13e2000, 237568, PROT_NONE) = 0
mmap(0x79f1c13e2000, 180224, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8000) = 0x79f1c13e2000
mmap(0x79f1c140e000, 53248, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x34000) = 0x79f1c140e000
mmap(0x79f1c141c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x41000) = 0x79f1c141c000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=166280, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 177672, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x79f1c13ae000
mprotect(0x79f1c13b4000, 139264, PROT_NONE) = 0
mmap(0x79f1c13b4000, 106496, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x79f1c13b4000
mmap(0x79f1c13ce000, 28672, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x20000) = 0x79f1c13ce000
mmap(0x79f1c13d6000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x27000) = 0x79f1c13d6000
mmap(0x79f1c13d8000, 5640, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x79f1c13d8000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\237\2\0\0\0\0\0"..., 832) = 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0 \0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0"..., 48, 848) = 48
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0I\17\357\204\3$\f\221\2039x\324\224\323\236S"..., 68, 896) = 68
newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=2220400, ...}, AT_EMPTY_PATH) = 0
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
mmap(NULL, 2264656, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x79f1c1000000
mprotect(0x79f1c1028000, 2023424, PROT_NONE) = 0
mmap(0x79f1c1028000, 1658880, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x28000) = 0x79f1c1028000
mmap(0x79f1c11bd000, 360448, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bd000) = 0x79f1c11bd000
mmap(0x79f1c1216000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x215000) = 0x79f1c1216000
mmap(0x79f1c121c000, 52816, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x79f1c121c000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libblkid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=220208, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 222128, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x79f1c1377000
mprotect(0x79f1c137e000, 172032, PROT_NONE) = 0
mmap(0x79f1c137e000, 131072, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x79f1c137e000
mmap(0x79f1c139e000, 36864, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x27000) = 0x79f1c139e000
mmap(0x79f1c13a8000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x30000) = 0x79f1c13a8000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=613064, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 615184, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x79f1c12e0000
mmap(0x79f1c12e2000, 438272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x79f1c12e2000
mmap(0x79f1c134d000, 163840, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6d000) = 0x79f1c134d000
mmap(0x79f1c1375000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x94000) = 0x79f1c1375000
close(3)                                = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x79f1c12de000
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x79f1c12db000
arch_prctl(ARCH_SET_FS, 0x79f1c12db800) = 0
set_tid_address(0x79f1c12dbad0)         = 178400
set_robust_list(0x79f1c12dbae0, 24)     = 0
rseq(0x79f1c12dc1a0, 0x20, 0, 0x53053053) = 0
mprotect(0x79f1c1216000, 16384, PROT_READ) = 0
mprotect(0x79f1c1375000, 4096, PROT_READ) = 0
mprotect(0x79f1c13a8000, 20480, PROT_READ) = 0
mprotect(0x79f1c13d6000, 4096, PROT_READ) = 0
mprotect(0x79f1c141c000, 4096, PROT_READ) = 0
mprotect(0x651b3ebb0000, 4096, PROT_READ) = 0
mprotect(0x79f1c1461000, 8192, PROT_READ) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
munmap(0x79f1c141e000, 34323)           = 0
statfs("/sys/fs/selinux", 0x7ffc6eb32d50) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7ffc6eb32d50)      = -1 ENOENT (No such file or directory)
getrandom("\x79\x10\xbd\x05\xb0\x49\x85\x2c", 8, GRND_NONBLOCK) = 8
brk(NULL)                               = 0x651b3f3e4000
brk(0x651b3f405000)                     = 0x651b3f405000
openat(AT_FDCWD, "/proc/filesystems", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0444, st_size=0, ...}, AT_EMPTY_PATH) = 0
read(3, "nodev\tsysfs\nnodev\ttmpfs\nnodev\tbd"..., 1024) = 466
read(3, "", 1024)                       = 0
close(3)                                = 0
access("/etc/selinux/config", F_OK)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=3048928, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 3048928, PROT_READ, MAP_PRIVATE, 3, 0) = 0x79f1c0c00000
close(3)                                = 0
getuid()                                = 0
geteuid()                               = 0
access("/sys/fs/smackfs", F_OK)         = -1 ENOENT (No such file or directory)
readlink("/data", 0x7ffc6eb31160, 1023) = -1 EINVAL (Invalid argument)
readlink("/data/nas07", 0x7ffc6eb31160, 1023) = -1 EINVAL (Invalid argument)
newfstatat(AT_FDCWD, "/sbin/mount.nfs", {st_mode=S_IFREG|S_ISUID|0755, st_size=101048, ...}, 0) = 0
getuid()                                = 0
geteuid()                               = 0
getgid()                                = 0
getegid()                               = 0
prctl(PR_GET_DUMPABLE)                  = 1 (SUID_DUMP_USER)
newfstatat(AT_FDCWD, "/run", {st_mode=S_IFDIR|0755, st_size=1420, ...}, 0) = 0
newfstatat(AT_FDCWD, "/run/mount/utab", {st_mode=S_IFREG|0644, st_size=1482, ...}, AT_SYMLINK_NOFOLLOW) = 0
newfstatat(AT_FDCWD, "/run/mount/utab", {st_mode=S_IFREG|0644, st_size=1482, ...}, 0) = 0
geteuid()                               = 0
getegid()                               = 0
getuid()                                = 0
getgid()                                = 0
access("/run/mount/utab", R_OK|W_OK)    = 0
clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x79f1c12dbad0) = 178404
wait4(178404,

Logs 3-2: tcpdump at truenas server: still not get request from client, only get regular nfs packets.

root@truenas[/home/admin]# tcpdump -XX -i any "host 172.22.4.3"
tcpdump: data link type LINUX_SLL2
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on any, link-type LINUX_SLL2 (Linux cooked v2), snapshot length 262144 bytes
15:00:52.033002 enp138s0 In  IP 172.22.4.3.695 > 172.22.5.7.nfs: Flags [.], ack 1018003136, win 502, options [nop,nop,TS val 1452606743 ecr 3646206022], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 36b9 4000 4006 a2d4  M...E..46.@.@...
        0x0020:  ac16 0403 ac16 0507 02b7 0801 889b c9e8  ................
        0x0030:  3cad 7ec0 8010 01f6 159f 0000 0101 080a  <.~.............
        0x0040:  5695 0517 d954 b046                      V....T.F
15:00:52.033033 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.695: Flags [.], ack 1, win 243, options [nop,nop,TS val 3646236742 ecr 1452206935], length 0
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0034 7f6f 4000 4006 5a1e  4...E..4.o@.@.Z.
        0x0020:  ac16 0507 ac16 0403 0801 02b7 3cad 7ec0  ............<.~.
        0x0030:  889b c9e9 8010 00f3 615d 0000 0101 080a  ........a]......
        0x0040:  d955 2846 568e eb57                      .U(FV..W
15:00:57.063844 enp138s0 Out ARP, Request who-has 172.22.4.3 tell 172.22.5.7, length 28
        0x0000:  0806 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 0001 0800 0604 0001 e41d 2ddf  4.............-.
        0x0020:  3480 ac16 0507 0000 0000 0000 ac16 0403  4...............
15:00:57.063897 enp138s0 In  ARP, Reply 172.22.4.3 is-at 6c:92:bf:13:4d:ca (oui Unknown), length 46
        0x0000:  0806 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 0001 0800 0604 0002 6c92 bf13  M...........l...
        0x0020:  4dca ac16 0403 e41d 2ddf 3480 ac16 0507  M.......-.4.....
        0x0030:  0000 0000 0000 0000 0000 0000 0000 0000  ................
        0x0040:  0000                                     ..
4

Attemp 4: reboot the client:

Result: still get hang

But according to Logs 4-2, the client send nfs packet normally, But the truenas not respond correctly causes client hang.

Logs 4-1: strace mount from client

strace mount -t nfs -o rw,noatime,rsize=1048576,wsize=1048576,nofail,nodev,nosuid,_netdev,soft 172.22.5.7:/mnt/MainPool/DS/PublicData /data/nas07
execve("/usr/bin/mount", ["mount", "-t", "nfs", "-o", "rw,noatime,rsize=1048576,wsize=1"..., "172.22.5.7:/mnt/MainPool/DS/Publ"..., "/data/nas07"], 0x7ffcba0a1060 /* 32 vars */) = 0
brk(NULL)                               = 0x606b00d31000
arch_prctl(0x3001 /* ARCH_??? */, 0x7fffce1874a0) = -1 EINVAL (Invalid argument)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x787dac990000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=34323, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 34323, PROT_READ, MAP_PRIVATE, 3, 0) = 0x787dac987000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libmount.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=273064, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 274984, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x787dac943000
mprotect(0x787dac94b000, 237568, PROT_NONE) = 0
mmap(0x787dac94b000, 180224, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8000) = 0x787dac94b000
mmap(0x787dac977000, 53248, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x34000) = 0x787dac977000
mmap(0x787dac985000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x41000) = 0x787dac985000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=166280, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 177672, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x787dac917000
mprotect(0x787dac91d000, 139264, PROT_NONE) = 0
mmap(0x787dac91d000, 106496, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x787dac91d000
mmap(0x787dac937000, 28672, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x20000) = 0x787dac937000
mmap(0x787dac93f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x27000) = 0x787dac93f000
mmap(0x787dac941000, 5640, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x787dac941000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\237\2\0\0\0\0\0"..., 832) = 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0 \0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0"..., 48, 848) = 48
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0I\17\357\204\3$\f\221\2039x\324\224\323\236S"..., 68, 896) = 68
newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=2220400, ...}, AT_EMPTY_PATH) = 0
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
mmap(NULL, 2264656, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x787dac600000
mprotect(0x787dac628000, 2023424, PROT_NONE) = 0
mmap(0x787dac628000, 1658880, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x28000) = 0x787dac628000
mmap(0x787dac7bd000, 360448, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bd000) = 0x787dac7bd000
mmap(0x787dac816000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x215000) = 0x787dac816000
mmap(0x787dac81c000, 52816, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x787dac81c000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libblkid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=220208, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 222128, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x787dac8e0000
mprotect(0x787dac8e7000, 172032, PROT_NONE) = 0
mmap(0x787dac8e7000, 131072, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x787dac8e7000
mmap(0x787dac907000, 36864, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x27000) = 0x787dac907000
mmap(0x787dac911000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x30000) = 0x787dac911000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=613064, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 615184, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x787dac849000
mmap(0x787dac84b000, 438272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x787dac84b000
mmap(0x787dac8b6000, 163840, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6d000) = 0x787dac8b6000
mmap(0x787dac8de000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x94000) = 0x787dac8de000
close(3)                                = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x787dac847000
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x787dac844000
arch_prctl(ARCH_SET_FS, 0x787dac844800) = 0
set_tid_address(0x787dac844ad0)         = 24190
set_robust_list(0x787dac844ae0, 24)     = 0
rseq(0x787dac8451a0, 0x20, 0, 0x53053053) = 0
mprotect(0x787dac816000, 16384, PROT_READ) = 0
mprotect(0x787dac8de000, 4096, PROT_READ) = 0
mprotect(0x787dac911000, 20480, PROT_READ) = 0
mprotect(0x787dac93f000, 4096, PROT_READ) = 0
mprotect(0x787dac985000, 4096, PROT_READ) = 0
mprotect(0x606b00c5f000, 4096, PROT_READ) = 0
mprotect(0x787dac9ca000, 8192, PROT_READ) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
munmap(0x787dac987000, 34323)           = 0
statfs("/sys/fs/selinux", 0x7fffce1874e0) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7fffce1874e0)      = -1 ENOENT (No such file or directory)
getrandom("\x36\xe1\xde\xf5\x9c\xc0\xb4\x41", 8, GRND_NONBLOCK) = 8
brk(NULL)                               = 0x606b00d31000
brk(0x606b00d52000)                     = 0x606b00d52000
openat(AT_FDCWD, "/proc/filesystems", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0444, st_size=0, ...}, AT_EMPTY_PATH) = 0
read(3, "nodev\tsysfs\nnodev\ttmpfs\nnodev\tbd"..., 1024) = 466
read(3, "", 1024)                       = 0
close(3)                                = 0
access("/etc/selinux/config", F_OK)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=3048928, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 3048928, PROT_READ, MAP_PRIVATE, 3, 0) = 0x787dac200000
close(3)                                = 0
getuid()                                = 0
geteuid()                               = 0
access("/sys/fs/smackfs", F_OK)         = -1 ENOENT (No such file or directory)
readlink("/data", 0x7fffce1858f0, 1023) = -1 EINVAL (Invalid argument)
readlink("/data/nas07", 0x7fffce1858f0, 1023) = -1 EINVAL (Invalid argument)
newfstatat(AT_FDCWD, "/sbin/mount.nfs", {st_mode=S_IFREG|S_ISUID|0755, st_size=101048, ...}, 0) = 0
getuid()                                = 0
geteuid()                               = 0
getgid()                                = 0
getegid()                               = 0
prctl(PR_GET_DUMPABLE)                  = 1 (SUID_DUMP_USER)
newfstatat(AT_FDCWD, "/run", {st_mode=S_IFDIR|0755, st_size=1300, ...}, 0) = 0
newfstatat(AT_FDCWD, "/run/mount/utab", {st_mode=S_IFREG|0644, st_size=1482, ...}, AT_SYMLINK_NOFOLLOW) = 0
newfstatat(AT_FDCWD, "/run/mount/utab", {st_mode=S_IFREG|0644, st_size=1482, ...}, 0) = 0
geteuid()                               = 0
getegid()                               = 0
getuid()                                = 0
getgid()                                = 0
access("/run/mount/utab", R_OK|W_OK)    = 0
clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x787dac844ad0) = 24191
wait4(24191,

Logs 4-2: tcpdump fom truenas server

listening on any, link-type LINUX_SLL2 (Linux cooked v2), snapshot length 262144 bytes
15:12:22.229194 enp138s0 B   ARP, Request who-has 172.22.5.7 tell 172.22.4.3, length 46
        0x0000:  0806 0000 0000 0004 0001 0106 6c92 bf13  ............l...
        0x0010:  4dca 0000 0001 0800 0604 0001 6c92 bf13  M...........l...
        0x0020:  4dca ac16 0403 0000 0000 0000 ac16 0507  M...............
        0x0030:  0000 0000 0000 0000 0000 0000 0000 0000  ................
        0x0040:  0000                                     ..
15:12:22.229210 enp138s0 Out ARP, Reply 172.22.5.7 is-at e4:1d:2d:df:34:80 (oui Unknown), length 28
        0x0000:  0806 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 0001 0800 0604 0002 e41d 2ddf  4.............-.
        0x0020:  3480 ac16 0507 6c92 bf13 4dca ac16 0403  4.....l...M.....
15:12:22.229314 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [S], seq 1452212832, win 64240, options [mss 1460,sackOK,TS val 3917017947 ecr 0,nop,wscale 7], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 003c be3e 4000 4006 1b47  M...E..<.>@.@..G
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0260  ............V..`
        0x0030:  0000 0000 a002 faf0 aa8c 0000 0204 05b4  ................
        0x0040:  0402 080a e978 f35b 0000 0000 0103 0307  .....x.[........
15:12:22.229370 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [S.], seq 1164082070, ack 1452212833, win 31856, options [mss 1460,sackOK,TS val 3646926938 ecr 3917017947,nop,wscale 7], length 0
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 003c 0000 4000 4006 d985  4...E..<..@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7b96  ............Eb{.
        0x0030:  568f 0261 a012 7c70 6165 0000 0204 05b4  V..a..|pae......
        0x0040:  0402 080a d95f b05a e978 f35b 0103 0307  ....._.Z.x.[....
15:12:22.229427 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 1, win 502, options [nop,nop,TS val 3917017947 ecr 3646926938], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 be3f 4000 4006 1b4e  M...E..4.?@.@..N
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0261  ............V..a
        0x0030:  4562 7b97 8010 01f6 878f 0000 0101 080a  Eb{.............
        0x0040:  e978 f35b d95f b05a                      .x.[._.Z
15:12:22.229468 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 1:45, ack 1, win 502, options [nop,nop,TS val 3917017947 ecr 3646926938], length 44: NFS request xid 1633660111 40 null
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0060 be40 4000 4006 1b21  M...E..`.@@.@..!
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0261  ............V..a
        0x0030:  4562 7b97 8018 01f6 7259 0000 0101 080a  Eb{.....rY......
        0x0040:  e978 f35b d95f b05a 8000 0028 615f accf  .x.[._.Z...(a_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0000 0000 0000 0000 0000 0000 0000  ................
        0x0070:  0000 0000                                ....
15:12:22.229484 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [.], ack 45, win 249, options [nop,nop,TS val 3646926938 ecr 3917017947], length 0
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0034 0c65 4000 4006 cd28  4...E..4.e@.@..(
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7b97  ............Eb{.
        0x0030:  568f 028d 8010 00f9 615d 0000 0101 080a  V.......a]......
        0x0040:  d95f b05a e978 f35b                      ._.Z.x.[
15:12:22.229539 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 1:29, ack 45, win 249, options [nop,nop,TS val 3646926938 ecr 3917017947], length 28: NFS reply xid 1633660111 reply ok 24 null
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0050 0c66 4000 4006 cd0b  4...E..P.f@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7b97  ............Eb{.
        0x0030:  568f 028d 8018 00f9 6179 0000 0101 080a  V.......ay......
        0x0040:  d95f b05a e978 f35b 8000 0018 615f accf  ._.Z.x.[....a_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000                                ....
15:12:22.229584 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 29, win 502, options [nop,nop,TS val 3917017948 ecr 3646926938], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 be41 4000 4006 1b4c  M...E..4.A@.@..L
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 028d  ............V...
        0x0030:  4562 7bb3 8010 01f6 8746 0000 0101 080a  Eb{......F......
        0x0040:  e978 f35c d95f b05a                      .x.\._.Z
15:12:22.229724 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 45:321, ack 29, win 502, options [nop,nop,TS val 3917017948 ecr 3646926938], length 276: NFS request xid 1650437327 272 getattr fh 0,2/42
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0148 be42 4000 4006 1a37  M...E..H.B@.@..7
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 028d  ............V...
        0x0030:  4562 7bb3 8018 01f6 ca6c 0000 0101 080a  Eb{......l......
        0x0040:  e978 f35c d95f b05a 8000 0110 625f accf  .x.\._.Z....b_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002a 17e7 88f9 1766 5d86  .......*.....f].
        0x00b0:  0000 001b 4c69 6e75 7820 4e46 5376 342e  ....Linux.NFSv4.
        0x00c0:  3220 6465 6570 7365 7276 6572 2d30 3300  2.deepserver-03.
        0x00d0:  0000 0103 0000 0000 0000 0001 0000 000a  ................
        0x00e0:  6b65 726e 656c 2e6f 7267 0000 0000 005e  kernel.org.....^
        0x00f0:  4c69 6e75 7820 362e 352e 302d 3434 2d67  Linux.6.5.0-44-g
        0x0100:  656e 6572 6963 2023 3434 7e32 322e 3034  eneric.#44~22.04
        0x0110:  2e31 2d55 6275 6e74 7520 534d 5020 5052  .1-Ubuntu.SMP.PR
        0x0120:  4545 4d50 545f 4459 4e41 4d49 4320 5475  EEMPT_DYNAMIC.Tu
        0x0130:  6520 4a75 6e20 3138 2031 343a 3336 3a31  e.Jun.18.14:36:1
        0x0140:  3620 5554 4320 3220 7838 365f 3634 0000  6.UTC.2.x86_64..
        0x0150:  0000 0000 0000 0000 0000 0000            ............
15:12:22.229847 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 29:133, ack 321, win 249, options [nop,nop,TS val 3646926939 ecr 3917017948], length 104: NFS reply xid 1650437327 reply ok 100 getattr NON 1 ids 0/181905510 sz 3011753937
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 009c 0c67 4000 4006 ccbe  4...E....g@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7bb3  ............Eb{.
        0x0030:  568f 03a1 8018 00f9 61c5 0000 0101 080a  V.......a.......
        0x0040:  d95f b05b e978 f35c 8000 0064 625f accf  ._.[.x.\...db_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 0000 0000 0000 0000 0001  ................
        0x0070:  0000 002a 0000 0000 0ad7 a866 b383 b7d1  ...*.......f....
        0x0080:  0000 0001 0002 0001 0000 0000 0000 0000  ................
        0x0090:  0000 0000 0000 0007 7472 7565 6e61 7300  ........truenas.
        0x00a0:  0000 0007 7472 7565 6e61 7300 0000 0000  ....truenas.....
15:12:22.229998 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 321:597, ack 133, win 502, options [nop,nop,TS val 3917017948 ecr 3646926939], length 276: NFS request xid 1667214543 272 getattr fh 0,2/42
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0148 be43 4000 4006 1a36  M...E..H.C@.@..6
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 03a1  ............V...
        0x0030:  4562 7c1b 8018 01f6 c7ef 0000 0101 080a  Eb|.............
        0x0040:  e978 f35c d95f b05b 8000 0110 635f accf  .x.\._.[....c_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002a 17e7 88f9 1766 5d86  .......*.....f].
        0x00b0:  0000 001b 4c69 6e75 7820 4e46 5376 342e  ....Linux.NFSv4.
        0x00c0:  3220 6465 6570 7365 7276 6572 2d30 3300  2.deepserver-03.
        0x00d0:  0000 0103 0000 0000 0000 0001 0000 000a  ................
        0x00e0:  6b65 726e 656c 2e6f 7267 0000 0000 005e  kernel.org.....^
        0x00f0:  4c69 6e75 7820 362e 352e 302d 3434 2d67  Linux.6.5.0-44-g
        0x0100:  656e 6572 6963 2023 3434 7e32 322e 3034  eneric.#44~22.04
        0x0110:  2e31 2d55 6275 6e74 7520 534d 5020 5052  .1-Ubuntu.SMP.PR
        0x0120:  4545 4d50 545f 4459 4e41 4d49 4320 5475  EEMPT_DYNAMIC.Tu
        0x0130:  6520 4a75 6e20 3138 2031 343a 3336 3a31  e.Jun.18.14:36:1
        0x0140:  3620 5554 4320 3220 7838 365f 3634 0000  6.UTC.2.x86_64..
        0x0150:  0000 0000 0000 0000 0000 0000            ............
15:12:22.230097 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 133:237, ack 597, win 249, options [nop,nop,TS val 3646926939 ecr 3917017948], length 104: NFS reply xid 1667214543 reply ok 100 getattr NON 1 ids 0/181905510 sz 3028531153
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 009c 0c68 4000 4006 ccbd  4...E....h@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7c1b  ............Eb|.
        0x0030:  568f 04b5 8018 00f9 61c5 0000 0101 080a  V.......a.......
        0x0040:  d95f b05b e978 f35c 8000 0064 635f accf  ._.[.x.\...dc_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 0000 0000 0000 0000 0001  ................
        0x0070:  0000 002a 0000 0000 0ad7 a866 b483 b7d1  ...*.......f....
        0x0080:  0000 0001 0002 0001 0000 0000 0000 0000  ................
        0x0090:  0000 0000 0000 0007 7472 7565 6e61 7300  ........truenas.
        0x00a0:  0000 0007 7472 7565 6e61 7300 0000 0000  ....truenas.....
15:12:22.230195 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 597:813, ack 237, win 502, options [nop,nop,TS val 3917017948 ecr 3646926939], length 216: NFS request xid 1683991759 212 getattr fh 0,2/43
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 010c be44 4000 4006 1a71  M...E....D@.@..q
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 04b5  ............V...
        0x0030:  4562 7c83 8018 01f6 656c 0000 0101 080a  Eb|.....el......
        0x0040:  e978 f35c d95f b05b 8000 00d4 645f accf  .x.\._.[....d_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002b 0ad7 a866 b483 b7d1  .......+...f....
        0x00b0:  0000 0001 0000 0003 0000 0000 0010 0414  ................
        0x00c0:  0010 0388 0000 1da0 0000 0008 0000 0040  ...............@
        0x00d0:  0000 0000 0000 0000 0000 1000 0000 1000  ................
        0x00e0:  0000 0000 0000 0002 0000 0010 0000 0000  ................
        0x00f0:  4000 0000 0000 0001 0000 0001 1766 5d86  @............f].
        0x0100:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0110:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
15:12:22.230290 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 237:285, ack 813, win 249, options [nop,nop,TS val 3646926939 ecr 3917017948], length 48: NFS reply xid 1683991759 reply ok 44 getattr ERROR: Request couldn't be completed in time
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0064 0c69 4000 4006 ccf4  4...E..d.i@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7c83  ............Eb|.
        0x0030:  568f 058d 8018 00f9 618d 0000 0101 080a  V.......a.......
        0x0040:  d95f b05b e978 f35c 8000 002c 645f accf  ._.[.x.\...,d_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 2718 0000 0000 0000 0001  ......'.........
        0x0070:  0000 002b 0000 2718                      ...+..'.
15:12:22.273576 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 285, win 502, options [nop,nop,TS val 3917017992 ecr 3646926939], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 be45 4000 4006 1b48  M...E..4.E@.@..H
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 058d  ............V...
        0x0030:  4562 7cb3 8010 01f6 8319 0000 0101 080a  Eb|.............
        0x0040:  e978 f388 d95f b05b                      .x..._.[
15:12:23.257614 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 813:1029, ack 285, win 502, options [nop,nop,TS val 3917018976 ecr 3646926939], length 216: NFS request xid 1700768975 212 getattr fh 0,2/43
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 010c be46 4000 4006 1a6f  M...E....F@.@..o
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 058d  ............V...
        0x0030:  4562 7cb3 8018 01f6 5f60 0000 0101 080a  Eb|....._`......
        0x0040:  e978 f760 d95f b05b 8000 00d4 655f accf  .x.`._.[....e_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002b 0ad7 a866 b483 b7d1  .......+...f....
        0x00b0:  0000 0001 0000 0003 0000 0000 0010 0414  ................
        0x00c0:  0010 0388 0000 1da0 0000 0008 0000 0040  ...............@
        0x00d0:  0000 0000 0000 0000 0000 1000 0000 1000  ................
        0x00e0:  0000 0000 0000 0002 0000 0010 0000 0000  ................
        0x00f0:  4000 0000 0000 0001 0000 0001 1766 5d86  @............f].
        0x0100:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0110:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
15:12:23.257762 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 285:333, ack 1029, win 249, options [nop,nop,TS val 3646927966 ecr 3917018976], length 48: NFS reply xid 1700768975 reply ok 44 getattr ERROR: Request couldn't be completed in time
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0064 0c6a 4000 4006 ccf3  4...E..d.j@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7cb3  ............Eb|.
        0x0030:  568f 0665 8018 00f9 618d 0000 0101 080a  V..e....a.......
        0x0040:  d95f b45e e978 f760 8000 002c 655f accf  ._.^.x.`...,e_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 2718 0000 0000 0000 0001  ......'.........
        0x0070:  0000 002b 0000 2718                      ...+..'.
15:12:23.257818 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 333, win 502, options [nop,nop,TS val 3917018976 ecr 3646927966], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 be47 4000 4006 1b46  M...E..4.G@.@..F
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0665  ............V..e
        0x0030:  4562 7ce3 8010 01f6 7a36 0000 0101 080a  Eb|.....z6......
        0x0040:  e978 f760 d95f b45e                      .x.`._.^
15:12:24.281642 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 1029:1245, ack 333, win 502, options [nop,nop,TS val 3917020000 ecr 3646927966], length 216: NFS request xid 1717546191 212 getattr fh 0,2/43
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 010c be48 4000 4006 1a6d  M...E....H@.@..m
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0665  ............V..e
        0x0030:  4562 7ce3 8018 01f6 5555 0000 0101 080a  Eb|.....UU......
        0x0040:  e978 fb60 d95f b45e 8000 00d4 665f accf  .x.`._.^....f_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002b 0ad7 a866 b483 b7d1  .......+...f....
        0x00b0:  0000 0001 0000 0003 0000 0000 0010 0414  ................
        0x00c0:  0010 0388 0000 1da0 0000 0008 0000 0040  ...............@
        0x00d0:  0000 0000 0000 0000 0000 1000 0000 1000  ................
        0x00e0:  0000 0000 0000 0002 0000 0010 0000 0000  ................
        0x00f0:  4000 0000 0000 0001 0000 0001 1766 5d86  @............f].
        0x0100:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0110:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
15:12:24.281827 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 333:381, ack 1245, win 249, options [nop,nop,TS val 3646928991 ecr 3917020000], length 48: NFS reply xid 1717546191 reply ok 44 getattr ERROR: Request couldn't be completed in time
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0064 0c6b 4000 4006 ccf2  4...E..d.k@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7ce3  ............Eb|.
        0x0030:  568f 073d 8018 00f9 618d 0000 0101 080a  V..=....a.......
        0x0040:  d95f b85f e978 fb60 8000 002c 665f accf  ._._.x.`...,f_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 2718 0000 0000 0000 0001  ......'.........
        0x0070:  0000 002b 0000 2718                      ...+..'.
15:12:24.281880 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 381, win 502, options [nop,nop,TS val 3917020000 ecr 3646928991], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 be49 4000 4006 1b44  M...E..4.I@.@..D
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 073d  ............V..=
        0x0030:  4562 7d13 8010 01f6 712d 0000 0101 080a  Eb}.....q-......
        0x0040:  e978 fb60 d95f b85f                      .x.`._._
15:12:25.305613 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 1245:1461, ack 381, win 502, options [nop,nop,TS val 3917021024 ecr 3646928991], length 216: NFS request xid 1734323407 212 getattr fh 0,2/43
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 010c be4a 4000 4006 1a6b  M...E....J@.@..k
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 073d  ............V..=
        0x0030:  4562 7d13 8018 01f6 4b4c 0000 0101 080a  Eb}.....KL......
        0x0040:  e978 ff60 d95f b85f 8000 00d4 675f accf  .x.`._._....g_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002b 0ad7 a866 b483 b7d1  .......+...f....
        0x00b0:  0000 0001 0000 0003 0000 0000 0010 0414  ................
        0x00c0:  0010 0388 0000 1da0 0000 0008 0000 0040  ...............@
        0x00d0:  0000 0000 0000 0000 0000 1000 0000 1000  ................
        0x00e0:  0000 0000 0000 0002 0000 0010 0000 0000  ................
        0x00f0:  4000 0000 0000 0001 0000 0001 1766 5d86  @............f].
        0x0100:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0110:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
15:12:25.305799 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 381:429, ack 1461, win 249, options [nop,nop,TS val 3646930015 ecr 3917021024], length 48: NFS reply xid 1734323407 reply ok 44 getattr ERROR: Request couldn't be completed in time
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0064 0c6c 4000 4006 ccf1  4...E..d.l@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7d13  ............Eb}.
        0x0030:  568f 0815 8018 00f9 618d 0000 0101 080a  V.......a.......
        0x0040:  d95f bc5f e978 ff60 8000 002c 675f accf  ._._.x.`...,g_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 2718 0000 0000 0000 0001  ......'.........
        0x0070:  0000 002b 0000 2718                      ...+..'.
15:12:25.305865 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 429, win 502, options [nop,nop,TS val 3917021024 ecr 3646930015], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 be4b 4000 4006 1b42  M...E..4.K@.@..B
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0815  ............V...
        0x0030:  4562 7d43 8010 01f6 6825 0000 0101 080a  Eb}C....h%......
        0x0040:  e978 ff60 d95f bc5f                      .x.`._._
15:12:26.329652 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 1461:1677, ack 429, win 502, options [nop,nop,TS val 3917022048 ecr 3646930015], length 216: NFS request xid 1751100623 212 getattr fh 0,2/43
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 010c be4c 4000 4006 1a69  M...E....L@.@..i
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0815  ............V...
        0x0030:  4562 7d43 8018 01f6 4144 0000 0101 080a  Eb}C....AD......
        0x0040:  e979 0360 d95f bc5f 8000 00d4 685f accf  .y.`._._....h_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002b 0ad7 a866 b483 b7d1  .......+...f....
        0x00b0:  0000 0001 0000 0003 0000 0000 0010 0414  ................
        0x00c0:  0010 0388 0000 1da0 0000 0008 0000 0040  ...............@
        0x00d0:  0000 0000 0000 0000 0000 1000 0000 1000  ................
        0x00e0:  0000 0000 0000 0002 0000 0010 0000 0000  ................
        0x00f0:  4000 0000 0000 0001 0000 0001 1766 5d86  @............f].
        0x0100:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0110:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
15:12:26.329831 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 429:477, ack 1677, win 249, options [nop,nop,TS val 3646931039 ecr 3917022048], length 48: NFS reply xid 1751100623 reply ok 44 getattr ERROR: Request couldn't be completed in time
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0064 0c6d 4000 4006 ccf0  4...E..d.m@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7d43  ............Eb}C
        0x0030:  568f 08ed 8018 00f9 618d 0000 0101 080a  V.......a.......
        0x0040:  d95f c05f e979 0360 8000 002c 685f accf  ._._.y.`...,h_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 2718 0000 0000 0000 0001  ......'.........
        0x0070:  0000 002b 0000 2718                      ...+..'.
15:12:26.329887 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 477, win 502, options [nop,nop,TS val 3917022048 ecr 3646931039], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 be4d 4000 4006 1b40  M...E..4.M@.@..@
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 08ed  ............V...
        0x0030:  4562 7d73 8010 01f6 5f1d 0000 0101 080a  Eb}s...._.......
        0x0040:  e979 0360 d95f c05f                      .y.`._._
15:12:27.353613 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 1677:1893, ack 477, win 502, options [nop,nop,TS val 3917023072 ecr 3646931039], length 216: NFS request xid 1767877839 212 getattr fh 0,2/43
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 010c be4e 4000 4006 1a67  M...E....N@.@..g
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 08ed  ............V...
        0x0030:  4562 7d73 8018 01f6 373c 0000 0101 080a  Eb}s....7<......
        0x0040:  e979 0760 d95f c05f 8000 00d4 695f accf  .y.`._._....i_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002b 0ad7 a866 b483 b7d1  .......+...f....
        0x00b0:  0000 0001 0000 0003 0000 0000 0010 0414  ................
        0x00c0:  0010 0388 0000 1da0 0000 0008 0000 0040  ...............@
        0x00d0:  0000 0000 0000 0000 0000 1000 0000 1000  ................
        0x00e0:  0000 0000 0000 0002 0000 0010 0000 0000  ................
        0x00f0:  4000 0000 0000 0001 0000 0001 1766 5d86  @............f].
        0x0100:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0110:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
15:12:27.353761 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 477:525, ack 1893, win 249, options [nop,nop,TS val 3646932062 ecr 3917023072], length 48: NFS reply xid 1767877839 reply ok 44 getattr ERROR: Request couldn't be completed in time
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0064 0c6e 4000 4006 ccef  4...E..d.n@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7d73  ............Eb}s
        0x0030:  568f 09c5 8018 00f9 618d 0000 0101 080a  V.......a.......
        0x0040:  d95f c45e e979 0760 8000 002c 695f accf  ._.^.y.`...,i_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 2718 0000 0000 0000 0001  ......'.........
        0x0070:  0000 002b 0000 2718                      ...+..'.
15:12:27.353822 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 525, win 502, options [nop,nop,TS val 3917023072 ecr 3646932062], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 be4f 4000 4006 1b3e  M...E..4.O@.@..>
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 09c5  ............V...
        0x0030:  4562 7da3 8010 01f6 5616 0000 0101 080a  Eb}.....V.......
        0x0040:  e979 0760 d95f c45e                      .y.`._.^
15:12:28.377611 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 1893:2109, ack 525, win 502, options [nop,nop,TS val 3917024096 ecr 3646932062], length 216: NFS request xid 1784655055 212 getattr fh 0,2/43
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 010c be50 4000 4006 1a65  M...E....P@.@..e
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 09c5  ............V...
        0x0030:  4562 7da3 8018 01f6 2d35 0000 0101 080a  Eb}.....-5......
        0x0040:  e979 0b60 d95f c45e 8000 00d4 6a5f accf  .y.`._.^....j_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002b 0ad7 a866 b483 b7d1  .......+...f....
        0x00b0:  0000 0001 0000 0003 0000 0000 0010 0414  ................
        0x00c0:  0010 0388 0000 1da0 0000 0008 0000 0040  ...............@
        0x00d0:  0000 0000 0000 0000 0000 1000 0000 1000  ................
        0x00e0:  0000 0000 0000 0002 0000 0010 0000 0000  ................
        0x00f0:  4000 0000 0000 0001 0000 0001 1766 5d86  @............f].
        0x0100:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0110:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
15:12:28.377794 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 525:573, ack 2109, win 249, options [nop,nop,TS val 3646933087 ecr 3917024096], length 48: NFS reply xid 1784655055 reply ok 44 getattr ERROR: Request couldn't be completed in time
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 8429 4500 0064 0c6f 4000 4006 ccee  4..)E..d.o@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7da3  ............Eb}.
        0x0030:  568f 0a9d 8018 00f9 618d 0000 0101 080a  V.......a.......
        0x0040:  d95f c85f e979 0b60 8000 002c 6a5f accf  ._._.y.`...,j_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 2718 0000 0000 0000 0001  ......'.........
        0x0070:  0000 002b 0000 2718                      ...+..'.
15:12:28.377851 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 573, win 502, options [nop,nop,TS val 3917024096 ecr 3646933087], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 2eaf 4500 0034 be51 4000 4006 1b3c  M...E..4.Q@.@..<
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0a9d  ............V...
        0x0030:  4562 7dd3 8010 01f6 4d0d 0000 0101 080a  Eb}.....M.......
        0x0040:  e979 0b60 d95f c85f                      .y.`._._
15:12:29.401567 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 2109:2325, ack 573, win 502, options [nop,nop,TS val 3917025120 ecr 3646933087], length 216: NFS request xid 1801432271 212 getattr fh 0,2/43
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 010c be52 4000 4006 1a63  M...E....R@.@..c
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0a9d  ............V...
        0x0030:  4562 7dd3 8018 01f6 232c 0000 0101 080a  Eb}.....#,......
        0x0040:  e979 0f60 d95f c85f 8000 00d4 6b5f accf  .y.`._._....k_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002b 0ad7 a866 b483 b7d1  .......+...f....
        0x00b0:  0000 0001 0000 0003 0000 0000 0010 0414  ................
        0x00c0:  0010 0388 0000 1da0 0000 0008 0000 0040  ...............@
        0x00d0:  0000 0000 0000 0000 0000 1000 0000 1000  ................
        0x00e0:  0000 0000 0000 0002 0000 0010 0000 0000  ................
        0x00f0:  4000 0000 0000 0001 0000 0001 1766 5d86  @............f].
        0x0100:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0110:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
15:12:29.401757 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 573:621, ack 2325, win 249, options [nop,nop,TS val 3646934110 ecr 3917025120], length 48: NFS reply xid 1801432271 reply ok 44 getattr ERROR: Request couldn't be completed in time
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 f35b 4500 0064 0c70 4000 4006 cced  4..[E..d.p@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7dd3  ............Eb}.
        0x0030:  568f 0b75 8018 00f9 618d 0000 0101 080a  V..u....a.......
        0x0040:  d95f cc5e e979 0f60 8000 002c 6b5f accf  ._.^.y.`...,k_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 2718 0000 0000 0000 0001  ......'.........
        0x0070:  0000 002b 0000 2718                      ...+..'.
15:12:29.401809 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 621, win 502, options [nop,nop,TS val 3917025120 ecr 3646934110], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 6800 4500 0034 be53 4000 4006 1b3a  M.h.E..4.S@.@..:
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0b75  ............V..u
        0x0030:  4562 7e03 8010 01f6 4406 0000 0101 080a  Eb~.....D.......
        0x0040:  e979 0f60 d95f cc5e                      .y.`._.^
15:12:30.425611 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 2325:2541, ack 621, win 502, options [nop,nop,TS val 3917026144 ecr 3646934110], length 216: NFS request xid 1818209487 212 getattr fh 0,2/43
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 010c be54 4000 4006 1a61  M...E....T@.@..a
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0b75  ............V..u
        0x0030:  4562 7e03 8018 01f6 1925 0000 0101 080a  Eb~......%......
        0x0040:  e979 1360 d95f cc5e 8000 00d4 6c5f accf  .y.`._.^....l_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002b 0ad7 a866 b483 b7d1  .......+...f....
        0x00b0:  0000 0001 0000 0003 0000 0000 0010 0414  ................
        0x00c0:  0010 0388 0000 1da0 0000 0008 0000 0040  ...............@
        0x00d0:  0000 0000 0000 0000 0000 1000 0000 1000  ................
        0x00e0:  0000 0000 0000 0002 0000 0010 0000 0000  ................
        0x00f0:  4000 0000 0000 0001 0000 0001 1766 5d86  @............f].
        0x0100:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0110:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
15:12:30.425744 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 621:669, ack 2541, win 249, options [nop,nop,TS val 3646935134 ecr 3917026144], length 48: NFS reply xid 1818209487 reply ok 44 getattr ERROR: Request couldn't be completed in time
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0064 0c71 4000 4006 ccec  4...E..d.q@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7e03  ............Eb~.
        0x0030:  568f 0c4d 8018 00f9 618d 0000 0101 080a  V..M....a.......
        0x0040:  d95f d05e e979 1360 8000 002c 6c5f accf  ._.^.y.`...,l_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 2718 0000 0000 0000 0001  ......'.........
        0x0070:  0000 002b 0000 2718                      ...+..'.
15:12:30.425797 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 669, win 502, options [nop,nop,TS val 3917026144 ecr 3646935134], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 be55 4000 4006 1b38  M...E..4.U@.@..8
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0c4d  ............V..M
        0x0030:  4562 7e33 8010 01f6 3afe 0000 0101 080a  Eb~3....:.......
        0x0040:  e979 1360 d95f d05e                      .y.`._.^
15:12:31.449617 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 2541:2757, ack 669, win 502, options [nop,nop,TS val 3917027168 ecr 3646935134], length 216: NFS request xid 1834986703 212 getattr fh 0,2/43
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 010c be56 4000 4006 1a5f  M...E....V@.@.._
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0c4d  ............V..M
        0x0030:  4562 7e33 8018 01f6 0f1d 0000 0101 080a  Eb~3............
        0x0040:  e979 1760 d95f d05e 8000 00d4 6d5f accf  .y.`._.^....m_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002b 0ad7 a866 b483 b7d1  .......+...f....
        0x00b0:  0000 0001 0000 0003 0000 0000 0010 0414  ................
        0x00c0:  0010 0388 0000 1da0 0000 0008 0000 0040  ...............@
        0x00d0:  0000 0000 0000 0000 0000 1000 0000 1000  ................
        0x00e0:  0000 0000 0000 0002 0000 0010 0000 0000  ................
        0x00f0:  4000 0000 0000 0001 0000 0001 1766 5d86  @............f].
        0x0100:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0110:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
15:12:31.449800 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 669:717, ack 2757, win 249, options [nop,nop,TS val 3646936159 ecr 3917027168], length 48: NFS reply xid 1834986703 reply ok 44 getattr ERROR: Request couldn't be completed in time
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0064 0c72 4000 4006 cceb  4...E..d.r@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7e33  ............Eb~3
        0x0030:  568f 0d25 8018 00f9 618d 0000 0101 080a  V..%....a.......
        0x0040:  d95f d45f e979 1760 8000 002c 6d5f accf  ._._.y.`...,m_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 2718 0000 0000 0000 0001  ......'.........
        0x0070:  0000 002b 0000 2718                      ...+..'.
15:12:31.449856 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 717, win 502, options [nop,nop,TS val 3917027168 ecr 3646936159], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 be57 4000 4006 1b36  M...E..4.W@.@..6
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0d25  ............V..%
        0x0030:  4562 7e63 8010 01f6 31f5 0000 0101 080a  Eb~c....1.......
        0x0040:  e979 1760 d95f d45f                      .y.`._._
15:12:32.473615 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 2757:2973, ack 717, win 502, options [nop,nop,TS val 3917028192 ecr 3646936159], length 216: NFS request xid 1851763919 212 getattr fh 0,2/43
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 010c be58 4000 4006 1a5d  M...E....X@.@..]
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0d25  ............V..%
        0x0030:  4562 7e63 8018 01f6 0514 0000 0101 080a  Eb~c............
        0x0040:  e979 1b60 d95f d45f 8000 00d4 6e5f accf  .y.`._._....n_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002b 0ad7 a866 b483 b7d1  .......+...f....
        0x00b0:  0000 0001 0000 0003 0000 0000 0010 0414  ................
        0x00c0:  0010 0388 0000 1da0 0000 0008 0000 0040  ...............@
        0x00d0:  0000 0000 0000 0000 0000 1000 0000 1000  ................
        0x00e0:  0000 0000 0000 0002 0000 0010 0000 0000  ................
        0x00f0:  4000 0000 0000 0001 0000 0001 1766 5d86  @............f].
        0x0100:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0110:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
15:12:32.473721 enp138s0 Out IP 172.22.5.7.nfs > 172.22.4.3.903: Flags [P.], seq 717:765, ack 2973, win 249, options [nop,nop,TS val 3646937182 ecr 3917028192], length 48: NFS reply xid 1851763919 reply ok 44 getattr ERROR: Request couldn't be completed in time
        0x0000:  0800 0000 0000 0004 0001 0406 e41d 2ddf  ..............-.
        0x0010:  3480 0000 4500 0064 0c73 4000 4006 ccea  4...E..d.s@.@...
        0x0020:  ac16 0507 ac16 0403 0801 0387 4562 7e63  ............Eb~c
        0x0030:  568f 0dfd 8018 00f9 618d 0000 0101 080a  V.......a.......
        0x0040:  d95f d85e e979 1b60 8000 002c 6e5f accf  ._.^.y.`...,n_..
        0x0050:  0000 0001 0000 0000 0000 0000 0000 0000  ................
        0x0060:  0000 0000 0000 2718 0000 0000 0000 0001  ......'.........
        0x0070:  0000 002b 0000 2718                      ...+..'.
15:12:32.473774 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [.], ack 765, win 502, options [nop,nop,TS val 3917028192 ecr 3646937182], length 0
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 0034 be59 4000 4006 1b34  M...E..4.Y@.@..4
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0dfd  ............V...
        0x0030:  4562 7e93 8010 01f6 28ee 0000 0101 080a  Eb~.....(.......
        0x0040:  e979 1b60 d95f d85e                      .y.`._.^
15:12:33.497660 enp138s0 In  IP 172.22.4.3.903 > 172.22.5.7.nfs: Flags [P.], seq 2973:3189, ack 765, win 502, options [nop,nop,TS val 3917029216 ecr 3646937182], length 216: NFS request xid 1868541135 212 getattr fh 0,2/43
        0x0000:  0800 0000 0000 0004 0001 0006 6c92 bf13  ............l...
        0x0010:  4dca 0000 4500 010c be5a 4000 4006 1a5b  M...E....Z@.@..[
        0x0020:  ac16 0403 ac16 0507 0387 0801 568f 0dfd  ............V...
        0x0030:  4562 7e93 8018 01f6 fb0c 0000 0101 080a  Eb~.............
        0x0040:  e979 1f60 d95f d85e 8000 00d4 6f5f accf  .y.`._.^....o_..
        0x0050:  0000 0000 0000 0002 0001 86a3 0000 0004  ................
        0x0060:  0000 0001 0000 0001 0000 0024 0000 0000  ...........$....
        0x0070:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0080:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
        0x0090:  0000 0000 0000 0000 0000 0000 0000 0002  ................
        0x00a0:  0000 0001 0000 002b 0ad7 a866 b483 b7d1  .......+...f....
        0x00b0:  0000 0001 0000 0003 0000 0000 0010 0414  ................
        0x00c0:  0010 0388 0000 1da0 0000 0008 0000 0040  ...............@
        0x00d0:  0000 0000 0000 0000 0000 1000 0000 1000  ................
        0x00e0:  0000 0000 0000 0002 0000 0010 0000 0000  ................
        0x00f0:  4000 0000 0000 0001 0000 0001 1766 5d86  @............f].
        0x0100:  0000 000d 6465 6570 7365 7276 6572 2d30  ....deepserver-0
        0x0110:  3300 0000 0000 0000 0000 0000 0000 0000  3...............
5

Attemp 5: restart service at truenas server:

Result 5: it stucks, and /usr/sbin/rpc.nfsd becomes D state

# /usr/sbin/rpc.nfsd is not in D state
systemctl restart nfs-server
# command hang
# /usr/sbin/rpc.nfsd becomes D state

service nfs-kernel-server restart
# command hang

By the way, I found some messages from dmesg -T from truenas server, may related to this. Only nfsd crashs, access file from termal works

And I found there are some ``nfsd` stucks at D state. (See Logs 5-2)

Logs 5-1 part 1: dmesg -T from truenas server

[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:06:25 2024] megaraid_sas 0000:d8:00.0: 35547 (775656359s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:06:31 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:06:31 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:06:31 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:06:31 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 0
[Tue Jul 30 20:06:40 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:06:58 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:09:01 2024] ses 0:0:60:0: tag#70 CDB: Receive Diagnostic 1c 01 01 ff fc 00
[Tue Jul 30 20:09:50 2024] INFO: task kworker/17:1:626 blocked for more than 120 seconds.
[Tue Jul 30 20:09:50 2024]       Tainted: P           OE      6.6.20-production+truenas #1
[Tue Jul 30 20:09:50 2024] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Tue Jul 30 20:09:50 2024] task:kworker/17:1    state:D stack:0     pid:626   ppid:2      flags:0x00004000
[Tue Jul 30 20:09:50 2024] Workqueue: events megasas_aen_polling [megaraid_sas]
[Tue Jul 30 20:09:50 2024] Call Trace:
[Tue Jul 30 20:09:50 2024]  <TASK>
[Tue Jul 30 20:09:50 2024]  __schedule+0x349/0x950
[Tue Jul 30 20:09:50 2024]  schedule+0x5b/0xa0
[Tue Jul 30 20:09:50 2024]  schedule_timeout+0x98/0x160
[Tue Jul 30 20:09:50 2024]  ? __pfx_process_timeout+0x10/0x10
[Tue Jul 30 20:09:50 2024]  megasas_issue_blocked_cmd+0x1f5/0x240 [megaraid_sas]
[Tue Jul 30 20:09:50 2024]  ? __pfx_autoremove_wake_function+0x10/0x10
[Tue Jul 30 20:09:50 2024]  megasas_get_pd_list+0x14a/0x3b0 [megaraid_sas]
[Tue Jul 30 20:09:50 2024]  ? psi_task_switch+0xd6/0x230
[Tue Jul 30 20:09:50 2024]  megasas_aen_polling+0x420/0x740 [megaraid_sas]
[Tue Jul 30 20:09:50 2024]  process_one_work+0x178/0x350
[Tue Jul 30 20:09:50 2024]  worker_thread+0x303/0x450
[Tue Jul 30 20:09:50 2024]  ? __pfx_worker_thread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  kthread+0xe5/0x120
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork+0x31/0x50
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork_asm+0x1b/0x30
[Tue Jul 30 20:09:50 2024]  </TASK>
[Tue Jul 30 20:09:50 2024] INFO: task kworker/u160:2:1197 blocked for more than 120 seconds.
[Tue Jul 30 20:09:50 2024]       Tainted: P           OE      6.6.20-production+truenas #1
[Tue Jul 30 20:09:50 2024] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Tue Jul 30 20:09:50 2024] task:kworker/u160:2  state:D stack:0     pid:1197  ppid:2      flags:0x00004000
[Tue Jul 30 20:09:50 2024] Workqueue: poll_megasas0_status megasas_fault_detect_work [megaraid_sas]
[Tue Jul 30 20:09:50 2024] Call Trace:
[Tue Jul 30 20:09:50 2024]  <TASK>
[Tue Jul 30 20:09:50 2024]  __schedule+0x349/0x950
[Tue Jul 30 20:09:50 2024]  schedule+0x5b/0xa0
[Tue Jul 30 20:09:50 2024]  schedule_preempt_disabled+0x15/0x30
[Tue Jul 30 20:09:50 2024]  __mutex_lock.constprop.0+0x399/0x700
[Tue Jul 30 20:09:50 2024]  megasas_reset_fusion+0x41/0x11a0 [megaraid_sas]
[Tue Jul 30 20:09:50 2024]  ? psi_task_switch+0xd6/0x230
[Tue Jul 30 20:09:50 2024]  ? finish_task_switch.isra.0+0x90/0x2d0
[Tue Jul 30 20:09:50 2024]  megasas_fault_detect_work+0xb8/0x340 [megaraid_sas]
[Tue Jul 30 20:09:50 2024]  process_one_work+0x178/0x350
[Tue Jul 30 20:09:50 2024]  worker_thread+0x303/0x450
[Tue Jul 30 20:09:50 2024]  ? __pfx_worker_thread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  kthread+0xe5/0x120
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork+0x31/0x50
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork_asm+0x1b/0x30
[Tue Jul 30 20:09:50 2024]  </TASK>
[Tue Jul 30 20:09:50 2024] INFO: task txg_sync:5265 blocked for more than 120 seconds.
[Tue Jul 30 20:09:50 2024]       Tainted: P           OE      6.6.20-production+truenas #1
[Tue Jul 30 20:09:50 2024] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Tue Jul 30 20:09:50 2024] task:txg_sync        state:D stack:0     pid:5265  ppid:2      flags:0x00004000
[Tue Jul 30 20:09:50 2024] Call Trace:
[Tue Jul 30 20:09:50 2024]  <TASK>
[Tue Jul 30 20:09:50 2024]  __schedule+0x349/0x950
[Tue Jul 30 20:09:50 2024]  schedule+0x5b/0xa0
[Tue Jul 30 20:09:50 2024]  schedule_timeout+0x98/0x160
[Tue Jul 30 20:09:50 2024]  ? __pfx_process_timeout+0x10/0x10
[Tue Jul 30 20:09:50 2024]  io_schedule_timeout+0x50/0x80
[Tue Jul 30 20:09:50 2024]  __cv_timedwait_common+0x12a/0x160 [spl]
[Tue Jul 30 20:09:50 2024]  ? __pfx_autoremove_wake_function+0x10/0x10
[Tue Jul 30 20:09:50 2024]  __cv_timedwait_io+0x19/0x20 [spl]
[Tue Jul 30 20:09:50 2024]  zio_wait+0x10f/0x220 [zfs]
[Tue Jul 30 20:09:50 2024]  vdev_config_sync+0xe0/0x290 [zfs]
[Tue Jul 30 20:09:50 2024]  spa_sync_rewrite_vdev_config+0x1ae/0x1d0 [zfs]
[Tue Jul 30 20:09:50 2024]  spa_sync+0x315/0x5e0 [zfs]
[Tue Jul 30 20:09:50 2024]  txg_sync_thread+0x1e8/0x260 [zfs]
[Tue Jul 30 20:09:50 2024]  ? __pfx_txg_sync_thread+0x10/0x10 [zfs]
[Tue Jul 30 20:09:50 2024]  ? __pfx_thread_generic_wrapper+0x10/0x10 [spl]
[Tue Jul 30 20:09:50 2024]  thread_generic_wrapper+0x5b/0x70 [spl]
[Tue Jul 30 20:09:50 2024]  kthread+0xe5/0x120
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork+0x31/0x50
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork_asm+0x1b/0x30
[Tue Jul 30 20:09:50 2024]  </TASK>
[Tue Jul 30 20:09:50 2024] INFO: task nfsd:7103 blocked for more than 120 seconds.
[Tue Jul 30 20:09:50 2024]       Tainted: P           OE      6.6.20-production+truenas #1
[Tue Jul 30 20:09:50 2024] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Tue Jul 30 20:09:50 2024] task:nfsd            state:D stack:0     pid:7103  ppid:2      flags:0x00004000
[Tue Jul 30 20:09:50 2024] Call Trace:
[Tue Jul 30 20:09:50 2024]  <TASK>
[Tue Jul 30 20:09:50 2024]  __schedule+0x349/0x950
[Tue Jul 30 20:09:50 2024]  schedule+0x5b/0xa0
[Tue Jul 30 20:09:50 2024]  cv_wait_common+0xf0/0x130 [spl]
[Tue Jul 30 20:09:50 2024]  ? __pfx_autoremove_wake_function+0x10/0x10
[Tue Jul 30 20:09:50 2024]  dmu_buf_hold_array_by_dnode+0x41a/0x580 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_read_uio_dnode+0x60/0x110 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_read_uio_dbuf+0x46/0x60 [zfs]
[Tue Jul 30 20:09:50 2024]  zfs_read+0x123/0x2f0 [zfs]
[Tue Jul 30 20:09:50 2024]  zpl_iter_read+0xc0/0x130 [zfs]
[Tue Jul 30 20:09:50 2024]  copy_splice_read+0x1b8/0x2d0
[Tue Jul 30 20:09:50 2024]  splice_direct_to_actor+0xb5/0x260
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_direct_splice_actor+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_splice_read+0x6b/0x100 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_splice_read+0x58/0x100 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_read+0x106/0x150 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_operation+0xa6/0x280 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_proc_compound+0x1d4/0x680 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_dispatch+0xee/0x200 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process_common+0x2f5/0x6f0 [sunrpc]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_dispatch+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process+0x131/0x180 [sunrpc]
[Tue Jul 30 20:09:50 2024]  nfsd+0x84/0xd0 [nfsd]
[Tue Jul 30 20:09:50 2024]  kthread+0xe5/0x120
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork+0x31/0x50
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork_asm+0x1b/0x30
[Tue Jul 30 20:09:50 2024]  </TASK>
[Tue Jul 30 20:09:50 2024] INFO: task nfsd:7104 blocked for more than 120 seconds.
[Tue Jul 30 20:09:50 2024]       Tainted: P           OE      6.6.20-production+truenas #1
[Tue Jul 30 20:09:50 2024] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Tue Jul 30 20:09:50 2024] task:nfsd            state:D stack:0     pid:7104  ppid:2      flags:0x00004000
[Tue Jul 30 20:09:50 2024] Call Trace:
[Tue Jul 30 20:09:50 2024]  <TASK>
[Tue Jul 30 20:09:50 2024]  __schedule+0x349/0x950
[Tue Jul 30 20:09:50 2024]  schedule+0x5b/0xa0
[Tue Jul 30 20:09:50 2024]  schedule_timeout+0x98/0x160
[Tue Jul 30 20:09:50 2024]  ? __pfx_process_timeout+0x10/0x10
[Tue Jul 30 20:09:50 2024]  io_schedule_timeout+0x50/0x80
[Tue Jul 30 20:09:50 2024]  __cv_timedwait_common+0x12a/0x160 [spl]
[Tue Jul 30 20:09:50 2024]  ? __pfx_autoremove_wake_function+0x10/0x10
[Tue Jul 30 20:09:50 2024]  __cv_timedwait_io+0x19/0x20 [spl]
[Tue Jul 30 20:09:50 2024]  zio_wait+0x10f/0x220 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_buf_hold_array_by_dnode+0x3c8/0x580 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_read_uio_dnode+0x60/0x110 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_read_uio_dbuf+0x46/0x60 [zfs]
[Tue Jul 30 20:09:50 2024]  zfs_read+0x123/0x2f0 [zfs]
[Tue Jul 30 20:09:50 2024]  zpl_iter_read+0xc0/0x130 [zfs]
[Tue Jul 30 20:09:50 2024]  copy_splice_read+0x1b8/0x2d0
[Tue Jul 30 20:09:50 2024]  splice_direct_to_actor+0xb5/0x260
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_direct_splice_actor+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_splice_read+0x6b/0x100 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_splice_read+0x58/0x100 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_read+0x106/0x150 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_operation+0xa6/0x280 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_proc_compound+0x1d4/0x680 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_dispatch+0xee/0x200 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process_common+0x2f5/0x6f0 [sunrpc]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_dispatch+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process+0x131/0x180 [sunrpc]
[Tue Jul 30 20:09:50 2024]  nfsd+0x84/0xd0 [nfsd]
[Tue Jul 30 20:09:50 2024]  kthread+0xe5/0x120
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork+0x31/0x50
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork_asm+0x1b/0x30
[Tue Jul 30 20:09:50 2024]  </TASK>
[Tue Jul 30 20:09:50 2024] INFO: task nfsd:7107 blocked for more than 120 seconds.
[Tue Jul 30 20:09:50 2024]       Tainted: P           OE      6.6.20-production+truenas #1
[Tue Jul 30 20:09:50 2024] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Tue Jul 30 20:09:50 2024] task:nfsd            state:D stack:0     pid:7107  ppid:2      flags:0x00004000
[Tue Jul 30 20:09:50 2024] Call Trace:
[Tue Jul 30 20:09:50 2024]  <TASK>
[Tue Jul 30 20:09:50 2024]  __schedule+0x349/0x950
[Tue Jul 30 20:09:50 2024]  schedule+0x5b/0xa0
[Tue Jul 30 20:09:50 2024]  cv_wait_common+0xf0/0x130 [spl]
[Tue Jul 30 20:09:50 2024]  ? __pfx_autoremove_wake_function+0x10/0x10
[Tue Jul 30 20:09:50 2024]  dmu_buf_hold_array_by_dnode+0x41a/0x580 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_read_uio_dnode+0x60/0x110 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_read_uio_dbuf+0x46/0x60 [zfs]
[Tue Jul 30 20:09:50 2024]  zfs_read+0x123/0x2f0 [zfs]
[Tue Jul 30 20:09:50 2024]  zpl_iter_read+0xc0/0x130 [zfs]
[Tue Jul 30 20:09:50 2024]  copy_splice_read+0x1b8/0x2d0
[Tue Jul 30 20:09:50 2024]  splice_direct_to_actor+0xb5/0x260
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_direct_splice_actor+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_splice_read+0x6b/0x100 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_splice_read+0x58/0x100 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_read+0x106/0x150 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_operation+0xa6/0x280 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_proc_compound+0x1d4/0x680 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_dispatch+0xee/0x200 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process_common+0x2f5/0x6f0 [sunrpc]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_dispatch+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process+0x131/0x180 [sunrpc]
[Tue Jul 30 20:09:50 2024]  nfsd+0x84/0xd0 [nfsd]
[Tue Jul 30 20:09:50 2024]  kthread+0xe5/0x120
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork+0x31/0x50
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork_asm+0x1b/0x30
[Tue Jul 30 20:09:50 2024]  </TASK>
[Tue Jul 30 20:09:50 2024] INFO: task nfsd:7108 blocked for more than 120 seconds.
[Tue Jul 30 20:09:50 2024]       Tainted: P           OE      6.6.20-production+truenas #1
[Tue Jul 30 20:09:50 2024] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Tue Jul 30 20:09:50 2024] task:nfsd            state:D stack:0     pid:7108  ppid:2      flags:0x00004000
[Tue Jul 30 20:09:50 2024] Call Trace:
[Tue Jul 30 20:09:50 2024]  <TASK>
[Tue Jul 30 20:09:50 2024]  __schedule+0x349/0x950
[Tue Jul 30 20:09:50 2024]  schedule+0x5b/0xa0
[Tue Jul 30 20:09:50 2024]  schedule_timeout+0x98/0x160
[Tue Jul 30 20:09:50 2024]  ? __pfx_process_timeout+0x10/0x10
[Tue Jul 30 20:09:50 2024]  io_schedule_timeout+0x50/0x80
[Tue Jul 30 20:09:50 2024]  __cv_timedwait_common+0x12a/0x160 [spl]
[Tue Jul 30 20:09:50 2024]  ? __pfx_autoremove_wake_function+0x10/0x10
[Tue Jul 30 20:09:50 2024]  __cv_timedwait_io+0x19/0x20 [spl]
[Tue Jul 30 20:09:50 2024]  zio_wait+0x10f/0x220 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_buf_hold_array_by_dnode+0x3c8/0x580 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_read_uio_dnode+0x60/0x110 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_read_uio_dbuf+0x46/0x60 [zfs]
[Tue Jul 30 20:09:50 2024]  zfs_read+0x123/0x2f0 [zfs]
[Tue Jul 30 20:09:50 2024]  zpl_iter_read+0xc0/0x130 [zfs]
[Tue Jul 30 20:09:50 2024]  copy_splice_read+0x1b8/0x2d0
[Tue Jul 30 20:09:50 2024]  splice_direct_to_actor+0xb5/0x260
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_direct_splice_actor+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_splice_read+0x6b/0x100 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_splice_read+0x58/0x100 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_read+0x106/0x150 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_operation+0xa6/0x280 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_proc_compound+0x1d4/0x680 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_dispatch+0xee/0x200 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process_common+0x2f5/0x6f0 [sunrpc]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_dispatch+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process+0x131/0x180 [sunrpc]
[Tue Jul 30 20:09:50 2024]  nfsd+0x84/0xd0 [nfsd]
[Tue Jul 30 20:09:50 2024]  kthread+0xe5/0x120
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork+0x31/0x50
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork_asm+0x1b/0x30
[Tue Jul 30 20:09:50 2024]  </TASK>
[Tue Jul 30 20:09:50 2024] INFO: task nfsd:7109 blocked for more than 120 seconds.
[Tue Jul 30 20:09:50 2024]       Tainted: P           OE      6.6.20-production+truenas #1
[Tue Jul 30 20:09:50 2024] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Tue Jul 30 20:09:50 2024] task:nfsd            state:D stack:0     pid:7109  ppid:2      flags:0x00004000
[Tue Jul 30 20:09:50 2024] Call Trace:
[Tue Jul 30 20:09:50 2024]  <TASK>
[Tue Jul 30 20:09:50 2024]  __schedule+0x349/0x950
[Tue Jul 30 20:09:50 2024]  schedule+0x5b/0xa0
[Tue Jul 30 20:09:50 2024]  cv_wait_common+0xf0/0x130 [spl]
[Tue Jul 30 20:09:50 2024]  ? __pfx_autoremove_wake_function+0x10/0x10
[Tue Jul 30 20:09:50 2024]  dmu_buf_hold_array_by_dnode+0x41a/0x580 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_read_uio_dnode+0x60/0x110 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_read_uio_dbuf+0x46/0x60 [zfs]
[Tue Jul 30 20:09:50 2024]  zfs_read+0x123/0x2f0 [zfs]
[Tue Jul 30 20:09:50 2024]  zpl_iter_read+0xc0/0x130 [zfs]
[Tue Jul 30 20:09:50 2024]  copy_splice_read+0x1b8/0x2d0
[Tue Jul 30 20:09:50 2024]  splice_direct_to_actor+0xb5/0x260
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_direct_splice_actor+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_splice_read+0x6b/0x100 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_splice_read+0x58/0x100 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_read+0x106/0x150 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_operation+0xa6/0x280 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_proc_compound+0x1d4/0x680 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_dispatch+0xee/0x200 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process_common+0x2f5/0x6f0 [sunrpc]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_dispatch+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process+0x131/0x180 [sunrpc]
[Tue Jul 30 20:09:50 2024]  nfsd+0x84/0xd0 [nfsd]
[Tue Jul 30 20:09:50 2024]  kthread+0xe5/0x120
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork+0x31/0x50
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork_asm+0x1b/0x30
[Tue Jul 30 20:09:50 2024]  </TASK>
[Tue Jul 30 20:09:50 2024] INFO: task nfsd:7110 blocked for more than 120 seconds.
[Tue Jul 30 20:09:50 2024]       Tainted: P           OE      6.6.20-production+truenas #1
[Tue Jul 30 20:09:50 2024] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Tue Jul 30 20:09:50 2024] task:nfsd            state:D stack:0     pid:7110  ppid:2      flags:0x00004000
[Tue Jul 30 20:09:50 2024] Call Trace:
[Tue Jul 30 20:09:50 2024]  <TASK>
[Tue Jul 30 20:09:50 2024]  __schedule+0x349/0x950
[Tue Jul 30 20:09:50 2024]  schedule+0x5b/0xa0
[Tue Jul 30 20:09:50 2024]  schedule_timeout+0x98/0x160
[Tue Jul 30 20:09:50 2024]  ? __pfx_process_timeout+0x10/0x10
[Tue Jul 30 20:09:50 2024]  io_schedule_timeout+0x50/0x80
[Tue Jul 30 20:09:50 2024]  __cv_timedwait_common+0x12a/0x160 [spl]
[Tue Jul 30 20:09:50 2024]  ? __pfx_autoremove_wake_function+0x10/0x10
[Tue Jul 30 20:09:50 2024]  __cv_timedwait_io+0x19/0x20 [spl]
[Tue Jul 30 20:09:50 2024]  zio_wait+0x10f/0x220 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_buf_hold_array_by_dnode+0x3c8/0x580 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_read_uio_dnode+0x60/0x110 [zfs]
[Tue Jul 30 20:09:50 2024]  dmu_read_uio_dbuf+0x46/0x60 [zfs]
[Tue Jul 30 20:09:50 2024]  zfs_read+0x123/0x2f0 [zfs]
[Tue Jul 30 20:09:50 2024]  zpl_iter_read+0xc0/0x130 [zfs]
[Tue Jul 30 20:09:50 2024]  copy_splice_read+0x1b8/0x2d0
[Tue Jul 30 20:09:50 2024]  splice_direct_to_actor+0xb5/0x260
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_direct_splice_actor+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_splice_read+0x6b/0x100 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_splice_read+0x58/0x100 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_read+0x106/0x150 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_encode_operation+0xa6/0x280 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_proc_compound+0x1d4/0x680 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_dispatch+0xee/0x200 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process_common+0x2f5/0x6f0 [sunrpc]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_dispatch+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process+0x131/0x180 [sunrpc]
[Tue Jul 30 20:09:50 2024]  nfsd+0x84/0xd0 [nfsd]
[Tue Jul 30 20:09:50 2024]  kthread+0xe5/0x120
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork+0x31/0x50
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork_asm+0x1b/0x30
[Tue Jul 30 20:09:50 2024]  </TASK>
[Tue Jul 30 20:09:50 2024] INFO: task nfsd:7114 blocked for more than 120 seconds.
[Tue Jul 30 20:09:50 2024]       Tainted: P           OE      6.6.20-production+truenas #1
[Tue Jul 30 20:09:50 2024] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Tue Jul 30 20:09:50 2024] task:nfsd            state:D stack:0     pid:7114  ppid:2      flags:0x00004000
[Tue Jul 30 20:09:50 2024] Call Trace:
[Tue Jul 30 20:09:50 2024]  <TASK>
[Tue Jul 30 20:09:50 2024]  __schedule+0x349/0x950
[Tue Jul 30 20:09:50 2024]  schedule+0x5b/0xa0
[Tue Jul 30 20:09:50 2024]  cv_wait_common+0xf0/0x130 [spl]
[Tue Jul 30 20:09:50 2024]  ? __pfx_autoremove_wake_function+0x10/0x10
[Tue Jul 30 20:09:50 2024]  zil_commit_waiter+0x8b/0x100 [zfs]
[Tue Jul 30 20:09:50 2024]  zil_commit_impl+0x71/0xd0 [zfs]
[Tue Jul 30 20:09:50 2024]  zfs_write+0x985/0xb50 [zfs]
[Tue Jul 30 20:09:50 2024]  zpl_iter_write+0x102/0x140 [zfs]
[Tue Jul 30 20:09:50 2024]  do_iter_readv_writev+0xf3/0x160
[Tue Jul 30 20:09:50 2024]  do_iter_write+0x88/0x200
[Tue Jul 30 20:09:50 2024]  nfsd_vfs_write+0x1cc/0x700 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_write+0x13a/0x1d0 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd4_proc_compound+0x356/0x680 [nfsd]
[Tue Jul 30 20:09:50 2024]  nfsd_dispatch+0xee/0x200 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process_common+0x2f5/0x6f0 [sunrpc]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd_dispatch+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[Tue Jul 30 20:09:50 2024]  svc_process+0x131/0x180 [sunrpc]
[Tue Jul 30 20:09:50 2024]  nfsd+0x84/0xd0 [nfsd]
[Tue Jul 30 20:09:50 2024]  kthread+0xe5/0x120
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork+0x31/0x50
[Tue Jul 30 20:09:50 2024]  ? __pfx_kthread+0x10/0x10
[Tue Jul 30 20:09:50 2024]  ret_from_fork_asm+0x1b/0x30
[Tue Jul 30 20:09:50 2024]  </TASK>
[Tue Jul 30 20:09:50 2024] Future hung task reports are suppressed, see sysctl kernel.hung_task_warnings
[Tue Jul 30 20:10:00 2024] sd 0:0:65:0: [sde] tag#854 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e8 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:73:0: [sdm] tag#867 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e0 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:65:0: [sde] tag#827 CDB: Read(16) 88 00 00 00 00 02 cb f7 ce 80 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:73:0: [sdm] tag#466 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:65:0: [sde] tag#419 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:65:0: [sde] tag#236 CDB: Read(16) 88 00 00 00 00 03 a1 d8 27 b8 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:73:0: [sdm] tag#235 CDB: Read(16) 88 00 00 00 00 03 a1 d8 27 b0 00 00 00 18 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:65:0: [sde] tag#105 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:73:0: [sdm] tag#113 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:65:0: [sde] tag#102 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:68:0: [sdh] tag#857 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e8 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:68:0: [sdh] tag#96 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:68:0: [sdh] tag#39 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:66:0: [sdc] tag#858 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e8 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:71:0: [sdi] tag#856 CDB: Read(16) 88 00 00 00 00 03 d0 56 1e e8 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:66:0: [sdc] tag#420 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:66:0: [sdc] tag#234 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:71:0: [sdi] tag#98 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:66:0: [sdc] tag#95 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:71:0: [sdi] tag#42 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:63:0: [sdb] tag#860 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e8 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:63:0: [sdb] tag#568 CDB: Read(16) 88 00 00 00 00 03 a1 d8 27 b8 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:63:0: [sdb] tag#424 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:63:0: [sdb] tag#92 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:63:0: [sdb] tag#69 CDB: Read(16) 88 00 00 00 00 02 7c a5 4a 58 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:63:0: [sdb] tag#40 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:69:0: [sdj] tag#826 CDB: Read(16) 88 00 00 00 00 02 cb f7 ce 80 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:69:0: [sdj] tag#814 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:69:0: [sdj] tag#689 CDB: Read(16) 88 00 00 00 00 02 64 bc 9c 90 00 00 00 18 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:69:0: [sdj] tag#418 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:69:0: [sdj] tag#104 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:67:0: [sdg] tag#855 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e8 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:67:0: [sdg] tag#815 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:67:0: [sdg] tag#691 CDB: Read(16) 88 00 00 00 00 03 a1 d8 27 b8 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:67:0: [sdg] tag#100 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:82:0: [sdq] tag#864 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e0 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:82:0: [sdq] tag#822 CDB: Read(16) 88 00 00 00 00 03 a1 d8 27 b0 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:82:0: [sdq] tag#566 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:82:0: [sdq] tag#122 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:72:0: [sdl] tag#824 CDB: Read(16) 88 00 00 00 00 02 cb f7 ce 80 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:72:0: [sdl] tag#692 CDB: Read(16) 88 00 00 00 00 03 60 9a e3 58 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:72:0: [sdl] tag#425 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:72:0: [sdl] tag#140 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:72:0: [sdl] tag#108 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:80:0: [sdo] tag#865 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e0 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:80:0: [sdo] tag#690 CDB: Read(16) 88 00 00 00 00 03 a1 d8 27 b0 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:80:0: [sdo] tag#233 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:80:0: [sdo] tag#119 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:81:0: [sdp] tag#863 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e0 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:81:0: [sdp] tag#677 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:81:0: [sdp] tag#318 CDB: Read(16) 88 00 00 00 00 03 a1 d8 27 b0 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:81:0: [sdp] tag#125 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:64:0: [sdd] tag#861 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e8 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:64:0: [sdd] tag#569 CDB: Read(16) 88 00 00 00 00 03 a1 d8 27 b8 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:64:0: [sdd] tag#565 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:64:0: [sdd] tag#90 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:78:0: [sdn] tag#866 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e0 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:78:0: [sdn] tag#823 CDB: Read(16) 88 00 00 00 00 03 a1 d8 27 b0 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:78:0: [sdn] tag#576 CDB: Read(16) 88 00 00 00 00 03 76 4a 40 28 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:78:0: [sdn] tag#316 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:78:0: [sdn] tag#116 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:78:0: [sdn] tag#49 CDB: Read(16) 88 00 00 00 00 03 be c5 24 88 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:62:0: [sda] tag#859 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e8 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:62:0: [sda] tag#836 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:62:0: [sda] tag#421 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:62:0: [sda] tag#94 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:76:0: [sdk] tag#868 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e0 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:76:0: [sdk] tag#637 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:76:0: [sdk] tag#570 CDB: Read(16) 88 00 00 00 00 03 a1 d8 27 b0 00 00 00 18 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:76:0: [sdk] tag#423 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:76:0: [sdk] tag#110 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:70:0: [sdf] tag#825 CDB: Read(16) 88 00 00 00 00 02 cb f7 ce 80 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:70:0: [sdf] tag#638 CDB: Read(16) 88 00 00 00 00 03 60 9a e3 58 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:70:0: [sdf] tag#417 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:70:0: [sdf] tag#383 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:70:0: [sdf] tag#106 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:83:0: [sdr] tag#862 CDB: Read(16) 88 00 00 00 00 03 d0 96 1e e0 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:83:0: [sdr] tag#853 CDB: Read(16) 88 00 00 00 00 00 e8 fd 6c e8 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:83:0: [sdr] tag#688 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:00 2024] sd 0:0:83:0: [sdr] tag#639 CDB: Read(16) 88 00 00 00 00 03 a1 d8 27 b0 00 00 00 10 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:83:0: [sdr] tag#422 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:83:0: [sdr] tag#319 CDB: Read(16) 88 00 00 00 00 02 b2 16 3e b8 00 00 00 08 00 00
[Tue Jul 30 20:10:00 2024] sd 0:0:83:0: [sdr] tag#66 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:10:02 2024] ses 0:0:61:0: tag#88 CDB: Receive Diagnostic 1c 01 0a 01 f8 00
[Tue Jul 30 20:10:02 2024] ses 0:0:60:0: tag#567 CDB: Receive Diagnostic 1c 01 07 00 20 00
[Tue Jul 30 20:10:02 2024] ses 0:0:61:0: tag#88 OCR is requested due to IO timeout!!
[Tue Jul 30 20:10:02 2024] ses 0:0:61:0: tag#88 SCSI host state: 5  SCSI host busy: 86  FW outstanding: 86
[Tue Jul 30 20:10:02 2024] ses 0:0:61:0: tag#88 scmd: (0x00000000dac6b847)  retries: 0x0  allowed: 0x1
[Tue Jul 30 20:10:02 2024] ses 0:0:61:0: tag#88 CDB: Receive Diagnostic 1c 01 0a 01 f8 00
[Tue Jul 30 20:10:02 2024] ses 0:0:61:0: tag#88 Request descriptor details:
[Tue Jul 30 20:10:02 2024] ses 0:0:61:0: tag#88 RequestFlags:0x0  MSIxIndex:0x2  SMID:0x59  LMID:0x0  DevHandle:0x1e
[Tue Jul 30 20:10:02 2024] IO request frame:
[Tue Jul 30 20:10:02 2024] 00000000: f100001e 00000000 00000000 6c2e2100 00600002 00000020 00000000 000001f8
[Tue Jul 30 20:10:02 2024] 00000020: 00000000 00000006 00000000 00000000 00000000 00000000 00000000 02000000
[Tue Jul 30 20:10:02 2024] 00000040: 010a011c 000000f8 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000060: 001e0000 003d0000 00000000 00000000 00000000 00000000 00000110 00000000
[Tue Jul 30 20:10:02 2024] 00000080: 52ed1a00 00000001 000001f8 40000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 000000a0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 000000c0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 000000e0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] Chain frame:
[Tue Jul 30 20:10:02 2024] 00000000: e0073000 00000011 00001000 00000000 9b8eb000 00000010 00001000 00000000
[Tue Jul 30 20:10:02 2024] 00000020: d60e7000 00000010 00001000 00000000 1b247000 00000001 00001000 00000000
[Tue Jul 30 20:10:02 2024] 00000040: 2aae0000 00000002 00001000 00000000 01155000 00000002 00001000 00000000
[Tue Jul 30 20:10:02 2024] 00000060: f9f71000 00000001 00001000 00000000 e51f7000 00000010 00001000 00000000
[Tue Jul 30 20:10:02 2024] 00000080: 99ba0000 00000011 00001000 00000000 03176000 00000002 00001000 00000000
[Tue Jul 30 20:10:02 2024] 000000a0: 1ccd8000 00000002 00001000 00000000 293a7000 00000002 00001000 00000000
[Tue Jul 30 20:10:02 2024] 000000c0: 21732000 00000011 00001000 00000000 22157000 00000011 00001000 00000000
[Tue Jul 30 20:10:02 2024] 000000e0: 02a9b000 00000002 00001000 00000000 02312000 00000002 00001000 00000000
[Tue Jul 30 20:10:02 2024] 00000100: e1b9c000 00000011 00001000 00000000 019d2000 00000002 00001000 40000000
[Tue Jul 30 20:10:02 2024] 00000120: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000140: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000160: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000180: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 000001a0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 000001c0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 000001e0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000200: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000220: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000240: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000260: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000280: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 000002a0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 000002c0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 000002e0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000300: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000320: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000340: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000360: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 00000380: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 000003a0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 000003c0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:10:02 2024] 000003e0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
6

Logs 5-1 part 2:

[Tue Jul 30 20:10:08 2024] megaraid_sas 0000:d8:00.0: DCMD(opcode: 0x2010100) is timed out, func:megasas_issue_blocked_cmd
[Tue Jul 30 20:10:08 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:10:08 2024] megaraid_sas 0000:d8:00.0: MFI command is timed out
[Tue Jul 30 20:10:08 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:10:08 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 38
[Tue Jul 30 20:10:17 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: 35599 (775656381s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:10:36 2024] megaraid_sas 0000:d8:00.0: 35651 (775656403s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:10:46 2024] sd 0:0:83:0: [sdr] tag#688 FAILED Result: hostbyte=DID_REQUEUE driverbyte=DRIVER_OK cmd_age=249s
[Tue Jul 30 20:10:46 2024] sd 0:0:83:0: [sdr] tag#688 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:46 2024] sd 0:0:70:0: [sdf] tag#383 FAILED Result: hostbyte=DID_REQUEUE driverbyte=DRIVER_OK cmd_age=249s
[Tue Jul 30 20:10:46 2024] sd 0:0:70:0: [sdf] tag#383 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:46 2024] sd 0:0:76:0: [sdk] tag#637 FAILED Result: hostbyte=DID_REQUEUE driverbyte=DRIVER_OK cmd_age=249s
[Tue Jul 30 20:10:46 2024] sd 0:0:76:0: [sdk] tag#637 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:46 2024] sd 0:0:62:0: [sda] tag#836 FAILED Result: hostbyte=DID_REQUEUE driverbyte=DRIVER_OK cmd_age=249s
[Tue Jul 30 20:10:46 2024] sd 0:0:62:0: [sda] tag#836 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:46 2024] sd 0:0:78:0: [sdn] tag#316 FAILED Result: hostbyte=DID_REQUEUE driverbyte=DRIVER_OK cmd_age=249s
[Tue Jul 30 20:10:46 2024] sd 0:0:78:0: [sdn] tag#316 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:46 2024] sd 0:0:64:0: [sdd] tag#565 FAILED Result: hostbyte=DID_REQUEUE driverbyte=DRIVER_OK cmd_age=249s
[Tue Jul 30 20:10:46 2024] sd 0:0:64:0: [sdd] tag#565 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:46 2024] sd 0:0:81:0: [sdp] tag#677 FAILED Result: hostbyte=DID_REQUEUE driverbyte=DRIVER_OK cmd_age=249s
[Tue Jul 30 20:10:46 2024] sd 0:0:81:0: [sdp] tag#677 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:46 2024] sd 0:0:80:0: [sdo] tag#233 FAILED Result: hostbyte=DID_REQUEUE driverbyte=DRIVER_OK cmd_age=249s
[Tue Jul 30 20:10:46 2024] sd 0:0:80:0: [sdo] tag#233 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:46 2024] sd 0:0:72:0: [sdl] tag#140 FAILED Result: hostbyte=DID_REQUEUE driverbyte=DRIVER_OK cmd_age=249s
[Tue Jul 30 20:10:46 2024] sd 0:0:72:0: [sdl] tag#140 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:46 2024] sd 0:0:82:0: [sdq] tag#566 FAILED Result: hostbyte=DID_REQUEUE driverbyte=DRIVER_OK cmd_age=249s
[Tue Jul 30 20:10:46 2024] sd 0:0:82:0: [sdq] tag#566 CDB: ATA command pass through(16) 85 06 20 00 00 00 00 00 00 00 00 00 00 40 e3 00
[Tue Jul 30 20:10:47 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:10:47 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:10:47 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:10:47 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 38
[Tue Jul 30 20:10:47 2024] ses 0:0:60:0: tag#70 timing out command, waited 120s
[Tue Jul 30 20:10:47 2024] ses 0:0:61:0: tag#88 timing out command, waited 60s
[Tue Jul 30 20:10:56 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:11:14 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:11:15 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:11:15 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:11:15 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:11:15 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:11:15 2024] megaraid_sas 0000:d8:00.0: 35703 (775656436s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:11:23 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:11:23 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:11:23 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:11:23 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 5
[Tue Jul 30 20:11:32 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:11:50 2024] megaraid_sas 0000:d8:00.0: 35755 (775656466s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#902 CDB: Read(16) 88 00 00 00 00 03 89 07 56 80 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#881 CDB: Write(16) 8a 00 00 00 00 04 d5 40 33 90 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#831 CDB: Write(16) 8a 00 00 00 00 04 d5 40 36 b0 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#632 CDB: Read(16) 88 00 00 00 00 04 15 a8 94 28 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#439 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#206 CDB: Read(16) 88 00 00 00 00 02 29 c5 6b 60 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#161 CDB: Read(16) 88 00 00 00 00 04 16 d9 8e 78 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#135 CDB: Read(16) 88 00 00 00 00 02 ea 51 1c 10 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#132 CDB: Read(16) 88 00 00 00 00 02 25 1e 3f 58 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#128 CDB: Read(16) 88 00 00 00 00 03 5e 36 d5 90 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#124 CDB: Read(16) 88 00 00 00 00 02 cb 7d e5 70 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#114 CDB: Read(16) 88 00 00 00 00 01 42 16 57 58 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:73:0: [sdm] tag#36 CDB: Read(16) 88 00 00 00 00 02 2e a8 88 98 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:70:0: [sdf] tag#823 CDB: Read(16) 88 00 00 00 00 02 2f 98 cf c0 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#817 CDB: Read(16) 88 00 00 00 00 02 2f 98 cf c0 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#879 CDB: Write(16) 8a 00 00 00 00 02 2b ea ef f0 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:70:0: [sdf] tag#446 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#758 CDB: Write(16) 8a 00 00 00 00 02 2c 5c b9 30 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#816 CDB: Read(16) 88 00 00 00 00 02 b6 4b b4 40 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:70:0: [sdf] tag#317 CDB: Read(16) 88 00 00 00 00 03 7c 0b 46 78 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#633 CDB: Read(16) 88 00 00 00 00 04 15 a8 94 28 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#429 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:70:0: [sdf] tag#312 CDB: Read(16) 88 00 00 00 00 02 9d 5f 97 28 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#434 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#383 CDB: Read(16) 88 00 00 00 00 04 16 d9 8e 80 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#775 CDB: Write(16) 8a 00 00 00 00 05 33 b1 fe 00 00 00 01 30 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:70:0: [sdf] tag#308 CDB: Read(16) 88 00 00 00 00 03 5e 36 d5 40 00 00 00 60 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#905 CDB: Read(16) 88 00 00 00 00 02 2f 95 e8 20 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#392 CDB: Read(16) 88 00 00 00 00 02 ea 51 1c 10 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#678 CDB: Write(16) 8a 00 00 00 00 04 d5 40 36 b0 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#378 CDB: Write(16) 8a 00 00 00 00 02 2b ea ee 98 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#509 CDB: Write(16) 8a 00 00 00 00 02 2b ea ef e0 00 00 00 18 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:70:0: [sdf] tag#212 CDB: Read(16) 88 00 00 00 00 04 16 d9 8e 78 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#900 CDB: Write(16) 8a 00 00 00 00 02 2e 79 18 70 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#391 CDB: Read(16) 88 00 00 00 00 04 16 d9 8e 78 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#437 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#257 CDB: Read(16) 88 00 00 00 00 02 6a ea f6 50 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#438 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:70:0: [sdf] tag#208 CDB: Read(16) 88 00 00 00 00 02 80 14 5c c0 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#887 CDB: Read(16) 88 00 00 00 00 02 6a ea f6 48 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#390 CDB: Read(16) 88 00 00 00 00 02 80 14 5c c0 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#301 CDB: Read(16) 88 00 00 00 00 03 5e 36 d5 40 00 00 00 60 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#254 CDB: Read(16) 88 00 00 00 00 01 c6 7b d7 58 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#393 CDB: Read(16) 88 00 00 00 00 02 b6 4b b4 38 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:70:0: [sdf] tag#183 CDB: Read(16) 88 00 00 00 00 02 6a ea f6 50 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#788 CDB: Write(16) 8a 00 00 00 00 02 2b ea ee 90 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#182 CDB: Read(16) 88 00 00 00 00 02 6a ea f6 50 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#255 CDB: Read(16) 88 00 00 00 00 02 ea 51 1c 10 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#220 CDB: Read(16) 88 00 00 00 00 02 ea 51 1c 18 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#322 CDB: Read(16) 88 00 00 00 00 02 2f 98 e0 38 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:70:0: [sdf] tag#155 CDB: Read(16) 88 00 00 00 00 01 c6 7b d7 50 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#432 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#145 CDB: Read(16) 88 00 00 00 00 03 f0 4d c1 a8 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#207 CDB: Read(16) 88 00 00 00 00 02 29 c5 6b 60 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#189 CDB: Read(16) 88 00 00 00 00 03 5e 36 d5 48 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#205 CDB: Read(16) 88 00 00 00 00 02 29 c5 6b 60 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:70:0: [sdf] tag#147 CDB: Read(16) 88 00 00 00 00 03 f0 4d c1 a8 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#264 CDB: Read(16) 88 00 00 00 00 01 c6 7b d7 50 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#144 CDB: Read(16) 88 00 00 00 00 02 25 1e 3f 58 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#191 CDB: Read(16) 88 00 00 00 00 03 f0 4d c1 a8 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#169 CDB: Read(16) 88 00 00 00 00 03 84 18 a2 10 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#190 CDB: Write(16) 8a 00 00 00 00 04 d5 40 33 90 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:70:0: [sdf] tag#140 CDB: Read(16) 88 00 00 00 00 02 ea 51 1c 10 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#262 CDB: Read(16) 88 00 00 00 00 03 b0 35 cb e8 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#116 CDB: Read(16) 88 00 00 00 00 01 42 16 57 58 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#184 CDB: Read(16) 88 00 00 00 00 02 6a ea f6 50 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#25 CDB: Read(16) 88 00 00 00 00 02 80 14 5c c0 00 00 00 18 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#187 CDB: Read(16) 88 00 00 00 00 01 c6 7b d7 50 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:70:0: [sdf] tag#139 CDB: Read(16) 88 00 00 00 00 02 2e a8 88 98 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#252 CDB: Read(16) 88 00 00 00 00 03 84 18 a2 08 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#63 CDB: Write(16) 8a 00 00 00 00 02 2b ea ef e8 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#167 CDB: Write(16) 8a 00 00 00 00 02 2b ea ef e8 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#16 CDB: Read(16) 88 00 00 00 00 03 b0 35 cb f0 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#174 CDB: Read(16) 88 00 00 00 00 03 5e 36 d5 90 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#192 CDB: Read(16) 88 00 00 00 00 02 9d 5f 97 20 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:72:0: [sdl] tag#40 CDB: Read(16) 88 00 00 00 00 02 2e a8 88 98 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#115 CDB: Read(16) 88 00 00 00 00 01 42 16 57 58 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:64:0: [sdd] tag#15 CDB: Read(16) 88 00 00 00 00 02 9d 5f 97 28 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#160 CDB: Read(16) 88 00 00 00 00 02 2f 98 82 28 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#176 CDB: Read(16) 88 00 00 00 00 02 b6 4b b4 38 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#62 CDB: Read(16) 88 00 00 00 00 01 c6 7b d7 50 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#622 CDB: Write(16) 8a 00 00 00 00 02 2c 99 f7 20 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#149 CDB: Read(16) 88 00 00 00 00 04 16 d9 8e 78 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#168 CDB: Read(16) 88 00 00 00 00 02 2c e5 ee a8 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#38 CDB: Read(16) 88 00 00 00 00 02 2e a8 88 98 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#512 CDB: Read(16) 88 00 00 00 00 02 2e a8 8a 60 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#148 CDB: Read(16) 88 00 00 00 00 02 ea 51 1c 10 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#163 CDB: Read(16) 88 00 00 00 00 02 80 14 5c b8 00 00 00 18 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#9 CDB: Read(16) 88 00 00 00 00 04 16 d9 8e 78 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#441 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#35 CDB: Read(16) 88 00 00 00 00 02 2e a8 88 98 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#156 CDB: Read(16) 88 00 00 00 00 03 5e 36 d5 40 00 00 00 60 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:76:0: [sdk] tag#7 CDB: Read(16) 88 00 00 00 00 02 80 14 5c c0 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#318 CDB: Read(16) 88 00 00 00 00 01 c6 7b d7 50 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#13 CDB: Read(16) 88 00 00 00 00 02 80 14 5c c0 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:83:0: [sdr] tag#8 CDB: Read(16) 88 00 00 00 00 02 2f 7b a0 c8 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#311 CDB: Read(16) 88 00 00 00 00 03 84 18 a2 08 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:78:0: [sdn] tag#11 CDB: Read(16) 88 00 00 00 00 02 6a ea f6 50 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#260 CDB: Read(16) 88 00 00 00 00 02 ea 51 1c 10 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#253 CDB: Write(16) 8a 00 00 00 00 04 d5 40 33 90 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#210 CDB: Read(16) 88 00 00 00 00 03 f0 4d c1 a0 00 00 00 18 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#202 CDB: Read(16) 88 00 00 00 00 04 16 d9 8e 78 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#166 CDB: Read(16) 88 00 00 00 00 02 b6 4b b4 38 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#157 CDB: Write(16) 8a 00 00 00 00 05 33 b1 fd 60 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#130 CDB: Read(16) 88 00 00 00 00 02 2f 98 82 28 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#104 CDB: Read(16) 88 00 00 00 00 02 2b 5a 9a d0 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:81:0: [sdp] tag#21 CDB: Read(16) 88 00 00 00 00 02 9d 5f 97 20 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#899 CDB: Write(16) 8a 00 00 00 00 02 2b ea ef 68 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#820 CDB: Read(16) 88 00 00 00 00 02 2a 6d b8 88 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#782 CDB: Read(16) 88 00 00 00 00 02 a7 12 8a b8 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#756 CDB: Write(16) 8a 00 00 00 00 02 2b ea ef e0 00 00 00 18 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#681 CDB: Read(16) 88 00 00 00 00 02 2f 98 e0 38 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#631 CDB: Read(16) 88 00 00 00 00 04 15 a8 94 28 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#436 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#204 CDB: Read(16) 88 00 00 00 00 02 29 c5 6b 60 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#138 CDB: Read(16) 88 00 00 00 00 02 2f 98 82 28 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#129 CDB: Read(16) 88 00 00 00 00 02 ea 51 1c 10 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#123 CDB: Read(16) 88 00 00 00 00 02 cb 7d e5 70 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#113 CDB: Read(16) 88 00 00 00 00 01 42 16 57 58 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#890 CDB: Read(16) 88 00 00 00 00 02 b6 4b b4 40 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:80:0: [sdo] tag#5 CDB: Read(16) 88 00 00 00 00 04 16 d9 8e 78 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#822 CDB: Read(16) 88 00 00 00 00 02 6a ea f6 50 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#784 CDB: Write(16) 8a 00 00 00 00 02 2b ea ee 98 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#675 CDB: Write(16) 8a 00 00 00 00 02 2c 5c b9 28 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#444 CDB: Synchronize Cache(10) 35 00 00 00 00 00 00 00 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#323 CDB: Read(16) 88 00 00 00 00 02 80 14 5c c0 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#263 CDB: Read(16) 88 00 00 00 00 01 c6 7b d7 58 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#258 CDB: Read(16) 88 00 00 00 00 03 b0 35 cb f0 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#256 CDB: Read(16) 88 00 00 00 00 02 9d 5f 97 28 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#143 CDB: Read(16) 88 00 00 00 00 03 70 84 7a 48 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#142 CDB: Read(16) 88 00 00 00 00 03 84 18 a2 10 00 00 00 10 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#141 CDB: Read(16) 88 00 00 00 00 02 2f 98 b1 28 00 00 00 08 00 00
[Tue Jul 30 20:14:51 2024] sd 0:0:63:0: [sdb] tag#48 CDB: Read(16) 88 00 00 00 00 03 5e 36 d5 48 00 00 00 10 00 00
[Tue Jul 30 20:14:53 2024] ses 0:0:61:0: tag#236 CDB: Receive Diagnostic 1c 01 02 ff fc 00
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: DCMD(opcode: 0x2010100) is timed out, func:megasas_issue_blocked_cmd
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: MFI command is timed out
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 124
[Tue Jul 30 20:15:03 2024] ses 0:0:60:0: tag#707 timing out command, waited 60s
[Tue Jul 30 20:15:03 2024] ses 0:0:61:0: tag#783 timing out command, waited 60s
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: Controller is not OPERATIONAL,SCSI host:0
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: Controller is not OPERATIONAL,SCSI host:0
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: Controller is not OPERATIONAL,SCSI host:0
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: Controller is not OPERATIONAL,SCSI host:0
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: Controller is not OPERATIONAL,SCSI host:0
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: Controller is not OPERATIONAL,SCSI host:0
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: Controller is not OPERATIONAL,SCSI host:0
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: Controller is not OPERATIONAL,SCSI host:0
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: Controller is not OPERATIONAL,SCSI host:0
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: Controller is not OPERATIONAL,SCSI host:0
[Tue Jul 30 20:15:03 2024] megaraid_sas 0000:d8:00.0: Controller is not OPERATIONAL,SCSI host:0
[Tue Jul 30 20:15:03 2024] ses 0:0:61:0: tag#236 OCR is requested due to IO timeout!!
[Tue Jul 30 20:15:03 2024] ses 0:0:61:0: tag#236 SCSI host state: 5  SCSI host busy: 134  FW outstanding: 0
[Tue Jul 30 20:15:03 2024] ses 0:0:61:0: tag#236 scmd: (0x0000000049697c46)  retries: 0x0  allowed: 0x0
[Tue Jul 30 20:15:03 2024] ses 0:0:61:0: tag#236 CDB: Receive Diagnostic 1c 01 02 ff fc 00
[Tue Jul 30 20:15:03 2024] ses 0:0:61:0: tag#236 Request descriptor details:
[Tue Jul 30 20:15:03 2024] ses 0:0:61:0: tag#236 RequestFlags:0x0  MSIxIndex:0x43  SMID:0xed  LMID:0x0  DevHandle:0x1e
[Tue Jul 30 20:15:03 2024] IO request frame:
[Tue Jul 30 20:15:03 2024] 00000000: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000020: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000040: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000060: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000080: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000000a0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000000c0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000000e0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] Chain frame:
[Tue Jul 30 20:15:03 2024] 00000000: 4fca9000 00000001 00001000 00000000 4c966000 00000001 00001000 00000000
[Tue Jul 30 20:15:03 2024] 00000020: 3f6f0000 00000001 00001000 00000000 3f9b0000 00000001 00001000 00000000
[Tue Jul 30 20:15:03 2024] 00000040: 4c1ed000 00000001 00001000 00000000 3bfea000 00000001 00001000 00000000
[Tue Jul 30 20:15:03 2024] 00000060: 4bef8000 00000001 00001000 00000000 4e0de000 00000001 00001000 00000000
[Tue Jul 30 20:15:03 2024] 00000080: 336c6000 00000002 00000ffc 40000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000000a0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000000c0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000000e0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000100: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000120: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000140: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000160: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000180: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000001a0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000001c0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000001e0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000200: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000220: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000240: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000260: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000280: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000002a0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000002c0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000002e0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000300: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000320: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000340: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000360: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 00000380: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000003a0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000003c0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:03 2024] 000003e0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[Tue Jul 30 20:15:12 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:15:31 2024] megaraid_sas 0000:d8:00.0: 35807 (775656488s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:15:57 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:15:57 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:15:57 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:15:57 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 33
[Tue Jul 30 20:16:06 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:16:25 2024] megaraid_sas 0000:d8:00.0: 35859 (775656536s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:16:31 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:16:31 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:16:31 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:16:31 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 0
[Tue Jul 30 20:16:40 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:16:58 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:17:04 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:17:04 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:17:04 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:17:04 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 8
[Tue Jul 30 20:17:13 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:17:32 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:17:38 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:17:38 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:17:38 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:17:38 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 22
[Tue Jul 30 20:17:47 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:18:05 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:18:12 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:18:12 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:18:12 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:18:12 2024] ses 0:0:60:0: tag#53 timing out command, waited 120s
[Tue Jul 30 20:18:12 2024] ses 0:0:61:0: tag#144 timing out command, waited 60s
[Tue Jul 30 20:18:12 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 16
[Tue Jul 30 20:18:21 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: 35911 (775656558s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: 35963 (775656580s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: 36015 (775656602s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:18:39 2024] megaraid_sas 0000:d8:00.0: 36067 (775656624s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:18:48 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:18:48 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:18:48 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:18:48 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 0
[Tue Jul 30 20:18:57 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:19:15 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:19:15 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:19:15 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:19:15 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:19:15 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:19:15 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:19:16 2024] megaraid_sas 0000:d8:00.0: 36119 (775656655s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:19:22 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:19:22 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:19:22 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:19:22 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 0
[Tue Jul 30 20:19:31 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:19:49 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:19:55 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:19:55 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:19:55 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:19:55 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 21
[Tue Jul 30 20:20:04 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:20:23 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:20:29 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:20:29 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:20:29 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:20:29 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 7
[Tue Jul 30 20:20:38 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:20:56 2024] megaraid_sas 0000:d8:00.0: 36171 (775656677s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:21:02 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:21:02 2024] megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
[Tue Jul 30 20:21:02 2024] megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
[Tue Jul 30 20:21:02 2024] ses 0:0:60:0: tag#640 timing out command, waited 60s
[Tue Jul 30 20:21:02 2024] ses 0:0:61:0: tag#99 timing out command, waited 60s
[Tue Jul 30 20:21:02 2024] ses 0:0:60:0: tag#256 timing out command, waited 120s
[Tue Jul 30 20:21:02 2024] megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 30
[Tue Jul 30 20:21:11 2024] megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: FW now in Ready state
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: 36223 (775656699s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: 36275 (775656721s/0x0020/CRIT) - Controller encountered a fatal error and was reset
[Tue Jul 30 20:21:30 2024] megaraid_sas 0000:d8:00.0: 36327 (775656743s/0x0020/CRIT) - Controller encountered a fatal error and was reset

Logs 5-2:

root@truenas[/]# ps -aux | grep nfsd
root        7092  0.0  0.0      0     0 ?        I    Jul30   0:05 [nfsd]
root        7093  0.0  0.0      0     0 ?        I    Jul30   0:05 [nfsd]
root        7094  0.0  0.0      0     0 ?        I    Jul30   0:05 [nfsd]
root        7095  0.0  0.0      0     0 ?        I    Jul30   0:07 [nfsd]
root        7096  0.0  0.0      0     0 ?        I    Jul30   0:06 [nfsd]
root        7097  0.0  0.0      0     0 ?        I    Jul30   0:07 [nfsd]
root        7098  0.0  0.0      0     0 ?        I    Jul30   0:07 [nfsd]
root        7099  0.0  0.0      0     0 ?        I    Jul30   0:08 [nfsd]
root        7100  0.0  0.0      0     0 ?        I    Jul30   0:08 [nfsd]
root        7101  0.0  0.0      0     0 ?        I    Jul30   0:10 [nfsd]
root        7102  0.0  0.0      0     0 ?        I    Jul30   0:16 [nfsd]
root        7103  0.0  0.0      0     0 ?        I    Jul30   0:15 [nfsd]
root        7104  0.0  0.0      0     0 ?        I    Jul30   0:20 [nfsd]
root        7105  0.0  0.0      0     0 ?        I    Jul30   0:25 [nfsd]
root        7106  0.0  0.0      0     0 ?        I    Jul30   0:34 [nfsd]
root        7107  0.0  0.0      0     0 ?        I    Jul30   0:51 [nfsd]
root        7108  0.0  0.0      0     0 ?        I    Jul30   1:19 [nfsd]
root        7109  0.0  0.0      0     0 ?        I    Jul30   1:54 [nfsd]
root        7110  0.1  0.0      0     0 ?        I    Jul30   2:45 [nfsd]
root        7111  0.1  0.0      0     0 ?        I    Jul30   4:01 [nfsd]
root        7112  0.2  0.0      0     0 ?        I    Jul30   5:36 [nfsd]
root        7113  0.2  0.0      0     0 ?        I    Jul30   7:19 [nfsd]
root        7114  0.3  0.0      0     0 ?        I    Jul30   9:44 [nfsd]
root        7115  0.4  0.0      0     0 ?        D    Jul30  10:39 [nfsd]
root        7116  0.5  0.0      0     0 ?        I    Jul30  13:41 [nfsd]
root        7117  0.6  0.0      0     0 ?        I    Jul30  15:47 [nfsd]
root        7118  0.6  0.0      0     0 ?        D    Jul30  18:12 [nfsd]
root        7119  0.9  0.0      0     0 ?        I    Jul30  24:33 [nfsd]
root        7120  1.1  0.0      0     0 ?        I    Jul30  29:10 [nfsd]
root        7121  0.8  0.0      0     0 ?        D    Jul30  23:36 [nfsd]
root        7122  1.7  0.0      0     0 ?        I    Jul30  46:07 [nfsd]
root        7123  1.1  0.0      0     0 ?        D    Jul30  31:17 [nfsd]
root      631081  0.0  0.0      0     0 ?        D    15:12   0:00 [kworker/u161:1+nfsd4]
root      666053  0.0  0.0   2496   640 ?        Ds   15:30   0:00 /usr/sbin/rpc.nfsd 0
root      701413  0.0  0.0   3744  1280 pts/0    S+   15:49   0:00 grep nfsd
root     3387481  0.0  0.0      0     0 ?        D    02:16   0:01 [kworker/u160:1+nfsd4_callbacks]

root@truenas[/]# cat /proc/7118/status
Name:   nfsd
Umask:  0000
State:  D (disk sleep)
Tgid:   7118
Ngid:   0
Pid:    7118
PPid:   2
TracerPid:      0
Uid:    0       0       0       0
Gid:    0       0       0       0
FDSize: 64
Groups:
NStgid: 7118
NSpid:  7118
NSpgid: 0
NSsid:  0
Kthread:        1
Threads:        1
SigQ:   3/512769
SigPnd: 0000000000000000
ShdPnd: 0000000000000000
SigBlk: 0000000000000000
SigIgn: ffffffffffffffff
SigCgt: 0000000000000000
CapInh: 0000000000000000
CapPrm: 000001ffffffffff
CapEff: 000001ffffffffff
CapBnd: 000001ffffffffff
CapAmb: 0000000000000000
NoNewPrivs:     0
Seccomp:        0
Seccomp_filters:        0
Speculation_Store_Bypass:       thread vulnerable
SpeculationIndirectBranch:      conditional enabled
Cpus_allowed:   ffff,ffffffff,ffffffff
Cpus_allowed_list:      0-79
Mems_allowed:   00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000003
Mems_allowed_list:      0-1
voluntary_ctxt_switches:        17426173
nonvoluntary_ctxt_switches:     2277

Attemp 6: reboot nfs server

result: it finally works

The server just crash every week. This is really annoyings to me a lot. If there any other tests I can do, please replies me, many thinks.

1 Like
7

Above it looks like you have zpool on a RAID controller and at least one of hangs is related to the raid controller. How is the pool configured precisely?

8

The controller is configured as passthrough mode, and I uses raidz3 for it.

And meanwhile, the filesystem is not hang. I can access all files under the truenas terminal. Only NFS hangs forever

root@truenas[/home/admin]# zpool status -v
  pool: MainPool
 state: ONLINE
  scan: scrub repaired 700K in 08:40:39 with 0 errors on Sun Jun 30 23:44:04 2024
config:

        NAME                                      STATE     READ WRITE CKSUM
        MainPool                                  ONLINE       0     0     0
          raidz3-0                                ONLINE       0     0     0
            a9e39c43-b5e4-4e34-9a06-20abe4f2300d  ONLINE       0     0     0
            b57b4a7e-a732-49d1-83c5-ed38acacb515  ONLINE       0     0     0
            004dad74-dfaf-4c6a-a120-3f7b1042dc21  ONLINE       0     0     0
            a27a39a2-40e1-465b-ba92-e822339da2de  ONLINE       0     0     0
            f6a2c9d4-346b-4c0f-83cd-1e86724349c5  ONLINE       0     0     0
            67e4d1b7-2574-48bc-817c-8cbc4ebdf71e  ONLINE       0     0     0
            29c0f0ff-ca25-4d70-9635-18d43e50bcff  ONLINE       0     0     0
            7d76885c-4df1-45fa-9d51-7f8e2e7bf788  ONLINE       0     0     0
            9a005094-c661-4dba-ab6d-aab55d50efae  ONLINE       0     0     0
            0947cf61-d984-48de-be59-39509be08ec5  ONLINE       0     0     0
            8d8bb7c2-23b3-49a9-8170-2cd1a8f851bf  ONLINE       0     0     0
            71694af4-9951-4b53-b6da-d97b9e677897  ONLINE       0     0     0
            a02ab1fa-fb3f-46b6-b2ec-014dab3abf7b  ONLINE       0     0     0
            472a3d2c-bc1f-43b5-92f3-5d46a09bfb8a  ONLINE       0     0     0
            501740fe-022e-4ef6-9522-c46619f109b4  ONLINE       0     0     0
            c752a49f-4f45-432e-80c6-9cc777363f49  ONLINE       0     0     0
            d76cc3de-d18c-4baf-8e07-7af52ad2f6fe  ONLINE       0     0     0
            e85c464a-68fa-436a-a519-af522f7fe7af  ONLINE       0     0     0
        cache
          8bc6d4cc-5fe7-4820-93e3-76314be0a0de    ONLINE       0     0     0
          5ea0bbba-260d-4c98-a0ff-ed0d97b44f6f    ONLINE       0     0     0

errors: No known data errors

  pool: boot-pool
 state: ONLINE
  scan: scrub repaired 0B in 00:00:06 with 0 errors on Tue Jul 30 03:45:08 2024
config:

        NAME           STATE     READ WRITE CKSUM
        boot-pool      ONLINE       0     0     0
          mirror-0     ONLINE       0     0     0
            nvme2n1p3  ONLINE       0     0     0
            nvme3n1p3  ONLINE       0     0     0

errors: No known data errors

By the way, I found some similar issue occurs on upstream debian, maybe it’s a general issue from upstream nfs-kernel-server.
https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1071562

Does FreeBSD version of truenas has same issue? I may switch to Truenas core from Truenas scale if freebsd version works fine.

9

I find NFS on TrueNAS CORE 13.0 very stable.

10

NFS v4 or v3?

11

NFSv4

1 Like
12

If possible, please try NFSv3.

13

Tried everything, non of them works.
Purpose: put maching learning training data on it, access by computional node(5 node, ubuntu 22.04 server with GPU attached) by python. And all checkpoints will put on it while training.

NFSv3:
mount option: nfs nfsvers=3,rw,noatime,rsize=1048576,wsize=1048576,nofail,nodev,nosuid,_netdev,soft 0 0
Problem: I get bus error on client every day. The pytorch dataloader just random crash with bus error. It seems this caused by file mmap not works properly on NFSv3.

NFSv4:
mount option: nfs rw,noatime,rsize=1048576,wsize=1048576,nofail,nodev,nosuid,_netdev,soft 0 0
Problem: The nfsd just goes into D state every week and no ways to recover. I have to reboot the server and meanwhile all client on computional node dies.

root@truenas[~]# ps -eo ppid,pid,user,stat,pcpu,comm,wchan:32 | grep " D"

      2    8239 root     D     0.1 nfsd            flush_workqueue
      2    8240 root     D     0.1 nfsd            flush_workqueue
      2    8241 root     D     0.2 nfsd            flush_workqueue
      2    8242 root     D     0.3 nfsd            flush_workqueue
      2    8243 root     D     0.3 nfsd            flush_workqueue
      2  369380 root     D     0.0 kworker/u161:0+ flush_workqueue
      2  380560 root     D     0.0 kworker/u160:0+ rpc_shutdown_client
root@truenas[~]#

SMB:
mount option: credentials=/root/.ssh/.smbcred,uid=root,gid=root,file_mode=0777,dir_mode=0777
Problem: The smb process get killed by oom(128GB RAM is not enough?) every day, but I wrote a script to check and restart every minutes. But all running python on computional node which reading datas still dies with this restart.

[Wed Aug 21 02:34:27 2024] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0-1,global_oom,task_memcg=/system.slice/smbd.service,task=smbd[172.22.4.2,pid=408438,uid=0
[Wed Aug 21 02:34:27 2024] Out of memory: Killed process 408438 (smbd[172.22.4.2) total-vm:535951600kB, anon-rss:111094612kB, file-rss:26240kB, shmem-rss:12424kB, UID:0 pgtables:1048808kB oom_score_adj:0
[Wed Aug 21 02:34:32 2024] oom_reaper: reaped process 408438 (smbd[172.22.4.2), now anon-rss:0kB, file-rss:25600kB, shmem-rss:2596kB
[Wed Aug 21 10:24:47 2024] perf: interrupt took too long (3157 > 3141), lowering kernel.perf_event_max_sample_rate to 63250
[Wed Aug 21 11:00:01 2024] python.d.plugin invoked oom-killer: gfp_mask=0x140cca(GFP_HIGHUSER_MOVABLE|__GFP_COMP), order=0, oom_score_adj=-900

I just need a Network Storage Server, to store training data and training checkpoint on it, no need for fency features.
TrueNAS scale provides 3 way to share data, but non of them works as my expected. Is there any my setting wrong?

14

Until now, there is no ways to make TrueNAS Scale useable.
No matter NFSv3(bus error), NFSv4(hang forever) or SMB(oom killed).

The last option for me is switching to TrueNAS core 13.3, will report if it useable after the switch done.

15

Have you tried CORE 13.0? That’s the one I can confirm works well.

16

No, because the zpool is created at TrueNAS Scale with new zfs version and I have 50TB data in it now.

But the Core 13.0 can’t import the pool because it’s old zfs version, and I don’t have additional space to store the data and recreate the pool

17

Ah I see your issue.

18

I recently merged (for electric eel) a fix for the SMB OOM issues some users were seeing when using varieties of loopback mounts to our SMB server.

19

Thanks everyone, switched to TrueNAS CORE 13.3 now, so far so good.
I will report here if there is any issue futher.

by the way, there is no nfs session tab in TrueNAS CORE, how can I list all nfs client and last handshake?

20

Actually, I like TrueNAS Scale more, but it’s not useable
This is a collected briefing for this NFSv4 issue at linux now, hope it will get solved in the future:

Hardware:

  • OS: TrueNAS-SCALE-24.04.0
  • Product: ASUS RS540-E9-RS36-E
  • CPU:Intel(R) Xeon(R) Gold 6138 CPU *2
  • RAM: 128G ECC
  • 40G Mellanox NIC
  • 16T HDD disk * 18, RAIDZ3
  • HBA card: LSI 9364-8I, passthrough mode, no hardware raid.

Potential hardware issue:
LSI 9364-8I
Keeping get warning from dmesg(see below) But all disks works normally, ZFS shows all disk healthy and no disk error.
And there is no any issue on file operation on nas, that’s why I tend to believe it’s a software bug instead of hardware issue.

megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
megaraid_sas 0000:d8:00.0: FW in FAULT state Fault code:0x10000 subcode:0x0 func:megasas_wait_for_outstanding_fusion
megaraid_sas 0000:d8:00.0: resetting fusion adapter scsi0.
megaraid_sas 0000:d8:00.0: Outstanding fastpath IOs: 5
megaraid_sas 0000:d8:00.0: Waiting for FW to come to ready state
megaraid_sas 0000:d8:00.0: FW now in Ready state
megaraid_sas 0000:d8:00.0: FW now in Ready state
megaraid_sas 0000:d8:00.0: Current firmware supports maximum commands: 928    LDIO threshold: 0
megaraid_sas 0000:d8:00.0: Performance mode :Latency (latency index = 1)
megaraid_sas 0000:d8:00.0: FW supports sync cache    : No
megaraid_sas 0000:d8:00.0: megasas_disable_intr_fusion is called outbound_intr_mask:0x40000009
megaraid_sas 0000:d8:00.0: FW provided supportMaxExtLDs: 1   max_lds: 64
megaraid_sas 0000:d8:00.0: controller type   : MR(1024MB)
megaraid_sas 0000:d8:00.0: Online Controller Reset(OCR)      : Enabled
megaraid_sas 0000:d8:00.0: Secure JBOD support       : Yes
megaraid_sas 0000:d8:00.0: NVMe passthru support     : No
megaraid_sas 0000:d8:00.0: FW provided TM TaskAbort/Reset timeout    : 0 secs/0 secs
megaraid_sas 0000:d8:00.0: JBOD sequence map support : No
megaraid_sas 0000:d8:00.0: PCI Lane Margining support        : No
megaraid_sas 0000:d8:00.0: JBOD sequence map is disabled megasas_setup_jbod_map 5796
megaraid_sas 0000:d8:00.0: megasas_enable_intr_fusion is called outbound_intr_mask:0x40000000
megaraid_sas 0000:d8:00.0: Adapter is OPERATIONAL for scsi:0
megaraid_sas 0000:d8:00.0: Reset successful for scsi0.
megaraid_sas 0000:d8:00.0: 35755 (775656466s/0x0020/CRIT) - Controller encountered a fatal error and was reset

Software:
Server: TrueNAS, NFSv4 with default setting
Client: Ubuntu 22.04 *5
Client Mount option:

nfs rw,noatime,rsize=1048576,wsize=1048576,nofail,nodev,nosuid,_netdev,soft 0 0

Syndrome:
Bug will not affects to all client. Only affects to some client. But all client may occur, It’s randomly
Can happens on multiple client simultaneously.

While bug occured, both client and server hang. Both client and server needs to restart.
Client: Can no longer mount this nfs server, even the NFS server rebooted.
Server: Can no longer accept new NFS connection. Existing connection works normally until bug occured.

On server, we can see nfsd goes into “D” state and no ways to recover except reboot.

root@truenas[~]# ps -eo ppid,pid,user,stat,pcpu,comm,wchan:32 | grep " D"

      2    8239 root     D     0.1 nfsd            flush_workqueue
      2    8240 root     D     0.1 nfsd            flush_workqueue
      2    8241 root     D     0.2 nfsd            flush_workqueue
      2    8242 root     D     0.3 nfsd            flush_workqueue
      2    8243 root     D     0.3 nfsd            flush_workqueue
      2  369380 root     D     0.0 kworker/u161:0+ flush_workqueue
      2  380560 root     D     0.0 kworker/u160:0+ rpc_shutdown_client
root@truenas[~]#

Can always see this in the dmesg

INFO: task nfsd:7103 blocked for more than 120 seconds.
      Tainted: P           OE      6.6.20-production+truenas #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
task:nfsd            state:D stack:0     pid:7103  ppid:2      flags:0x00004000
Call Trace:
 <TASK>
 __schedule+0x349/0x950
 schedule+0x5b/0xa0
 cv_wait_common+0xf0/0x130 [spl]
 ? __pfx_autoremove_wake_function+0x10/0x10
 dmu_buf_hold_array_by_dnode+0x41a/0x580 [zfs]
 dmu_read_uio_dnode+0x60/0x110 [zfs]
 dmu_read_uio_dbuf+0x46/0x60 [zfs]
 zfs_read+0x123/0x2f0 [zfs]
 zpl_iter_read+0xc0/0x130 [zfs]
 copy_splice_read+0x1b8/0x2d0
 splice_direct_to_actor+0xb5/0x260
 ? __pfx_nfsd_direct_splice_actor+0x10/0x10 [nfsd]
 nfsd_splice_read+0x6b/0x100 [nfsd]
 nfsd4_encode_splice_read+0x58/0x100 [nfsd]
 nfsd4_encode_read+0x106/0x150 [nfsd]
 nfsd4_encode_operation+0xa6/0x280 [nfsd]
 nfsd4_proc_compound+0x1d4/0x680 [nfsd]
 nfsd_dispatch+0xee/0x200 [nfsd]
 ? __pfx_nfsd+0x10/0x10 [nfsd]
 svc_process_common+0x2f5/0x6f0 [sunrpc]
 ? __pfx_nfsd_dispatch+0x10/0x10 [nfsd]
 ? __pfx_nfsd+0x10/0x10 [nfsd]
 svc_process+0x131/0x180 [sunrpc]
 nfsd+0x84/0xd0 [nfsd]
 kthread+0xe5/0x120
 ? __pfx_kthread+0x10/0x10
 ret_from_fork+0x31/0x50
 ? __pfx_kthread+0x10/0x10
 ret_from_fork_asm+0x1b/0x30
 </TASK>

On the client, no error message at all. Even no nfs timeout message.
Access files in the nfs will hang forever. Access other nfs server works normal.
mount -l /data/nas07 and try mount again will always hang, even after the TrueNAS Server rebooted.
No ways to recover except reboot.