3

My Ubuntu server 20.04 keeps freezing after a few hours of operation to the point where the fan is running hot and I can't even establish a connection to ssh to it.

After rebooting it behaves normally with no CPU load spikes or memory issues. The server is a fresh install and only really running an empty MongoDB database.

I looked at the kern.log and found it flooded with these logs at the time before the reboot.

Dec 10 21:41:05 backend kernel: [102373.982171] rc rc0: receive overflow
Dec 10 21:41:05 backend kernel: [102373.986010] rc rc0: receive overflow

It just started happening again and I got a chance to see which processes are running hot.

  /usr/sbin/rsyslogd -n -iNONE
  /lib/systemd/systemd-journald
  /usr/lib/accountsservice/accounts-daemon

Do you have any idea why this keeps happening?

karel
  • 114,770
itchap
  • 29
  • 4

3 Answers3

3

I've had the same problem on an Intel NUC and strace-ing the process revealed a couple of interesting files:

openat(AT_FDCWD, "/run/log/journal/0322fcd160934520a68be3469a358ed3/system.journal", O_RDWR|O_NONBLOCK|O_CLOEXEC) = -1 ENOENT (No such file or directory)
gettid()                                = 393
timerfd_settime(17, TFD_TIMER_ABSTIME, {it_interval={tv_sec=0, tv_nsec=0}, it_value={tv_sec=13887, tv_nsec=175660000}}, NULL) = 0
epoll_wait(8, [], 62, 0)                = 0
ftruncate(37, 16777216)                 = 0
gettid()                                = 393
epoll_wait(8, [{events=EPOLLIN, data={u32=3603220400, u64=94565898176432}}], 62, -1) = 1
read(9, "4,1594792,13834610419,-;rc rc0: "..., 8192) = 79
access("/sys/subsystem/rc/devices/rc0", F_OK) = -1 ENOENT (No such file or directory)
access("/sys/bus/rc/devices/rc0", F_OK) = -1 ENOENT (No such file or directory)
access("/sys/class/rc/rc0", F_OK)       = 0
openat(AT_FDCWD, "/", O_RDONLY|O_CLOEXEC|O_PATH|O_DIRECTORY) = 23
openat(23, "sys", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 36
newfstatat(36, "", {st_mode=S_IFDIR|0555, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(23)                               = 0
openat(36, "class", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 23
newfstatat(23, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(36)                               = 0
openat(23, "rc", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 36
newfstatat(36, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(23)                               = 0
openat(36, "rc0", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 23
newfstatat(23, "", {st_mode=S_IFLNK|0777, st_size=0, ...}, AT_EMPTY_PATH) = 0
readlinkat(36, "rc0", "../../devices/pnp0/00:01/rc/rc0", 4096) = 31
close(23)                               = 0
openat(36, "..", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 23
close(36)                               = 0
openat(23, "..", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 36
close(23)                               = 0
openat(36, "devices", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 23
newfstatat(23, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(36)                               = 0
openat(23, "pnp0", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 36
newfstatat(36, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(23)                               = 0
openat(36, "00:01", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 23
newfstatat(23, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(36)                               = 0
openat(23, "rc", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 36
newfstatat(36, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(23)                               = 0
openat(36, "rc0", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 23
newfstatat(23, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(36)                               = 0
close(23)                               = 0
access("/sys/devices/pnp0/00:01/rc/rc0/uevent", F_OK) = 0
openat(AT_FDCWD, "/sys/devices/pnp0/00:01/rc/rc0/uevent", O_RDONLY|O_CLOEXEC) = 23
newfstatat(23, "", {st_mode=S_IFREG|0644, st_size=4096, ...}, AT_EMPTY_PATH) = 0
read(23, "NAME=rc-rc6-mce\nDRV_NAME=ite-cir"..., 4104) = 66
close(23)                               = 0
readlinkat(AT_FDCWD, "/sys/devices/pnp0/00:01/rc/rc0/subsystem", "../../../../../class/rc", 4096) = 23
openat(AT_FDCWD, "/run/udev/data/+rc:rc0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/log/journal/0322fcd160934520a68be3469a358ed3/system.journal", O_RDWR|O_NONBLOCK|O_CLOEXEC) = -1 ENOENT (No such file or directory)

And looking at these files gave me a way to reproduce it:

marius@nuc:~$ cat /sys/devices/pnp0/00\:01/rc/rc0/input4/name
ITE8708 CIR transceiver
marius@nuc:~$ cat /sys/devices/pnp0/00\:01/rc/rc0/protocols
rc-5 nec [rc-6] jvc sony rc-5-sz sanyo sharp mce_kbd xmp imon rc-mm [lirc]

The NUC sits beneath a TV, it has an IR receiver(had no idea about that) and it captures the signal from the TV remote. Pressing any button on the remote would trigger this problem.

I found the kernel module responsible for this using lsmod and disabled it with sudo modprobe -r ite_cir. This fix is temporary, I'll have to look into blacklisting the module.

1

I am running Linux Mint (not Ubuntu, but similar enough) on an Intel NUC with infrared receiver.

I found the solution here: https://support.tools/post/rc-rc0-overflow/

TLDR; The IR receiver, if not in use, can be disabled and blacklisted, solving the issue. You can do it as follows:

# Disable IR kernel module
sudo modprobe -r ite_cir

Check journal to make sure 'kernel: rc rc0: receive overflow' message is gone

sudo journalctl -f

If disabling the IR module resolved your issue, you have to blacklist it to make it permanent.

Edit this file: /etc/modprobe.d/blacklist.conf

sudo nano /etc/modprobe.d/blacklist.conf

Add to it the following line:

blacklist ite_cir

ctrl+o and return to save, ctrl+x to exit nano editor.

Update initramfs

sudo update-initramfs -u

This solved it for me (at least for the current kernel installed, might need to repeat steps after the next kernel update, fortunately, it is a small, repeatable operation.

Louis Parkin
  • 213
  • 1
  • 11
-1

Some methods I used for high CPU load issues on the NUC which worked.

#SOLUTION 1
sudo vim /etc/default/grub
# Add the following parameter to the GRUB_CMDLINE_LINUX_DEFAULT property
pci=nomsi
# And try also addin this parameter to the GRUB_CMDLINE_LINUX_DEFAULT and GRUB_CMDLINE_LINUX properties
pcie_aspm=off 
#
sudo update-grub
sudo reboot

#SOLUTION 2 - Update the bios firmware wget https://downloadmirror.intel.com/758735/BN0089.bio sudo cp BN0089.bio /boot/ sudo cp BN0089.bio /root/ sudo reboot

#SOLUTION 3 (Usually this fixes the problem)

Disable IR kernel module

sudo modprobe -r ite_cir

Check journal to make sure 'kernel: rc rc0: receive overflow' message is gone

sudo journalctl -f

If disabling the IR module resolved your issue, you have to blacklist it to make it permanent.

Edit this file: /etc/modprobe.d/blacklist.conf

sudo vi /etc/modprobe.d/blacklist.conf

Add to it the following line:

blacklist ite_cir

Update initramfs

sudo update-initramfs -u

itchap
  • 29
  • 4