Linux Troubleshooting

de facto

timeshift --help
sudo timeshift --list

df -h                            # to see space
dpkg --list | grep linux-image   # to see the kernel images
sudo apt purge IMG1 IMG2 IMG3    # to purge several of the oldest images

timeshift --help
sudo timeshift --list

sudo journalctl -f

sudo journalctl -b -1 -e    # see end of log for last boot
# Or:
sudo journalctl -b -1 >lastboot.log
tail lastboot.log    # maybe take it to a working machine

# Or:
sudo journalctl --boot=-1 --priority=3

sudo dmesg -xT
sudo dmesg --level=emerg,alert,crit,err,warn -T
sudo journalctl -b -1 -e    # see end of log for last boot
sudo journalctl --pager-end
sudo journalctl --since=today
sudo journalctl --since=today --priority=emerg..err
sudo journalctl --since=2019-12-31
less /var/log/Xorg.0.log

idle=nomwait acpi_backlight=vendor rcu_nocbs=0-7
# https://www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.html

sudo hdparm -i /dev/sda
sudo hdparm -I /dev/sda

# show areas:
sudo hdparm -N /dev/sda
sudo hdparm --dco-identify /dev/sda

# VERY dangerous:
sudo hdparm -N pSSSSSSSSSSS /dev/sdx  # remove HPA, set visible area/max sectors to SSSSSSSSSSS
sudo hdparm --dco-restore /dev/sdx    # revert DCO back to factory defaults; may brick drive
# Also, GParted has a feature to remove HPA ?  But I don't see it.

grep 'ACTIVE_CONSOLES=' /etc/default/console-setup	# see "/dev/tty[1-6]" or similar
sudo sed -i "s/#NAutoVTs=/NAutoVTs=/" /etc/systemd/logind.conf
# then reboot

df -khT
df -khT -x squashfs -x tmpfs		# don't show snap devices

sudo du -h --summarize --exclude=/home --exclude=/sys --exclude=/proc --exclude=/run /
sudo du -h --summarize -c --exclude=/home --exclude=/sys --exclude=/proc --exclude=/run /*

sudo du -h --summarize -c /home/*
du -sh $HOME/* | sort -h
du -sh $HOME/.[0-9a-z]* | sort -h

sudo df -ih        # inodes

sudo du -sh /var/log
journalctl --disk-usage

sudo systemctl status logrotate --full --lines 1000
grep logrotate /etc/crontab /etc/cron.*/* /etc/anacrontab /var/spool/cron/crontabs/*
cat /etc/logrotate.conf
man logrotate

# As a quick fix:
sudo rm /var/log/*.gz
sudo echo 0 >/var/log/syslog

# List installed packages sorted by size:
dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -rn | less
rpm -qa --queryformat '%10{size} - %-25{name} \t %{version}\n' | sort -n
pacgraph     # Arch

sudo bash -c "! pgrep -x flatpak && rm -r /var/tmp/flatpak-cache-*"

# show current kernel:
uname -r

# show all kernels except the one currently running:
sudo dpkg --list 'linux-image*'|awk '{ if ($1=="ii") print $2}'|grep -v `uname -r`

# For each image you want to remove, do:
sudo apt purge linux-image-NNNNNNNNNNNNN-generic

# These shouldn't be necessary:
sudo apt autoremove
sudo update-grub

# show current kernel:
uname -r

# show all kernels:
dnf list kernel --quiet
rpm -q kernel

# For each kernel you want to remove, do:
sudo dnf remove kernel-NNNNNNNNNNNNN

tmpfs  /tmp      tmpfs  noatime,nodiratime,nodev,nosuid,mode=1777,size=1G,defaults  0  0
tmpfs  /var/crash  tmpfs  noatime,nodiratime,nodev,nosuid,mode=1777,size=1G,defaults  0  0

# Download and extract the test package.
# Then:

sudo ./install-sh
sudo apt install php-cli php-xml pkg-config

phoronix-test-suite --help
# have to accept some license terms etc

# can run via cli or GUI
# GUI will give a CLI-type interface; type 'quit' to get out

# https://www.phoronix-test-suite.com/documentation/phoronix-test-suite.html

phoronix-test-suite list-available-suites
phoronix-test-suite list-available-tests | less

# I want: simple, few dependencies, run in a few minutes, test CPU RAM disk.

# Best to run tests after a fresh boot, no other apps running.
# Run each test 3 times and average them.

phoronix-test-suite benchmark pts/sysbench
# My 2011 Dell Inspiron N5010 with Kubuntu 20.10:
# (Intel Core i3 370M @ 2.39GHz, 8 GB RAM, 320 GB WD3200BEVT-7 with ext4/LUKS)
# gave CPU 2100, RAM 9.8 M
# My 2021 KDE Slimbook 15 with KDE Neon 5.20.5:
# (AMD Ryzen 7 4800H @ 2.9GHz, 16 GB RAM, 256 GB NVMe SSD with ext4)
# gave CPU 17600, RAM 13.6 M

phoronix-test-suite benchmark pts/leveldb
# My 2011 Dell Inspiron N5010 with Kubuntu 20.10:
# (Intel Core i3 370M @ 2.39GHz, 8 GB RAM, 320 GB WD3200BEVT-7 with ext4/LUKS)
# gave seq fill 12 MB/S, hot read 1.76 usec/op, random read 1.71 usec/op.
# My 2021 KDE Slimbook 15 with KDE Neon 5.20.5:
# (AMD Ryzen 7 4800H @ 2.9GHz, 16 GB RAM, 256 GB NVMe SSD with ext4)
# gave seq fill 23 MB/S, hot read 7 usec/op, random read 7 usec/op.

phoronix-test-suite benchmark pts/fio
# Choose sub-test 3 (sequential read), option 4 (Linux AIO), Buffered, Not Direct,
# 4 KB block size.
# My 2011 Dell Inspiron N5010 with Kubuntu 20.10:
# (Intel Core i3 370M @ 2.39GHz, 8 GB RAM, 320 GB WD3200BEVT-7 with ext4/LUKS)
# gave 48 MB/S.
# My 2021 KDE Slimbook 15 with KDE Neon 5.20.5:
# (AMD Ryzen 7 4800H @ 2.9GHz, 16 GB RAM, 256 GB NVMe SSD with ext4)
# gave 1132 MB/S.

hwinfo --sound
sudo inxi -A

aconnect -l
aplay -l
aplay -L
cat /proc/asound/cards
alsa-info.sh --stdout | less

amixer sget Master --device pulse   # get current volume and state

speaker-test -D default

pactl info
LANG=C pactl info | grep 'Server Name'

pactl list short sources
pactl list short sinks
pactl list short clients
pactl get-default-source

less /etc/pulse/daemon.conf

pavucontrol

# Increase speaker volume:
pactl set-sink-volume @DEFAULT_SINK@ +10%

systemctl --user status pipewire
pw-cli info all | grep process.binary
pw-cli info all | less
pw-record --list-targets

# If PipeWire gets screwed up:
systemctl --user restart pipewire.service

pw-cli list-objects Device
pw-top
wpctl status
wpctl get-volume @DEFAULT_SINK@

inxi -A             # to see most of the stack
fuser -v /dev/snd/* # to see what process is running sound system

less /etc/pipewire/pipewire.conf
# example == /usr/share/pipewire/pipewire.conf

sudo apt install pulseaudio-equalizer

# If speaker device disappears:
pulseaudio -k

# If no sound, maybe:
sudo alsa force-reload
# or:
systemctl --user restart pipewire

ps -ax | grep -E 'pulseaudio|pipewire' | grep -v grep
pactl info
apt list | grep alsa | grep installed
less /etc/modprobe.d/alsa-base.conf

# How to software-prevent any use of microphone ?  Maybe:
pactl suspend-source @DEFAULT_SOURCE@ true

# If you decide to give up on PipeWire and switch back to PulseAudio:
sudo dnf swap --allowerasing pipewire-pulseaudio pulseaudio

default-sample-format = float32ne  
default-sample-rate = 44100  
alternate-sample-rate = 48000  
default-sample-channels = 2    
default-fragments = 2  
default-fragment-size-msec = 125  
resample-method = soxr-vhq      
nice-level = -11    
realtime-priority = 9  
rlimit-rtprio = 9

xdg-open https://errors.ubuntu.com/user/$(sudo cat /var/lib/whoopsie/whoopsie-id)

apt install PACKAGENAME-dbgsym
# or:
apt install PACKAGENAME-dbg

gdb NAMEOFAPP 2>&1 | tee $HOME/gdb-NAMEOFAPP.txt
(gdb) run ARG1 ARG2 ARG3
# get app to crash/fail
(gdb) thread apply all bt
(gdb) q

sudo apt install v4l-utils qv4l2
qv4l2

guvcview --control_panel

# How many processes using webcam ?
lsmod | grep '^uvcvideo' | awk '{print $3}'

# See what's using webcam:
sudo fuser -v /dev/video*

# Software-prevent anything from using webcam:
sudo modprobe -r uvcvideo

# Make a fake webcam device:
sudo modprobe v4l2loopback
ls /dev/video*
ffmpeg -loop 1 -re -i MYIMAGE.jpg -f v4l2 -vcodec rawvideo -pix_fmt yuv420p /dev/video5
# Will show at speed of conversion, not speed in file:
ffmpeg -stream_loop 99 -i MYVIDEO.mp4 -f v4l2 -vcodec rawvideo -pix_fmt yuv420p /dev/video5

# May have to install package xorg-xinput

xinput --list
xinput --list DEVICEID
# Output shows 7 buttons, 2nd is middle button.
xinput set-button-map DEVICEID 1 0
# Test it, then add it as a startup application.
# Could put it in ~/.xinitrc or ~/.xprofile instead ?

# If ID of device keeps changing, do:
xinput set-button-map $(xinput list --id-only 'DEVICENAME') 1 0

# If full name and ID of device keep changing, do:
xinput set-button-map $(xinput list --id-only "$(xinput list --name-only | grep Touchpad)") 1 0

wl-paste -p --watch wl-copy -p ''

#gsettings set org.gnome.desktop.peripherals.touchpad middle-click-emulation true
#https://gist.github.com/poyotanp/d3aff65df96d0cf93c78a3e52fedd6fa

lsusb
lsusb -D /dev/bus/usb/BUSNUMBER/DEVICENUMBER
udevadm info /dev/bus/usb/BUSNUMBER/DEVICENUMBER
udevadm info --query=path /dev/bus/usb/BUSNUMBER/DEVICENUMBER
ls -l /sys/`udevadm info --query=path /dev/bus/usb/BUSNUMBER/DEVICENUMBER`

# see what speed has been negotiated with each of your USB devices:
lsusb -tv

usb-devices

sudo tlp-stat --usb

dmesg | grep 'New USB device found'
dmesg | grep 'usb '
sudo usbview

# Set a USB port to send out power or charge from external supply:
ls /sys/class/typec
echo 'source' >/sys/class/typec/port0/power_role
echo 'sink' >/sys/class/typec/port0/power_role
# Not in my openSUSE Tumbleweed system; need sysfs-class-typec driver ?
# https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-class-typec
# https://www.kernel.org/doc/html/latest/driver-api/usb/typec.html
grep -i typec /boot/config-$(uname -r)

lspci
lspci -k
sudo lspci -v

sudo dmidecode | grep -i pci

sudo lshw
sudo hwinfo

sudo tlp-stat --pcie

sudo dmidecode -s bios-version
sudo dmidecode -s bios-release-date
sudo dmidecode -s bios-revision
sudo dmidecode -t bios
sudo find /sys/devices/virtual/dmi/id/ -type f -exec bash -c "echo -n '{}: '; cat {}" \;
sudo biosdecode
sudo vpddecode

Linux Troubleshooting

Precautions

Problems and troubleshooting

Magic key-sequences

Clean up space on disk

Performance

Run Performance Tests

Network performance testing

GPU performance testing

Disk performance testing

Tweaking/fixing video

Tweaking/fixing mouse/trackpad/scrolling/video latency/tearing

Tweaking/fixing audio

Reporting Bugs

Devices

Webcam

Keyboard

Battery / Power Management

Display

Mouse

Touchpad

USB

PCI

Miscellaneous

Firmware

Firmware update that comes only in a Windows/DOS EXE format

Motherboard Firmware

Sleep / Suspend