Linux Notes


/etc/systemd/system - default location for system units \~/.config/systemd/user - default location for user units


# reload units and timers
        systemctl [--user] daemon-reload
        # show all units (including disabled)
        systemctl [--user] list-units -a
        # view logs for unit
        # also accepts:
        #   -f               | tail the log
        #   --user-unit foo  | target user unit instead of system
        #   --boot=0         | show logs from current boot (-1 for previous, etc)
        journalctl --unit foo


A simple service unit


        Description=example service



foobar.timer A simple timer unit

        Description=example timer

        # run every 15 minutes (aligns to the hour)
        # run timer immediately if script is enabled and is past due


Basic Arch Install


        timedatectl set-ntp true

        fdisk /dev/sda 

        # Create 300MB boot, 2GB swap, and leave the rest for root

        mkswp /dev/sda2

        mkfs.ext4 /dev/sda3

        mount /dev/sda3 /mnt

        swapon /dev/sda2

        # edit /etc/pacman.d/mirrorlist to change mirror order **

        pacstrap /mnt base

        genfstab -p /mnt >> /mnt/etc/fstab

        arch-chroot /mnt

        ln -s /usr/share/zoneinfo/America/Indianapolis /etc/localtime

        hwclock --systohc --utc

        # uncomment en_US locales in /etc/locale.gen **


        # enter hostname in /etc/hostname **

        mkinitcpio -p linux


        pacman -S grub

        grub-install --target=i386-pc --recheck --debug /dev/sda

        grub-mkconfig -o /boot/grub/grub.cfg



        pacman -S vim htop git openssh wget

        pacman -S xorg-server xf86-video-ati xorg-xinit

Generic Linux Install

# list all available block devices
        # Copy bootable image to flash drive (status=progress requires dd >= 8.24)
        # be careful not to mess up the of= argument, or you might overwrite your HD
        sudo dd if=foobar.iso of=/dev/sdX status=progress && sync


# show device connect/disconnects and fired rules
        udevadm monitor
        # list all attributes for a particular device (and parents)
        udevadm info --attribute-walk --path=/devices/...
# simple vendorid, productid example
        # note SYMLINK is only for block devices (I think, look it up yourself)
        SUBSYSTEMS=="usb", ATTRS{idProduct}=="3300", ATTRS{idVendor}=="1e10", MODE="0666", SYMLINK+="foobar"



# list all tables
        iptables -L -n -v
        # (fedora) save iptables rules and remember to disable firewalld
        iptables-save > /etc/sysconfig/iptables


# allow ssh
        # must allow incoming connection and response

        # append rule to input (-A INPUT) on input interface enp6s0f0 (-i enp6s0f0) 
        # with destination port 22 (--dport 22).  use 'state' module (-m state)
        # and allow new and established connections (--state NEW,ESTABLISHED)
        # jump to target ACCEPT (-j ACCEPT)
        iptables -A INPUT -i enp6s0f0 -p tcp --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT

        # append rule to output (-A OUTPUT) on output interface enp6s0f0 (-o enp6s0f0) 
        # with source port 22 (--sport 22).  use 'state' module (-m state)
        # and allow established connections (--state ESTABLISHED)
        # jump to target ACCEPT (-j ACCEPT)
        iptables -A OUTPUT -o enp6s0f0 -p tcp --sport 22 -m state --state ESTABLISHED -j ACCEPT
# filter table: flush all chains, and delete all user added chains
        iptables -F
        iptables -X
        # nat table: flush all chains, and delete all user added chains
        iptables -t nat -F
        iptables -t nat -X



# create new lv `foo` in group `foo_group`
        lvcreate -L 10G foo_group -n foo


lvremove /dev/[vgname]/[lvname]


# add all devices to filesystem
        btrfs device add /dev/sdb2 /dev/sdc2 /dev/sdd2 /
        # convert system to raid10
        btrfs balance start -dconvert=raid10 -mconvert=raid10 /
        # check balance progress
        btrfs balance status /
        # get rid of single chunks to get another shot at degraded,rw mount
        btrfs balance start -dconvert=raid10,soft -mconvert=raid10,soft  /mount


Config examples

/etc/lxc/lxc.conf - set path for containers to be stored (default /var/lib/lxc)

lxc.lxcpath = "/lxc"

/etc/lxc/default.conf - config options for all newly created containers to inherit = veth = lxcbr0 = up = 00:16:3e:xx:xx:xx = 1

        # address = =

        # memory
        lxc.cgroup.memory.limit_in_bytes = 512M

        # memory + swap
        lxc.cgroup.memory.memsw.limit_in_bytes = 1G

/etc/default/lxc-net - it may be necessary to add /etc/lxc/dnsasq.conf to the apparmor profile (/etc/apparmor.d/*dnsmasq*) with read privileges




iptables config

        ## Evan Widloski - 2016-11-11
        # Diode iptables rules

        # filter table: flush all chains, and delete all user added chains
        iptables -F
        iptables -X
        # nat table: flush all chains, and delete all user added chains
        iptables -t nat -F
        iptables -t nat -X

        # set default policies to DROP packets
        iptables -P INPUT DROP
        iptables -P OUTPUT DROP
        iptables -P FORWARD DROP

        # allow inbound outbound traffic on host 
        iptables -A OUTPUT -o enp6s0f0 -d -j ACCEPT 
        iptables -A INPUT -i enp6s0f0 -m state --state ESTABLISHED,RELATED -j ACCEPT

        # set up chain for sshguard
        iptables -N sshguard
        iptables -A INPUT -p tcp --dport 22 -j sshguard

        # allow ssh
        iptables -A INPUT -i enp6s0f0 -p tcp --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT
        iptables -A OUTPUT -o enp6s0f0 -p tcp --sport 22 -m state --state ESTABLISHED -j ACCEPT

        # allow mosh
        iptables -A INPUT -i enp6s0f0 -p udp --dport 60000:61000 -j ACCEPT
        iptables -A OUTPUT -o enp6s0f0 -p udp --sport 60000:61000 -j ACCEPT

        # allow connections to varnish service
        #iptables -A INPUT -i enp6s0f0 -p tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT
        #iptables -A OUTPUT -o enp6s0f0 -p tcp --sport 80 -m state --state ESTABLISHED -j ACCEPT

        # allow host to access LXC targets via network
        iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
        iptables -A OUTPUT -s -j ACCEPT

        # allow outbound traffic for lxc containers
        iptables -A FORWARD -i lxcbr0 -j ACCEPT
        iptables -t nat -A POSTROUTING -s -j MASQUERADE

        # after incoming packets have been NAT'ed (see below), allow them to pass through
        # the forward chain to their intended LXC target
        iptables -A FORWARD -m state --state NEW,ESTABLISHED,RELATED -j ACCEPT

        ##------------ evan --------------
        ## ssh
        iptables -t nat -A PREROUTING -p tcp --dport 20022 -j DNAT --to-destination


# list container statuses and ip addresses (fancy mode)
        lxc-ls -f
brctl show
        brctl delbr virbr0
        brctl addbr virbr0
        ip link set virbr0 down

New Container Setup

New LXC containers are very barebones and need a bit of setup to be useful. Here is an overview of steps for various distros.


Setup PATH

# add /bin, /sbin to path
        echo 'PATH=$PATH:/bin:/sbin'>>.bashrc

Install packages

# core commands
        apt-get install apt-utils vim man tar less iputils-ping

        # extra commands
        apt-get install git zip autojump wget htop ncdu nload


Install packages

# core commands
        dnf install vim man

        # core commands
        dnf install git zip autojump wget htop ncdu nload


# enable notifications for any messages in buffer (works for Android client, too)
        /buffer set highlight_regex .\ast{}.*


Checking state and simulating failure

# check RAID state
        cat /proc/mdstat  # look for failure, (F), after the drive name: sda1[0](F)

        # simulate a failed drive
        mdadm --manage --set-faulty /dev/md/pv00 /dev/sda1

        # remove faulty state by removing and readding
        mdadm --remove /dev/md/pv00 /dev/sda1
        mdadm --add /dev/md/pv00 /dev/sda1

Replacing a failed drive (sdc)

# set hard drive as failed
        # mark as failed and remove
        mdadm --manage /dev/md127 --fail /dev/sdc1
        mdadm --manage /dev/md127 --remove /dev/sdc1

        # write down serial number of failed drive
        hdparm -i /dev/sdc1 | grep -i serial
        shutdown -h now
        # remove broken harddrive, insert the new hardddrive

        # copy partition scheme from working harddrive to new harddrive
        sfdisk -d /dev/sda | sfdisk /dev/sdc

        # add new harddrive
        mdadm --manage /dev/md127 --add /dev/sdc1

        # verify that array is recovering
        cat /proc/mdstat

Notifying on harddrive failure (gmail)


# add this after `begin routers` in router config section
             driver = manualroute
             domains = ! +local_domains
             transport = gmail_smtp
             route_list = *
        # add this after `begin transports` in transports config section
             driver = smtp
             port = 587
             hosts_require_auth =
             hosts_require_tls =
        # add this after `begin authenaticators` in authentication config section
             driver = plaintext
             public_name = LOGIN
             client_send = : : password_in_plaintext_here


        AUTO +imsm +1.x -all
        ARRAY /dev/md/pv00 level=raid5 num-devices=4 UUID=1327a02b:b19f6696:0e3f8ac7:9615591c

Growing RAID size

This is useful if the RAID array needs to be grown by using up more free space (no added harddrive)

umount /dev/sda
        umount /dev/sdb
        umount /dev/sdc
        umount /dev/sdd

        # grow RAID array to 500GB (this will take a while)
        mdadm -G /dev/md127 -z 500G

        # resize physical volume to fit new RAID partition size
        pvresize /dev/md127

Accessing via Live CD

If the array gets screwed up somehow, you can try mounting it on a livecd.

apt install mdadm

        # assemble array from block devices
        mdadm --assemble --scan

        # mount array (assuming lvm)
        apt install lvm2

        # see if lv's are intact

        # mount lv
        mount /dev/[vgname]/[lvname] /mnt/foo

Installing GRUB on a Live CD Mounted System

# mount root lv
        mount /dev/[vgname]/root /mnt/root

        # mount live CD directories inside mounted lv
        for i in /dev /dev/pts /proc /sys /run; do sudo mount -B $i /mnt/root$i; done

        # chroot into root lv
        chroot /mnt/root

        # install grub to each device in array
        grub2-install /dev/sda
        grub2-install /dev/sdb
        grub2-install /dev/sdc
        grub2-install /dev/sdd

        # update grub config
        grub2-mkconfig -o /boot/grub2/grub.cfg

Mounting Images

# list the partitions on the image file
        fdisk -l /tmp/sdcard.img 
Disk /tmp/sdcard.img: 162 MiB, 169869824 bytes, 331777 sectors
        Units: sectors of 1 * 512 = 512 bytes
        Sector size (logical/physical): 512 bytes / 512 bytes
        I/O size (minimum/optimal): 512 bytes / 512 bytes
        Disklabel type: dos
        Disk identifier: 0x00000000

        Device           Boot Start    End Sectors  Size Id Type
        /tmp/sdcard.img1 *        1  65536   65536   32M  c W95 FAT32 (LBA)
        /tmp/sdcard.img2      65537 331776  266240  130M 83 Linux
# use the sector size and the partition start sector to calculate offset (512 * 65537)
        sudo mount -o loop,offset=33554944 /tmp/sdcard.img /mnt/tmp

Auto FS

Auto FS + SSHFS allows the system to mount ssh filesystems on access and then automatically unmount after a certain timeout. The necessary tools are autofs and sshfs.

/etc/auto.master or /etc/auto.master.d/foobar.autofs or /etc/autofs/auto.master

# mounts all the entries listed in /etc/auto.sshfs in /mnt/ with the given options
        # add the --verbose option here to debug mounting issues
        # set --timeout to control when sshfs mount is automatically unmounted
        /mnt /etc/auto.sshfs --timeout=180 --ghost


# make a mount to be used by auto.master
        foobar -fstype=fuse,rw,IdentityFile=/home/evan/.ssh/foobar,port=22,allow_other :sshfs\\:

AutoFS runs as root, so ensure that the host fingerprint has been added to root.ssh/known_hosts. You can add this easily by attempting to ssh login to foo\ from root.

su -
        # enter yes

Resizing LUKS encrypted LVM

# expand the block device with fdisk, if necessary

        # resize physical volume
        pvresize --setphysicalvolumesize 111.8G /dev/sdb2
        # be careful about using `-l +100%FREE`.  this broke /home until I manually shrank fedora--vg-home by a few GB
        lvextend -l 80G /dev/mapper/fedora--vg-home
        resize2fs /dev/mapper/fedora--vg-home

Fixing Nodejs


# Sync permissions only. (useful if you forgot `-p` option in cp)
        # Looks at filesize differences to determine if a copy is needed rather
        # than timestamp (which gets reset when `-p` is left out of cp.
        rsync --archive --size-only /src/foo /dest/bar

DNS Tunneling with iodine

Most of this was taken from

Domain Setup

On a domain you own (e.g., create an A record pointing to the ip of a server you own and an NS record pointing to

To verify the setup is working, you can do:

# on the server
        sudo nc -u -l -p 53

        # on another device
        dig +trace
        # you should see some stuff printed out in the console on the server

Server Setup

# set iptables rules
        iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
        iptables -A FORWARD -i eth0 -o dns0 -m state --state RELATED,ESTABLISHED -j ACCEPT
        iptables -A FORWARD -i dns0 -o eth0 -j ACCEPT

        # enable ip forwarding on the server
        #  unnecessary if you want to use `ssh -D 1234` for dynamic port forwarding 
        #  on the client (as opposed to setting default routes)
        echo 1 > /proc/sys/net/ipv4/ip_forward

        # install iodine
        dnf install iodine

        # run iodine (as root in a screen session)
        #  `password` is the password to use the tunnel
        #  `` is the ip of the server on the tunnel network
        iodined -c -P password -f

        # note that if iodined is behind a NAT, you'll need to port forward UDP 53 and
        #   use the `-p` option to tell iodined the server's external ip
        # iodined -c -P password -f -p [ ip]

Client Setup

Alternatively, you can download a script that does this part from

# launch iodine client and wait for a 'Connection setup complete' message
        sudo iodine -f

        # either use SSH for dynamic forwarding (one application at a time)  or set up routes

        # ssh
        ssh -D 1234
        # set Firefox to use socks proxy localhost on port 1234

        # set up routes
        # get the current gateway ip
        ip route
        # get the tunnel server ip
        # add a route for iodine to communicate with the outside world
        sudo ip route add [ IP] via [current gateway IP]
        # delete the default route for traffic
        sudo ip route delete default
        # add a default route so that all traffic is tunneled
        sudo ip route add default via
        # also DNS may not work, so you might have to manually set it to something reasonable
        sudo sh -c "echo nameserver > /etc/resolv.conf"


# lookup the hostname or domain name associated with ip address
        # also works for local machines in the search domain
        dig -x


block - smallest addressable unit of storage

Block size is defined in the hardware of a hard drive, but the OS can
        define a virtual block size which chains multiple blocks together.

There are three primary boot options involving UEFI and BIOS firmwares

GPT - GUID Partition Table

protective mbr - a small partition at the beginning of the GPT disk (where the MBR would normally be) that prevents older MBR tools from damaging the GPT formatting

This partition contains a fake partition record which spans the entirety of the disk. MBR programs will see that there is a partition of unknown type that spans the entire disk and will refuse to operate.

A GPT disk is formatted like so:

Protective MBR512B

|------------------------|------| | Protect MBR | 512B | | GPT Header | 512B | | GPT Partition TAble | 16KB | | Partitions | XXX | | Backup Partition Table | 16KB | | Backup Header | 512B | |------------------------+------|

So there should be 17KB and 16.5KB of free space at the beginning and end of a GPT disk.

Random facts

SMART Status

::: {.definitions} smartctl -a /dev/sdX

smartctl -t short /dev/sdX :::

Network interfaces and bridging

Simulating network disconnect

# add network namespaces (for network isolation)
        sudo ip netns add client-ns
        sudo ip netns add server-ns

        # create pairs of virtual interfaces
        sudo ip link add client type veth peer name client-bridge
        sudo ip link add server type veth peer name server-bridge
        # add virtual interfaces to namespace
        sudo ip link set client netns client-ns
        sudo ip link set server netns server-ns
        # give addresses to each virtual interface
        sudo ip netns exec client-ns ip addr add dev client
        sudo ip netns exec server-ns ip addr add dev server
        # set virtual interfaces up
        sudo ip netns exec client-ns ip link set client up
        sudo ip netns exec server-ns ip link set server up
        sudo ip link set client-bridge up
        sudo ip link set server-bridge up

        # add bridge interface
        sudo brctl addbr bridge0
        # link virtual interfaces to bridge
        sudo brctl add if bridge0 client-bridge
        sudo brctl add if bridge0 server-bridge
        # set bridge up
        sudo ip link set bridge0 up

        # (in a new terminal) do stuff on the virtual interface
        sudo ip netns exec client-ns ping

        # set bridge down (simulate network offline)
        sudo ip link set bridge0 down

        # sometimes you might need to use the loopback interface
        sudo ip netns exec client-ns ip link set lo up
        sudo ip netns exec server-ns ip link set lo up

Ubuntu VNC Server

# install x11vnc
        sudo apt install x11vnc
        # create a password
        x11vnc -storepasswd
        # start the server

To automatically run the server at login, add a startup script:


Add a new item called \"VNC\" with the command field set to

x11-vnc -forever -loop -noxdamage -repeat -rfbport 5900 -shared -usepw

Remember to set the machine to never suspend in the system power settings.


Useful options


List copr repos

dnf copr list

Removing copr repos

sudo dnf copr remove foo/bar

Broken TTY

sudo DISPLAY=:0 loadkeys us