Linux Tricks

* grep and awk

1. Simple grep for an error, then pull out the fifth field:

grep "ERROR - Unauthorized Access" error_log | awk '{print $5}' > error_report

2. Slightly more complex - pull a username out of a gzipped file, then grep again to pull out all 'PUT' operations, then print out the third and last field with a space between them, then sort, then uniq -c

zgrep -i infinitran customer_access_log.gz | grep " PUT - " | awk '{print $3 " " $NF}' | sort | uniq -c > infinitran_put_report

3. Fancy example - Think of a log file that looks like this:

20050223 10:45:10 jsmith,192.168.1.54,84.21,www.google.com,Y
20050223 10:53:02 jfrank,192.168.1.21,60.21,www.slashdot.org,Y
20050223 10:54:39 jsmith,192.168.1.54,22.93,www.cnn.com,Y

So, you want to grep for all instances of 'jsmith', then grab the comma separated line and pull just the useful stuff out. Since awk uses spaces by default, you have to tell it to split on commas instead. While I could have said 'print the first and fourth fields', I chose to say 'print the first and second-from-last fields', which comes in handy when lines get really long. Note the second sort, which will do a reverse numeric sort on the numbers printed by uniq -c. You'll end up with something like this:

29181 jsmith www.google.com
  398 jsmith www.cnn.com
   23 jsmith www.foo.com
    1 jsmith www.bar.net

grep jsmith example_log | awk '{print $3}' | awk -F , '{print $1 " " $(NF-1)}' | sort | uniq -c | sort -rn > example_report

4. Process killer:

ps -elf | egrep badprocessname | awk '{print $4}' | sudo xargs kill

5. Another awk example

tail -f time_log | awk '{print $6 " " $8}' | egrep -v "^[01] "

6. Quick 'who's being naughty?' ASA/PIX firewall reports:

grep "ASA-4-106023" 2007.05.14.local4 | awk '{print $9}' | awk -F : '{print $2}' | awk -F / '{print $1}' | sort | uniq -c | sort -rn
grep "ASA-4-106023" 2007.05.14.local4 | awk '{print $11}' | awk -F : '{print $2}' | awk -F / '{print $1}' | sort | uniq -c | sort -rn

7. Here's one more awk example:

grep "luckydragon.net" some_log_file | awk '{print $3}' | awk -F : '{print $1,$3}' | sort | uniq -c > report1

8. Watch an apache combined log for errors:

tail -f access_log |  awk '{print $10 " " $1 " " $2 " " $8}' | grep -v ^[2,3]0

9. Show the amount of memory each apache thread is using:

ps -eLo pid,user,comm,rss | grep 'httpd' | awk '{print $4/1024 " MB";}'

Average:

ps -eLo pid,user,comm,rss | grep 'httpd' | awk '{print $4/1024;}'  | awk '{avg += ($1 - avg) / NR;} END {print avg " MB";}'

10. grep for two different strings

ps -elf | egrep 'archiver|recovering' | grep -v grep

* sed

1. Basic sed substitution:

NEWNAME="gandalf"
sed -i -e "s/HOSTNAME.*/HOSTNAME=$NEWNAME.foo.com/" /etc/sysconfig/network

* find

# Dealing with files that have whitespace, slashes, or quotes:
find . -type f -print0 | xargs -0 rm -rf
find . -type f -print0 | xargs -0 -iFOO mv "FOO" /home/jsmith/tmp/
find . -type f -print0 -mtime +31 | xargs -0 -iFOO rm "FOO"

# quick copy:
tar -czf - httpd | dd bs=100k | ssh jsmith@server1 "cd /home/jsmith/foo; tar -xzf -"

# quick 'find and gzip' for big files:
find . -name Old -type d | xargs -iFOO find "FOO" -type f -size +10000k | xargs -iFOO gzip -v "FOO"

# find all files in a directory older than 60 days, then output the total amount of space those files consume in KB:
find . -type f -mtime +60 | xargs -iFOO ls -al "FOO" | awk '{sum+=$5} END{print sum}'

# find all files in a directory older than 31 days and delete them:
find /var/log/appname -type f -mtime +31 -exec rm {} \;

* rsync

# rsync options:
  -a = archive mode (same as -rlptgoD (no -H,-A,-X))
  -r = recursive
  -l = copy symlinks as symlinks
  -p = preserve permissions
  -t = preserve modification times
  -g = preserve group
  -o = preserve owner
  -D = preserve device files, preserve special files


# rsync across ssh using a shared key
rsync -avcuH --delete -e '/usr/bin/ssh -i /home/httpd/.ssh/id_dsa.us_prod_push' /home/httpd/foo/ prod-web01:/home/httpd/foo/

# rsync across ssh using regular password-based auth, one-time copy to a blank source, with rate limiting in KBytes / second
rsync -a --bwlimit=180 -e 'ssh -l jsmith' /data/foo/ 192.168.55.13:/home/jsmith/foo/

* tcpdump and friends

# listen for everything on all interfaces
tcpdump -i any

# listen for everything on all interfaces *except* the IP address from which you ssh'd in on
tcpdump -i any not host 192.168.31.210

# listen for incoming packets from 192.168.5.20 on tcp port 80
tcpdump -i eth0 host 192.168.2.124 and tcp src port 80

# listen for just DNS traffic
tcpdump -i eth0 -n -v port 53

# Most verbose output:
tcpdump -XX -vvv -i any

# Listen on eth0, no DNS, complete packet capture, log to a file called capture1:
tcpdump -i eth0 -n -U -s 0 -w capture1

# Show interface stats:
ethtool -S eth0

* ps

# Aliases:
alias p='ps -eLo pid,user,group,rss,args'
alias pg='ps -eLo pid,user,group,rss,args | grep '

# Top Ten Memory Users (Mem use in KB, PID, Command)
alias pm='ps -eLo rss,pid,args | sort -rn | head -10l'

* RPM

# List all installed rpms:
rpm -qa

# List all installed packages, sorted by install date, latest first:
rpm -qa --last

# Dump all installed rpms, name only (no version), into a sorted list:
rpm -qa --queryformat "%{name}\n" | sort > rpmlist

* cron jobs

# home directory disk usage report, twice a month
03 18 1,15 * * /usr/bin/du -ms /home/* | /bin/sort -rn | /bin/mail -s "Home Directory Disk Usage (MB)" reports@foo.com

* date and time

# date
date "+%Y-%m-%d %H:%M:%S"
2015-05-28 19:04:38

date "+%Y %m %d"
2015 05 28

TIMESTAMP=`date "+%Y-%m-%d %H:%M:%S"`

# convert unix time to human-readable time
$ date -d @1163110662
Thu Nov  9 16:17:42 CST 2006


# convert epoch to localtime:
perl -pe 's/(\d+)/localtime($1)/e' foo.log

* Random

# check entropy (should be > 100)
cat /proc/sys/kernel/random/entropy_avail

# make a 100 MB file with zeros:
dd if=/dev/zero of=testfile-100MB bs=1024 count=0 seek=$[1024*100]

# make a 1 GB file with random data:
dd if=/dev/urandom of=testfile-1GB bs=1024 count=0 seek=$[1024*1024]

# generating load 1:
compile n+1 kernels simulatneously, where n is the number of CPU cores

# generating load 2:
find . -type f| xargs -iFOO md5sum FOO

# generating load 3 (untested):
dd if=/dev/zero bs=100M | gzip | gzip -d | gzip | gzip -d | gzip | gzip -d > /dev/null & 

# random egrep example:
egrep "^/20[0-9][0-9]/[[:digit:]]{3}/[[:alnum:]]{17}/[[:alpha:]]{4,9}/[[:digit:]]{1,2}.jpg$" example.log
Matches:
/2012/472/33CPC3E198M134472/thumbnail/22.jpg
/2012/212/3YCBCVVV9851L4O7I/medium/54.jpg
/2013/960/J8YFN8EP6A851896K/full/9.jpg

# IPv4 address regex (from http://answers.oreilly.com/topic/318-how-to-match-ipv4-addresses-with-regular-expressions/)
egrep "^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" foo

# grep or
ps -elf | grep 'green\|red'

# sort du output
sort -k 2

# redirect both STDOUT and STDERR to the same file:
ls -al > /tmp/filename 2>&1

# redirect STDOUT and STDERR to different files:
ls -al > /tmp/stdout.log 2> stderr.log


# find what ports process id 1212 is listening on (you may need to grep 1212 /etc/services and use the resulting string instead):
lsof -i | grep -i listen | grep 1212
 OR
netstat -lpn | grep 1212

# Truncate a file to zero length (2 ways)
sudo truncate -s0  /var/log/tomcat/catalina.out
cat /dev/null > /var/log/tomcat/catalina.out

# List listening TCP processes, don't convert network/ports to names
lsof -nP -iTCP -sTCP:LISTEN

# List all open network connections (similar to netstat -a -p )
lsof -i

# list open network connections to/from a single host:
lsof -i @somehost.foo.com

# list open files associated with a PID
lsof -p 8324

# list open files in a directory
lsof +d /var/log/apache/ 

# undelete (make sure the output file is on a different partition than the one you are trying to recover from)
egrep -100 'ftp.+COL' /dev/hda3 > /tmp/foo
strings /tmp/foo | less

# find out what brand/version of DNS someone is using:
dig @64.57.182.179 txt chaos version.bind. | grep -i VERSION

# iso burning:
cdrecord -scanbus
cdrecord -v -dev 2,0,0 -speed 8 boot.iso


# Working with .cgz files...
zcat modules.cgz | cpio -id
find 2.2.16-22BOOT/*.* | cpio -o --format=crc |gzip -9 > modules.cgz

# Full packet capture:
tcpdump -n -nn -s 0 -w outputfile host 192.168.1.33

# hard drive partition mirror:
sfdisk -d /dev/hda | sfdisk -f /dev/hdc

# quick ramdisk
mount -o size=16G -t tmpfs none /mnt/tmpfs

* LVM2

# Create a new logical volume:
  lvcreate -L22G -n foo rootvg
  mkfs.ext3 -O dir_index /dev/rootvg/foo

# Expand an existing logical volume:
  /usr/sbin/lvextend -L+5G /dev/rootvg/usr

  ext2online /dev/rootvg/usr (good up through RedHat/CentOS 4.x)
  resize2fs /dev/rootvg/usr (good for RedHat/CentOS 5.x)

* Linux Tweaks

# enable DMA5 (64 + DMA Mode 5 = 69)
hdparm -c1 -X 69 /dev/hda

# increase max file descriptors
echo "32768" > /proc/sys/fs/file-max

# dump all kernel parameters
sysctl -A

# load settings from /etc/sysctl.conf
sysctl -p /etc/sysctl.conf

# setting kernel parameters
sysctl -w kernel.sem="250 32000 100 128"

* Other Stuff

# linux single user:
	linux single
	linux init=/bin/bash

# group sticky bit plus 775 == 2775

# case insensitivity for less:
less -I

# case insensitivity for vim:
:set ignorecase

# turn off auto-indenting in vim (useful when pasting in existing code/configuration):
:set paste
# re-enable auto-indenting:
:set nopaste

# nfs kickstart with a driver disk:
linux dd ks=nfs:192.168.50.100:/usr/local/kickstart/kickstart/ks1.cfg

# iso mount:
mount -t iso9660 -o loop foo.iso /mnt/tmp

# BIND nsupdate examples:
 update delete mail.foo.com. in mx
 update add foo.com. in mx 10 mail01.foo.com.
 update add foo.com. in mx 50 mail02.foo.com.