文本处理

1. sort

sort -u, --unique
sort -r, --reverse
sort -o, --output=FILE
sort -f, --ignore-case
sort -n,  --numeric-sort
sort -t, --field-separator=SEP
sort -k, --key=KEYDEF

sort -m, --merge

sort -o result.txt myfile   <=> sort myfile > result.txt 
sort -o myfile myfile

sh-4.2# cat myfile 
java,20,5 
python,18,7  
perl,12,3            
shell,27,7               
c,21,4 

sh-4.2# sort -n -t',' -k 2 myfile
perl,12,3                 
python,18,7              
java,20,5               
c,21,4                  
shell,27,7 

2. uniq

uniq -c, --count
uniq -i, --ignore-case

uniq -u, --unique       # only print unique lines
uniq -d, --repeated     # only print duplicate lines

sort myfile | uniq -c | sort -nr

3. cut

cut -c, --characters=LIST
cut -d, --delimiter=DELIM     # use DELIM instead of TAB for field delimiter, DELIM is just a character
cut -f, --fields=LIST

[root@localhost ~]# cut -d' ' -f1,2 /etc/mtab 
/dev/sda2 /
proc /proc
sysfs /sys
devpts /dev/pts
tmpfs /dev/shm
/dev/sda1 /boot
/dev/sdb1 /bkboot
/dev/mapper/LVM1-web1 /web1
/dev/mapper/LVM1-web2 /web2
none /proc/sys/fs/binfmt_misc
vmware-vmblock /var/run/vmblock-fuse

[root@localhost ~]# uname -a | cut -d' ' -f1,3,11,12
Linux 2.6.32-220.el6.i686 2011 i686

[root@localhost ~]# echo $PATH
/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin

[root@localhost ~]# echo $PATH | cut -d':' -f1,4-
/usr/local/sbin:/bin:/usr/sbin:/usr/bin:/root/bin

[root@localhost ~]# echo "abc123" | cut -c 2-4
bc1

4. head

head -c, --bytes=[-]K       # outputs a specified number of characters
head -n, --lines=[-]K

# generating 10-digit random numbers
[root@localhost ~]# head -c4 /dev/urandom | od -N4 -tu4 | sed -ne '1s/.* //p'
2908845609

# od: dump files in octal and other formats
# -N4 option limits output to 4 bytes
# -tu4 option selects unsigned decimal format for ouput

5. tail

tail -n, --lines=K
tail -f, --follow[={name|descriptor}]
tail -n N <=> tail -N (older way, deprecate)

6. wc

wc -c, --bytes
wc -m, --chars
wc -l, --lines
wc -w, --words
wc -L, --max-line-length

[root@localhost ~]# wc -l install.log
875 install.log
[root@localhost ~]# wc -c install.log
37361 install.log
[root@localhost ~]# wc -L install.log
108 install.log
[root@localhost ~]# cat install.log | awk 'BEGIN{max=0}{len=length($0); if(max<len) max=len}END{print max}'
108

7. grep

grep -E, --extended-regexp      # egrep
grep -F, --fixed-strings        # fgrep
grep -P, --perl-regexp
grep -e PATTERN, --regexp=PATTERN
grep -i, --ignore-case
grep -v, --invert-match
grep -w, --word-regexp
grep -l, --files-with-matches
grep -R, -r, --recursive

grep -n, --line-number
grep -z, --null-data

8. tr

tr -c, -C, --complement
tr -d, --delete
tr -s, --squeeze-repeats
tr -t, --truncate-set1

SET:
'\NNN'   # octal value
'\n'
'\r'
'\t'
'\\'
'CHAR1-CHAR2'
'[CHAR*]'

[root@localhost ~]# echo "abcdef" | tr -d b-d  
aef

[root@localhost ~]# echo "aabc123" | tr -s 'a'
abc123

[root@localhost ~]# echo "abcd123cba" | tr -c b-d +
+bcd+++cb++

tr -d '\015' < file1 > file2   # dos2unix

# variants
BSD/GUN: not use brackets (tr a-z A-Z)
SysV: tr '[a-z]' '[A-Z]'

9. expand/unexpand

expand       # converts tabs to spaces
unexpand     # converts spaces to tabs

expand -t, --tabs=NUMBER
expand -t, --tabs=LIST

unexpand -t, --tabs=N
unexpand -t, --tabs=LIST

[root@localhost ~]# cat -A myfile
abc^Iabc\$
^I123^I789\$
[root@localhost ~]# cat -A myfile2
abc^Iabc\$
       ^I123^I789\$

[root@localhost ~]# expand myfile > myfile2
[root@localhost ~]# cat -A myfile2
abc     abc\$
        123     789\$

[root@localhost ~]# expand -t 4 myfile > myfile3
[root@localhost ~]# cat myfile3
abc abc\$
    123 789\$

[root@localhost ~]# expand -t 1 myfile > myfile4
[root@localhost ~]# cat -A myfile4
abc abc\$
 123 789\$

10. join, the fields must have a common content, and the field is splited by single tab or space
-a FILENUM：除了显示匹配好的行另外将指定序号（1或2）文件中部匹配的行显示出来
-e EMPTY：将需要显示但是文件中不存在的域用此选项指定的字符代替
-i :忽略大小写
-j FIELD ：等同于 -1 FIELD -2 FIELD,-j指定一个域作为匹配字段
-o FORMAT：以指定格式输出
-t CHAR ：以指定字符作为输入输出的分隔符
join 默认以空白字符做分隔符（空格和\t）,可以使用 join -t $’\t’来指定使用tab做分隔符
-v FILENUM：与-a相似 但值显示文件中没匹配上的行
-1 FIELD：以file1中FIELD字段进行匹配
-2 FIELD：以file2中FIELD字段进行匹配

# 按第一列匹配
join a.txt b.txt
join -j 1 a.txt b.txt
join -1 1 -2 1 a.txt b.txt

# 第1个文件的第2列和第2个文件的第3列做匹配字段
join -1 2 -2 3 a.txt b.txt

# 以第1个文件为基准，输出文件的所有列
# -o, 指定输出列
# -a, 指定需要全部输出的文件
# -e, 指定不匹配项的默认输出
join -o 1.1 -o 1.2 -o 1.3 -o 2.1 -o 2.2 -o 2.3 -e 'NA' -a 1 a.txt b.txt

# 第1个文件的不匹配行
join -v 1 file1.txt file2.txt 


5. paste, merge lines of files
# use : instead of TAB
paste -d: a.txt b.txt

# paste one file at a time instead of in parallel
paste -s a.txt b.txt

# 3 files each row
ls | paste -d" " - - -

11. ^M: CTRL-V, ENTER

dos2uninx dosfile

sed -e 's/^M//' dosfile

col -bx < dosfile

tr -s "\r\n" "\n" < dosfile
tr -d "\015" < dosfile

# delete ^M in vim
:set ff=unix
:%s/\r//g
:s/^M//gc

12. 删除空行

sed "/^\s*$/d"
sed '/^$/d'
sed -i '/^$/d'
awk 'NF>0'
perl -i.backup -n -e "print if /\S/"
grep -v '^$'

13. echo
echo -n 取消行末换行
echo -E 关闭反斜线控制字符转换
echo -e 启动反斜线控制字符转换

\c 取消行末换行符
\n newline => \012
\r return
\t TAB => \011
\num ASCII八进制编码
\xnum ASCII十六进制编码