bash 3.

x
Advanced Shell Scripting
Michael Potter
Replatform Technologies, LLC
May 20th, 2007

WAVV Green Bay Copyright Michael Potter 1

May 20th, 2007 [email protected]
 Why bash?
• Simple to get started.
• Actively developed and ported.
• Includes advanced features.

WAVV Green Bay Copyright Michael Potter 2

May 20th, 2007 [email protected]
 Goals for Today

WAVV Green Bay Copyright Michael Potter 3

May 20th, 2007 [email protected]
 Understanding the Demos

WAVV Green Bay Copyright Michael Potter 4

May 20th, 2007 [email protected]
 The demo script

#!/opt/local/bin/bash
set -o option
echo "My process list:" >outputfile.txt

ps -ef 2>&1 |grep "^$USR" >outputfile.txt

echo "script finished: return code is $?"

WAVV Green Bay Copyright Michael Potter 5

May 20th, 2007 [email protected]
 noclobber demo

WAVV Green Bay Copyright Michael Potter 6

May 20th, 2007 [email protected]
 What did we learn?
•set -o noclobber
•used to avoid overlaying files
•set -o errexit
•used to exit upon error, avoiding cascading errors

WAVV Green Bay Copyright Michael Potter 7

May 20th, 2007 [email protected]
 command1 | command2

WAVV Green Bay Copyright Michael Potter 8

May 20th, 2007 [email protected]
 pipefail demo

WAVV Green Bay Copyright Michael Potter 9

May 20th, 2007 [email protected]
 What did we learn?
• set -o pipefail
– unveils hidden failures
• set -o errexit
– can exit silently
• trap command ERR
– corrects silent exits
• $LINENO
– enhances error reporting

WAVV Green Bay Copyright Michael Potter 10

May 20th, 2007 [email protected]
 nounset demo

WAVV Green Bay Copyright Michael Potter 11

May 20th, 2007 [email protected]
 What did we learn?
•set -o nounset
•exposes unset variables

WAVV Green Bay Copyright Michael Potter 12

May 20th, 2007 [email protected]
 the final demo script
#!/opt/local/bin/bash

set -o noclobber
set -o errexit
set -o pipefail
set -o nounset
trap 'echo error at about $LINENO' ERR

mv outputfile.txt outputfile.bak
echo "My process list:" >outputfile.txt
ps aux 2>&1 |grep "^$USER" >>outputfile.txt

echo "script finished: return code is $?"

WAVV Green Bay Copyright Michael Potter 13

May 20th, 2007 [email protected]
 the final demo script
#!/opt/local/bin/bash

. ./stringent.sh || exit 1

mv outputfile.txt outputfile.bak
echo "My process list:" >outputfile.txt
ps aux 2>&1 |grep "^$USER" >>outputfile.txt

echo "script finished: return code is $?"

WAVV Green Bay Copyright Michael Potter 14

May 20th, 2007 [email protected]
 the final demo script
#!/opt/local/bin/bash

source ./stringent.sh || exit 1

mv outputfile.txt outputfile.bak
echo "My process list:" >outputfile.txt
ps aux 2>&1 |grep "^$USER" >>outputfile.txt

echo "script finished: return code is $?"

WAVV Green Bay Copyright Michael Potter 15

May 20th, 2007 [email protected]
 stringent.sh
# stringent.sh

set -o errexit
download stringent.sh from
set -o noclobber
set -o nounset www.replatformtech.com
set -o pipefail

function traperr
{
echo "ERROR: ${BASH_SOURCE[1]} “ \
“at about line ${BASH_LINENO[0]}"
}

set -o errtrace
trap traperr ERR
WAVV Green Bay Copyright Michael Potter 16
May 20th, 2007 [email protected]
BASH_SOURCE/BASH_LINENO
echo "ERROR: ${BASH_SOURCE[1]} “ \
“at about line ${BASH_LINENO[0]}"

ERROR: ./buggy.sh at about line 7

${FUNCNAME[$i]} was called at

${BASH_LINENO[$i]}in ${BASH_SOURCE[$i]}

WAVV Green Bay Copyright Michael Potter 17

May 20th, 2007 [email protected]
 BASH_COMMAND

function traperr
{
E N
O K
echo "ERROR: $1 ${BASH_SOURCE[1]} “ \
“at about line ${BASH_LINENO[0]}"
}

B R
trap ‘traperr $BASH_COMMAND’ ERR

WAVV Green Bay Copyright Michael Potter 18

May 20th, 2007 [email protected]
 PIPESTATUS
bash-3.1$ ps -ef 2>&1 |grep "^$USR" >/dev/null
bash-3.1$ echo "PIPESTATUS = ${PIPESTATUS[*]} \$? = $?"
PIPESTATUS = 1 0 $? = 0
bash-3.1$ set -o pipefail
bash-3.1$ ps -ef 2>&1 |grep "^$USR" >/dev/null
bash-3.1$ echo "PIPESTATUS = ${PIPESTATUS[*]} \$? = $?"
PIPESTATUS = 1 0 $? = 1
bash-3.1$ ps aux 2>&1 |grep "^$USER" >/dev/null
bash-3.1$ echo "PIPESTATUS = ${PIPESTATUS[*]} \$? = $?"
PIPESTATUS = 0 0 $? = 0
bash-3.1$ echo "PIPESTATUS = ${PIPESTATUS[*]} \$? = $?"
PIPESTATUS = 0 $? = 0
WAVV Green Bay Copyright Michael Potter 19
May 20th, 2007 [email protected]
 capturing return code
#!/opt/local/bin/bash

source ./stringent.sh

errexitoff # part of stringent.sh

cmp /tmp/filea /tmp/fileb
cmpRtn=$?
errexiton # part of stringent.sh

if (( $cmpRtn == 0 ))
then
echo "file same"
elif (( $cmpRtn == 1 ))
then
echo "file different"
else
echo "cmp failed"
fi

WAVV Green Bay Copyright Michael Potter 20

May 20th, 2007 [email protected]
 Variables

WAVV Green Bay Copyright Michael Potter 21

May 20th, 2007 [email protected]
 Integer Demo

WAVV Green Bay Copyright Michael Potter 22

May 20th, 2007 [email protected]
 What did we learn
• stringent.sh
– Proven to be a good idea
• declare -i variable
• non-integer values caught sooner
• unset variables used as an int are 0
• unless caught with set -o nounset
• $(( ... ))
• arithmetic syntax

WAVV Green Bay Copyright Michael Potter 23

May 20th, 2007 [email protected]
 gotcha
declare -i MyInt1=012
declare -i MyInt2=0x12

echo "Value1 = $MyInt1"

echo "Value2 = $MyInt2"
printf "%o %x\n" $MyInt1 $MyInt2

Value1 = 10
Value2 = 18
12 12
WAVV Green Bay Copyright Michael Potter 24
May 20th, 2007 [email protected]
 Arithmetic Syntax
• intA=$(( ($intB + 5) * 2 ))
– Allowed anywhere a variable is allowed
• let "intA = ($intB + 5 ) * 2"
– returns 0 or 1
• (( intA = ($intB + 5) * 2 ))
– equivalent to let
• intA=\($intB+5\)*2
– no spaces allowed
– Special characters must be escaped
– intA must be declare -i
• intA=$[ ($intB + 5) * 2 ]
– deprecated

WAVV Green Bay Copyright Michael Potter 25

May 20th, 2007 [email protected]
 Two More
• eval
– Char=B
– eval “intA=\$(( (\$int$Char + 5) * 2 ))”
• external command
– intA=$(echo “($intB + 5) * 2” | bc)

WAVV Green Bay Copyright Michael Potter 26

May 20th, 2007 [email protected]
 local variables
• weak
• good enough
• not just local, local and below
• two ways to declare:
– declare
– local
• $1, $2, ... are not scoped the same

WAVV Green Bay Copyright Michael Potter 27

May 20th, 2007 [email protected]
 Scoping
declare Var=”Zero" function Func1 { function Func2 {
MyPrintVars 1 declare Var1=”One" declare Var3=”Tri"
Func1 declare Var2=”Pair" declare Var4=”Quad"
MyPrintVars 5 declare Var4="Four" Var2=”Two"
MyPrintVars 2 MyPrintVars 3
Func2 }
MyPrintVars 4
}

WAVV Green Bay Copyright Michael Potter 28

May 20th, 2007 [email protected]
 Handling undefined variables
function PrintVars {
echo -n “Var1=${Var1:-notset}”
echo -n “Var2=${Var2:-notset}”
echo -n “Var3=${Var3:-notset}”
echo -n “Var4=${Var4:-notset}”
echo “Var5=${Var5:-notset}"
}

WAVV Green Bay Copyright Michael Potter 29

May 20th, 2007 [email protected]
 Scoping
declare Var=”Zero" function Func1 { function Func2 {
MyPrintVars 1 declare Var1=”One" declare Var3=”Tri"
Func1 declare Var2=”Pair" declare Var4=”Quad"
MyPrintVars 5 declare Var4="Four" Var2=”Two"
MyPrintVars 2 MyPrintVars 3
Func2 }
MyPrintVars 4
}

1 Var=Zero Var1=notset Var2=notset Var3=notset Var4=notset

2 Var=Zero Var1=One Var2=Pair Var3=notset Var4=Four
3 Var=Zero Var1=One Var2=Two Var3=Tri Var4=Quad
4 Var=Zero Var1=One Var2=Two Var3=notset Var4=Four
5 Var=Zero Var1=notset Var2=notset Var3=notset Var4=notset

WAVV Green Bay Copyright Michael Potter 30

May 20th, 2007 [email protected]
 readonly variables
• Two ways to declare
– declare -r
– readonly
• One way trip
• Used with -i to create readonly integers
• readonly can be used on system variables
– e.g. keep users from changing their prompt
– not documented!

WAVV Green Bay Copyright Michael Potter 31

May 20th, 2007 [email protected]
 conditionals

if command
if (( ))
if [ ]
if test
if [[ ]]

WAVV Green Bay Copyright Michael Potter 32

May 20th, 2007 [email protected]
 if command
if grep Jim /etc/passwd
set +o errexit then
grep Jim /etc/passwd echo “Jim is a user”
declare -i Status=$? fi
set -o errexit ! grep Jim /etc/passwd
if (( $Status == 0 )) declare -i Status=$?
then if (( $Status != 0 ))
echo “Jim is a user” then
fi echo “Jim is a user”
fi
WAVV Green Bay Copyright Michael Potter 33
May 20th, 2007 [email protected]
 What did we learn?
• set +o errexit turns off errexit
• Save $? to a permanent variable
• ! turns off errexit for a single command
• zero is true, non-zero is false
• if (( )) used for numeric tests

WAVV Green Bay Copyright Michael Potter 34

May 20th, 2007 [email protected]
 gotcha
• if [[ $Age > 20 ]] # bad, 3 buys beer!
– > is a string comparison operator
• if [ $Age > 20 ] # bad, everyone buys beer!
– > is a redirection operator
• if [[ $Age -gt 20 ]] # good
– fails in strange ways if $Age is not numeric
• if (( $Age > 20 )) # best
– $ on Age is optional

WAVV Green Bay Copyright Michael Potter 35

May 20th, 2007 [email protected]
 test and [
bash-3.1$ which test
/bin/test
bash-3.1$ which [
/bin/[
bash-3.1$ ls -l /bin/[ /bin/test
-r-xr-xr-x 2 root wheel 18104 Aug 21 2005 /bin/[
-r-xr-xr-x 2 root wheel 18104 Aug 21 2005 /bin/test

WAVV Green Bay Copyright Michael Potter 36

May 20th, 2007 [email protected]
 So?

WAVV Green Bay Copyright Michael Potter 37

May 20th, 2007 [email protected]
 if [[ ]]

WAVV Green Bay Copyright Michael Potter 38

May 20th, 2007 [email protected]
 [ versus [[
• [[ $a == z* ]]
• True if $a starts with an "z”.
• [[ $a == "z*" ]]
• True if $a is exactly equal to “z*”.
• [ $a == z* ]
• Error if $a has a space.
• Error if more than one filename starts with z.
• True if a filename exists that starts with z and is exactly $a.
• True if no filenames exist that start with z and $a equals z*.
• [ "$a" == "z*" ]
• True if $a is exactly equal to z*.

WAVV Green Bay Copyright Michael Potter 39

May 20th, 2007 [email protected]
WAVV Green Bay Copyright Michael Potter 40
May 20th, 2007 [email protected]
 the rules
• use [
– when you “want” to use file globbing
• use ((
– when you want to do math
• use [[
– for everything else

WAVV Green Bay Copyright Michael Potter 41

May 20th, 2007 [email protected]
 regular expressions
• Introduced with version 3.0
• Implemented as part of [[ ]]
• Uses binary operator =~
• Supports extended regular expressions
• Supports parenthesized subexpressions

WAVV Green Bay Copyright Michael Potter 42

May 20th, 2007 [email protected]
 regular expression
declare MyStr="the quick brown fox"

[[ $MyStr == "the*" ]] # false: must be exact

[[ $MyStr == the* ]] # true: pattern match

[[ $MyStr =~ "the" ]] # true

[[ $MyStr =~ the ]] # true
[[ $MyStr =~ "the*" ]] # true

WAVV Green Bay Copyright Michael Potter 43

May 20th, 2007 [email protected]
 subexpressions
declare MyStr="the quick brown fox"

if [[ $MyStr =~ "the ([a-z]*) ([a-z]*)" ]]

then
echo "${BASH_REMATCH[0]}” # the quick brown
echo "${BASH_REMATCH[1]}” # quick
echo "${BASH_REMATCH[2]}” # brown
fi

WAVV Green Bay Copyright Michael Potter 44

May 20th, 2007 [email protected]
 bad expressions
declare MyStr="the quick brown fox"

if [[ $MyStr =~ "the [a-z) ([a-z*)" ]]

then
echo "got a match"
elif (( $? == 2 ))
: # no match, colon is no-op command
then
traperr "Assertion Error: Regular expression error"
exit 1
fi
WAVV Green Bay Copyright Michael Potter 45
May 20th, 2007 [email protected]
 gotcha
• cp $srcfile $dstfile
– broken if $srcfile has a space
• cp “$srcfile” “$dstfile”
– broken if srcfile begins with -
• cp -- “$srcfile” “$dstfile”

WAVV Green Bay Copyright Michael Potter 46

May 20th, 2007 [email protected]
 quoting
declare MyVar="bob"
echo ’ \\ $MyVar \x41'
echo $’ \\ $MyVar \x41’
echo ” \\ $MyVar \x41"

bash-3.1$ ./quoting.sh
\\ $MyVar \x41
\ $MyVar A
\ bob \x41

WAVV Green Bay Copyright Michael Potter 47

May 20th, 2007 [email protected]
 quoting recommendation
• quote variables liberally
– extra quotes likely to cause a consistent error
– missing quotes are likely to cause inconsistent
behavior
• Safe Exceptions
– within if [[ ]]
– Integer variables (define -i)
– within if (( ))

WAVV Green Bay Copyright Michael Potter 48

May 20th, 2007 [email protected]
 Handling undefined variables
function PrintVars {
echo -n “Var1=${Var1:-notset}”
echo -n “Var2=${Var2:-notset}”
echo -n “Var3=${Var3:-notset}”
echo -n “Var4=${Var4:-notset}”
echo -n “Var5=${Var5:-notset}"
}

WAVV Green Bay Copyright Michael Potter 49

May 20th, 2007 [email protected]
 unset & null variables
• ${parameter:-word}
– returns word
• ${parameter:+word}
– returns empty (returns word if set)
• ${parameter:=word}
– sets parameter to word, returns word
• ${parameter:?message}
– echos message and exits
WAVV Green Bay Copyright Michael Potter 50
May 20th, 2007 [email protected]
 unset & null variables
• ${parameter-word}
–.
• ${parameter+word}
–.
• ${parameter=word}
–.
• ${parameter?message}
–.
WAVV Green Bay Copyright Michael Potter 51
May 20th, 2007 [email protected]
 default variables
function MyDate
{
declare -i Year=${1:?"$0 Year is required"}
declare -i Month=${2:-1}
declare -i Day=${3:-1}

if (( $Month > 12 )); then

echo "Error Month > 12" >&2
exit 1
fi
if (( $Day > 31 )); then
echo "Error Day > 31" >&2
exit 1
fi

echo "$Year-$Month-$Day"
}
WAVV Green Bay Copyright Michael Potter 52
May 20th, 2007 [email protected]
 sub strings
declare MyStr="The quick brown fox"

echo "${MyStr:0:3}" # The

echo "${MyStr:4:5}" # quick
echo "${MyStr: -9:5}" # brown
echo "${MyStr: -3:3}" # fox
echo "${MyStr: -9}" # brown fox

WAVV Green Bay Copyright Michael Potter 53

May 20th, 2007 [email protected]
 substr by pattern
• ${Var#pattern}
• ${Var%pattern}
• ${Var##pattern}
• ${Var%%pattern}

WAVV Green Bay Copyright Michael Potter 54

May 20th, 2007 [email protected]
 a jingle

We are #1 because we give 110%

WAVV Green Bay Copyright Michael Potter 55

May 20th, 2007 [email protected]
 substr by pattern
declare MyStr="/home/pottmi/my.sample.sh"

echo "${MyStr##*/}" # my.sample.sh

echo "${MyStr%.*}" # /home/pottmi/my.sample
echo "${MyStr%/*}" # /home/pottmi

echo "${MyStr#*/}” #home/pottmi/my.sample.sh

echo "${MyStr%%.*}" # /home/pottmi/my

WAVV Green Bay Copyright Michael Potter 56

May 20th, 2007 [email protected]
 search and replace
• ${Var/pattern/replace}

WAVV Green Bay Copyright Michael Potter 57

May 20th, 2007 [email protected]
 substr by pattern
declare MyStr="the fox jumped the dog"

echo "${MyStr/the/a}"
# a fox jumped the dog
echo "${MyStr//the/a}”
# a fox jumped a dog
echo "${MyStr//the }”
# fox jumped dog

WAVV Green Bay Copyright Michael Potter 58

May 20th, 2007 [email protected]
 unintended subshells
declare -i Count=0
declare Lines

cat /etc/passwd | while read Lines

do
echo -n "."
((Count++))
done

echo " final count=$Count"

...................................... final count=0

WAVV Green Bay Copyright Michael Potter 59

May 20th, 2007 [email protected]
 unintended subshells
declare -i Count=0
declare Lines

while read Lines

do
echo -n "."
((Count++))
done </etc/passwd

echo " final count=$Count"

...................................... final count=38

WAVV Green Bay Copyright Michael Potter 60

May 20th, 2007 [email protected]
 unintended subshells
declare -i Count=0
declare Lines

while read Lines

do
echo -n "."
((Count++))
done < <(cat /etc/passwd)

echo " final count=$Count"

...................................... final count=38

WAVV Green Bay Copyright Michael Potter 61

May 20th, 2007 [email protected]
 running vi in a loop

while read FileName 0<&3

do
vi $FileName
done 3< <(ls *.sh)

WAVV Green Bay Copyright Michael Potter 62

May 20th, 2007 [email protected]
 Learn more
• man bash
• O’Reilly - ‘Learning the Bash shell’
• http://bashdb.sourceforge.net/bashref.html
• http://www.faqs.org/docs/abs/HTML/
• Ask me to help!

WAVV Green Bay Copyright Michael Potter 63

May 20th, 2007 [email protected]
 Contact Information

Michael Potter
PO Box 469
Lisle, IL 60532

+1 630 926 8133

[email protected]

WAVV Green Bay Copyright Michael Potter 64

May 20th, 2007 [email protected]
 Copyright Notice
This presentation is copyright Michael Potter 2006.

No duplication is allowed without my permission.

Contact me for permission, you just might get it.

You are welcome to view and link to this

presentation on the UniForum website.

You are welcome to copy stringent.sh from the

slide, but please add a comment with a link to
this presentation in the source.

WAVV Green Bay Copyright Michael Potter 65

May 20th, 2007 [email protected]