post-install/runTeraGenSort.sh
#!/bin/bash
# jbenninghoff 2013-Mar-8 vi: set ai et sw=3 tabstop=3:

usage() {
cat << EOF
Usage: $0 [-d -t] <int>
-d To enable debug output
-t To specify thin map tasks

This script runs TeraGen and TeraSort using MR2 to measure big data
sort performance.  Specify a reduce task per node count as an
argument to the script. Modern multicore servers will run best with
8 or more reduce tasks per node.  Some experimentation is required
to achieve peak performance.

EOF
}

# Handle script options
DBG=""; mtask=fat
while getopts "dt" opt; do
  case $opt in
    d) DBG=true ;;
    t) mtask=thin ;;
    \?) usage; exit ;;
  esac
done
[ -n "$DBG" ] && set -x

if [[ $mtask == "thin" ]]; then
   chunksize=$((1*256*1024*1024)) # default size
else
   chunksize=$((3*256*1024*1024)) # Fat map tasks
fi
### TeraGen size (specify size using 100 Byte records)
size=$((10*1000*1000*1000)) #1TB for full TeraSort run
size=$((1000*1000*1000))  # 1B records (100GB) for quick runs when tuning
bytes=$((size*100)) #Convert size to bytes

#Define a map count for TeraGen resulting in no sharded/chunked files
#Bash does not do floating point, round up using modulo
maps=$(( (bytes/chunksize) + (bytes % chunksize > 0) ))

#find latest hadoop installed
hdphome=$(find /opt/mapr/hadoop -maxdepth 1 -type d -name hadoop-\* \
         |sort -n |tail -1)
hdpjar=$(find "$hdphome" -name hadoop-mapreduce-examples\*[0-9].jar)
#tbpath=/benchmarks/100tb                                                        

#Run TeraGen
if (hadoop fs -stat /benchmarks/tera/in); then
   echo Using existing TeraGen data
   mesg="Use: hadoop fs -rm -r /benchmarks/tera/in, "
   mesg+="to remove and trigger new TeraGen"
   echo "$mesg"
   sleep 3
else
   #Delete previous data generated by teragen
   hadoop fs -rm -r /benchmarks/tera/in
   maprcli volume create -name benchmarks -path /benchmarks -replication 1
   hadoop fs -chmod 777 /benchmarks
   hadoop fs -mkdir /benchmarks/tera
   #Set MFS chunksize to 768MB for fat map tasks
   if ! hadoop mfs -setchunksize $chunksize /benchmarks/tera; then
      echo "setchunksize failed"
      echo "set chunksize on /benchmarks/tera to $chunksize manually"
      exit
   fi
   # Run TeraGen
   hadoop jar "$hdpjar" teragen \
   -Dmapreduce.job.maps=$maps \
   -Dmapreduce.map.disk=0 \
   -Dmapreduce.map.cpu.vcores=0 \
   -Dmapreduce.map.speculative=false \
   $size /benchmarks/tera/in
   sleep 3
fi
hadoop mfs -ls /benchmarks/tera/in | grep ^-rwx | tail 

# maprcli config load -keys cldb.balancer.role.paused
# maprcli config load -keys cldb.balancer.disk.paused
# maprcli config save -values '{"cldb.balancer.disk.paused":"1"}'
# maprcli config save -values '{"cldb.balancer.role.paused":"1"}'
# Define vars for TeraSort run
logname=terasort-$(date "+%FT%T").log
nodes=$(maprcli node list -columns service |grep -c nodemanager)
# Start with 2 reduce tasks per node, reduce tasks per node limited by RAM
((rtasks=nodes*${1:-2}))
echo nodes="$nodes" | tee "$logname"
echo rtasks="$rtasks" | tee -a "$logname"
hadoop fs -rm -r /benchmarks/tera/out

# Run TeraSort with fat or thin map tasks, depending on chunksize
# Set mapreduce disk and vcores to 0 to size yarn containers by RAM only
case $chunksize in
   $((3*256*1024*1024)) )
      echo "Running TeraSort (size=$size) using 'fat' map tasks"
      echo "Uses fewer map tasks, reduces MxR shuffle"
      sleep 2
      hadoop jar "$hdpjar" terasort \
      -Dmapreduce.map.disk=0 \
      -Dmapreduce.map.cpu.vcores=0 \
      -Dmapreduce.map.output.compress=false \
      -Dmapreduce.map.sort.spill.percent=0.99 \
      -Dmapreduce.map.memory.mb=2000 \
      -Dmapreduce.map.java.opts="-Xmx1900m -Xms1900m" \
      -Dmapreduce.task.io.sort.mb=1500 \
      -Dmapreduce.task.io.sort.factor=100 \
      -Dmapreduce.reduce.disk=0 \
      -Dmapreduce.reduce.cpu.vcores=0 \
      -Dmapreduce.reduce.shuffle.parallelcopies="$nodes" \
      -Dmapreduce.reduce.merge.inmem.threshold=0 \
      -Dmapreduce.job.reduces=$rtasks \
      -Dmapreduce.job.reduce.slowstart.completedmaps=0.85 \
      -Dyarn.app.mapreduce.am.log.level=ERROR \
      -Dyarn.app.mapreduce.am.resource.mb=4000 \
      -Dyarn.app.mapreduce.am.command-opts="-Xmx3200M -Xms3200M" \
      /benchmarks/tera/in /benchmarks/tera/out 2>&1 | tee terasort.tmp
      # AM resized to handle 10 and 100TB runs
      ;;
   $((1*256*1024*1024)) )
      echo "Running TeraSort (size=$size) using thin map tasks(more map tasks)"
      sleep 2
      hadoop jar "$hdpjar" terasort \
      -Dmapreduce.map.disk=0 \
      -Dmapreduce.map.cpu.vcores=0 \
      -Dmapreduce.map.output.compress=false \
      -Dmapreduce.map.sort.spill.percent=0.99 \
      -Dmapreduce.reduce.disk=0 \
      -Dmapreduce.reduce.cpu.vcores=0 \
      -Dmapreduce.reduce.shuffle.parallelcopies="$nodes" \
      -Dmapreduce.reduce.merge.inmem.threshold=0 \
      -Dmapreduce.task.io.sort.mb=480 \
      -Dmapreduce.task.io.sort.factor=100 \
      -Dmapreduce.job.reduces=$rtasks \
      -Dmapreduce.job.reduce.slowstart.completedmaps=0.55 \
      -Dyarn.app.mapreduce.am.log.level=ERROR \
      /benchmarks/tera/in /benchmarks/tera/out 2>&1 | tee terasort.tmp
      ;;
   *) echo Undefined chunk size!; exit ;;
esac

#-Dmapreduce.map.speculative=false \
#-Dmapreduce.reduce.speculative=false \
#-Dmapreduce.reduce.memory.mb=3000 \

# Post-process the TeraSort job output
sleep 3
# Capture the job history log
myd=$(date +'%Y/%m/%d')
myj=$(grep 'INFO mapreduce.Job: Running job' terasort.tmp |awk '{print $7}')
myhist="/var/mapr/cluster/yarn/rm/staging/history/done/$myd/00*/$myj*.jhist"
until (hadoop fs -stat $myhist); do
  echo Waiting 20 sec for "$myhist"; sleep 20
done
myhist=$(hadoop fs -ls "$myhist" |awk '{print $NF}')                            
#echo "HISTORY FILE: $myhist"

mapred job -history "$myhist" >> "$logname"  # capture the run log
cat "$0" >> "$logname" # append actual script run to the log
head -22 "$logname"  # show the top of the log with elapsed time, etc
echo; echo View "$logname" for full job stats
cat terasort.tmp >> "$logname"; rm terasort.tmp
#./mapr-audit.sh >> $logname

# To validate TeraSort output, uncomment below and change output folder
# hadoop jar /opt/mapr/hadoop/hadoop-0.20.2/hadoop-0.20.2-dev-examples.jar
# teravalidate /benchmarks/tera/out /benchmarks/tera/validate
