#!/bin/bash

################################################################################
#Script Description: 
#Script Name: curl_malware_downloader.sh
#Script Version: 
#Script Author: Sheldon Johnson
#Script Sponsor: Selex-ES
#Script Release Date: 22/05/2015
#1.1 Release Date: 26/05/2015
#
#This program is free software: you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation, either version 3 of the License, or
#(at your option) any later version.
#
#This program is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#GNU General Public License for more details.
#
#You should have received a copy of the GNU General Public License
#along with this program.  If not, see <http://www.gnu.org/licenses/>.
################################################################################

################################################################################
#Background
#I like using curl to grab samples from the interwebs
#I also like to know if different user agents and times mean different outcomes
#the idea of this tool is that it allows you to do different user agents
#at different times to see the difference.
#just run it, follow the instructions and let it go...
#sadly you have to let it complete running or it will not output its findings,
#however they will still be downloaded in the right location.
#Oh yeah, when putting a time delay in, it will add a variance of 0-30% so it
#isnt too obvious its scripted.
#I also added a function to poll a website an arbitrary number of times
#using a single user agent over a period of time and compare the output
#to look for changes.
################################################################################

################################################################################
#Notes
#I've tried to be clean with my bash, but bash is a dirty language and I am not
#a coder!
################################################################################

################################################################################
##########################PLACE YOUR USER AGENTS HERE!##########################
################################################################################

UASTRING[${#UASTRING[@]}]="Mozilla/5.0 (compatible, MSIE 10, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko"
UASTRING[${#UASTRING[@]}]="Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko"
UASTRING[${#UASTRING[@]}]="Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0"
#UASTRING[${#UASTRING[@]}]="EXAMPLE USERAGENT"

################################################################################
##########################PLACE YOUR USER AGENTS HERE!##########################
################################################################################

#*******DO NOT EDIT BEYOND THIS POINT UNLESS YOU KNOW WHAT YOU ARE DOING!*******

################################################################################
#CONSTANTS
################################################################################

RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m' # No Color

SLEEPTIME=0
TRIES=1

INCIDENTNO=$1
URL=$2

URLHASH=$(echo $URL | sha256sum | cut -d " " -f 1)
INITDATE=$(date +%F)
CURRENTDATE=$(date +%z_%F_%X)


#variables for folder structure.
WORKDIR="$HOME/work"
CURLDOWNLOADS="$WORKDIR/curl_downloads"
CURLINCIDENT="$CURLDOWNLOADS/$INCIDENTNO"
CURLURLHASH="$CURLINCIDENT/$URLHASH"
CURLURLHASHOUTPUT="$CURLURLHASH""/""$CURRENTDATE"

OUTFILESTRING="Incident=""$INCIDENTNO""_Date=""$CURRENTDATE""_URLHash=""$URLHASH""_Section="
#used to create and access the output html
INDEXNAME="index.Incident=""$INCIDENTNO""_Date=""$CURRENTDATE""_URLHash=""$URLHASH"".html"
FULLINDEXPATH="$CURLURLHASH"/"$INDEXNAME"

################################################################################
#FUNCTIONS
################################################################################

function UsageInstructions(){ #Number_Of_Script_Arguments
	#Quick and dirty check to ensure the script is at least provided 
	#with the right number of arguments.
	if [[ $1 != 2 ]]; then 
		echo "Usage: $0 incident_number url"	
		exit
	else
		echo -e "Welcome to the automated curl URL grabber.\n"
	fi
}

function MakeRequiredDirectories(){ #$WorkDirectory
	#Creates the basic folder structure
	if [ ! -e $WORKDIR ]; then mkdir $WORKDIR; fi
	if [ ! -e $CURLDOWNLOADS ]; then mkdir $CURLDOWNLOADS; fi
	if [ ! -e $CURLINCIDENT ]; then mkdir $CURLINCIDENT; fi
	if [ ! -e $CURLURLHASH ]; then mkdir $CURLURLHASH; fi
	if [ ! -e $CURLURLHASHOUTPUT ]; then mkdir $CURLURLHASHOUTPUT; fi
}

function SelectionError(){ clear; echo -e "$RED""Please select a valid option.\n""$NC"; }
function SelectItem(){ echo -e "\nPlease select the appropriate numeric option."; }

function Selector(){ #CONTENTSTRING $CONTENTARRAY
	#simple function to give you selection menu
	#with minor error checking.
	local INITVAL=0
	unset X

	while [[ $X != 1 ]]; do
		local ARRAYINDEX=$INITVAL
		echo -e $1
		for i in ${!2}; do
			local ARRAYINDEX=$[$ARRAYINDEX+1]
			echo $ARRAYINDEX - $i
		done
		SelectItem
		read INPUT
		local LOCALINPUT=$INPUT
		unset INPUT

		if [[ $LOCALINPUT -gt $INITVAL ]] && [[ $LOCALINPUT -le $ARRAYINDEX ]]; then X=1
		else SelectionError; fi
	done
	local LOCALINPUT=$[$LOCALINPUT-1] #the array starts at 0, so we have to account for that.
	return $LOCALINPUT
}

function SelectUsage(){ #UASTRING
	local USAGESTRING="Do you want to attempt the download once with a single user agent string,\n"
	local USAGESTRING+="Or would you prefer to download the webpage with all configured user agents?\n"
	local SELECTIONSTRING

	local SELECTIONSTRING[${#SELECTIONSTRING[@]}]="Single Download/User Agent"
	local SELECTIONSTRING[${#SELECTIONSTRING[@]}]="Multiple Downloads/All User Agents"

	Selector $USAGESTRING SELECTIONSTRING[@]
	
	return $?
}

function SingleUASelector(){ #UASTRING
	local USAGESTRING="Please Select the User Agent string you want to use:\n"
	local SELECTIONSTRING

	for i in ${!1}; do SELECTIONSTRING[${#SELECTIONSTRING[@]}]=$i; done

	Selector $USAGESTRING SELECTIONSTRING[@]
		
	return $?
}

function TimeDelayQuestion1(){
	local USAGESTRING="Would you like a time delay between each download?\n"
	local SELECTIONSTRING

	local SELECTIONSTRING[${#SELECTIONSTRING[@]}]="Yes"
	local SELECTIONSTRING[${#SELECTIONSTRING[@]}]="No"
	
	Selector $USAGESTRING SELECTIONSTRING[@]

	return $?
}

function TimeDelayQuestion2(){
	local USAGESTRING="How long of a time delay would you like between each download?\n"
	local USAGESTRING+="This will include a small randomisation variance.\n"
	local SELECTIONSTRING

	local SELECTIONSTRING[${#SELECTIONSTRING[@]}]="No Delay"
	local SELECTIONSTRING[${#SELECTIONSTRING[@]}]="10 Seconds"
	local SELECTIONSTRING[${#SELECTIONSTRING[@]}]="1 Minute"
	local SELECTIONSTRING[${#SELECTIONSTRING[@]}]="10 Minutes"
	local SELECTIONSTRING[${#SELECTIONSTRING[@]}]="30 Minutes"
	local SELECTIONSTRING[${#SELECTIONSTRING[@]}]="1 hour"
	
	Selector $USAGESTRING SELECTIONSTRING[@]

	return $?
}

function PollQuestion1(){
	local USAGESTRING="Would you like to repeatedly poll the website?\n"
	local SELECTIONSTRING

	local SELECTIONSTRING[${#SELECTIONSTRING[@]}]="No"
	local SELECTIONSTRING[${#SELECTIONSTRING[@]}]="Yes"
	
	Selector $USAGESTRING SELECTIONSTRING[@]

	return $?
}

function PollQuestion2(){
	echo -e "Please enter the number of times you wish to poll the site:"
}

function IsInt(){ #INPUT
	local X=1
	if [[ $1 =~ ^-?[0-9]+$ ]]; then local X=0; 
	else 
		echo -e $RED"Please enter a number."$NC;
		echo "Press any key to continue..."
		read
		clear
	fi


	return $X
}


function TimeDelayImplemtation(){ #$ARRAYSELECTION
	#this function exists since bash returns can only return 0-255
	#so instead you pass the variable 0-X which selects 
	#the appropriate time from the array and echoes it out
	#to capture this value do x=$(TimeDelayImplemtation $VAR)
	local TIMEDELAY
	local TIMEDELAY[${#TIMEDELAY[@]}]="0"
	local TIMEDELAY[${#TIMEDELAY[@]}]="10"
	local TIMEDELAY[${#TIMEDELAY[@]}]="60"
	local TIMEDELAY[${#TIMEDELAY[@]}]="360"
	local TIMEDELAY[${#TIMEDELAY[@]}]="1080"
	local TIMEDELAY[${#TIMEDELAY[@]}]="2160"

	local OUTPUT=${TIMEDELAY[$1]}

	local PSEUDORETURN=0
	
	if [[ $1 -ge 0 ]] && [[ $1 -lt ${#TIMEDELAY[@]} ]]; then
		local PSEUDORETURN=${TIMEDELAY[$1]}
	fi

	#this is the pseudo return.
	echo $PSEUDORETURN
}

function TimeVariance(){ #$SLEEPCOUNTER
	#creates a random increase up to 30% on the sleep time
	#this is so that when grabbing multiple times from the same location
	#it isnt so obvious it is an automated tool.

	#creates a float, then mangles it to make it an int again.
	local VARIANCEMAX=$(echo "$1 * 1.3" | bc | cut -d "." -f 1)

	#as before this is a pseudoreturn, so
	##to capture this value do x=$(TimeVariance $VAR)
	local PSEUDORETURN=$(shuf -i $1-$VARIANCEMAX -n 1)
	echo $PSEUDORETURN
}

function SleepyTime(){ #$SLEEPCOUNTER
	#little sleep timer with a message so the user doesnt think it is broken!

	if [[ $1 -gt 0 ]]; then
		echo -e "\nSleeping for $1 seconds.\n"
		sleep $1
	fi
}

function CurlURL(){ #"$OUTFILESTRING" $CURLURLHASH $URL $FULLINDEXPATH $SLEEPCOUNTER $UASTRING[@] $CURRENTDATE
	#create the output pathing and file strings here
	#since they will be passed to all the funtions that require them from here.
	#and by segregating them we can use relative positions on the file system
	#when building the html links.
	local HEADSTR="headers_UserAgent="
	local BODYSTR="body_UserAgent="
	local OUTFILEHEAD="$1""$HEADSTR"
	local OUTFILEBODY="$1""$BODYSTR"
	local USERAGENTARRAY
	for i in ${!6}; do USERAGENTARRAY[${#USERAGENTARRAY[@]}]=$i; done

	#first loops creates all the required arrays and values
	#then performs a curl using said values.
	local X=0
	for i in ${USERAGENTARRAY[@]}; do
		clear
		echo -e "Downloading file using user agent string "$[$X+1]"/"${#USERAGENTARRAY[@]}":\n"
		echo -e "$i\n"

		local FULLOUTFILEHEADER
		local FULLOUTFILEHEADER[${#FULLOUTFILEHEADER[@]}]="$7""/""$OUTFILEHEAD""$X"".txt"
		
		local FULLOUTFILEBODY
		local FULLOUTFILEBODY[${#FULLOUTFILEBODY[@]}]="$7""/""$OUTFILEBODY""$X"".txt"

		local FULLPATHOUTFILEHEADER
		local FULLPATHOUTFILEHEADER[${#FULLPATHOUTFILEHEADER[@]}]="$2""/""${FULLOUTFILEHEADER[$X]}"
		
		local FULLPATHOUTFILEBODY
		local FULLPATHOUTFILEBODY[${#FULLPATHOUTFILEBODY[@]}]="$2""/""${FULLOUTFILEBODY[$X]}"

		local FULLPATHARRAY
		local FULLPATHARRAY[${#FULLPATHARRAY[@]}]=${FULLPATHOUTFILEBODY[$X]}
		
		local NEWDATE
		local NEWDATE[${#NEWDATE[@]}]=$(date +%z_%F_%X)	
		
		curl -L -A "$i" -D ${FULLPATHOUTFILEHEADER[$X]} -o ${FULLPATHOUTFILEBODY[$X]} $3

		local X=$[$X+1]
		if [[ $X -ne ${#USERAGENTARRAY[@]} ]]; then
			local SLEEPTIMMER=$(TimeVariance $5)
			SleepyTime $SLEEPTIMMER
		fi
	done

	#the second loop creates the output pages (inlcuding the fuzzy hash process and their output)
	local X=0
	for i in ${UASTRING[@]}; do
		CreateIndexLink $4 $i ${FULLOUTFILEHEADER[$X]} ${FULLOUTFILEBODY[$X]} $X ${NEWDATE[$X]}
		HashBodyContent $4 ${FULLPATHOUTFILEBODY[$X]}
		FuzzyHashComparison FULLPATHARRAY[@] $X $4

		local X=$[$X+1]
	done
}

function SingleUAHack(){ #$INDEX
	#recreates the array with just a single entry
	#since passing vaiables over bash functions is hellish.
	#this is a hacky but simple solution.
	for i in ${!1}; do USERAGENTARRAY[${#USERAGENTARRAY[@]}]=$i; done
	echo ${UASTRING[$2]}
}

function MakeHTMLIndex(){ #$FULLINDEXPATH $INCIDENTNO $URL $URLHASH $CURRENTDATE $USAGE
	#simple html index landing page with some simple information.
	echo "<html>" > $1
	echo -e "\t<body>" >> $1
	echo -e "\t\t<h1>Incident Information:</h1>" >> $1
	echo -e "\t\t\t<p>Incident Number: $2</p>" >> $1
	echo -e "\t\t\t<p>Target URL: $3</p>" >> $1
	echo -e "\t\t\t<p>URL SHA256 Hash: $4</p>" >> $1
	echo -e "\t\t\t<p>Analysis Date: $5</p>" >> $1
	echo -e "\t\t<h1>Curl Downloads:</h1>" >> $1
	echo -e "\t\t<table>" >> $1
}

function CreateIndexLink(){ #$FULLINDEXPATH $UASTRING $FULLOUTFILEHEADER $FULLOUTFILEBODY $RUNNUMBER $NEWDATE
	#content to be added to the index.
	TableRow "$1" "<hr>" "<hr>" 
	TableRow "$1" "Download Number:" "$[$5+1]"
	TableRow "$1" "Download Time/Date:" "$6"
	TableRow "$1" "User Agent String:" "$2"
	TableRow "$1" "Content:" "<a href=\"./$3\">Header</a> <a href=\"./$4\">Body</a>"
}

function HashBodyContent(){ #$FULLINDEXPATH $FULLOUTFILEBODY
	#hashes all of the data to be added to the start of the landing page.
	local X=$(md5sum "$2" | cut -d " " -f 1)
	TableRow $1 "Body MD5 Hash" $X
	unset IFS
	for i in `echo "1 224 256 384 512"`; do 
		#$SHASUM -a "$i" "$2" | cut -d " " -f 1
		local X=$(shasum -a "$i" "$2" | cut -d " " -f 1)
		TableRow $1 "Body SHA$i Hash" $X
	done
	local X=$(ssdeep -s "$2" | cut -d "," -f 1)

	#ssdeep adds in a \n, this is a way of dealing with it
	local Y=$(echo $X | cut -d " " -f 2)

	TableRow $1 "SSDeep Hash" $Y
	IFS=$'\n'
}

function TableRow(){ #$FULLINDEXPATH $CONTENTTITLE $CONTENTS
	#creates a table row
	echo -e "\t\t\t<tr>" >> $1
	echo -e "\t\t\t\t<td>$2</td>" >> $1
	echo -e "\t\t\t\t<td>$3</td>" >> $1
	echo -e "\t\t\t</tr>" >> $1
}

function FuzzyHashComparison(){ #FULLPATHARRAY[@] $COUNTER
	#hashes the output of one user agent string against the contents of all of the others.
	#loops through and does this for every page.
	#yes there will be duplicates, yes this was on purpose for ease of reading
	for i in ${!1}; do 
		local COMPARISONFILES
		COMPARISONFILES[${#COMPARISONFILES[@]}]=$i
	done
	local SOURCEFILE=${COMPARISONFILES[$2]}
	COMPARISONFILES[$2]=""

	for i in ${COMPARISONFILES[@]}; do 
		#pseudoreturn
		#ssdeep -lrd $i $SOURCEFILE
		local SSDEEPMATCH=$(ssdeep -lrd $i $SOURCEFILE 2> /dev/null | sed -n -e 's/^.*matches//p')
		TableRow "$3" "SSDeep matches on:" "$SSDEEPMATCH"
	done
}
	
function CloseHTMLIndex(){ #$FULLINDEXPATH
	echo -e "\t\t</table>" >> $1
	echo -e "\t</body>" >> $1
	echo "</html>" >> $1
}

function CheckDependencies(){
	which ssdeep > /dev/null
	if [[ $? != 0 ]]; then 
		echo -e $RED"This program requires ssdeep to be installed."$NC
		echo -e "You can get it here: http://ssdeep.sourceforge.net/#download"
		exit
	fi
}

################################################################################
#init
################################################################################

UsageInstructions $#
CheckDependencies

MakeRequiredDirectories $WORKDIR

IFS=$'\n'
SelectUsage
USAGE=$?

if [[ $USAGE == 0 ]]; then
	SingleUASelector UASTRING[@]
	NEWUASTRING=$(SingleUAHack UASTRING[@] $?)
	unset UASTRING
	UASTRING[${#UASTRING[@]}]="$NEWUASTRING"

	#place a polling option here which changes $USAGE to 1
	PollQuestion1
	USAGE=$?
	if [[ $USAGE == 1 ]]; then
		i=1
		while [[ $i != 0 ]]; do
			PollQuestion2
			read TRIES
			IsInt $TRIES; 
			i=$?; 
		done
		unset i
		echo $TRIES
		for i in $(seq 2 $TRIES); do
			UASTRING[${#UASTRING[@]}]=$NEWUASTRING
		done
	fi
	unset NEWUASTRING
fi
if [[ $USAGE == 1 ]]; then
	TimeDelayQuestion1
	if [[ $? == 0 ]]; then
		TimeDelayQuestion2
		TIMESELECTOR=$?
		SLEEPTIME=$(TimeDelayImplemtation $TIMESELECTOR)
	fi
fi

MakeHTMLIndex $FULLINDEXPATH $INCIDENTNO $URL $URLHASH $CURRENTDATE $USAGE
CurlURL "$OUTFILESTRING" "$CURLURLHASH" "$URL" "$FULLINDEXPATH" $SLEEPTIME UASTRING[@] "$CURRENTDATE"
CloseHTMLIndex $FULLINDEXPATH

xdg-open $FULLINDEXPATH &> /dev/null &

clear
