OWASP Log Parser

This project started in 2016 and it has been lying dormant in my archive of unfinished code, i’m going to finish it in the near future since it’s a pain reading thru the log files and trying to fix false positives.

The first problem was that GeoIP is no longer working in the same way, without it you’d miss out on a lot of the cool features like longitude/latitude for use with Google Map API and also the little country flags on each row.

I’m working on the code when i have some time over so please be patient, the parsing class needs some more cleaning up before i release it to the world.

In the meanwhile, here is a very useful bash-script for parsing Apache OWASP logs, it even has a tail function to follow the logs in real-time.

#!/usr/bin/env bash
#
# @License   GNU GENERAL PUBLIC LICENSE Version 3
# @Author    Nimpen <nimpen@asbra.nu>
# @Copyright 2016 ASBRA AB

logfile="$2"

if ! command -v geoiplookup &> /dev/null
then echo "Please install the geoiplookup command!" ; exit 1
fi

country_code() {
	[[ $(geoiplookup $1 | head -n 1) =~ :[[:space:]]([A-Z]+) ]] && echo ${BASH_REMATCH[1]}
}

filter() {
	line=$*
	[[ $line =~ ^\[([^\]]*)\] ]]                       && OWASP_DATE="${BASH_REMATCH[1]}"
	[[ $line =~ \[id[[:space:]]\"([^\"]*)\"\] ]]       && OWASP_ID="${BASH_REMATCH[1]}"
	[[ $line =~ \[client[[:space:]]([^:]*)\] ]]        && OWASP_CLIENT="${BASH_REMATCH[1]}"
	[[ $line =~ \[msg[[:space:]]\"([^\"]*)\"\] ]]      && OWASP_MSG="${BASH_REMATCH[1]}"
	[[ $line =~ \[hostname[[:space:]]\"([^\"]*)\"\] ]] && OWASP_SITE="${BASH_REMATCH[1]}"
	[[ $line =~ \[uri[[:space:]]\"([^\"]*)\"\] ]]      && OWASP_URI="${BASH_REMATCH[1]}"
	[[ $line =~ \[severity[[:space:]]\"([^\"]*)\"\] ]] && OWASP_SEVERITY="${BASH_REMATCH[1]}"

	if [[ $showcolor == 'true' ]] ; then
		[[ $OWASP_SEVERITY == 'CRITICAL' ]] && echo -n -e "\e[41m"
		[[ $OWASP_SEVERITY == 'WARNING' ]] && echo -n -e "\e[33m"
		[[ $OWASP_SEVERITY == 'NOTICE' ]] && echo -n -e "\e[34m"
	fi

	for var in "${OPTIONS[@]}" ; do
		case ${var} in
		'showdate')     printf "%-32s"  "$OWASP_DATE" ;;
		'showcountry')  printf "%-4s"  "$(country_code $OWASP_CLIENT)" ;;
		'showclient')   printf "%-17s" "$OWASP_CLIENT" ;;
		'showid')       printf "%-10s" "$OWASP_ID" ;;
		'showseverity') printf "%-10s" "$OWASP_SEVERITY" ;;
		'showsite')     printf "%-${colssite}s" "$OWASP_SITE" ;;
		'showuri')      printf "%-${colsuri}s" "$OWASP_URI" ;;
		'showmsg')      printf "%-${colsmsg}s"  "$OWASP_MSG" ;;
		esac
	done
	printf "%s"  " "

	echo -e "\e[0m"
}

syntax() {
cat<<EOF

AsbraFW - OWASP Log Parser v1.0
Copyright 2016 Asbra AB - GPLv3

Arguments:
    -f Logfile
    -t type [tail,parse]
    -d Date (Apache)
    -i Owasp ID
    -g GeoIP Country Code
    -l Severity level
    -c Client IP-Address
    -s [cols] Site name
    -u [cols] URI
    -m [cols] Message
    -a Ansi colorized output based on severity

Example:
    $0 -t parse -f "/var/log/apache2/*error.log" -d -i -g -l -c -s 20 -u 30 -m 30 -a | more
    $0 -t tail -f "/var/log/apache2/*error.log" -d -i -g -l -c -s 20 -u 30 -m 30 -a

EOF
}

declare -a OPTIONS
counter=0

while getopts "t:f:dicm:s:u:lag" o; do
	case "${o}" in
		t) type=${OPTARG} ;
			if [ -z "${OPTARG}" -o "${OPTARG:0:1}" = "-" ]
			then echo "-f requires an argument"; syntax; exit 1
			fi
			;;
		f) logfile=${OPTARG} ;
			if [ -z "${OPTARG}" -o "${OPTARG:0:1}" = "-" ]
			then echo "-f requires an argument"; syntax; exit 1
			fi
			;;
		d) OPTIONS[$counter]='showdate'  ; counter=$((counter+1)) ;;
		i) OPTIONS[$counter]='showid'    ; counter=$((counter+1)) ;;
		c) OPTIONS[$counter]='showclient'; counter=$((counter+1)) ;;
		m) OPTIONS[$counter]='showmsg'   ; counter=$((counter+1)) ;
			if [ -z "${OPTARG}" -o "${OPTARG:0:1}" = "-" ]
			then echo "-m requires an argument"; syntax; exit 1
			else colsmsg=${OPTARG}
			fi
			;;
		s) OPTIONS[$counter]='showsite'  ; counter=$((counter+1)) ;
			if [ -z "${OPTARG}" -o "${OPTARG:0:1}" = "-" ]
			then echo "-s requires an argument"; syntax; exit 1
			else colssite=${OPTARG}
			fi
			;;
		u) OPTIONS[$counter]='showuri' ; counter=$((counter+1)) ;
			if [ -z "${OPTARG}" -o "${OPTARG:0:1}" = "-" ]
			then echo "-u requires an argument"; syntax; exit 1
			else colsuri=${OPTARG}
			fi
			;;
		l) OPTIONS[$counter]='showseverity'; counter=$((counter+1)) ;;
		g) OPTIONS[$counter]='showcountry' ; counter=$((counter+1)) ;;
		a) showcolor='true' ;;
		*) syntax; exit 1 ;;
	esac
counter=$((counter+1))
done

[[ -z "$logfile" ]] && { syntax ; exit 1 ; }
[[ -z "$type" ]] && { syntax ; exit 1 ; }

case "$type" in
	"parse")
		IFS=$'\n';
		for line in $(cat $logfile | grep -i OWASP)
		do filter $line
		done
		;;
	"tail")
		while read -r line
		do filter $line
		done < <(tail --silent -n 10 -f $logfile | grep "ModSecurity")
		;;
	*)
		syntax
		exit 1
esac

exit 0