Categories
Uncategorized

Blacklisting with Ulogd2 & nftables

This script is a part of the Asbra Firewall Project which is a set of utilities for managing a Linux Netfilter Firewall.

Notice that there is 15,754,925 (Fifteen million, seven hundred and fifty-four thousand and nine hundred and twenty-five) IP-addresses in the blacklist array. They are all loaded into memory for quick access.

Since all netfilter logging is done via a named pipe there is no IO access involved in parsing the traffic, block logging is optional and can handle both logging to syslog and custom output.

Evil ports

Evil ports can be specified for both TCP and UDP, combined with a max hit counter you can block unwanted traffic to illegal ports.

Ulogd2 is a logging daemon written for Netfliter, the script utilizes the JSON plugin in combination with the jq utility for collecting and analyzing data. One reason for this is that it offers more info on the network packets and more accurate field extraction with jq than with Bash Regexp.

Read more about how to setup Ulogd2 for JSON.

The script will also fetch 3 different blacklists and merge them into one. This can be changed in the global config /etc/asbrafw-blacklist.conf

  • https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt
  • https://iplists.firehol.org/files/firehol_level1.netset
  • https://myip.ms/files/blacklist/csf/latest_blacklist.txt
# apt update && apt install ulogd2 ulogd2-json jq wget grepcidr
#!/bin/bash
#
# AsbraFW - ulog_blacklist.sh
# This script is a part of the Asbra Firewall Project
#

LOG_PIPE="/afw/ulog/ulogd.json.pipe"

LOG="false"
LOG_FILE="/var/log/asbrafw-blacklist.log"
SYSLOG="true"
SILENT="false"

# Blacklisting on first seen event
BLACKLIST_URLS="\
	https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt\
	https://iplists.firehol.org/files/firehol_level1.netset\
	https://myip.ms/files/blacklist/csf/latest_blacklist.txt"
BLACKLIST_FILE="/tmp/blacklist.txt"
BLACKLIST_CMD="nft add element ip NAT BLACK"
BLACKLIST_TIMEOUT="24h"

# Evil Ports setting uses the BLOCKLIST_* values
EVIL_PORTS="true"
EVIL_PORTS_MAX_HIT=5
EVIL_TCP_PORTS="20 21 22 222 2222 23 81 445 514 587 3128 3306 8080 8081 8181 31 1170 1234 1243 1981 2001 2023 2989 3024 3150 3700 4950 6346 6400 6667 6670 12345 12346 16660 20034 20432 27374 27665 30100 31337 33270 33567 33568 40421 60008 65000"
EVIL_UDP_PORTS="2140 18753 20433 27444 31335"

BLOCKLIST_CMD="nft add element ip NAT BLOCK"
BLOCKLIST_TIMEOUT="1h"

VERBOSE="false"

# We can use ip addresses as key if the array is declared with -A
declare -A IP_HITS

bluebg="\e[44;1;37m"
yellow="\e[1;33m"
white="\e[1;37m"
reset="\e[0m"

##
# Check /etc for a config file
#
[[ -f "/etc/asbrafw-blacklist.conf" ]] && source /etc/asbrafw-blacklist.conf

##
# Usage instructions
#
function help() {
	cat <<- END
	Syntax: $0
	    -f Fetch & merge blacklists
	    -q Quiet/Silent, no output
	    -t <ports> Evil TCP Ports (Blocking timeout is applied)
	    -u <ports> Evil UDP Ports
		-v Verbose output, all traffic
	    -p <pipe> Name of pipe to read log from
	    -l <logfile> Name of file to write log messages tog
		-s Log to syslog
	END
	exit 0;
}

##
# Print message and die
#
function die() { echo "$1" ; exit 1; }

##
# Process command line parameters
#
while getopts "hfqvst:u:l:p:" opt
do
	case $opt in
		h) help ;;
		f) UPDATE="true" ;;  # Fetch update
		q) SILENT="true" ;;  # No output to STDOUT or STDERR
		v) VERBOSE="true" ;; # Show all traffic
		s) SYSLOG="true" ;;  # Write messages to syslog
		t) EVIL_TCP_PORTS="true" # Check for evil ports
			[[ ! -z "$optarg" ]] && EVIL_TCP_PORTS=$optarg
			;;
		u) EVIL_UDP_PORTS="true"
			[[ ! -z "$optarg" ]] && EVIL_UDP_PORTS=$optarg
			;;
		l) LOG="true"
			if [[ -z "$OPTARG" ]] ; then
				LOG_FILE="/var/log/asbrafw-blacklist.log"
				echo "No logfile given, using: $LOG_FILE"
			else
				LOG_FILE="$OPTARG"
			fi
		;;
		p) [[ -z "$OPTARG" ]] && die "Missing filename for pipe."
		   LOG_PIPE="$OPTARG"
		;;
	esac
done

##
# Fetch the latest blacklists
#
function update_blacklist()
{

	mv $BLACKLIST_FILE ${BLACKLIST_FILE}.bak

	for url in $BLACKLIST_URLS
	do
		rm /tmp/$(basename $url)
		wget --directory-prefix=/tmp/ $url
		cat /tmp/$(basename $url) >> $BLACKLIST_FILE
	done

	cat $BLACKLIST_FILE | sort | uniq > ${BLACKLIST_FILE}.sorted
	mv ${BLACKLIST_FILE}.sorted ${BLACKLIST_FILE}
}

##
# Look for match in blocklist array
#
function ip_is_evil()
{
	if grepcidr "$EVIL_IPS" <(echo "$1") >/dev/null
		then return 0 # True
		else return 1 # False
	fi
}

##
# Count total number of ips in blacklist
#
function blacklist_counter()
{
	local IP_LIST="$1"
	local IP_COUNTER=0
	local IP_ROWS=0

	while IPS='' read -r line ; do

		case $line in
			*/30) addr=2 ;;
			*/29) addr=6 ;;
			*/28) addr=14 ;;
			*/27) addr=30 ;;
			*/26) addr=62 ;;
			*/25) addr=126 ;;
			*/24) addr=254 ;;
			*/23) addr=510 ;;
			*/22) addr=1022 ;;
			*/21) addr=2046 ;;
			*/20) addr=4094 ;;
			*/19) addr=8190 ;;
			*/18) addr=16382 ;;
			*/17) addr=32766 ;;
			*/16) addr=65534 ;;
			*)	  addr=1 ;;
		esac

		IP_COUNTER=$(($IP_COUNTER+$addr))
		IP_ROWS=$(($IP_ROWS+1))

	done <<< $IP_LIST
	printf "${yellow}Evil hosts: ${reset}%'d IP Addresses in %'d rows...\n" "$IP_COUNTER" "$IP_ROWS"
}


##
# Have we seen ip before?
#
function hit_counter()
{
	ip=$1
	if [[ ! -z "${IP_HITS[$ip]}" ]]
	then IP_HITS[$ip]=$(( ${IP_HITS[$ip]}+1 )) # Increment
	else IP_HITS[$ip]=1 # First hit
	fi
}


##
# Parse JSON from the UlogD Pipe
#
function parse()
{
	[[ -z "$1" ]] && { echo "parse() missing named pipe as argument 1"; exit 1; }
	local pipe="$1"
	local logfile="$2"

	# Read Pipe Loop
	while read line <$pipe
	do
		# Split JSON string from pipe into local variables, protocol has to be quoted because of the period.
		json_vals=$(echo $line | jq -r '.src_ip, .src_port, .dest_ip, .dest_port, ."ip.protocol"')
		read -r _src_ip _src_port _dest_ip _dest_port _ip_protocol < <( echo $json_vals )

		# Give IP protocol a name
		case "$_ip_protocol" in
			1)  _ip_protocol="icmp" ;;
			6)  _ip_protocol="tcp"  ;;
			17) _ip_protocol="udp"  ;;
		esac

		# Declare vars
		_status="" ;_block="false" ; _black="false" ; _color="\e[0m"

		# is IP Address Evil
		if ip_is_evil $_src_ip ; then

			_status="${_status}Blacklisted host! "
			_color="\e[31m"
			_black="true"

		# Check config if we should look for evil ports
		elif [[ $EVIL_PORTS == "true" ]] ; then

			# TCP Protocol
			if [[ $_ip_protocol == "tcp" ]] ; then

				# Loop evil TCP ports
				for PORT in $EVIL_TCP_PORTS
				do
					if [[ $_dest_port == $PORT ]]
					then
						hit_counter $_src_ip
						_color="\e[33m"
						_status="${_status}Evil TCP port, hit: ${IP_HITS[$_src_ip]} of $EVIL_PORTS_MAX_HIT"

						if [[ ${IP_HITS[$_src_ip]} -ge $EVIL_PORTS_MAX_HIT ]]
						then
							_color="\e[31m"
							_block="true"
							_status="Evil TCP max hit count, blocking for: $BLOCKLIST_TIMEOUT"
						fi
					fi
				done

			# Loop evil UDP ports
			elif [[ $_ip_protocol == "udp" ]] ; then

				# Loop evil UDP ports
				for PORT in $EVIL_UDP_PORTS
				do
					if [[ $_dest_port == $PORT ]]
					then
						hit_counter $_src_ip
						_color="\e[33m"
						_status="${_status}Evil UDP port, hit: ${IP_HITS[$_src_ip]} of $EVIL_PORTS_MAX_HIT"

						if [[ ${IP_HITS[$_src_ip]} -ge $EVIL_PORTS_MAX_HIT ]]
						then
							_color="\e[31m"
							_block="true"
							_status="Evil UDP max hit count, blocking for: $BLOCKLIST_TIMEOUT"
						fi
					fi
				done

			fi # TCP or UDP

		fi # Check for evil ports

		if [[ -z $_status && "$VERBOSE" == "true" ]] ; then
			_color="\e[0m"
			_status="Pass"
		fi

		#SERVICE=$(getent services $_dest_port | cut -f1 -d" ")
		#[[ -n $SERVICE ]] && _status="${_status}(${SERVICE}) "

		# Block or blacklist the IP
		if   [[ $_black == "true" ]] ; then $BLACKLIST_CMD "{ $_src_ip timeout $BLACKLIST_TIMEOUT }"
		elif [[ $_block == "true" ]] ; then $BLOCKLIST_CMD "{ $_src_ip timeout $BLOCKLIST_TIMEOUT }"
		fi

		if [[ ! -z $_status ]] ; then

			# Date string
			DATE=$(date +"%Y-%m-%d %H:%M")

			# Print on screen if not silent
			[[ $SILENT != "true" ]] && \
				printf "${_color}%-16s | %-15s | %-6s | %-15s | %-6s | %-4s | %-s \e[0m\n" "$DATE" "$_src_ip" "$_src_port" "$_dest_ip" "$_dest_port" "$_ip_protocol" "$_status"

			# Write to log
			[[ $LOG == "true" && -n $logfile ]] && echo "$_src_ip - $_src_port - $_dest_ip - $_dest_port - $_status" >> $logfile
			[[ $SYSLOG == "true" ]] && logger --id=$$ "[AsbraFW] ${_src_ip}:${_src_port} ${_dest_ip}:${_dest_port} - $_status"

		fi

		_status=''
	done
}

# Update blacklist
[[ $UPDATE == "true" ]] && update_blacklist

EVIL_IPS=$(grep -v '^$\|^\s*\#' $BLACKLIST_FILE)

if [[ -z "$EVIL_IPS" ]]
then
	die "Blacklist seems empty!"
fi

##
# Be verbose if not silent
#
if [[ "$SILENT" == "false" ]]
then

	# Title
	echo -e "\n\e[31mAsbraFW\e[0m - Blacklist\n"

	echo -e "${yellow}Verbose:    ${reset}${VERBOSE}"
	echo -e "${yellow}Syslog:     ${reset}${SYSLOG}"

	# Log file
	[[ $LOG == "true" ]] && echo -e "${yellow}Log File:   ${reset}$LOG_FILE"

	# Pipe to read from
	echo -e "${yellow}JSON Pipe:  ${reset}$LOG_PIPE"

	# Name of blacklist file
	echo -e "${yellow}Blacklist:  ${reset}$BLACKLIST_FILE"

	# Blacklist timeout
	echo -e "${yellow}Timeout:    ${reset}$BLACKLIST_TIMEOUT for blacklisting, $BLOCKLIST_TIMEOUT for blocking"

	# Evil ports to block
	[[ $EVIL_PORTS == "true" ]] && echo -e "${yellow}Evil ports: ${reset}$EVIL_TCP_PORTS"

	# Show statistics, number och ips in blacklist array
	blacklist_counter "$EVIL_IPS"

	printf "\n\e[1;37;46m%-16s   %-15s   %-6s   %-15s   %-6s   %-4s   %-s \e[K\e[0m\n" "Timestamp" "Src IP" "SPort" "DstIP" "DPort" "Prot" "Status"
fi

# Parse traffic and block matches
parse $LOG_PIPE $LOG_FILE

And a simple firewall could look something like this.

#!/usr/sbin/nft

flush ruleset

define NIC_WAN = eno1
define NIC_LAN = eno2
define NIC_VPN = tun1

table arp FW {

	# Accept ARP requests and replys
	chain IN { type filter hook input priority 0; policy drop;
		arp operation { request, reply } counter accept
	}

	# Accept ARP requests and replys
	chain OUT { type filter hook output priority 0; policy drop;
		arp operation { request, reply } counter accept
	}

	# Dont forward ARP
	chain FORW { type filter hook forward priority 0; policy drop; }
}

table ip NAT {

	# Whitelisted trusted ip-addresses
	set WHITE { type ipv4_addr;
		elements = { 192.168.1.160, 192.168.10.10 }
	}

	# Blacklist, timeout 1 day
	set BLACK { type ipv4_addr; timeout 1d; }

	# Blocked for 4 hour
	set BLOCK { type ipv4_addr; timeout 4h; }

	chain PREROUTING { type nat hook prerouting priority -150;

		ip saddr @WHITE counter accept
		ip saddr @BLACK counter drop
		ip saddr @BLOCK counter drop
  }
}

table inet FW {

	chain IN { type filter hook input priority 0; policy drop;

		# Configure ulogd for JSON output on group 1
		iif $NIC_WAN ip protocol tcp ct state new tcp flags syn log prefix "[AsbraFW] TCP new in." group 1 counter counter
		iif $NIC_WAN ip protocol udp log prefix "[AsbraFW] UDP new in." group 1 counter counter

		# if the connection is NEW and is not SYN then drop
		tcp flags != syn ct state new log prefix "[AsbraFW] First packet is not SYN." group 0 counter drop

		# new and sending FIN the connection? DROP!
		tcp flags & (fin|syn) == (fin|syn) log prefix "[AsbraFW] Incompatible Flags." group 0 counter drop

		# i don't think we've met but you're sending a reset?
		tcp flags & (syn|rst) == (syn|rst) log prefix "[AsbraFW] Uninitiated Request." group 0 counter drop

		# 0 attack?
		tcp flags & (fin|syn|rst|psh|ack|urg) < (fin) log prefix "[AsbraFW] 0 Attack." group 0 counter drop

		# xmas attack. lights up everything
		tcp flags & (fin|syn|rst|psh|ack|urg) == (fin|psh|urg) log prefix "[AsbraFW] Xmas Attack." group 0 counter drop

		# if the ctstate is invalid
		#ct state invalid flags all log prefix "[AsbraFW] Invalid conntrack state" group 0 counter drop

		# Accept all from LAN, VPN and LO interface
		iif $NIC_LAN counter accept
		iif $NIC_VPN counter accept
		iif lo accept

		# accept traffic originated from us
		ct state established,related counter accept

		# accept local services
		tcp dport { 80, 443 } ct state new counter accept
		tcp dport 53 ct state new counter accept
		udp dport 53 ct state new counter accept
	}
}

By Nimpen J. Nordström

System Developer and Network Security Enthusiast