Commit fc3d911f authored by bunkerity's avatar bunkerity
Browse files

improved blacklist/whitelist/dnsbl with lua

parent ef7d842f
......@@ -10,6 +10,7 @@ COPY confs/ /opt/confs
COPY scripts/ /opt/scripts
COPY fail2ban/ /opt/fail2ban
COPY logs/ /opt/logs
COPY lua/ /opt/lua
RUN apk --no-cache add php7-fpm certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog && \
chmod +x /opt/entrypoint.sh /opt/scripts/* && \
......
......@@ -10,6 +10,7 @@ COPY confs/ /opt/confs
COPY scripts/ /opt/scripts
COPY fail2ban/ /opt/fail2ban
COPY logs/ /opt/logs
COPY lua/ /opt/lua
RUN apk --no-cache add php7-fpm certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog && \
chmod +x /opt/entrypoint.sh /opt/scripts/* && \
......
......@@ -17,6 +17,7 @@ COPY confs/ /opt/confs
COPY scripts/ /opt/scripts
COPY fail2ban/ /opt/fail2ban
COPY logs/ /opt/logs
COPY lua/ /opt/lua
RUN apk --no-cache add php7-fpm certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog && \
chmod +x /opt/entrypoint.sh /opt/scripts/* && \
......
......@@ -17,6 +17,7 @@ COPY confs/ /opt/confs
COPY scripts/ /opt/scripts
COPY fail2ban/ /opt/fail2ban
COPY logs/ /opt/logs
COPY lua/ /opt/lua
RUN apk --no-cache add php7-fpm certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog && \
chmod +x /opt/entrypoint.sh /opt/scripts/* && \
......
......@@ -10,6 +10,7 @@ COPY confs/ /opt/confs
COPY scripts/ /opt/scripts
COPY fail2ban/ /opt/fail2ban
COPY logs/ /opt/logs
COPY lua/ /opt/lua
RUN apk --no-cache add php7-fpm certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog && \
chmod +x /opt/entrypoint.sh /opt/scripts/* && \
......
access_by_lua_block {
-- get client IP
local ip = ngx.var.remote_addr
-- check if IP is in cache
local cached = ngx.shared.dnsblcache:get(ip)
if cached ~= nil then
if cached == "ok" then
ngx.exit(ngx.OK)
else
ngx.exit(ngx.HTTP_FORBIDDEN)
end
end
-- get the reverse DNS
local rdns = ""
local both = false
local resolver = require "resty.dns.resolver"
local resolvers = {%DNSBL_RESOLVERS%}
local r, err = resolver:new{nameservers=resolvers, retrans=2, timeout=2000}
if not r then
ngx.exit(ngx.OK)
end
local answers, err = r:reverse_query(ip)
if not answers.errcode then
for ak, av in ipairs(answers) do
if av.ptrdname then
rdns = av.ptrdname
break
end
end
end
if rdns ~= "" then
local answers, err, tries = r:query(rdns, nil, {})
for ak, av in ipairs(answers) do
if av.address and av.address == ip then
both = true
break
end
end
end
-- check if it's a legitimate SE crawler
local ips = {"23.21.227.69", "40.88.21.235", "50.16.241.113", "50.16.241.114", "50.16.241.117", "50.16.247.234", "52.204.97.54", "52.5.190.19", "54.197.234.188", "54.208.100.253", "54.208.102.37", "107.21.1.8"}
local domains = {".googlebot.com", ".google.com", ".search.msn.com", ".crawl.yahoot.net", ".crawl.baidu.jp", ".crawl.baidu.com", ".yandex.com", ".yandex.ru", ".yandex.net"}
for k, v in pairs(ips) do
if v == ip then
ngx.shared.dnsblcache:set(ip, "ok", 86400)
ngx.exit(ngx.OK)
end
end
if both and rdns ~= "" then
for k, v in pairs(domains) do
if rdns:sub(-#v) == v then
ngx.shared.dnsblcache:set(ip, "ok", 86400)
ngx.exit(ngx.OK)
end
end
end
-- dnsbl check
local dnsbls = {%DNSBL_LIST%}
for k, v in pairs(dnsbls) do
local name = resolver.arpa_str(ip)
name = name:gsub("%.in%-addr%.arpa", ""):gsub("%.ip6%.arpa", "") .. "." .. v
local answers, err, tries = r:query(name, nil, {})
if not answers.errcode then
for ak, av in ipairs(answers) do
if av.address then
a,b,c,d = av.address:match("([%d]+).([%d]+).([%d]+).([%d]+)")
if a == "127" then
ngx.shared.dnsblcache:set(ip, "dnsbl", 86400)
ngx.exit(ngx.HTTP_FORBIDDEN)
end
end
end
end
end
-- legitimate user
ngx.shared.dnsblcache:set(ip, "ok", 86400)
ngx.exit(ngx.OK)
}
access_by_lua_block {
local use_whitelist_ip = %USE_WHITELIST_IP%
local use_whitelist_reverse = %USE_WHITELIST_REVERSE%
local use_blacklist_ip = %USE_BLACKLIST_IP%
local use_blacklist_reverse = %USE_BLACKLIST_REVERSE%
local use_dnsbl = %USE_DNS%
-- include LUA code
local whitelist = require "whitelist"
local blacklist = require "blacklist"
local dnsbl = require "dnsbl"
-- check if already in whitelist cache
if use_whitelist_ip and whitelist.ip_cached_ok() then
ngx.exit(ngx.OK)
end
if use_whitelist_reverse and whitelist.reverse_cached_ok() then
ngx.exit(ngx.OK)
end
-- check if already in blacklist cache
if use_blacklist_ip and blacklist.ip_cached_ko() then
ngx.exit(ngx.HTTP_FORBIDDEN)
end
if use_blacklist_reverse and blacklist.reverse_cached_ko() then
ngx.exit(ngx.HTTP_FORBIDDEN)
end
-- check if already in dnsbl cache
if use_dnsbl and dnsbl.cached_ko() then
ngx.exit(ngx.HTTP_FORBIDDEN)
end
-- check if IP is whitelisted (only if not in cache)
if use_whitelist_ip and not whitelist.ip_cached() then
if whitelist.check_ip() then
ngx.exit(ngx.OK)
end
end
-- check if reverse is whitelisted (only if not in cache)
if use_whitelist_reverse and not whitelist.reverse_cached() then
if whitelist.check_reverse() then
ngx.exit(ngx.OK)
end
end
-- check if IP is blacklisted (only if not in cache)
if use_blacklist_ip and not blacklist.ip_cached() then
if blacklist.check_ip() then
ngx.exit(ngx.HTTP_FORBIDDEN)
end
end
-- check if reverse is blacklisted (only if not in cache)
if use_blacklist_reverse and not blacklist.reverse_cached() then
if blacklist.check_reverse() then
ngx.exit(ngx.HTTP_FORBIDDEN)
end
end
-- check if IP is in DNSBLs (only if not in cache)
if use_dnsbl and not dnsbl.cached() then
if dnsbl.check() then
ngx.exit(ngx.HTTP_FORBIDDEN)
end
end
ngx.exit(ngx.OK)
}
......@@ -65,10 +65,14 @@ http {
access_log syslog:server=unix:/dev/log,nohostname,facility=local0,severity=notice combined;
error_log syslog:server=unix:/dev/log,nohostname,facility=local0 warn;
# lua path
# lua path and dicts
lua_package_path "/usr/local/lib/lua/?.lua;;";
%WHITELIST_IP_CACHE%
%WHITELIST_REVERSE_CACHE%
%BLACKLIST_IP_CACHE%
%BLACKLIST_REVERSE_CACHE%
%DNSBL_CACHE%
# shared memory zone for limit_req
%LIMIT_REQ_ZONE%
......
server {
include /server-confs/*.conf;
include /etc/nginx/main-lua.conf;
%LISTEN_HTTP%
%USE_HTTPS%
%REDIRECT_HTTP_TO_HTTPS%
......@@ -11,7 +12,6 @@ server {
return 405;
}
%LIMIT_REQ%
%DNSBL%
%AUTH_BASIC%
%USE_PHP%
%HEADER_SERVER%
......
......@@ -52,6 +52,7 @@ cp -r /opt/confs/owasp-crs /etc/nginx
cp /opt/confs/php.ini /etc/php7/php.ini
cp /opt/logs/rsyslog.conf /etc/rsyslog.conf
cp /opt/logs/logrotate.conf /etc/logrotate.conf
cp /opt/lua/*.lua /usr/local/lib/lua
# remove cron jobs
echo "" > /etc/crontabs/root
......@@ -123,9 +124,16 @@ USE_CUSTOM_HTTPS="${USE_CUSTOM_HTTPS-no}"
ROOT_FOLDER="${ROOT_FOLDER-/www}"
LOGROTATE_MINSIZE="${LOGROTATE_MINSIZE-10M}"
LOGROTATE_MAXAGE="${LOGROTATE_MAXAGE-7}"
DNS_RESOLVERS="${DNS_RESOLVERS-8.8.8.8 8.8.4.4}"
USE_WHITELIST_IP="${USE_WHITELIST_IP-yes}"
WHITELIST_IP_LIST="${WHITELIST_IP_LIST-23.21.227.69 40.88.21.235 50.16.241.113 50.16.241.114 50.16.241.117 50.16.247.234 52.204.97.54 52.5.190.19 54.197.234.188 54.208.100.253 54.208.102.37 107.21.1.8}"
USE_WHITELIST_REVERSE="${USE_WHITELIST_REVERSE-yes}"
WHITELIST_REVERSE_LIST="${WHITELIST_REVERSE_LIST-.googlebot.com .google.com .search.msn.com .crawl.yahoot.net .crawl.baidu.jp .crawl.baidu.com .yandex.com .yandex.ru .yandex.net}"
USE_BLACKLIST_IP="${USE_BLACKLIST_IP-yes}"
BLACKLIST_IP_LIST="${BLACKLIST_IP_LIST-}"
USE_BLACKLIST_REVERSE="${USE_BLACKLIST_REVERSE-yes}"
BLACKLIST_REVERSE_LIST="${BLACKLIST_REVERSE_LIST-.shodan.io}"
USE_DNSBL="${USE_DNSBL-yes}"
DNSBL_CACHE="${DNSBL_CACHE-10m}"
DNSBL_RESOLVERS="${DNSBL_RESOLVERS-8.8.8.8 8.8.4.4}"
DNSBL_LIST="${DNSBL_LIST-bl.blocklist.de problems.dnsbl.sorbs.net sbl.spamhaus.org xbl.spamhaus.org}"
USE_LIMIT_REQ="${USE_LIMIT_REQ-yes}"
LIMIT_REQ_RATE="${LIMIT_REQ_RATE-20r/s}"
......@@ -423,17 +431,66 @@ if [ "$USE_AUTH_BASIC" = "yes" ] ; then
else
replace_in_file "/etc/nginx/server.conf" "%AUTH_BASIC%" ""
fi
# lua resolvers
resolvers=$(spaces_to_lua "$DNS_RESOLVERS")
replace_in_file "/usr/local/lib/lua/dns.lua" "%DNS_RESOLVERS%" "$resolvers"
# whitelist IP
if [ "$USE_WHITELIST_IP" = "yes" ] ; then
replace_in_file "/etc/nginx/nginx.conf" "%WHITELIST_IP_CACHE%" "lua_shared_dict whitelist_ip_cache 10m;"
replace_in_file "/etc/nginx/main-lua.conf" "%USE_WHITELIST_IP%" "true"
else
replace_in_file "/etc/nginx/nginx.conf" "%WHITELIST_IP_CACHE%" ""
replace_in_file "/etc/nginx/main-lua.conf" "%USE_WHITELIST_IP%" "false"
fi
list=$(spaces_to_lua "$WHITELIST_IP_LIST")
replace_in_file "/usr/local/lib/lua/whitelist.lua" "%WHITELIST_IP_LIST%" "$list"
# whitelist rDNS
if [ "$USE_WHITELIST_REVERSE" = "yes" ] ; then
replace_in_file "/etc/nginx/nginx.conf" "%WHITELIST_REVERSE_CACHE%" "lua_shared_dict whitelist_reverse_cache 10m;"
replace_in_file "/etc/nginx/main-lua.conf" "%USE_WHITELIST_REVERSE%" "true"
else
replace_in_file "/etc/nginx/nginx.conf" "%WHITELIST_REVERSE_CACHE%" ""
replace_in_file "/etc/nginx/main-lua.conf" "%USE_WHITELIST_REVERSE%" "false"
fi
list=$(spaces_to_lua "$WHITELIST_REVERSE_LIST")
replace_in_file "/usr/local/lib/lua/whitelist.lua" "%WHITELIST_REVERSE_LIST%" "$list"
# blacklist IP
if [ "$USE_BLACKLIST_IP" = "yes" ] ; then
replace_in_file "/etc/nginx/nginx.conf" "%BLACKLIST_IP_CACHE%" "lua_shared_dict blacklist_ip_cache 10m;"
replace_in_file "/etc/nginx/main-lua.conf" "%USE_BLACKLIST_IP%" "true"
else
replace_in_file "/etc/nginx/nginx.conf" "%BLACKLIST_IP_CACHE%" ""
replace_in_file "/etc/nginx/main-lua.conf" "%USE_BLACKLIST_IP%" "false"
fi
list=$(spaces_to_lua "$BLACKLIST_IP_LIST")
replace_in_file "/usr/local/lib/lua/blacklist.lua" "%BLACKLIST_IP_LIST%" "$list"
# blacklist rDNS
if [ "$USE_BLACKLIST_REVERSE" = "yes" ] ; then
replace_in_file "/etc/nginx/nginx.conf" "%BLACKLIST_REVERSE_CACHE%" "lua_shared_dict blacklist_reverse_cache 10m;"
replace_in_file "/etc/nginx/main-lua.conf" "%USE_BLACKLIST_REVERSE%" "true"
else
replace_in_file "/etc/nginx/nginx.conf" "%BLACKLIST_REVERSE_CACHE%" ""
replace_in_file "/etc/nginx/main-lua.conf" "%USE_BLACKLIST_REVERSE%" "false"
fi
list=$(spaces_to_lua "$BLACKLIST_REVERSE_LIST")
replace_in_file "/usr/local/lib/lua/blacklist.lua" "%BLACKLIST_REVERSE_LIST%" "$list"
# DNSBL
if [ "$USE_DNSBL" = "yes" ] ; then
replace_in_file "/etc/nginx/nginx.conf" "%DNSBL_CACHE%" "lua_shared_dict dnsblcache $DNSBL_CACHE;"
replace_in_file "/etc/nginx/server.conf" "%DNSBL%" "include /etc/nginx/dnsbl.conf;"
resolvers=$(spaces_to_lua "$DNSBL_RESOLVERS")
list=$(spaces_to_lua "$DNSBL_LIST")
replace_in_file "/etc/nginx/dnsbl.conf" "%DNSBL_RESOLVERS%" "$resolvers"
replace_in_file "/etc/nginx/dnsbl.conf" "%DNSBL_LIST%" "$list"
replace_in_file "/etc/nginx/nginx.conf" "%DNSBL_CACHE%" "lua_shared_dict dnsbl_cache 10m;"
replace_in_file "/etc/nginx/main-lua.conf" "%USE_DNSBL%" "true"
else
replace_in_file "/etc/nginx/nginx.conf" "%DNSBL_CACHE%" ""
replace_in_file "/etc/nginx/server.conf" "%DNSBL%" ""
replace_in_file "/etc/nginx/main-lua.conf" "%USE_DNSBL%" "false"
fi
list=$(spaces_to_lua "$DNSBL_LIST")
replace_in_file "/usr/local/lib/lua/dnsbl.lua" "%DNSBL_LIST%" "$list"
if [ "$USE_LIMIT_REQ" = "yes" ] ; then
replace_in_file "/etc/nginx/nginx.conf" "%LIMIT_REQ_ZONE%" "limit_req_zone \$binary_remote_addr zone=limit:${LIMIT_REQ_CACHE} rate=${LIMIT_REQ_RATE};"
replace_in_file "/etc/nginx/server.conf" "%LIMIT_REQ%" "include /etc/nginx/limit-req.conf;"
......
local dns = require "dns"
local ip_list = {%BLACKLIST_IP_LIST%}
local reverse_list = {%BLACKLIST_REVERSE_LIST%}
local ip = ngx.var.remote_addr
function ip_cached_ko ()
return ngx.shared.blacklist_ip_cache:get(ip) == "ko"
end
function reverse_cached_ko ()
return ngx.shared.blacklist_reverse_cache:get(ip) == "ko"
end
function ip_cached ()
return ngx.shared.blacklist_ip_cache:get(ip) ~= nil
end
function reverse_cached ()
return ngx.shared.blacklist_reverse_cache:get(ip) ~= nil
end
function check_ip ()
for k, v in ipairs(ip_list) do
if v == ip then
ngx.shared.blacklist_ip_cache:set(ip, "ko", 86400)
return true
end
end
ngx.shared.blacklist_ip_cache:set(ip, "ok", 86400)
return false
end
function check_reverse ()
local rdns = dns.get_reverse()
if rdns ~= "" then
for k, v in ipairs(reverse_list) do
if rdns:sub(-#v) == v then
ngx.shared.blacklist_reverse_cache:set(ip, "ko", 86400)
return true
end
end
end
ngx.shared.blacklist_reverse_cache:set(ip, "ok", 86400)
return false
end
local resolver = require "resty.dns.resolver"
local resolvers = {%DNS_RESOLVERS%}
local ip = ngx.var.remote_addr
function get_reverse()
local r, err = resolver:new{nameservers=resolvers, retrans=2, timeout=2000}
if not r then
return ""
end
local rdns = ""
local answers, err = r:reverse_query(ip)
if not answers.errcode then
for ak, av in ipairs(answers) do
if av.ptrdname then
rdns = av.ptrdname
break
end
end
end
return rdns
end
function get_ips(fqdn)
local r, err = resolver:new{nameservers=resolvers, retrans=2, timeout=2000}
if not r then
return ""
end
local ips = {}
local answers, err, tries = r:query(fqdn, nil, {})
for ak, av in ipairs(answers) do
if av.address then
table.insert(ips, av.address)
end
end
return ips
end
function ip_to_arpa()
return resolver.arpa_str(ip):gsub("%.in%-addr%.arpa", ""):gsub("%.ip6%.arpa", "")
end
local dns = require "dns"
local dnsbls = {%DNSBL_LIST%}
local ip = ngx.var.remote_addr
function cached_ko ()
return ngx.shared.dnsbl_cache:get(ip) == "ko"
end
function cached ()
return ngx.shared.dnsbl_cache:get(ip) ~= nil
end
function check ()
local rip = dns.ip_to_arpa()
for k, v in ipairs(dnsbls) do
local req = rip .. "." .. v
local ips = dns.get_ips(req)
for k2, v2 in ipairs(ips) do
a,b,c,d = v2:match("([%d]+).([%d]+).([%d]+).([%d]+)")
if a == "127" then
ngx.shared.dnsbl_cache:set(ip, "ko", 86400)
return true
end
end
end
ngx.shared.dnsbl_cache:set(ip, "ok", 86400)
return false
end
local dns = require "dns"
local ip_list = {%WHITELIST_IP_LIST%}
local reverse_list = {%WHITELIST_REVERSE_LIST%}
local ip = ngx.var.remote_addr
function ip_cached_ok ()
return ngx.shared.whitelist_ip_cache:get(ip) == "ok"
end
function reverse_cached_ok ()
return ngx.shared.whitelist_reverse_cache:get(ip) == "ok"
end
function ip_cached ()
return ngx.shared.whitelist_ip_cache:get(ip) ~= nil
end
function reverse_cached ()
return ngx.shared.whitelist_reverse_cache:get(ip) ~= nil
end
function check_ip ()
for k, v in ipairs(ip_list) do
if v == ip then
ngx.shared.whitelist_ip_cache:set(ip, "ok", 86400)
return true
end
end
ngx.shared.whitelist_ip_cache:set(ip, "ko", 86400)
return false
end
function check_reverse ()
local rdns = dns.get_reverse()
if rdns ~= "" then
local whitelisted = false
for k, v in ipairs(reverse_list) do
if rdns:sub(-#v) == v then
whitelisted = true
break
end
end
if whitelisted then
local ips = dns.get_ips(rdns)
for k, v in ipairs(ips) do
if v == ip then
ngx.shared.whitelist_reverse_cache:set(ip, "ok", 86400)
return true
end
end
end
end
ngx.shared.whitelist_reverse_cache:set(ip, "ko", 86400)
return false
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment