home July 01, 2021

HAProxy Configuration


A HAProxy High Performance TCP, UDP, HTTPS Load Balancer

HAProxy is free, open source software providing a high availability load balancer and proxy server for TCP and HTTP-based applications spreading requests across one(1) or more backend servers. HAProxy is written in C and has the reputation for being fast and efficient in terms of processor and memory usage. HAProxy was written in 2000 by Willy Tarreau, a core contributor to the Linux kernel and Willy still maintains the HAProxy project.

HAProxy Configuration

The following HAProxy configuration is a proof of concept and a fully working example. HAProxy is running as a non-privileged user in a chroot with logging enabled. The "torii" frontend is a TCP router which analyzes the client traffic and proxies the connection to the correct back end. The frontend torii manages Lets Encrypt renewal traffic by looking at the time and day as well as the Application-Layer Protocol Negotiation (ALPN) Transport Layer Security (TLS) extension. Traffic meant for the main website is directed to the https service. Finally, private ssh traffic using an https tunnel is checked and forwarded to an sshd server. Clients who fail the previous test are silently dropped just like a firewall would.

Although the following example is a bit complicated, wanted to provide a variety of examples in a working configuration so you can pick and choose. Take a look at the following haproxy.conf as well as the thoughts and insights highlighted later on this page.

#
# moneyslow.com  -|-  July 2021
#
# HAProxy v2.4.x - https://moneyslow.com/html/webconf/haproxy_https_proxy_ssl_chroot.html
#

global
   daemon
   user  haproxy
   group haproxy
   strict-limits
   ssl-mode-async
   stats socket /usr/local/etc/haproxy/haproxy_stats.sock mode 600 level admin
   ssl-default-bind-ciphersuites TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256
   ssl-default-bind-options force-tlsv13 no-tls-tickets
   tune.h2.max-concurrent-streams 25
   expose-experimental-directives
   chroot /usr/local/etc/haproxy
   log /dev/log local0 info

defaults
   log global
	 option abortonclose
   timeout client     2m
   timeout client-fin 100ms
   timeout connect    1h
   timeout server     1h

cache web_cache
   total-max-size 128 # MBytes
   max-object-size 524288 # Bytes
   max-age 864000 # 10 days

backend gatekeeper
   stick-table type ipv6 size 1m expire 10m store gpc0,conn_rate(5m),conn_cur,http_req_rate(5m),http_err_rate(5m)

frontend torii
   mode tcp
   option tcplog
   bind 127.0.0.1:443 defer-accept
   rate-limit sessions 50
   tcp-request inspect-delay 100ms
   log-format "%ci %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"
   #
   # Track requests keyed by client ip after tcp handshake
   tcp-request connection track-sc0 src table gatekeeper
   #
   # ACL - DoS protection - silently drop "uncivilized" clients, trigger ban
   acl uncivilized src_conn_rate(gatekeeper) ge 250
   acl uncivilized src_conn_cur(gatekeeper)  ge  75
   acl triggerban  src_inc_gpc0(gatekeeper)  gt   0
   acl banned      src_get_gpc0(gatekeeper)  gt   0
   tcp-request connection silent-drop if uncivilized triggerban
   tcp-request connection silent-drop if banned
   #
   # ACL - confirm client side support in the TLS stream of the request buffer 
   acl client_hello req.ssl_hello_type 1  # valid SSL hello message
   acl client_ecc   req.ssl_ec_ext 1      # ECC certificate compatable
   acl client_tls   req.ssl_ver ge 3.3    # TLSv1.2 / TLSv1.3 support
   tcp-request content silent-drop unless client_hello client_ecc client_tls
   #
   # Collect current system local time and date for ACLs
   tcp-request content set-var(req.current_day)  date,ltime(%d)
   tcp-request content set-var(req.current_hour) date,ltime(%H)
   #
   # ACL - Let's Encrypt TLS-ALPN (tls-alpn-01)
   acl acme_day  var(req.current_day)  eq 15  # 15th day of the month
   acl acme_hour var(req.current_hour) eq 13  # 1pm to 1:59pm
   acl acme_alpn req.ssl_alpn acme-tls/1      # ALPN string
   use_backend   rt_acmesh if acme_day acme_hour acme_alpn
   #
   # ACL - https://moneyslow.com/html/webconf  
   acl calomelorg_sni req.ssl_sni moneyslow.com
   use_backend rt_https if calomelorg_sni
   #
   # ACL - private ssh tunneled inside https, limit access between 10am-6:59pm local time
   acl ssh_addr src 11.22.33.44
   acl ssh_sni  req.ssl_sni ssh.moneyslow.com
   acl ssh_hour var(req.current_hour) eq 10 || 11 || 12 || 13 || 14 || 15 || 16 || 17 || 18
   use_backend  rt_httpssh if ssh_addr ssh_sni ssh_hour
   #
   # DENY - silently drop unmatched queries
   default_backend rt_silent_drop

backend rt_acmesh
   server acmesh 127.0.0.1:8443

backend rt_https
   server fe_https unix@haproxy_https.sock send-proxy-v2

backend rt_httpssh
   server fe_httpssh unix@haproxy_httpssh.sock send-proxy-v2

backend rt_silent_drop
   acl triggerban src_inc_gpc0(gatekeeper) gt 0
   tcp-request content silent-drop if triggerban
   tcp-request content silent-drop

#
# moneyslow.com
#

frontend fe_https
   bind unix@/usr/local/etc/haproxy/haproxy_https.sock ssl alpn h2,http/1.1 strict-sni crt /acme/.acme.sh/moneyslow.com_ecc/moneyslow.com.pem accept-proxy user haproxy mode 600
   default_backend bk_https
   mode http
   option http-buffer-request
   option httpclose
   option httplog
   timeout client 5s
   log-format "%ci %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r %sslv %sslc"
   #
   # Track requests keyed by client ip once the https session is negotiated 
   tcp-request session track-sc0 src table gatekeeper
   #
   # ACL - HTTPS DoS protection - silently drop "uncivilized" clients, trigger ban
   acl uncivilized src_http_req_rate(gatekeeper) ge 150
   acl uncivilized src_http_err_rate(gatekeeper) ge  50
   acl triggerban  src_inc_gpc0(gatekeeper)      gt   0
   acl banned      src_get_gpc0(gatekeeper)      gt   0
   http-request silent-drop if uncivilized triggerban
   http-request silent-drop if banned
   #
   # Sanitize client requests
   http-request silent-drop unless METH_GET
   http-request silent-drop unless HTTP_2.0 || HTTP_1.1
   http-request silent-drop if { req.hdr(Range) -m found }
   http-request del-header cache-control
   http-request del-header connection
   http-request del-header pragma
   http-request del-header referer
   http-request del-header user-agent
   http-request set-uri %[path]
   http-request normalize-uri fragment-strip if TRUE
   http-request normalize-uri percent-decode-unreserved strict if TRUE
   http-request normalize-uri path-strip-dotdot full if TRUE
   http-request normalize-uri path-strip-dot if TRUE
   http-request normalize-uri path-merge-slashes if TRUE
   #
   # ACL - inconsequential icons
   acl icons path     /favicon.ico
   acl icons path_beg /apple-touch
   http-request redirect code 301 location https://www.apple.com/apple-touch-icon.png if icons
   #
   # ACL - allowed paths
   acl path_valid  path     /file1.sh /file2.tar
   acl path_valid  path_beg /statistics/
   acl path_valid  path_end .html image.jpg
   acl path_length path_len le 100
   acl path_pct    path_sub %
   http-request deny deny_status 410 unless !path_pct path_length path_valid
   #
   # ACL - restricted paths
   acl network_allowed src 11.22.33.44 10.20.30.40
   acl restricted_page path /file1.sh /file2.tar
   acl restricted_page path_beg /statistics/
   http-request silent-drop if restricted_page !network_allowed
   #
   # ACL - brotli static compression with caching support
   acl br_static_hdr  req.hdr(Accept-Encoding) -m sub br
   acl br_static_path path_end .html .xml
   http-request set-path %[path].br if br_static_hdr br_static_path
   acl br_static_root path_end /
   http-request set-path %[path]index.html.br if br_static_hdr br_static_root
   acl br_static path_end .html.br
   http-request set-var(txn.brotli_trigger) int(121212) if br_static
   #
   # Security headers returned to the client
   http-response del-header age
   http-response del-header expires
   http-response del-header server
  #http-response set-header alt-svc "h3=\":443\"; ma=31536000; persist=1"
   http-response set-header link "</calomel_image.webp>; rel=preload; as=image"
   http-response set-header content-security-policy "upgrade-insecure-requests; default-src 'none'; style-src 'unsafe-inline'; img-src 'self' data:"
   http-response set-header strict-transport-security "max-age=31536000; includesubdomains; preload"
   http-response set-header cache-control "max-age=604800, s-maxage=604800, immutable"
   http-response set-header expect-ct "max-age=31536000, enforce"
   http-response set-header referrer-policy "no-referrer"
   #
   # errors
   http-response return status 410 default-errorfiles if { status 403 }
   http-response return status 410 default-errorfiles if { status 404 }

backend bk_https
   mode http
   server .nginx. unix@nginx.sock
   #
   # ACL - web cache
   acl no_cache  path_beg    /statistics
   http-request  cache-use   web_cache unless no_cache
   http-response cache-store web_cache
   #
   # ACL - brotli response headers
   acl br_static var(txn.brotli_trigger) equal 121212
   http-response set-header content-type "text/html; charset=utf-8" if br_static
   http-response set-header content-encoding "br" if br_static
   http-response set-header vary "accept-encoding" if br_static

#
# ssh through an https tunnel , TLSv1.3 , HTTP/2
#

frontend fe_httpssh
   bind unix@/usr/local/etc/haproxy/haproxy_httpssh.sock ssl alpn h2 strict-sni crt /acme/.acme.sh/moneyslow.com_ecc/moneyslow.com.pem accept-proxy user haproxy mode 600
   mode tcp
   option tcplog
   tcp-request inspect-delay 100ms
   log-format "%ci %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"
   #
   # ACL - validate client side ssh version in tcp payload
   acl ssh_banner req.payload(0,19) -m str "SSH-2.0-OpenSSH_8.6"
   tcp-request content set-var(req.ssh_trigger) int(343434) if ssh_banner
   acl ssh_allow var(req.ssh_trigger) equal 343434
   use_backend bk_sshd if ssh_allow
   #
   # DENY - silently drop unmatched queries
   default_backend rt_silent_drop

backend bk_sshd
   mode tcp
   server localhost-sshd 127.0.0.1:22

### EOF ###



HAPorxy Thoughts and Insights

In the following sections we will talk about and highlight some of the special or abstract sections of our haproxy.conf .

HAProxy default deny like a firewall

HAProxy is setup with the default deny mentality of a firewall. The client ip must provide the correct infomation and credentials else the connection will be dropped.

The "frontend torii" is the TCP router which looks at the client TCP request extracting information from the unencrypted TCP stream. HAProxy directs clients who provide "acme-tls/1" in the Application-Layer Protocol Negotiation (ALPN) to the backend "rt_acmesh". rt_acmesh is ment for the Let's Encrypt servers to connect to the acme.sh client to validate ssl certificate renewal. Clients who provide a server name indication (SNI) of moneyslow.com are directed to rt_https which then sends the requests to the fe_https frontend of the web server. Private ssh connections from specified ip addresses and alternate hostname proceed to the sshd server. All other clients who do not provide any of the previous credentials are sent to rt_silent_drop which initilizes a "triggerban" and drops the connection. The client ip is banned for the "gatekeeper" stick table expiration time of ten(10) minutes.

frontend torii
  ...
  # ACL - Let's Encrypt TLS-ALPN
  acl acme_alpn req.ssl_alpn acme-tls/1
  use_backend   rt_acmesh if acme_alpn
  #
  # ACL - https://moneyslow.com/html/webconf  
  acl calomelorg_sni req.ssl_sni moneyslow.com
  use_backend rt_https if calomelorg_sni
  #
  # ACL - private ssh tunneled inside https 
  acl ssh_ip  src 11.22.33.44 
  acl ssh_sni req.ssl_sni ssh.moneyslow.com
  use_backend rt_httpssh if ssh_ip ssh_sni
  #
  # DENY - silently drop unmatched queries
  default_backend rt_silent_drop

backend rt_acmesh
  server acmesh 127.0.0.1:8443

backend rt_https
  server fe_https unix@haproxy_https.sock send-proxy-v2

backend rt_httpssh
  server fe_httpssh unix@haproxy_httpssh.sock send-proxy-v2

backend rt_silent_drop
  acl triggerban src_inc_gpc0(gatekeeper) gt 0
  tcp-request content silent-drop if triggerban
  tcp-request content silent-drop

DoS Protection using stick tables

Denial of Service (DoS) protection works by storing the incoming client ip address in the stick table called "gatekeeper" with a total expiration time of ten(10) minutes. The stick table stores the global peer counter, client connection rate averaged over five(5) minutes, total concurrent client connections, http(s) request rate averaged over five(5) minutes and finally the http(s) error rate avergaed over five(5) minutes.

The "frontend torii" is the TCP router or director which can only record actions during the initial TCP connection. In "frontend torii" the TCP connection rate and the number current TCP connections per client ip address are recorded in the gatekeeper stick table. The "frontend fe_https" only tracks HTTP requests once the TLS session and ssl certificate have been negotiated. In HTTP(S) mode, HAProxy can record the rate of client HTTP requests and the amount of http errors the client has triggered. Within both DoS sections the totals are tallied and when the client violates any of the counters the "triggerban" ACL is triggered and the global peer counter, gcp(0), keyed by client ip address is incremented. When ANY client ip has a global peer counter, gcp(0), of greater than 0 the connection is silently dropped. In essence the client is "banned" for stick table expire time of ten(10) minutes. Every time a currently "banned" client ip reconnects, the banned expiration timer is reset again to ten(10) minutes. This results in the "uncivilized" belligerent client ip adding more jail time by their own actions. Karma.


backend gatekeeper
  stick-table type ipv6 size 1m expire 10m store gpc0,conn_rate(5m),conn_cur,http_req_rate(5m),http_err_rate(5m)

frontend torii
  ...
  # Track requests keyed by client ip after tcp handshake
  tcp-request connection track-sc0 src table gatekeeper
  #
  # ACL - DoS protection - silently drop "uncivilized" clients, trigger ban
  acl uncivilized src_conn_rate(gatekeeper) ge 250
  acl uncivilized src_conn_cur(gatekeeper)  ge  75
  acl triggerban  src_inc_gpc0(gatekeeper)  gt   0
  acl banned      src_get_gpc0(gatekeeper)  gt   0
  tcp-request connection silent-drop if uncivilized triggerban
  tcp-request connection silent-drop if banned
  ...

frontend fe_https
  ...
  # Track requests keyed by client ip once the https session is negotiated 
  tcp-request session track-sc0 src table gatekeeper
  #
  # ACL - HTTPS DoS protection - silently drop "uncivilized" clients, trigger ban
  acl uncivilized src_http_req_rate(gatekeeper) ge 150
  acl uncivilized src_http_err_rate(gatekeeper) ge  50
  acl triggerban  src_inc_gpc0(gatekeeper)      gt   0
  acl banned      src_get_gpc0(gatekeeper)      gt   0
  http-request silent-drop if uncivilized triggerban
  http-request silent-drop if banned
  ...

Poor Man's static compressed brotli support

Brotli is a compression algorithm developed by Google which works very well for text compression. Haproxy can be configured to look for client side brotil support by looking for the "br" string in the Accept-Encoding header which signals client side Brotli compression is supported.

On our sites we precompress both HTML and XML pages using the brotli binary. This means the web root directory contains both the uncompressed ".html" and the precompressed ".html.br" versions of the same page. If a client claims brotil support and requests either an html or xml files we re-write their request to end with ".html.br" or .xml.br" respectively. If a brotli client requests the support page "/" then the path is re-written to /index.html.br . Because of the way haproxy processes the request we need to set a trigger using a variable so the response contains the correct headers to send back to the client. Our brotli trigger is called txn.brotli_trigger and is defined as the integer "121212". This number is just made up, you can use any value.

On the backend HAproxy looks to see if the txn.brotli_trigger variable is equal to 121212 and if so, the http response is set to the correct server response headers.

NOTE: statically compressing the pages means HAProxy can locally cache both .html and .html.br files.

frontend fe_https
  ...
  # ACL - brotli static compression with caching support
  acl br_static_hdr  req.hdr(Accept-Encoding) -m sub br
  acl br_static_path path_end .html .xml
  http-request set-path %[path].br if br_static_hdr br_static_path
  acl br_static_root path_end /
  http-request set-path %[path]index.html.br if br_static_hdr br_static_root
  acl br_static path_end .html.br
  http-request set-var(txn.brotli_trigger) int(121212) if br_static
  ...

backend bk_https
  ...
  # ACL - brotli response headers
  acl br_static var(txn.brotli_trigger) equal 121212
  http-response set-header content-type "text/html; charset=utf-8" if br_static
  http-response set-header content-encoding "br" if br_static
  http-response set-header vary "accept-encoding" if br_static
  ...

Sanitizing client headers

Some of the headers a client send to the server can be harmful or unnecessary. Before doing any meaningful processing on the client request HAProxy should do some basic sanity checks.

Check if the client is making a GET or HEAD request and that the HTTP version is http/2.0 or http/1.1 . The client using range header is completely unnecessary on a server providing small static web pages. If any of these conditions are tru the connection is silently dropped.

The next section deletes some of the client headers. Cache-Control and Pragma can both be used by the client to override the local HAProxy cache settings and force the http request to go to the backend nginx server. The Referer and User-Agent headers can be spoofed by and manipulated by the client so we can not trust them in any way. The client Connection header can be ignored since the header is only set for http/1.1 and only to either "close" or "keep-alive". Our server will be in control of when the client is disconnected to this header is unnecessary for our use case.

Set the URI to just the host and the path removing the query string. A query is the string after a question mark "?".

HAProxy 2.4.x introduced the normalize-uri directives. Two(2) normalize-uri are important to us. percent-decode-unreserved will convert unreserved percent-encoded characters to their normalized character scheme. This means that a percent encoded letter "a" will be transformed from "%61" to the standard letter "a". Why force convert percent-encoded characters? Percent encoding is a common vector for reverse proxy attacks. The last normalize-uri is path-merge-slashes which simply merges all forward slashes together, i.e. "///" to "/", which is another vector for attack and aides in log readability.

frontend fe_https
  ...
  # Sanitize client requests
  http-request silent-drop unless METH_GET
  http-request silent-drop unless { req.ver 2.0 } || { req.ver 1.1 }
  http-request silent-drop if { req.hdr(Range) -m found }
  http-request del-header cache-control
  http-request del-header connection
  http-request del-header pragma
  http-request del-header referer
  http-request del-header user-agent
  http-request set-uri %[path]
  http-request normalize-uri percent-decode-unreserved if TRUE
  http-request normalize-uri path-merge-slashes if TRUE
  ...

The backend Nginx configuration

In the main https server HAProxy will proxy to the backend nginx server called ".nginx." listening on a unix socket. Make special note that the Nginx daemon is listening to the unix socket inside of the HAProxy chroot path, /usr/local/etc/haproxy . The following configuration was used for testing which resulted in an extremely low latency backend service as a unix socket does not have any TCP protocol overhead compared to a listening port on localhost. Notice the location block setting of the cache-control header (s-maxage=3600) for the file /calomel_image.webp which overrides the expiration time in the native haproxy cache (max-age 864000 # 10 days) allowing web developers to set their own cache timeouts per object or per path.

#
# moneyslow.com  -|-  July 2021
#

user www;
worker_processes  1;

events { }

http {
  charset       utf-8;
  default_type  application/octet-stream;
  expires       off;
  include       mime.types;
  max_ranges    0;
  server_tokens off;

  map "$time_local:$msec" $time_local_w_ms { ~(^\S+)(\s+\S+):\d+\.(\d+)$ $1.$3; }
  log_format main '[$time_local_w_ms] $status $request $body_bytes_sent' ;
  access_log /var/log/nginx/access.log main;
  error_log  /var/log/nginx/error.log warn;

  server {
    listen unix:/usr/local/etc/haproxy/nginx.sock;

    location / {
      root /var/www;
        index index.html;
        location /calomel_image.webp {add_header cache-control "s-maxage=3600, public";}
    }
  }
}

Setup OpenSSH to tunnel through HTTPS

Please check out our HAproxy tunnel ssh through https tutoral for more details.

Script to poll the HAProxy unix socket for cache statistics

The following script will poll the haproxy_stats.sock unix socket and print out all of the objects in the HAProxy cache as well as each object's cache timeout.

#!/bin/sh
set -euf -o pipefail

#
# moneyslow.com  -|-  July 2021
#

while true; do clear; \
/bin/echo "show table gatekeeper" | \
/usr/bin/nc -U /usr/local/etc/haproxy/haproxy_stats.sock | \
/usr/bin/sed 's/(300000)//g' | \
/usr/bin/sed 's/key=::ffff://' | \
/usr/bin/sed 's/=0/=/g' | \
/usr/bin/awk '{print $2" "$4" "$5" "$6" "$7" "$8" "$9}' | \
/usr/bin/sort -nk 1 | \
/usr/bin/column -t; \
sleep 10; \
done

## EOF ##

How to setup chroot logging on FreeBSD

In order to allow HAProxy to log to syslog we must tell syslogd to create a log device inside of the HAProxy chroot path. Our HAProxy configuration defines the chroot as "chroot /usr/local/etc/haproxy" and the log device as "log /dev/log local0". It is a bit confusing, but the HAPRoxy log device defined at /dev/log is inheriting the chroot path from "chroot /usr/local/etc/haproxy" so the true log device path is located at /usr/local/etc/haproxy/dev/log . With this full path we can configure syslogd_flags on FreeBSD to create the special log device on reboot with the following configuration in /etc/rc.conf .

#
# moneyslow.com  -|-  July 2021
#
# https://moneyslow.com/html/webconf/freebsd_network_tuning.html
#
...
haproxy_enable="YES"
syslogd_flags="-ss -cc -C -l /usr/local/etc/haproxy/dev/log"
...