Slow file transfer speeds with haproxy reverse proxy and nextcloud on freenas

Nextcloud version (eg, 18.0.2) : 19.0.1
Operating system and version (eg, Ubuntu 20.04) : FreeBSD 11.3
Apache or nginx version (eg, Apache 2.4.25) : nginx 1.18.0_20,2
PHP version (eg, 7.1) : 7.4

Hi all,

I’m running nextcloud in a freenas jail with an haproxy reverse proxy in front of it (running in a separate jail on the same server).

My problem is that the file transfer speeds are extremely slow when going through the haproxy connection.

  • When I connect to nextcloud via the haproxy server, using the full domain name, I’m seeing ~200KB/s upload speeds max
  • When I connect directly to the nextcloud server via IP address, bypassing the reverse proxy, I’m seeing 5-10MB/s upload speeds. Same server, same everything else.

I am terminating the SSL connection at haproxy, and connecting via IP address to the server also uses SSL.

Based on those side-by-side tests using the same file set, I’m assuming that the problem is either in the freenas jail configuration of the haproxy jail or in the haproxy config itself. Maybe some performance tuning on haproxy?

Here’s my haproxy.conf

    maxconn 500   
    tune.h2.initial-window-size 1048576
    tune.ssl.default-dh-param 2048
    ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
    ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
    log len 8096 local0 err
    stats socket /var/run/haproxy.sock mode 600 level admin

    log global
    option dontlognull
    option forwardfor
    option http-buffer-request
    option log-health-checks
    timeout connect 30s
    timeout client 2h
    timeout server 2h
    timeout check 5s
    timeout http-request 15s
    timeout http-keep-alive 15s

frontend default
    mode tcp

    http-request capture req.hdr(Host) len 30
    http-request capture req.hdr(User-Agent) len 200
    http-request capture req.hdr(Referer) len 800

    tcp-request inspect-delay 5s
    tcp-request content accept if { req.ssl_hello_type 1 }

    use_backend http-traffic-redirect if { req.ssl_hello_type 1 }

    default_backend no-match-tcp

frontend main-http
    mode http

    log len 8096 local0 info
    log-format '{"clientIP": "%ci","clientPort": "%cp","dateTime": "%t","frontendNameTransport": "%ft","backend": "%b","serverName": "%s","Tw": %Tw,"Tc": %Tc,"Tt": %Tt,"bytesRead": %B,"terminationState": "%ts","actconn": %ac,"FrontendCurrentConn": %fc,"backendCurrentConn": %bc,"serverConcurrentConn": %sc,"retries": %rc,"srvQueue": %sq,"backendQueue": %bq,"backendSourceIP": "%bi","backendSourcePort": "%bp","statusCode": "%ST","serverIP": "%si","serverPort": "%sp","frontendIP": "%fi","frontendPort": "%fp","httpRequest": "%r","capturedRequestHeaders": "%hr"}'

    capture request header Host len 30
    capture request header User-Agent len 200
    capture request header Referer len 800
    capture request header X-Forwarded-For len 20

    bind ssl crt /usr/local/etc/ssl/certs/ accept-proxy alpn h2,http/1.1

    http-request deny if HTTP_1.0

    acl plex hdr(host) -i
    acl nextcloud hdr(host) -i

    use_backend if plex
    use_backend if nextcloud

    default_backend no-match-http

backend http-traffic-redirect
    mode tcp
    server haproxy maxconn 200 send-proxy

    mode http
    option http-server-close
    http-response set-header Strict-Transport-Security "max-age=63072000; incluudeSubDomains; preload;"
    http-response set-header X-Frame-Options DENY
    http-response set-header X-XSS-Protection 1;mode=block
    http-response set-header X-Content-Type-Options nosniff
    http-response set-header Referrer-Policy no-referrer-when-downgrade
    server plex maxconn 20 check ssl verify none

    mode http
    option http-server-close
    fullconn 100
    timeout connect 30s
    timeout client 2h
    timeout server 2h
    http-request set-header X-Forwarded-Port %[dst_port]
    http-request add-header X-Forwarded-Proto https if { ssl_fc }
    option httpchk HEAD / HTTP/1.1\r\nHost:localhost
    http-response set-header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload;"
    http-response set-header X-Frame-Options SAMEORIGIN
    http-response set-header X-XSS-Protection 1;mode=block
    http-response set-header X-Content-Type-Options nosniff
    http-response set-header Referrer-Policy no-referrer-when-downgrade
    server nextcloud maxconn 100 check ssl verify none

backend no-match-http
    mode http
    http-request deny deny_status 503

backend no-match-tcp
    mode tcp
    tcp-request content reject

listen stats
    mode http
    bind :9000
    maxconn 20
    timeout client 30s
    timeout server 30s
    timeout connect 30s
    timeout queue 100s
    stats enable
    stats uri /stats
    stats hide-version
    stats refresh 30s
    stats show-node

I do have a few other backends on this server, running on other jails, but I removed them from the above config as I don’t think they are relevant.

There’s no errors in the logs, so it appears to be working correctly, just very slowly.

Thank you for your help!

I partially solved my own problem. I moved the haproxy server to the firewall, and added this line to the global section of the haproxy.conf:

nbthread 4

This drastically improved performance for the large uploads.