Nextcloud behind HAProxy - OPNsense configuration

otherego · March 22, 2023, 7:25am

I’m moving from a working nginx reverse proxy configuration to HAProxy on OPNsense.
After using two different guides i still get “too_many_redirects” error.
1st guide i followed: https://forum.opnsense.org/index.php?topic=23339.0
2nd guide https://community.spiceworks.com/how_to/177181-opnsense-haproxy-as-reverse-proxy-for-self-hosted-services
these are relevant parts of my configuration:

my host file

<VirtualHost *:80>  
  DocumentRoot "/usr/local/www/nextcloud"  
 ServerName nextcloud.mysite.com
<FilesMatch \.php$>
        SetHandler "proxy:fcgi://127.0.0.1:9000/"  
  </FilesMatch>   
 DirectoryIndex /index.php index.php
</VirtualHost>

my config.php

{
    "system": {
        "instanceid": "***REMOVED SENSITIVE VALUE***",
        "passwordsalt": "***REMOVED SENSITIVE VALUE***",
        "secret": "***REMOVED SENSITIVE VALUE***",
        "trusted_domains": [
            "10.0.0.48",
            "nextcloud.mysite.com"
        ],
        "datadirectory": "***REMOVED SENSITIVE VALUE***",
        "dbtype": "mysql",
        "version": "25.0.4.1",
        "overwrite.cli.url": "https:\/\/nextcloud.mysite.com",
        "dbname": "***REMOVED SENSITIVE VALUE***",
        "dbhost": "***REMOVED SENSITIVE VALUE***",
        "dbport": "",
        "dbtableprefix": "oc_",
        "mysql.utf8mb4": true,
        "dbuser": "***REMOVED SENSITIVE VALUE***",
        "dbpassword": "***REMOVED SENSITIVE VALUE***",
        "installed": true,
        "trusted_proxies": "***REMOVED SENSITIVE VALUE***",
        "overwritehost": "nextcloud.mysite.com",
        "overwriteprotocol": "https",
        "overwritecondaddr": "^10\\.0\\.0\\.1$",
        "redis": {
            "host": "***REMOVED SENSITIVE VALUE***",
            "port": 0
        },
        "memcache.local": "\\OC\\Memcache\\APCu",
        "memcache.locking": "\\OC\\Memcache\\Redis",
        "maintenance": false,
        "theme": "",
        "loglevel": 2,
        "updater.release.channel": "stable",
        "default_phone_region": "IT",
        "app_install_overwrite": [],
        "ldapProviderFactory": "OCA\\User_LDAP\\LDAPProviderFactory",
        "twofactor_enforced": "true",
        "twofactor_enforced_groups": [
            "admin"
        ],
        "twofactor_enforced_excluded_groups": [],
        "data-fingerprint": "d1c023081e0c9b662bc8049cf295c443"
    }
}

haproxy.conf obtained with first guide

global
    uid                         80
    gid                         80
    chroot                      /var/haproxy
    daemon
    stats                       socket /var/run/haproxy.socket group proxy mode 775 level admin
    nbthread                    4
    hard-stop-after             60s
    no strict-limits
    maxconn                     10000
    tune.ssl.default-dh-param   4096
    spread-checks               2
    tune.bufsize                16384
    tune.lua.maxmem             0
    log                         /var/run/log local0 info
    lua-prepend-path            /tmp/haproxy/lua/?.lua

defaults
    log     global
    option redispatch -1
    maxconn 5000
    timeout client 30s
    timeout connect 30s
    timeout server 30s
    retries 3
    default-server init-addr last,libc
    default-server maxconn 5000

# autogenerated entries for ACLs

# autogenerated entries for config in backends/frontends

# autogenerated entries for stats

# Frontend: 0_SNI_frontend (listening on 0.0.0.0:80. 0.0.0.0:443)
frontend 0_SNI_frontend
    bind 0.0.0.0:443 name 0.0.0.0:443
    bind 0.0.0.0:80 name 0.0.0.0:80
    mode tcp
    default_backend SSL_backend

    # logging options

# Frontend: 1_HTTP_frontend (Listening on 127.4.4.3:80)
frontend 1_HTTP_frontend
    bind 127.4.4.3:80 name 127.4.4.3:80 accept-proxy
    mode http
    option http-keep-alive
    option forwardfor

    # logging options
    # ACL: NoSSL_condition
    acl acl_64188d5dce2390.01132494 ssl_fc

    # ACTION: HTTPtoHTTPS_rule
    http-request redirect scheme https code 301 if !acl_64188d5dce2390.01132494

# Frontend: 1_HTTPS_frontend (Lisening on 127.4.4.3:443)
frontend 1_HTTPS_frontend
    http-response set-header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"
    bind 127.4.4.3:443 name 127.4.4.3:443 accept-proxy ssl curves secp384r1  no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256 ciphersuites TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 alpn h2,http/1.1 crt-list /tmp/haproxy/ssl/64189270e357f4.63771565.certlist
    mode http
    option http-keep-alive
    option forwardfor
    timeout client 15m

    # logging options

    # ACTION: PUBLIC_SUBDOMAINS_rule
    # NOTE: actions with no ACLs/conditions will always match
    use_backend %[req.hdr(host),lower,map_dom(/tmp/haproxy/mapfiles/64188dd26c8986.37023969.txt)]

# Backend: SSL_backend ()
backend SSL_backend
    # health checking is DISABLED
    mode tcp
    balance source
    # stickiness
    stick-table type ip size 50k expire 30m 
    stick on src
    server SSL_server 127.4.4.3 send-proxy-v2 check-send-proxy

# Backend: Nextcloud_backend ()
backend Nextcloud_backend
    # health checking is DISABLED
    mode http
    balance source
    # stickiness
    stick-table type ip size 50k expire 30m 
    stick on src
    http-reuse safe
    server Nextcloud_server 10.0.0.48:80 ssl verify none

haproxy.conf obtained with second guide

#
# Automatically generated configuration.
# Do not edit this file manually.
#

#
# NOTE: HAProxy is currently DISABLED
#
global
    uid                         80
    gid                         80
    chroot                      /var/haproxy
    daemon
    stats                       socket /var/run/haproxy.socket group proxy mode 775 level admin
    nbthread                    4
    hard-stop-after             60s
    no strict-limits
    maxconn                     10000
    tune.ssl.default-dh-param   4096
    spread-checks               2
    tune.bufsize                16384
    tune.lua.maxmem             0
    log                         /var/run/log local0 info
    lua-prepend-path            /tmp/haproxy/lua/?.lua

defaults
    log     global
    option redispatch -1
    maxconn 5000
    timeout client 30s
    timeout connect 30s
    timeout server 30s
    retries 3
    default-server init-addr last,libc
    default-server maxconn 5000

# autogenerated entries for ACLs

# autogenerated entries for config in backends/frontends

# autogenerated entries for stats

# Frontend: proxy ()
frontend proxy
    http-response set-header Strict-Transport-Security "max-age=15768000"
    bind 10.0.0.1:443 name 10.0.0.1:443 ssl prefer-client-ciphers ssl-min-ver TLSv1.2 ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256 ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 crt-list /tmp/haproxy/ssl/641aa0529dfc73.39354734.certlist 
    mode http
    option http-keep-alive

    # logging options
    # ACL: nextcloud
    acl acl_641a9d77b0a028.21861222 hdr(host) -i nextcloud.aivallecrosia.com
    # ACL: nc_caldav
    acl acl_641a9da12437f1.67374553 path_end -i /.well-known/caldav
    # ACL: nc_nodeinfo
    acl acl_641a9dca00e732.15641297 path /.well-known/nodeinfo
    # ACL: nc_carddav
    acl acl_641a9db24c6f94.95375979 path_end -i /.well-known/carddav
    # ACL: nc_webfinger
    acl acl_641a9de27e94c3.01726161 path /.well-known/webfinger

    # ACTION: nextcloud
    use_backend Nextclolud if acl_641a9d77b0a028.21861222
    # ACTION: nc_caldav
    http-request redirect code 301 location /remote.php/dav if acl_641a9da12437f1.67374553 acl_641a9d77b0a028.21861222
    # ACTION: nc_nodeinfo
    http-request redirect code 301 location /index.php/%[capture.req.uri] if acl_641a9dca00e732.15641297 acl_641a9d77b0a028.21861222
    # ACTION: nc_carddav
    http-request redirect code 301 location /remote.php/dav if acl_641a9db24c6f94.95375979 acl_641a9d77b0a028.21861222
    # ACTION: nc_webfinger
    http-request redirect code 301 location /index.php/%[capture.req.uri] if acl_641a9de27e94c3.01726161 acl_641a9d77b0a028.21861222

# Backend: Nextclolud ()
backend Nextclolud
    # health checking is DISABLED
    mode http
    balance source
    # stickiness
    stick-table type ip size 50k expire 30m  
    stick on src
    http-reuse safe
    server Nextcloud 10.0.0.48:80 ssl verify none

# statistics are DISABLED

i suppose something is wired with my php.conf but i can’t unserstand where

Kerasit · March 22, 2023, 9:58am

If you had to guess what this one does, then please. I can give you a hint: Listens to ANYTHING on port 80 and 443 including localhost:

# Frontend: 0_SNI_frontend (listening on 0.0.0.0:80. 0.0.0.0:443)
frontend 0_SNI_frontend
    bind 0.0.0.0:443 name 0.0.0.0:443
    bind 0.0.0.0:80 name 0.0.0.0:80
    mode tcp
    default_backend SSL_backend

    # logging options

# Backend: SSL_backend ()
backend SSL_backend
    # health checking is DISABLED
    mode tcp
    balance source
    # stickiness
    stick-table type ip size 50k expire 30m 
    stick on src
    server SSL_server 127.4.4.3 send-proxy-v2 check-send-proxy

server SSL_server 127.4.4.3 send-proxy-v2 check-send-proxy - Where is port definition? And using an address in the loopback address range? This would and should never work but it does beacuse there is no protection if the loopback addresses, as these are never exposed.

Let us just say that I like your second HAProxy config WAY more than the first.

frontend proxy
    http-response set-header Strict-Transport-Security "max-age=15768000"
    bind *:443 ssl prefer-client-ciphers ssl-min-ver TLSv1.2 ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256 ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 crt-list /tmp/haproxy/ssl/641aa0529dfc73.39354734.certlist

And use ONLY your second HAProxy config. Having both, will indeed loop around.
Your HAProxy script should work, but is very crude (sorry).

Your NGINX did not only serve as a reverse proxy. It was also used to server PHP-FPM. You will need to setup same in HAProxy - which it can now support in newer versions - but you will ahve to do all that url rewriting in HAProxy as well, or have a spererate webserver on your nextcloud instance, and then seriously just do haproxy backends for each matching hdr host and then keep it simple.

otherego · March 22, 2023, 2:04pm

Thank you for helping. i’m not using both config, i just posted two different haproxy config i’ve got following 2 different guides.

for now i’ve modified the secong haproxy following your suggestion (i hope so because i’m not familiar with Opnsense at all) and now it looks like this:

# Frontend: proxy ()
frontend proxy
    http-response set-header Strict-Transport-Security "max-age=15768000"
    bind *:443 name *:443 ssl prefer-client-ciphers ssl-min-ver TLSv1.2 ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256 ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 crt-list /tmp/haproxy/ssl/641aa0529dfc73.39354734.certlist 
    mode http
    option http-keep-alive

regardin PHP-FMP part i will have to investigate more…
btw thak you for helping again

TheHellSite · April 28, 2023, 1:51am

Hey,

author of the first guide here.
I found this thread while researching something and wanted to explain the reasons for why I built my tutorial this way.

Please note I am not trying to offend you in any way! I am just trying to explain the “whys”.

Keep in mind: The tutorial is a “get the basics going” guide that is meant for everyone so I tried to cover any possible setup and therefore didn’t apply any strict settings.

Why is the SNI_frontend listening on “ANYTHING”?
This is because, …

many people have dynamic WAN IPs.
it makes it easier to enable access to the services behind HAProxy from local IPs by using local DNS host overrides.
again, to cover any possible setup.

Of course I could have written “use your static WAN IP if you have one”, “use your local WAN interface IP if you have one”, …
But I decided the easiest option is to just listen on anything since LAN has access to anything by default anyway, additional VLANs/Subnets/Interfaces have no access to anything by default and are probably configured with restricted access anyways (where necessary).
So I don’t see any issues with listening on anything. Especially since you still have the main aspect of OPNsense as the firewall blocking which is meant to restrict access before anything else gets access to the traffic.

The SSL_server (127.4.4.3) is a virtual IP and indeed in the loopback subnet.
It is also the IP on which the HTTP/HTTPS_frontend is listening on for SNI and SSL-offloading.
This isn’t an issue per se since, …

you are correct, they are never exposed as nothing outside of the OPNsense has access to the data flowing over the localhost / this subnet.
it enables us to do many advanced setups with HAProxy (2-level-SNI, see below).

Where is the port definition?
This isn’t a mistake! There is none because there is none necessary.
If you are familiar with the HAProxy documentation than you will know that if no port is defined in the server settings the incoming port will be used instead to access the actual server.
This is exactly what you would want if you are aiming/preparing for an advanced HAProxy setup.
Why? Read the next part.

Why would you set up a SNI_frontend in TCP mode that is just passing traffic through without doing anything with it in the first place?
Because it enables a lot more (future) possibilities!
You can route SSL traffic based on SNI without doing SSL-offloading. Again, why?
This way you can access services like OpenVPN (or other reverse proxies) on port 443 while also running HAProxy with HTTPS_frontends on port 443 for SSL-offloading, …
And since we didn’t set any port in the SSL_server you can run services that need/support TCP SNI routing without SSL-offloading on any port.

There are a lot of people looking for ways to do this (OpenVPN over port 443, reverse proxies behind HAProxy, …). My tutorial enables them to do so, if they want/need to.
And even if they don’t need it, it won’t hurt them at all to have it set up like this because who knows, maybe one day they will need it and then they only have to add/change a few things.

The to many redirects issue OP is facing has nothing to do with my HAProxy tutorial.
Others including me have configured Nextcloud without any issues using my tutorial.

There is/was simply some sort of misconfiguration on his end.
Luckily he found the solution himself.

Kind Regards
TheHellSite

otherego · April 28, 2023, 7:10am

hello, thank you for reply, many thanks again for your guide, because i’ve got all working (nexcloud and other website i’m hosting).
The problem was not related to your configuration, but it comes from Cloudflare configuration.
When i’ve changed SSL/TLS encryption mode to Full (strict), my problems where solved keeping your configuration.
Thank you again

Kerasit · May 1, 2023, 7:51am

Fair points. And it makes good sense to use the virtual IP methods. And yes, SNI Preread for proxying without termination, is very cool. However you will miss the opportunity of mixing modes.

Example: Some services will require “SSL passthrough” / SNI preread for hostname extraction, others that SSL is terminated on HAProxy side. I do agree that your config makes it very versatile and enables virtually every possible scenarios, however even I as highly skilled HAProxy admin, was unable to correctly verify and/or pinpoint accurately, what excactly happens. Would I be able to use your solution in my setup if I did it from scratch, and build upon it? Yes I would. However only after testing and playing with it for a while. For new users it is to complex to start with. For new users I always recommends “follow the yellow brick road” until they are adapt enough to start tweaking. It is no different than software development. You have to fully understand classes and objects, before you can start understanding inheritance.

I do not like any service which essentially exsposes any and all ports on my local host. If the HAProxy runs as a privileged user (I do not know enough about OPNsense to know how it runs apps, but on other router firmware, than services sometimes runs in highest privileges). Example: Hit the outer IP on any given port with any given protocol. It will be routed to localhost address range. Only 127.0.0.1 will truly result in router OS commands, however any kind of command that can trigger a native protocol, will be able to route this through the OS kernel. Imagine any zero day vulnerability like the log4j.