Excrutiatingly slow upload speeds over both the app and the web interface

The Basics

  • Nextcloud Server version (e.g., 29.x.x):
    • 31.0.6
  • Operating system and version (e.g., Ubuntu 24.04):
    • FreeBSD 14.3
  • Web server and version (e.g, Apache 2.4.25):
    • nginx 1.28.0
  • Reverse proxy and version _(e.g. nginx 1.27.2)
    • haproxy 3.2.2
  • PHP version (e.g, 8.3):
    • 8.4
  • Is this the first time you’ve seen this error? (Yes / No):
    • Yes
  • When did this problem seem to first start?
    • Switch from OBHttpd to nginx likely
  • Installation method (e.g. AlO, NCP, Bare Metal/Archive, etc.)
    • bare metal
  • Are you using CloudfIare, mod_security, or similar? (Yes / No)
    • No

Summary of the issue you are facing:

Excrutiatingly slow upload speeds over both the app and the web interface. I am getting between 60 and 350 kbps.

I can rule out both the network speed - both the server and my machine are on the same gigabit LAN connected to the same router - and interface issues; uploads to other programs on that server, as well as e.g. rsync transfers to it are orders of magnitude faster.

Steps to replicate it (hint: details matter!):

  1. Upload a file, experience the slowness

Log entries

Not applicable

Configuration

Nextcloud

{
    "system": {
        "allow_local_remote_servers": true,
        "apps_paths": [
            {
                "path": "\/usr\/local\/www\/nextcloud\/apps",
                "url": "\/apps",
                "writable": true
            },
            {
                "path": "\/usr\/local\/www\/nextcloud\/apps-pkg",
                "url": "\/apps-pkg",
                "writable": false
            }
        ],
        "logfile": "\/var\/log\/nextcloud\/nextcloud.log",
        "memcache.local": "\\OC\\Memcache\\APCu",
        "memcache.distributed": "\\OC\\Memcache\\APCu",
        "memcache.locking": "\\OC\\Memcache\\APCu",
        "instanceid": "***REMOVED SENSITIVE VALUE***",
        "passwordsalt": "***REMOVED SENSITIVE VALUE***",
        "secret": "***REMOVED SENSITIVE VALUE***",
        "trusted_domains": [
            "nextcloud.lyrion.ch"
        ],
        "trusted_proxies": "***REMOVED SENSITIVE VALUE***",
        "datadirectory": "***REMOVED SENSITIVE VALUE***",
        "dbtype": "pgsql",
        "version": "31.0.6.2",
        "overwrite.cli.url": "https:\/\/nextcloud.lyrion.ch\/",
        "dbname": "***REMOVED SENSITIVE VALUE***",
        "dbhost": "***REMOVED SENSITIVE VALUE***",
        "dbport": "5432",
        "dbtableprefix": "oc_",
        "dbuser": "***REMOVED SENSITIVE VALUE***",
        "dbpassword": "***REMOVED SENSITIVE VALUE***",
        "installed": true,
        "mail_smtpmode": "smtp",
        "mail_smtpauth": true,
        "mail_sendmailmode": "smtp",
        "tempdirectory": "\/tmp\/nextcloud",
        "theme": "",
        "loglevel": 0,
        "maintenance": false,
        "maintenance_window_start": 3,
        "check_data_directory_permissions": false,
        "mail_from_address": "***REMOVED SENSITIVE VALUE***",
        "mail_domain": "***REMOVED SENSITIVE VALUE***",
        "mail_smtphost": "***REMOVED SENSITIVE VALUE***",
        "mail_smtpport": "465",
        "mail_smtpname": "***REMOVED SENSITIVE VALUE***",
        "mail_smtppassword": "***REMOVED SENSITIVE VALUE***",
        "memories.exiftool_no_local": true,
        "memories.vod.path": "\/usr\/local\/www\/nextcloud\/apps\/memories\/bin-ext\/go-vod-amd64",
        "memories.vod.ffmpeg": "\/usr\/local\/bin\/ffmpeg",
        "memories.vod.ffprobe": "\/usr\/local\/bin\/ffprobe",
        "enabledPreviewProviders": {
            "0": "OC\\Preview\\Image",
            "1": "OC\\Preview\\HEIC",
            "2": "OC\\Preview\\TIFF",
            "3": "OC\\Preview\\AVIF",
            "4": "OC\\Preview\\JXL",
            "20": "OC\\Preview\\Movie"
        },
        "preview_max_x": 1024,
        "preview_max_y": 1024,
        "preview_max_memory": 1024,
        "preview_max_filesize_image": 100,
        "app_install_overwrite": [
            "camerarawpreviews",
            "news",
            "maps"
        ],
        "memories.db.triggers.fcu": true,
        "mail_smtpsecure": "ssl",
        "allow_user_to_change_display_name": false,
        "lost_password_link": "disabled",
        "user_oidc": {
            "auto_provision": true,
            "soft_auto_provision": true,
            "enrich_login_id_token_with_userinfo": true
        },
        "files.chunked_upload.max_size": 20971520
    }
}

Nginx

e# Version 2024-07-17

upstream php-handler {
    server 127.0.0.1:9000;
    #server unix:/run/php/php8.2-fpm.sock;
}

# Set the `immutable` cache control options only for assets with a cache busting `v` argument
map $arg_v $asset_immutable {
    "" "";
    default ", immutable";
}

server {
    listen 80;
    listen [::]:80;
    server_name nextcloud.lyrion.ch;

    # Path to the root of your installation
    root /usr/local/www/nextcloud;

    # Prevent nginx HTTP Server Detection
    server_tokens off;

    # HSTS settings
    # WARNING: Only add the preload option once you read about
    # the consequences in https://hstspreload.org/. This option
    # will add the domain to a hardcoded list that is shipped
    # in all major browsers and getting removed from this list
    # could take several months.
    #add_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" always;

    # set max upload size and increase upload timeout:
    client_max_body_size 16G;
    client_body_timeout 300s;
    #fastcgi_buffers 64 4K;
    fastcgi_request_buffering off;

    # Enable gzip but do not remove ETag headers
    gzip on;
    gzip_vary on;
    gzip_comp_level 4;
    gzip_min_length 256;
    gzip_proxied expired no-cache no-store private no_last_modified no_etag auth;
    gzip_types application/atom+xml text/javascript application/javascript application/json application/ld+json application/manifest+json application/rss+xml application/vnd.geo+json application/vnd.ms-fontobject application/wasm application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard text/vnd.rim.location.xloc text/vtt text/x-component text/x-cross-domain-policy;

    # Pagespeed is not supported by Nextcloud, so if your server is built
    # with the `ngx_pagespeed` module, uncomment this line to disable it.
    #pagespeed off;

    # The settings allows you to optimize the HTTP2 bandwidth.
    # See https://blog.cloudflare.com/delivering-http-2-upload-speed-improvements/
    # for tuning hints
    client_body_buffer_size 512k;

    # HTTP response headers borrowed from Nextcloud `.htaccess`
    add_header Referrer-Policy                   "no-referrer"       always;
    add_header X-Content-Type-Options            "nosniff"           always;
    add_header X-Frame-Options                   "SAMEORIGIN"        always;
    add_header X-Permitted-Cross-Domain-Policies "none"              always;
    add_header X-Robots-Tag                      "noindex, nofollow" always;
    add_header X-XSS-Protection                  "1; mode=block"     always;
    add_header X-Accel-Buffering		"no"		    always;

    # Remove X-Powered-By, which is an information leak
    fastcgi_hide_header X-Powered-By;

    # Set .mjs and .wasm MIME types
    # Either include it in the default mime.types list
    # and include that list explicitly or add the file extension
    # only for Nextcloud like below:
    include mime.types;
    types {
        text/javascript mjs;
	image/jxl jxl;
    }

    # Specify how to handle directories -- specifying `/index.php$request_uri`
    # here as the fallback means that Nginx always exhibits the desired behaviour
    # when a client requests a path that corresponds to a directory that exists
    # on the server. In particular, if that directory contains an index.php file,
    # that file is correctly served; if it doesn't, then the request is passed to
    # the front-end controller. This consistent behaviour means that we don't need
    # to specify custom rules for certain paths (e.g. images and other assets,
    # `/updater`, `/ocs-provider`), and thus
    # `try_files $uri $uri/ /index.php$request_uri`
    # always provides the desired behaviour.
    index index.php index.html /index.php$request_uri;

    # Rule borrowed from `.htaccess` to handle Microsoft DAV clients
    location = / {
        if ( $http_user_agent ~ ^DavClnt ) {
            return 302 /remote.php/webdav/$is_args$args;
        }
    }

    location = /robots.txt {
        allow all;
        log_not_found off;
        access_log off;
    }

    # Make a regex exception for `/.well-known` so that clients can still
    # access it despite the existence of the regex rule
    # `location ~ /(\.|autotest|...)` which would otherwise handle requests
    # for `/.well-known`.
    location ^~ /.well-known {
        # The rules in this block are an adaptation of the rules
        # in `.htaccess` that concern `/.well-known`.

        location = /.well-known/carddav { return 301 /remote.php/dav/; }
        location = /.well-known/caldav  { return 301 /remote.php/dav/; }

        location /.well-known/acme-challenge    { try_files $uri $uri/ =404; }
        location /.well-known/pki-validation    { try_files $uri $uri/ =404; }

        # Let Nextcloud's API for `/.well-known` URIs handle all other
        # requests by passing them to the front-end controller.
        return 301 /index.php$request_uri;
    }

    # Rules borrowed from `.htaccess` to hide certain paths from clients
    location ~ ^/(?:build|tests|config|lib|3rdparty|templates|data)(?:$|/)  { return 404; }
    location ~ ^/(?:\.|autotest|occ|issue|indie|db_|console)                { return 404; }

    # Ensure this block, which passes PHP files to the PHP process, is above the blocks
    # which handle static assets (as seen below). If this block is not declared first,
    # then Nginx will encounter an infinite rewriting loop when it prepends `/index.php`
    # to the URI, resulting in a HTTP 500 error response.
    location ~ \.php(?:$|/) {
        # Required for legacy support
        rewrite ^/(?!index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|ocs-provider\/.+|.+\/richdocumentscode(_arm64)?\/proxy) /index.php$request_uri;

        fastcgi_split_path_info ^(.+?\.php)(/.*)$;
        set $path_info $fastcgi_path_info;

        try_files $fastcgi_script_name =404;

        include fastcgi_params;
        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
        fastcgi_param PATH_INFO $path_info;
        fastcgi_param HTTPS on;

        fastcgi_param modHeadersAvailable true;         # Avoid sending the security headers twice
        fastcgi_param front_controller_active true;     # Enable pretty urls
        fastcgi_pass php-handler;

        fastcgi_intercept_errors on;
        fastcgi_request_buffering off;

        fastcgi_max_temp_file_size 0;
    }

    # Serve static files
    location ~ \.(?:css|js|mjs|svg|gif|ico|jpg|png|webp|avif|jxl|wasm|tflite|map|ogg|flac)$ {
        try_files $uri /index.php$request_uri;
        # HTTP response headers borrowed from Nextcloud `.htaccess`
        add_header Cache-Control                     "public, max-age=15778463$asset_immutable";
        add_header Referrer-Policy                   "no-referrer"       always;
        add_header X-Content-Type-Options            "nosniff"           always;
        add_header X-Frame-Options                   "SAMEORIGIN"        always;
        add_header X-Permitted-Cross-Domain-Policies "none"              always;
        add_header X-Robots-Tag                      "noindex, nofollow" always;
        add_header X-XSS-Protection                  "1; mode=block"     always;
        access_log off;     # Optional: Don't log access to assets
    }

    location ~ \.(otf|woff2?)$ {
        try_files $uri /index.php$request_uri;
        expires 7d;         # Cache-Control policy borrowed from `.htaccess`
        access_log off;     # Optional: Don't log access to assets
    }

    # Rule borrowed from `.htaccess`
    location /remote {
        return 301 /remote.php$request_uri;
    }

    location / {
        try_files $uri $uri/ /index.php$request_uri;
    }
}

Haproxy

global
  tune.h2.fe.rxbuf 134217728

crt-store crtstore
  crt-base /usr/local/share/certs/acme/HOSTNAME
  key-base /usr/local/share/certs/acme/private/HOSTNAME
  load crt "cert.pem" key "key.pem"

frontend main
  mode http
  bind :443 ssl crt "@crtstore/cert.pem"
  http-response set-header Strict-Transport-Security "max-age=16000000; includeSubDomains; preload;"
  http-response set-header Content-Security-Policy "frame-ancestors https://*.HOSTNAME"
  http-request set-header X-Forwarded-Proto "https"
  http-request set-header X-Real-IP %[src]
  http-request set-header X-Forwarded-For %[src]
  use_backend nextcloud if { req.hdr(host) -i nextcloud.HOSTNAME }

backend nextcloud
  mode http
  server nextcloud 10.1.1.7:80 check

Apps

Enabled:

  • activity: 4.0.0
  • admin_audit: 1.21.0
  • announcementcenter: 7.1.4
  • app_api: 5.0.2
  • bruteforcesettings: 4.0.0
  • calendar: 5.3.5
  • camerarawpreviews: 0.8.7
  • chores: 0.0.6
  • circles: 31.0.0
  • cloud_federation_api: 1.14.0
  • cloud_py_api: 0.2.0
  • comments: 1.21.0
  • contacts: 7.1.5
  • contactsinteraction: 1.12.0
  • cookbook: 0.11.3
  • cospend: 3.0.11
  • dashboard: 7.11.0
  • dav: 1.33.0
  • deck: 1.15.1
  • event_update_notification: 2.6.1
  • external: 6.0.2
  • externalpassword: 1.1.2
  • federatedfilesharing: 1.21.0
  • federation: 1.21.0
  • files: 2.3.1
  • files_downloadlimit: 4.0.0
  • files_external: 1.23.0
  • files_pdfviewer: 4.0.0
  • files_reminders: 1.4.0
  • files_sharing: 1.23.1
  • files_trashbin: 1.21.0
  • fileslibreofficeedit: 2.0.1
  • firstrunwizard: 4.0.0
  • forms: 5.1.2
  • gpoddersync: 3.12.0
  • groupfolders: 19.1.2
  • integration_dropbox: 4.0.3
  • integration_paperless: 1.0.6
  • intros: 1.1.2
  • logreader: 4.0.0
  • lookup_server_connector: 1.19.0
  • mail: 5.1.7
  • mediadc: 0.4.0
  • memories: 7.6.0
  • music: 2.2.0
  • news: 26.0.2
  • nextcloud_announcements: 3.0.0
  • nextframe: 0.0.7
  • nextpod: 0.7.7
  • notes: 4.12.2
  • notifications: 4.0.0
  • oauth2: 1.19.1
  • ownershiptransfer: 1.2.1
  • ownpad: 0.13.0
  • password_policy: 3.0.0
  • photos: 4.0.0-dev.1
  • polls: 8.1.2
  • previewgenerator: 5.9.0
  • privacy: 3.0.0
  • profile: 1.0.0
  • provisioning_api: 1.21.0
  • recommendations: 4.0.0
  • related_resources: 2.0.0
  • serverinfo: 3.0.0
  • settings: 1.14.0
  • sharebymail: 1.21.0
  • side_menu: 5.1.1
  • spreed: 21.1.1
  • support: 3.0.0
  • survey_client: 3.0.0
  • suspicious_login: 9.0.1
  • systemtags: 1.21.1
  • tables: 0.9.4
  • tasks: 0.16.1
  • text: 5.0.0
  • theming: 2.6.1
  • twofactor_backupcodes: 1.20.0
  • twofactor_totp: 13.0.0-dev.0
  • unsplash: 3.1.0
  • updatenotification: 1.21.0
  • user_oidc: 7.2.0
  • user_status: 1.11.0
  • viewer: 4.0.0
  • weather_status: 1.11.0
  • webhook_listeners: 1.2.0
  • workflowengine: 2.13.0
    Disabled:
  • encryption: 2.19.0 (installed 2.19.0)
  • files_versions: 1.24.0 (installed 1.24.0)
  • maps: 0.1.10 (installed 1.5.0)
  • twofactor_nextcloud_notification: 5.0.0 (installed 5.0.0)
  • user_ldap: 1.22.0 (installed 1.22.0)