Mariadb locks up

For the past two or so months I have been struggling with database problems. Out of the blue the database locks up and a systemctl restart mariadb will not do. It’ll do nothing. Also a reboot will not help as the mariadb service cannot properly shut down. All I can do is a power cycle to reboot my database server.

Looking into the current state of affairs with MariaDB [(none)]> show processlist; gives me a list of very long running processes that somehow get stuck. They vary. Sometimes it’s something related to preferences, sometimes to file caches but the list gets quite long as I think all of them were started at some point but couldn’t finish.

This behaviour started I think with NC 28 but could have been the last version before that. I always updgrade as soon as updates are available. Will give more details further on.

I started with Mariadb 10 on the same server as nextcloud but then moved my database to a different server when I thought my initial db was corrupt. I tehn upgraded to mariadb 11 assuming it might help.

Before I move on to migrate all my data to Postgresql I thought I might ask here if there is any help available to fix this.

You will see that I’m running the enterprise version which is correct. We have a license but it’s without support since we’re a school and don’t have paid support.

There are about 100 active users of the nextcloud.

My Mariadb:

Server version: 11.2.3-MariaDB-1:11.2.3+maria~ubu2204-log mariadb.org binary distribution

My NextCloud:

sudo -u www-data php occ config:list system
{
    "system": {
        "instanceid": "***REMOVED SENSITIVE VALUE***",
        "passwordsalt": "***REMOVED SENSITIVE VALUE***",
        "secret": "***REMOVED SENSITIVE VALUE***",
        "trusted_domains": [ some servers ],
        "datadirectory": "***REMOVED SENSITIVE VALUE***",
        "dbtype": "mysql",
        "version": "28.0.3.3",
        "dbname": "***REMOVED SENSITIVE VALUE***",
        "dbhost": "***REMOVED SENSITIVE VALUE***",
        "dbport": "",
        "dbtableprefix": "oc_",
        "dbuser": "***REMOVED SENSITIVE VALUE***",
        "dbpassword": "***REMOVED SENSITIVE VALUE***",
        "logtimezone": "UTC",
        "installed": true,
        "ldapIgnoreNamingRules": false,
        "ldapProviderFactory": "\\OCA\\User_LDAP\\LDAPProviderFactory",
        "memcache.local": "\\OC\\Memcache\\APCu",
        "memcache.distributed": "\\OC\\Memcache\\Redis",
        "memcache.locking": "\\OC\\Memcache\\Redis",
        "redis": {
            "host": "***REMOVED SENSITIVE VALUE***",
            "port": 6379
        },
        "maintenance": false,
        "theme": "",
        "loglevel": 0,
        "log_rotate_size": 104857600,
        "mail_from_address": "***REMOVED SENSITIVE VALUE***",
        "mail_smtpmode": "smtp",
        "mail_sendmailmode": "smtp",
        "mail_domain": "***REMOVED SENSITIVE VALUE***",
        "mail_smtpsecure": "ssl",
        "mail_smtphost": "***REMOVED SENSITIVE VALUE***",
        "mail_smtpport": "465",
        "mail_smtpauthtype": "LOGIN",
        "mail_smtpauth": 1,
        "mail_smtpname": "***REMOVED SENSITIVE VALUE***",
        "mail_smtppassword": "***REMOVED SENSITIVE VALUE***",
        "mysql.utf8mb4": true,
        "app_install_overwrite": [
            "external",
            "cms_pico",
            "files_readmemd",
            "githubmergetracker",
            "calendar",
            "bruteforcesettings",
            "bbb"
        ],
        "overwrite.cli.url": "https:\/\/nextcloud.example.de",
        "htaccess.RewriteBase": "\/",
        "apps_paths": [
            {
                "path": "\/var\/www\/nextcloud\/apps",
                "url": "\/apps",
                "writable": false
            },
            {
                "path": "\/var\/www\/nextcloud\/apps2",
                "url": "\/apps2",
                "writable": true
            }
        ],
        "updater.release.channel": "enterprise",
        "default_phone_region": "DE",
        "updater.server.url": "https:\/\/updates.nextcloud.com\/customers\/J3CWG-94D1Q-QCLLJ-A6V9Y-PJSIR\/",
        "maintenance_window_start": "1"
    }
}

and my.cnf:

[client]
default-character-set = utf8mb4
port = 3306
socket = /var/run/mysqld/mysqld.sock
[mysqld_safe]
log_error=/var/log/mysql/mysql_error.log
nice = 0
socket = /var/run/mysqld/mysqld.sock
[mysqld]
# performance_schema=ON
basedir = /usr
# bind-address = 127.0.0.1
bind-address = 0.0.0.0
binlog_format = ROW
character-set-server = utf8mb4
collation-server = utf8mb4_general_ci
datadir = /var/lib/mysql
default_storage_engine = InnoDB
expire_logs_days = 2
general_log_file = /var/log/mysql/mysql.log
innodb_buffer_pool_size = 2G
innodb_log_buffer_size = 32M
innodb_log_file_size = 512M
innodb_read_only_compressed=OFF
join_buffer_size = 2M
key_buffer_size = 512M
lc_messages_dir = /usr/share/mysql
lc_messages = en_US
log_bin = /var/log/mysql/mariadb-bin
log_bin_index = /var/log/mysql/mariadb-bin.index
log_error = /var/log/mysql/mysql_error.log
log_slow_verbosity = query_plan
log_warnings = 2
long_query_time = 1
max_connections = 100
max_heap_table_size = 64M
myisam_sort_buffer_size = 512M
port = 3306
pid-file = /var/run/mysqld/mysqld.pid
query_cache_limit = 0
query_cache_size = 0
read_buffer_size = 2M
read_rnd_buffer_size = 2M
skip-name-resolve
socket = /var/run/mysqld/mysqld.sock
sort_buffer_size = 2M
table_open_cache = 400
table_definition_cache = 800
tmp_table_size = 32M
tmpdir = /tmp
transaction_isolation = READ-COMMITTED
user = mysql
wait_timeout = 600
[mysqldump]
max_allowed_packet = 16M
quick
quote-names
[isamchk]
key_buffer = 16M

If I forgot something plase let me know so I can update this post. Thank you very much for looking into this.

Hi @sme2

I’m not sure if this is the same problem you’re experiencing, or what the exact reasons are for this behaviour in the first place, but I noticed something similar after installing NC28 on a test instance and then uploading a large folder with about 1800 files. Nextcloud/PHP-FPM literally bruteforced the backend until after a certain time, I got 500 or 503 errors in the WebUI and Nextcloud became completely unresponsive and the RAM was fully utilised with MariaDB processes.

I then changed pm = dynamic to pm = ondemand in /etc/php/$PHPVERSION/fpm/pool.d/www.conf and I was able to upload the same folder without any issues and the RAM usage never exceeded 30%.

I also made this change on my production instance after upgrading it to 28.0.3 and did not notice any significant performance impact, if any at all, so I think I will leave it that way, at least for now. Note: I only run a home server with two users, so your mileage may vary :wink:

I’m using mod_php instead of PHP-FPM. Just followed the regular setup on this. So I don’t think this applies here.

Nevertheless I don’t think this is what’s causing any issues here. The MariaDB server is fully responsive, no memory or cpu problems at all. top doesn’t show anything out of the ordinary. It just crashes the database and completely locks it up. I do have another postgres server running along which keeps serving my other sites just fine all the while.

Thank you for your reply :heart_decoration:

1 Like

Could still be the same underlying issue (or maybe a change/bug in Nextcloud?) that shows somewhat different symptoms with mod_php. Unfortunately, I’m not expert enough to provide definitive answers, but perhaps others here have some more ideas…

BTW, this is how a typical lockup process list looks like:


MariaDB [(none)]> show processlist;
+------+-----------+------------------+-----------+---------+------+-----------------+------------------------------------------------------------------------------------------------------+----------+
| Id   | User      | Host             | db        | Command | Time | State           | Info                                                                                                 | Progress |
+------+-----------+------------------+-----------+---------+------+-----------------+------------------------------------------------------------------------------------------------------+----------+
| 3618 | nextcloud | 100.100.100.100:34870 | nextcloud | Query   | 4408 | Update          | INSERT INTO `oc_preferences` (`userid`, `appid`, `configkey`, `configvalue`) VALUES('user2', 'calendar |    0.000 |
| 3672 | nextcloud | 100.100.100.100:47208 | nextcloud | Query   | 4165 | Statistics      | SELECT `userid` FROM `oc_preferences` WHERE (`appid` = 'settings') AND (`configkey` = 'email') AND ( |    0.000 |
| 3683 | nextcloud | 100.100.100.100:35800 | nextcloud | Query   | 4124 | Statistics      | SELECT `userid`, `configvalue` FROM `oc_preferences` WHERE (`appid` = 'core') AND (`configkey` = 'ti |    0.000 |
| 3706 | nextcloud | 100.100.100.100:45858 | nextcloud | Query   | 4018 | init for update | DELETE FROM `oc_preferences` WHERE (`userid` = 'user1') AND (`appid` = 'login_token') AND (`configkey` |    0.000 |
| 3747 | nextcloud | 100.100.100.100:43100 | nextcloud | Query   | 3867 | init for update | DELETE FROM `oc_preferences` WHERE (`userid` = 'user1') AND (`appid` = 'login_token') AND (`configkey` |    0.000 |
| 3808 | nextcloud | 100.100.100.100:44312 | nextcloud | Query   | 3529 | Update          | INSERT INTO `oc_preferences` (`userid`, `appid`, `configkey`, `configvalue`) VALUES('user3', 'user_l |    0.000 |
| 3833 | nextcloud | 100.100.100.100:37306 | nextcloud | Query   | 3455 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 3848 | nextcloud | 100.100.100.100:53370 | nextcloud | Query   | 3450 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 3852 | nextcloud | 100.100.100.100:53412 | nextcloud | Query   | 3447 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 3859 | nextcloud | 100.100.100.100:57432 | nextcloud | Query   | 3406 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 3874 | nextcloud | 100.100.100.100:49292 | nextcloud | Query   | 3327 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 3894 | nextcloud | 100.100.100.100:40150 | nextcloud | Query   | 3207 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 3899 | nextcloud | 100.100.100.100:53522 | nextcloud | Query   | 3170 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 3909 | nextcloud | 100.100.100.100:36166 | nextcloud | Query   | 3104 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 3923 | nextcloud | 100.100.100.100:47196 | nextcloud | Query   | 3038 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 3961 | nextcloud | 100.100.100.100:44566 | nextcloud | Query   | 2882 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 3977 | nextcloud | 100.100.100.100:57420 | nextcloud | Query   | 2791 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 3987 | nextcloud | 100.100.100.100:43296 | nextcloud | Query   | 2734 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4002 | nextcloud | 100.100.100.100:49558 | nextcloud | Query   | 2649 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4013 | nextcloud | 100.100.100.100:37938 | nextcloud | Query   | 2578 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4029 | nextcloud | 100.100.100.100:33692 | nextcloud | Query   | 2473 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4063 | nextcloud | 100.100.100.100:54914 | nextcloud | Query   | 2282 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4077 | nextcloud | 100.100.100.100:58138 | nextcloud | Query   | 2204 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4085 | nextcloud | 100.100.100.100:32910 | nextcloud | Query   | 2152 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4097 | nextcloud | 100.100.100.100:51348 | nextcloud | Query   | 2089 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4101 | nextcloud | 100.100.100.100:32930 | nextcloud | Query   | 2063 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4128 | nextcloud | 100.100.100.100:42542 | nextcloud | Query   | 1970 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4218 | nextcloud | 100.100.100.100:57442 | nextcloud | Query   | 1737 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4223 | nextcloud | 100.100.100.100:38002 | nextcloud | Query   | 1731 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4228 | nextcloud | 100.100.100.100:60380 | nextcloud | Query   | 1696 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4230 | nextcloud | 100.100.100.100:40130 | nextcloud | Query   | 1680 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4252 | nextcloud | 100.100.100.100:42854 | nextcloud | Query   | 1494 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |
| 4269 | nextcloud | 100.100.100.100:59144 | nextcloud | Query   | 1374 | Statistics      | SELECT COUNT(*) FROM `oc_preferences` WHERE (`appid` = 'login') AND (`configkey` = 'lastLogin')      |    0.000 |