[mod] uWSGI config: configuring uwsgi for production

As stated in .. and other posts, the defaults of uWSGI not suitable for a
productive environment.  To give just one example, the workers run indefinitely
and the memory leaks aggregate.

- "Configuring uWSGI for Production: The defaults are all wrong" EuroPython 2019 [1]
- "Configuring uWSGI for Production Deployment" [2]
- "When Paul has tested some PR on his instance, we could clearly see a memory
  leak over a week: the memory never dropped to the initial value. Same for my
  instance using Docker." [3]

[1] https://av.tib.eu/media/44810
[2] https://www.bloomberg.com/company/stories/configuring-uwsgi-production-deployment/
[3] https://github.com/searxng/searxng/pull/3443#issuecomment-2094347004

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

dockerfiles/uwsgi.ini

@@ -1,3 +1,4 @@
+# -*- mode: conf-unix; coding: utf-8  -*-
 [uwsgi]
 # Who will run the code
 uid = searxng
@@ -6,6 +7,13 @@ gid = searxng
 # Number of workers (usually CPU count)
 # default value: %k (= number of CPU core, see Dockerfile)
 workers = $(UWSGI_WORKERS)
+harakiri = 60
+max-requests = 1000                  # Restart workers after this many requests
+max-worker-lifetime = 3600           # Restart workers after this many seconds
+reload-on-rss = 2048                 # Restart workers after this much resident memory
+worker-reload-mercy = 60             # How long to wait before forcefully killing workers
+die-on-term = true                   # Shutdown when receiving SIGTERM (default is respawn)
+py-callos-afterfork = true           # allow workers to trap signals
 
 # Number of threads per worker
 # default value: 4 (see Dockerfile)
@@ -17,6 +25,9 @@ chmod-socket = 666
 # Plugin to use and interpreter config
 single-interpreter = true
 master = true
+strict = true
+vacuum = true                        # Delete sockets during shutdown
+need-app = true
 plugin = python3
 lazy-apps = true
 enable-threads = true
@@ -33,7 +44,8 @@ auto-procname = true
 
 # Disable request logging for privacy
 disable-logging = true
-log-5xx = true
+log-4xx = true                       # but log 4xx's anyway
+log-5xx = true                       # and 5xx's
 
 # Set the max size of a request (request-body excluded)
 buffer-size = 8192

utils/templates/etc/uwsgi/apps-archlinux/searxng.ini

@@ -1,4 +1,4 @@
-# -*- mode: conf; coding: utf-8  -*-
+# -*- mode: conf-unix; coding: utf-8  -*-
 [uwsgi]
 
 # uWSGI core
@@ -24,6 +24,8 @@ env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH}
 # disable logging for privacy
 logger = systemd
 disable-logging = true
+log-4xx = true                       # but log 4xx's anyway
+log-5xx = true                       # and 5xx's
 
 # The right granted on the created socket
 chmod-socket = 666
@@ -33,6 +35,9 @@ single-interpreter = true
 
 # enable master process
 master = true
+strict = true
+vacuum = true                        # Delete sockets during shutdown
+need-app = true
 
 # load apps in each worker instead of the master
 lazy-apps = true
@@ -50,6 +55,13 @@ enable-threads = true
 # Number of workers (usually CPU count)
 workers = ${UWSGI_WORKERS:-%k}
 threads = ${UWSGI_THREADS:-4}
+harakiri = 60
+max-requests = 1000                  # Restart workers after this many requests
+max-worker-lifetime = 3600           # Restart workers after this many seconds
+reload-on-rss = 2048                 # Restart workers after this much resident memory
+worker-reload-mercy = 60             # How long to wait before forcefully killing workers
+die-on-term = true                   # Shutdown when receiving SIGTERM (default is respawn)
+py-callos-afterfork = true           # allow workers to trap signals
 
 # plugin: python
 # --------------

utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket

@@ -1,4 +1,4 @@
-# -*- mode: conf; coding: utf-8  -*-
+# -*- mode: conf-unix; coding: utf-8  -*-
 [uwsgi]
 
 # uWSGI core
@@ -24,6 +24,8 @@ env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH}
 # disable logging for privacy
 logger = systemd
 disable-logging = true
+log-4xx = true                       # but log 4xx's anyway
+log-5xx = true                       # and 5xx's
 
 # The right granted on the created socket
 chmod-socket = 666
@@ -33,6 +35,9 @@ single-interpreter = true
 
 # enable master process
 master = true
+strict = true
+vacuum = true                        # Delete sockets during shutdown
+need-app = true
 
 # load apps in each worker instead of the master
 lazy-apps = true
@@ -50,6 +55,13 @@ enable-threads = true
 # Number of workers (usually CPU count)
 workers = ${UWSGI_WORKERS:-%k}
 threads = ${UWSGI_THREADS:-4}
+harakiri = 60
+max-requests = 1000                  # Restart workers after this many requests
+max-worker-lifetime = 3600           # Restart workers after this many seconds
+reload-on-rss = 2048                 # Restart workers after this much resident memory
+worker-reload-mercy = 60             # How long to wait before forcefully killing workers
+die-on-term = true                   # Shutdown when receiving SIGTERM (default is respawn)
+py-callos-afterfork = true           # allow workers to trap signals
 
 # plugin: python
 # --------------

utils/templates/etc/uwsgi/apps-available/searxng.ini

@@ -1,4 +1,4 @@
-# -*- mode: conf; coding: utf-8  -*-
+# -*- mode: conf-unix; coding: utf-8  -*-
 [uwsgi]
 
 # uWSGI core
@@ -27,6 +27,8 @@ env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH}
 
 # disable logging for privacy
 disable-logging = true
+log-4xx = true                       # but log 4xx's anyway
+log-5xx = true                       # and 5xx's
 
 # The right granted on the created socket
 chmod-socket = 666
@@ -36,6 +38,9 @@ single-interpreter = true
 
 # enable master process
 master = true
+strict = true
+vacuum = true                        # Delete sockets during shutdown
+need-app = true
 
 # load apps in each worker instead of the master
 lazy-apps = true
@@ -53,6 +58,13 @@ enable-threads = true
 # Number of workers (usually CPU count)
 workers = ${UWSGI_WORKERS:-%k}
 threads = ${UWSGI_THREADS:-4}
+harakiri = 60
+max-requests = 1000                  # Restart workers after this many requests
+max-worker-lifetime = 3600           # Restart workers after this many seconds
+reload-on-rss = 2048                 # Restart workers after this much resident memory
+worker-reload-mercy = 60             # How long to wait before forcefully killing workers
+die-on-term = true                   # Shutdown when receiving SIGTERM (default is respawn)
+py-callos-afterfork = true           # allow workers to trap signals
 
 # plugin: python
 # --------------

utils/templates/etc/uwsgi/apps-available/searxng.ini:socket

@@ -1,4 +1,4 @@
-# -*- mode: conf; coding: utf-8  -*-
+# -*- mode: conf-unix; coding: utf-8  -*-
 [uwsgi]
 
 # uWSGI core
@@ -27,6 +27,8 @@ env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH}
 
 # disable logging for privacy
 disable-logging = true
+log-4xx = true                       # but log 4xx's anyway
+log-5xx = true                       # and 5xx's
 
 # The right granted on the created socket
 chmod-socket = 666
@@ -36,6 +38,9 @@ single-interpreter = true
 
 # enable master process
 master = true
+strict = true
+vacuum = true                        # Delete sockets during shutdown
+need-app = true
 
 # load apps in each worker instead of the master
 lazy-apps = true
@@ -53,6 +58,13 @@ enable-threads = true
 # Number of workers (usually CPU count)
 workers = ${UWSGI_WORKERS:-%k}
 threads = ${UWSGI_THREADS:-4}
+harakiri = 60
+max-requests = 1000                  # Restart workers after this many requests
+max-worker-lifetime = 3600           # Restart workers after this many seconds
+reload-on-rss = 2048                 # Restart workers after this much resident memory
+worker-reload-mercy = 60             # How long to wait before forcefully killing workers
+die-on-term = true                   # Shutdown when receiving SIGTERM (default is respawn)
+py-callos-afterfork = true           # allow workers to trap signals
 
 # plugin: python
 # --------------