Changes from the installation tests on (all) LXC containers.

Tested and fixed HTTP & uWSGI installation on:

  ubu1604 ubu1804 ubu1910 ubu2004 fedora31 archlinux

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2020-04-08 18:38:36 +02:00
parent ee39a098ac
commit f693149cde
13 changed files with 263 additions and 25 deletions

View file

@ -19,20 +19,28 @@ developers.
.. _toolboxing common: .. _toolboxing common:
Common commands Common commands & environment
=============== =============================
Scripts to maintain services often dispose of common commands and environments. Scripts to maintain services often dispose of common commands and environments.
``shell``: ``shell`` : command
Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for
troubleshooting. troubleshooting.
``inspect service``: ``inspect service`` : command
Shows status and log of the service, most often you have a option to enable Shows status and log of the service, most often you have a option to enable
more verbose debug logs. Very helpful for debugging, but be careful not to more verbose debug logs. Very helpful for debugging, but be careful not to
enable debugging in a production environment! enable debugging in a production environment!
``FORCE_TIMEOUT`` : environment
Sets timeout for interactive prompts. If you want to run a script in batch
job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a
reverse proxy for filtron on all containers of the :ref:`searx suite
<lxc-searx.env>` use ::
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install
.. _toolboxing setup: .. _toolboxing setup:
Tooling box setup Tooling box setup

View file

@ -88,7 +88,11 @@ WEB-Browser::
[searx-fedora31] INFO: (eth0) filtron: http://n.n.n.18:4004/ [searx-fedora31] INFO: (eth0) filtron: http://n.n.n.18:4004/
[searx-archlinux] INFO: (eth0) filtron: http://n.n.n.12:4004/ [searx-archlinux] INFO: (eth0) filtron: http://n.n.n.12:4004/
To install a reverse proxy for filtron and morty use::
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh apache install
Running commands Running commands
================ ================

View file

@ -23,7 +23,6 @@ FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \
[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/ [[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/
FILTRON_ETC="/etc/filtron" FILTRON_ETC="/etc/filtron"
FILTRON_RULES="$FILTRON_ETC/rules.json" FILTRON_RULES="$FILTRON_ETC/rules.json"
FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"
@ -447,7 +446,8 @@ This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SIT
install_apache install_apache
fi fi
echo "${REPO_ROOT}/utils/searx.sh" install uwsgi
apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}" apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"
info_msg "testing public url .." info_msg "testing public url .."
@ -465,11 +465,12 @@ This removes apache site ${APACHE_FILTRON_SITE}."
! apache_is_installed && err_msg "Apache is not installed." ! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?"; then if ! ask_yn "Do you really want to continue?" Yn; then
return return
fi fi
apache_remove_site "$APACHE_FILTRON_SITE" apache_remove_site "$APACHE_FILTRON_SITE"
} }
rst-doc() { rst-doc() {

View file

@ -389,7 +389,7 @@ install_template() {
local chmod="${pos_args[4]-644}" local chmod="${pos_args[4]-644}"
info_msg "install (eval=$do_eval): ${dst}" info_msg "install (eval=$do_eval): ${dst}"
[[ -n $variant ]] && info_msg "variant: ${variant}" [[ -n $variant ]] && info_msg "variant --> ${variant}"
if [[ ! -f "${template_origin}" ]] ; then if [[ ! -f "${template_origin}" ]] ; then
err_msg "${template_origin} does not exists" err_msg "${template_origin} does not exists"
@ -777,6 +777,7 @@ apache_dissable_site() {
ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}" ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}"
;; ;;
esac esac
apache_reload
} }
# uWSGI # uWSGI
@ -846,7 +847,7 @@ uWSGI_restart() {
if uWSGI_app_available "${CONF}"; then if uWSGI_app_available "${CONF}"; then
systemctl restart "uwsgi@${CONF%.*}" systemctl restart "uwsgi@${CONF%.*}"
else else
info_msg "in systemd template mode: ${CONF} not installed (nothing to restart)" info_msg "[uWSGI:systemd-template] ${CONF} not installed (no need to restart)"
fi fi
;; ;;
fedora-*) fedora-*)
@ -854,7 +855,7 @@ uWSGI_restart() {
if uWSGI_app_enabled "${CONF}"; then if uWSGI_app_enabled "${CONF}"; then
touch "${uWSGI_APPS_ENABLED}/${CONF}" touch "${uWSGI_APPS_ENABLED}/${CONF}"
else else
info_msg "in uWSGI emperor mode: ${CONF} not installed (nothing to restart)" info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)"
fi fi
;; ;;
*) *)
@ -864,6 +865,32 @@ uWSGI_restart() {
esac esac
} }
uWSGI_prepare_app() {
# usage: uWSGI_prepare_app <myapp.ini>
local APP="${1%.*}"
if [[ -z $APP ]]; then
err_msg "uWSGI_prepare_app: missing arguments"
return 42
fi
case $DIST_ID-$DIST_VERS in
fedora-*)
# in emperor mode, the uwsgi user is the owner of the sockets
info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}"
mkdir -p "/run/uwsgi/app/${APP}"
chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}"
;;
*)
info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}"
mkdir -p "/run/uwsgi/app/${APP}"
chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}"
;;
esac
}
uWSGI_app_available() { uWSGI_app_available() {
# usage: uWSGI_app_available <myapp.ini> # usage: uWSGI_app_available <myapp.ini>
local CONF="$1" local CONF="$1"
@ -888,6 +915,7 @@ uWSGI_install_app() {
*) pos_args+=("$i");; *) pos_args+=("$i");;
esac esac
done done
uWSGI_prepare_app "${pos_args[1]}"
mkdir -p "${uWSGI_APPS_AVAILABLE}" mkdir -p "${uWSGI_APPS_AVAILABLE}"
install_template "${template_opts[@]}" \ install_template "${template_opts[@]}" \
"${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \ "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \
@ -1281,3 +1309,30 @@ global_IPs(){
ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p' ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p'
} }
primary_ip() {
case $DIST_ID in
arch)
echo "$(ip -o addr show \
| sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' \
| head -n 1)"
;;
*) echo "$(hostname -I | cut -d' ' -f1)" ;;
esac
}
# URL
# ---
url_replace_hostname(){
# usage: url_replace_hostname <url> <new hostname>
# to replace hostname by primary IP::
#
# url_replace_hostname http://searx-ubu1604/morty $(primary_ip)
# http://10.246.86.250/morty
echo "$1" | sed "s|\(http[s]*://\)[^/]*\(.*\)|\1$2\2|"
}

View file

@ -438,7 +438,6 @@ lxc_cmd() {
else else
info_msg "lxc $* $i" info_msg "lxc $* $i"
lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "
echo
fi fi
done done
} }

View file

@ -14,12 +14,16 @@ in_container && lxc_set_suite_env
# config # config
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}"
PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty}"
PUBLIC_URL_MORTY="$(echo "$PUBLIC_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}"
MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"
PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty}"
SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
PUBLIC_URL_MORTY="$(echo "$SEARX_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}"
if in_container; then
# container hostnames do not have a DNS entry, use primary IP
PUBLIC_URL_MORTY="$(url_replace_hostname "$PUBLIC_URL_MORTY" "$(primary_ip)")"
fi
# shellcheck disable=SC2034 # shellcheck disable=SC2034
MORTY_TIMEOUT=5 MORTY_TIMEOUT=5
@ -425,7 +429,7 @@ This removes apache site ${APACHE_MORTY_SITE}."
! apache_is_installed && err_msg "Apache is not installed." ! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?"; then if ! ask_yn "Do you really want to continue?" Yn; then
return return
fi fi

View file

@ -748,6 +748,10 @@ excessively bot queries."
apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}" apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}"
rst_title "Install searx's uWSGI app (searx.ini)" section
echo
uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP"
if ! service_is_available "${PUBLIC_URL}"; then if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!" err_msg "Public service at ${PUBLIC_URL} is not available!"
fi fi
@ -762,11 +766,15 @@ This removes apache site ${APACHE_SEARX_SITE}."
! apache_is_installed && err_msg "Apache is not installed." ! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?"; then if ! ask_yn "Do you really want to continue?" Yn; then
return return
fi fi
apache_remove_site "${APACHE_SEARX_SITE}" apache_remove_site "${APACHE_SEARX_SITE}"
rst_title "Remove searx's uWSGI app (searx.ini)" section
echo
uWSGI_remove_app "$SEARX_UWSGI_APP"
} }
rst-doc() { rst-doc() {

View file

@ -2,7 +2,7 @@
LoadModule headers_module ${APACHE_MODULES}/mod_headers.so LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy_http.so LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so #LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
# SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog # SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog

View file

@ -2,7 +2,7 @@
LoadModule headers_module ${APACHE_MODULES}/mod_headers.so LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy_http.so LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so #LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog # SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog

View file

@ -74,7 +74,7 @@ http = ${SEARX_INTERNAL_HTTP}
# #
# On some distributions you need to create the app folder for the sockets:: # On some distributions you need to create the app folder for the sockets::
# #
# mkdir -p /run/uwsgi/app/searx/socket # mkdir -p /run/uwsgi/app/searx
# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx/socket # chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
# #
# socket = /run/uwsgi/app/searx/socket # socket = /run/uwsgi/app/searx/socket

View file

@ -0,0 +1,80 @@
[uwsgi]
# uWSGI core
# ----------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
# Who will run the code
uid = ${SERVICE_USER}
gid = ${SERVICE_GROUP}
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}/searx
# searx configuration (settings.yml)
env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
# disable logging for privacy
logger = systemd
disable-logging = true
# The right granted on the created socket
chmod-socket = 666
# Plugin to use and interpretor config
single-interpreter = true
# enable master process
master = true
# load apps in each worker instead of the master
lazy-apps = true
# load uWSGI plugins
plugin = python
# By default the Python plugin does not initialize the GIL. This means your
# app-generated threads will not run. If you need threads, remember to enable
# them with enable-threads. Running uWSGI in multithreading mode (with the
# threads options) will automatically enable threading support. This *strange*
# default behaviour is for performance reasons.
enable-threads = true
# plugin: python
# --------------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
# load a WSGI module
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SEARX_SRC}
# speak to upstream
# -----------------
#
# Activate the 'http' configuration for filtron or activate the 'socket'
# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
# using IP:
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
# http = ${SEARX_INTERNAL_HTTP}
# using unix-sockets:
#
# On some distributions you need to create the app folder for the sockets::
#
# mkdir -p /run/uwsgi/app/searx
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
#
socket = /run/uwsgi/app/searx/socket

View file

@ -73,7 +73,7 @@ http = ${SEARX_INTERNAL_HTTP}
# #
# On some distributions you need to create the app folder for the sockets:: # On some distributions you need to create the app folder for the sockets::
# #
# mkdir -p /run/uwsgi/app/searx/socket # mkdir -p /run/uwsgi/app/searx
# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx/socket # chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
# #
# socket = /run/uwsgi/app/searx/socket # socket = /run/uwsgi/app/searx/socket

View file

@ -0,0 +1,79 @@
[uwsgi]
# uWSGI core
# ----------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
# Who will run the code
uid = ${SERVICE_USER}
gid = ${SERVICE_GROUP}
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}/searx
# searx configuration (settings.yml)
env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
# disable logging for privacy
disable-logging = true
# The right granted on the created socket
chmod-socket = 666
# Plugin to use and interpretor config
single-interpreter = true
# enable master process
master = true
# load apps in each worker instead of the master
lazy-apps = true
# load uWSGI plugins
plugin = python3,http
# By default the Python plugin does not initialize the GIL. This means your
# app-generated threads will not run. If you need threads, remember to enable
# them with enable-threads. Running uWSGI in multithreading mode (with the
# threads options) will automatically enable threading support. This *strange*
# default behaviour is for performance reasons.
enable-threads = true
# plugin: python
# --------------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
# load a WSGI module
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SEARX_SRC}
# speak to upstream
# -----------------
#
# Activate the 'http' configuration for filtron or activate the 'socket'
# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
# using IP:
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
# http = ${SEARX_INTERNAL_HTTP}
# using unix-sockets:
#
# On some distributions you need to create the app folder for the sockets::
#
# mkdir -p /run/uwsgi/app/searx
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
#
socket = /run/uwsgi/app/searx/socket