searxng/manage.sh

276 lines
8.4 KiB
Bash
Raw Normal View History

#!/bin/sh
BASE_DIR="$(dirname -- "`readlink -f -- "$0"`")"
2018-08-01 09:53:08 +00:00
cd -- "$BASE_DIR"
set -e
# subshell
PYTHONPATH="$BASE_DIR"
SEARX_DIR="$BASE_DIR/searx"
ACTION="$1"
#
# Python
#
update_packages() {
pip install --upgrade pip
pip install --upgrade setuptools
pip install -Ur "$BASE_DIR/requirements.txt"
}
update_dev_packages() {
2016-01-07 18:56:59 +00:00
update_packages
pip install -Ur "$BASE_DIR/requirements-dev.txt"
}
install_geckodriver() {
echo '[!] Checking geckodriver'
# TODO : check the current geckodriver version
set -e
geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
set +e
if [ -z "$NOTFOUND" ]; then
2017-10-06 18:31:19 +00:00
return
fi
GECKODRIVER_VERSION="v0.24.0"
PLATFORM="`python -c "import six; import platform; six.print_(platform.system().lower(), platform.architecture()[0])"`"
case "$PLATFORM" in
2017-10-06 18:31:19 +00:00
"linux 32bit" | "linux2 32bit") ARCH="linux32";;
"linux 64bit" | "linux2 64bit") ARCH="linux64";;
"windows 32 bit") ARCH="win32";;
"windows 64 bit") ARCH="win64";;
"mac 64bit") ARCH="macos";;
esac
GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
if [ -z "$1" ]; then
2017-10-06 18:31:19 +00:00
if [ -z "$VIRTUAL_ENV" ]; then
2018-02-21 19:46:24 +00:00
printf "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n %s" "$GECKODRIVER_URL"
2017-10-06 18:31:19 +00:00
exit
else
GECKODRIVER_DIR="$VIRTUAL_ENV/bin"
fi
else
2017-10-06 18:31:19 +00:00
GECKODRIVER_DIR="$1"
mkdir -p -- "$GECKODRIVER_DIR"
fi
2018-02-21 19:46:24 +00:00
printf "Installing %s/geckodriver from\n %s" "$GECKODRIVER_DIR" "$GECKODRIVER_URL"
2017-10-06 18:31:19 +00:00
FILE="`mktemp`"
wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "$GECKODRIVER_DIR" -f "$FILE" geckodriver
rm -- "$FILE"
chmod 777 -- "$GECKODRIVER_DIR/geckodriver"
}
locales() {
pybabel compile -d "$SEARX_DIR/translations"
}
update_useragents() {
echo '[!] Updating user agent versions'
python utils/fetch_firefox_version.py
}
pep8_check() {
echo '[!] Running pep8 check'
2016-01-18 11:47:31 +00:00
# ignored rules:
# E402 module level import not at top of file
# W503 line break before binary operator
2017-02-12 14:06:01 +00:00
pep8 --exclude=searx/static --max-line-length=120 --ignore "E402,W503" "$SEARX_DIR" "$BASE_DIR/tests"
}
unit_tests() {
echo '[!] Running unit tests'
python -m nose2 -s "$BASE_DIR/tests/unit"
}
py_test_coverage() {
echo '[!] Running python test coverage'
PYTHONPATH="`pwd`" python -m nose2 -C --log-capture --with-coverage --coverage "$SEARX_DIR" -s "$BASE_DIR/tests/unit" \
2017-09-13 20:58:52 +00:00
&& coverage report \
&& coverage html
}
robot_tests() {
echo '[!] Running robot tests'
PYTHONPATH="`pwd`" python "$SEARX_DIR/testing.py" robot
}
tests() {
set -e
pep8_check
unit_tests
install_geckodriver
robot_tests
set +e
}
#
# Web
#
npm_path_setup() {
which npm || (printf 'Error: npm is not found\n'; exit 1)
export PATH="$(npm bin)":$PATH
}
npm_packages() {
npm_path_setup
echo '[!] install NPM packages'
cd -- "$BASE_DIR"
npm install less@2.7 less-plugin-clean-css grunt-cli
echo '[!] install NPM packages for oscar theme'
cd -- "$BASE_DIR/searx/static/themes/oscar"
npm install
2017-10-06 18:31:19 +00:00
echo '[!] install NPM packages for simple theme'
cd -- "$BASE_DIR/searx/static/themes/simple"
npm install
}
build_style() {
npm_path_setup
lessc --clean-css="--s1 --advanced --compatibility=ie9" "$BASE_DIR/searx/static/$1" "$BASE_DIR/searx/static/$2"
}
styles() {
npm_path_setup
echo '[!] Building legacy style'
build_style themes/legacy/less/style.less themes/legacy/css/style.css
build_style themes/legacy/less/style-rtl.less themes/legacy/css/style-rtl.css
echo '[!] Building courgette style'
build_style themes/courgette/less/style.less themes/courgette/css/style.css
build_style themes/courgette/less/style-rtl.less themes/courgette/css/style-rtl.css
echo '[!] Building pix-art style'
build_style themes/pix-art/less/style.less themes/pix-art/css/style.css
echo '[!] Building bootstrap style'
build_style less/bootstrap/bootstrap.less css/bootstrap.min.css
}
2016-01-02 11:21:40 +00:00
grunt_build() {
npm_path_setup
echo '[!] Grunt build : oscar theme'
grunt --gruntfile "$SEARX_DIR/static/themes/oscar/gruntfile.js"
2017-10-06 18:31:19 +00:00
echo '[!] Grunt build : simple theme'
grunt --gruntfile "$SEARX_DIR/static/themes/simple/gruntfile.js"
}
Update Docker image See #1561 , use uwsgi and Alpine Linux Volume: /var/log/uwsgi contains error log for 2 days (file uwsgi.log) /etc/searx contains the settings.yml and uwsgi.ini files. The docker image creates them if they don't exist. The two files can be modified after the first run. See below. Environement variables: MORTY_URL : external URL of Morty MORTY_KEY : base64 encoded key BASE_URL : external URL of Searx BIND_ADDRESS : internal HTTP port to listen to Labels : org.label-schema.schema.* Parameters: -h : display this help -d : will update the settings and quit immediately (settings.yml and uwsgi.ini) -f : always update the settings (previous version saved with suffix .old). without this parameter, the new settings are copied with suffix .new When the Docker image contains newer settings: - without -f parameter: the new versions are copied to /etc/searx/settings.yml.new and /etc/searx/uwsgi.ini.new. - with -f parameter: the old versions are renamed with .old suffix. The new version replaces /etc/searx/settings.yml and /etc/searx/uwsgi.ini Build using "./manage.sh docker_build", add "push" as parameter also push the Docker image. The script requires a git repository to work (it makes sure that the last git tag matches searx/version.py) "git describe" is used to create a meaningful version. Example : 0.15.0-90-49c5bcb4-dirty (dirty means that the docker image was made with uncommited changes). Use "docker inspect -f {{.Config.Labels.version}} searx" to get the version of an existing image. .dockerignore based on .gitignore .travis.yml: include docker stage
2019-06-29 09:59:13 +00:00
docker_build() {
# Check if it is a git repository
if [ ! -d .git ]; then
echo "This is not Git repository"
exit 1
fi
if [ ! -x "$(which git)" ]; then
echo "git is not installed"
exit 1
fi
if [ ! git remote get-url origin 2> /dev/null ]; then
echo "there is no remote origin"
exit 1
fi
# This is a git repository
# "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
# awk to remove the "v" and the "g"
SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); $3=substr($3, 2); print}')
# add the suffix "-dirty" if the repository has uncommited change
git update-index -q --refresh
if [ ! -z "$(git diff-index --name-only HEAD --)" ]; then
SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
fi
# Get the last git commit id, will be added to the Searx version (see Dockerfile)
VERSION_GITCOMMIT=$(echo $SEARX_GIT_VERSION | cut -d- -f2-4)
echo "Last commit : $VERSION_GITCOMMIT"
# Check consistency between the git tag and the searx/version.py file
# /!\ HACK : parse Python file with bash /!\
# otherwise it is not possible build the docker image without all Python dependencies ( version.py loads __init__.py )
# SEARX_PYTHON_VERSION=$(python -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
if [ $(echo "$SEARX_GIT_VERSION" | cut -d- -f1) != "$SEARX_PYTHON_VERSION" ]; then
echo "Inconsistency between the last git tag and the searx/version.py file"
echo "git tag: $SEARX_GIT_VERSION"
echo "searx/version.py: $SEARX_PYTHON_VERSION"
exit 1
fi
# define the docker image name
# /!\ HACK to get the user name /!\
GITHUB_USER=$(git remote get-url origin | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
SEARX_IMAGE_NAME="${GITHUB_USER:-searx}/searx"
# build Docker image
echo "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
sudo docker build \
--build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
--build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
--build-arg LABEL_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \
--build-arg LABEL_VCS_REF=$(git rev-parse HEAD) \
--build-arg LABEL_VCS_URL=$(git remote get-url origin) \
--build-arg TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- searx/settings.yml) \
--build-arg TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini) \
-t ${SEARX_IMAGE_NAME}:latest -t ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION} .
if [ "$1" = "push" ]; then
sudo docker push ${SEARX_IMAGE_NAME}:latest
sudo docker push ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}
fi
}
#
# Help
#
2016-01-02 15:04:32 +00:00
help() {
[ -z "$1" ] || printf 'Error: %s\n' "$1"
2016-01-02 15:04:32 +00:00
echo "Searx manage.sh help
Commands
========
help - This text
Build requirements
------------------
update_packages - Check & update production dependency changes
update_dev_packages - Check & update development and production dependency changes
install_geckodriver - Download & install geckodriver if not already installed (required for robot_tests)
npm_packages - Download & install npm dependencies
update_useragents - Update useragents.json with the most recent versions of Firefox
Build
-----
2016-01-02 15:04:32 +00:00
locales - Compile locales
styles - Build less files
grunt_build - Build files for themes
Update Docker image See #1561 , use uwsgi and Alpine Linux Volume: /var/log/uwsgi contains error log for 2 days (file uwsgi.log) /etc/searx contains the settings.yml and uwsgi.ini files. The docker image creates them if they don't exist. The two files can be modified after the first run. See below. Environement variables: MORTY_URL : external URL of Morty MORTY_KEY : base64 encoded key BASE_URL : external URL of Searx BIND_ADDRESS : internal HTTP port to listen to Labels : org.label-schema.schema.* Parameters: -h : display this help -d : will update the settings and quit immediately (settings.yml and uwsgi.ini) -f : always update the settings (previous version saved with suffix .old). without this parameter, the new settings are copied with suffix .new When the Docker image contains newer settings: - without -f parameter: the new versions are copied to /etc/searx/settings.yml.new and /etc/searx/uwsgi.ini.new. - with -f parameter: the old versions are renamed with .old suffix. The new version replaces /etc/searx/settings.yml and /etc/searx/uwsgi.ini Build using "./manage.sh docker_build", add "push" as parameter also push the Docker image. The script requires a git repository to work (it makes sure that the last git tag matches searx/version.py) "git describe" is used to create a meaningful version. Example : 0.15.0-90-49c5bcb4-dirty (dirty means that the docker image was made with uncommited changes). Use "docker inspect -f {{.Config.Labels.version}} searx" to get the version of an existing image. .dockerignore based on .gitignore .travis.yml: include docker stage
2019-06-29 09:59:13 +00:00
docker_build - Build Docker image
Tests
-----
2016-01-02 15:04:32 +00:00
unit_tests - Run unit tests
pep8_check - Pep8 validation
robot_tests - Run selenium tests
tests - Run all python tests (pep8, unit, robot_tests)
py_test_coverage - Unit test coverage
2016-01-02 15:04:32 +00:00
"
}
[ "$(command -V "$ACTION" | grep ' function$')" = "" ] \
&& help "action not found" \
|| "$ACTION" "$2"