pleroma installer and config versioning

- added DynamicSupervisor, which starts Pleroma deps and restarts config dependent deps
- added pleroma installer, where user can configure database credentials
and pleroma config. Settings are saved into file and in database
- added versioning for in database config. New version is created from
changes which are passed to config update/delete endpoint. Every version
contains backup with all changes added through update. Versioning
supports rollbacks with N steps. With a rollback, all versions that
come after the version on which the rollback was made are deleted.

.credo.exs

@@ -19,7 +19,7 @@
         #
         # You can give explicit globs or simply directories.
         # In the latter case `**/*.{ex,exs}` will be used.
-        included: ["lib/", "src/", "web/", "apps/", "test/"],
+        included: ["lib/", "src/", "web/", "apps/", "test/", "installer/pleroma"],
         excluded: [~r"/_build/", ~r"/deps/"]
       },
       #

.formatter.exs

@@ -1,3 +1,3 @@
 [
-  inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}", "priv/repo/migrations/*.exs", "priv/repo/optional_migrations/**/*.exs", "priv/scrubbers/*.ex"]
+  inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}", "priv/repo/migrations/*.exs", "priv/repo/optional_migrations/**/*.exs", "priv/scrubbers/*.ex", "installer/pleroma/**/*.{ex,exs}"]
 ]

.gitignore

@@ -13,6 +13,7 @@
 /doc
 /instance
 /priv/ssh_keys
+/installer/assets/node_modules/*
 
 # Prevent committing custom emojis
 /priv/static/emoji/custom/*
@@ -56,4 +57,7 @@ pleroma.iml
 
 # Editor temp files
 /*~
-/*#
+/*#
+
+# local iex
+.iex.exs

CHANGELOG.md

@@ -6,7 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+### Added
+
 - The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change.
+- Config Versioning.
+- Web installer.
 
 ## Unreleased (Patch)
 
@@ -51,7 +55,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Pleroma API: Reroute `/api/pleroma/*` to `/api/v1/pleroma/*`
 
 </details>
-- Improved hashtag timeline performance (requires a background migration). 
+- Improved hashtag timeline performance (requires a background migration).
 
 ### Added
 
@@ -88,6 +92,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: `/api/v1/accounts/:id` & `/api/v1/mutes` endpoints accept `with_relationships` parameter and return filled `pleroma.relationship` field.
 - Mastodon API: Endpoint to remove a conversation (`DELETE /api/v1/conversations/:id`).
 - Mastodon API: `expires_in` in the scheduled post `params` field on `/api/v1/statuses` and `/api/v1/scheduled_statuses/:id` endpoints.
+- Admin API: (`GET /api/pleroma/admin/config/versions`) - endpoint to get list of config versions.
+- Admin API: (`GET /api/pleroma/admin/config/versions/rollback/:id`) - endpoint to rollback config to specific version.
 </details>
 
 ### Fixed

config/config.exs

@@ -849,6 +849,25 @@ config :pleroma, ConcurrentLimiter, [
   {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}
 ]
 
+config :pleroma, Pleroma.InstallerWeb.Endpoint,
+  http: [port: 4001],
+  url: [host: "localhost"],
+  secret_key_base: "izidwIWRlyFL/YjJBH672OWpSFXkMTk3KOgFL/Gj5vO5parwfHnoAQ8ZL+2Mn1SH",
+  render_errors: [view: Pleroma.InstallerWeb.ErrorView, accepts: ~w(html)],
+  protocol: "http",
+  debug_errors: true,
+  code_reloader: true,
+  check_origin: false,
+  node: [
+    "node_modules/webpack/bin/webpack.js",
+    "--mode",
+    "development",
+    "--watch-stdin",
+    cd: Path.expand("../assets", __DIR__)
+  ]
+
+config :pleroma, :installer, psql_cmd_args: ["sudo", ["-Hu", "postgres", "psql", "-c"]]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/description.exs

@@ -1062,65 +1062,63 @@ config :pleroma, :config_description, [
         description:
           "Where logs will be sent, :console - send logs to stdout, { ExSyslogger, :ex_syslogger } - to syslog, Quack.Logger - to Slack.",
         suggestions: [:console, {ExSyslogger, :ex_syslogger}, Quack.Logger]
-      }
-    ]
-  },
-  %{
-    group: :logger,
-    type: :group,
-    key: :ex_syslogger,
-    label: "ExSyslogger",
-    description: "ExSyslogger-related settings",
-    children: [
-      %{
-        key: :level,
-        type: {:dropdown, :atom},
-        description: "Log level",
-        suggestions: [:debug, :info, :warn, :error]
       },
       %{
-        key: :ident,
-        type: :string,
-        description:
-          "A string that's prepended to every message, and is typically set to the app name",
-        suggestions: ["pleroma"]
+        key: :ex_syslogger,
+        type: :keyword,
+        label: "ExSyslogger",
+        description: "ExSyslogger-related settings",
+        children: [
+          %{
+            key: :level,
+            type: {:dropdown, :atom},
+            description: "Log level",
+            suggestions: [:debug, :info, :warn, :error]
+          },
+          %{
+            key: :ident,
+            type: :string,
+            description:
+              "A string that's prepended to every message, and is typically set to the app name",
+            suggestions: ["pleroma"]
+          },
+          %{
+            key: :format,
+            type: :string,
+            description: "Default: \"$date $time [$level] $levelpad$node $metadata $message\"",
+            suggestions: ["$metadata[$level] $message"]
+          },
+          %{
+            key: :metadata,
+            type: {:list, :atom},
+            suggestions: [:request_id]
+          }
+        ]
       },
       %{
-        key: :format,
-        type: :string,
-        description: "Default: \"$date $time [$level] $levelpad$node $metadata $message\"",
-        suggestions: ["$metadata[$level] $message"]
-      },
-      %{
-        key: :metadata,
-        type: {:list, :atom},
-        suggestions: [:request_id]
-      }
-    ]
-  },
-  %{
-    group: :logger,
-    type: :group,
-    key: :console,
-    label: "Console Logger",
-    description: "Console logger settings",
-    children: [
-      %{
-        key: :level,
-        type: {:dropdown, :atom},
-        description: "Log level",
-        suggestions: [:debug, :info, :warn, :error]
-      },
-      %{
-        key: :format,
-        type: :string,
-        description: "Default: \"$date $time [$level] $levelpad$node $metadata $message\"",
-        suggestions: ["$metadata[$level] $message"]
-      },
-      %{
-        key: :metadata,
-        type: {:list, :atom},
-        suggestions: [:request_id]
+        key: :console,
+        type: :keyword,
+        label: "Console Logger",
+        description: "Console logger settings",
+        children: [
+          %{
+            key: :level,
+            type: {:dropdown, :atom},
+            description: "Log level",
+            suggestions: [:debug, :info, :warn, :error]
+          },
+          %{
+            key: :format,
+            type: :string,
+            description: "Default: \"$date $time [$level] $levelpad$node $metadata $message\"",
+            suggestions: ["$metadata[$level] $message"]
+          },
+          %{
+            key: :metadata,
+            type: {:list, :atom},
+            suggestions: [:request_id]
+          }
+        ]
       }
     ]
   },
@@ -1840,19 +1838,13 @@ config :pleroma, :config_description, [
   },
   %{
     group: :pleroma,
+    key: :admin_token,
     label: "Pleroma Admin Token",
-    type: :group,
+    type: :string,
     description:
       "Allows setting a token that can be used to authenticate requests with admin privileges without a normal user account token. Append the `admin_token` parameter to requests to utilize it. (Please reconsider using HTTP Basic Auth or OAuth-based authentication if possible)",
-    children: [
-      %{
-        key: :admin_token,
-        type: :string,
-        description: "Admin token",
-        suggestions: [
-          "Please use a high entropy string or UUID"
-        ]
-      }
+    suggestions: [
+      "Please use a high entropy string or UUID"
     ]
   },
   %{
@@ -2153,16 +2145,11 @@ config :pleroma, :config_description, [
   },
   %{
     group: :pleroma,
+    key: Pleroma.Web.Auth.Authenticator,
     label: "Pleroma Authenticator",
-    type: :group,
+    type: :module,
     description: "Authenticator",
-    children: [
-      %{
-        key: Pleroma.Web.Auth.Authenticator,
-        type: :module,
-        suggestions: [Pleroma.Web.Auth.PleromaAuthenticator, Pleroma.Web.Auth.LDAPAuthenticator]
-      }
-    ]
+    suggestions: [Pleroma.Web.Auth.PleromaAuthenticator, Pleroma.Web.Auth.LDAPAuthenticator]
   },
   %{
     group: :pleroma,

config/dev.exs

@@ -62,5 +62,7 @@ else
   )
 end
 
+config :pleroma, :installer, psql_cmd_args: ["psql", ["-c"]]
+
 if File.exists?("./config/dev.exported_from_db.secret.exs"),
   do: import_config("dev.exported_from_db.secret.exs")

config/prod.exs

@@ -63,7 +63,7 @@ config :logger, :ex_syslogger, level: :info
 
 # Finally import the config/prod.secret.exs
 # which should be versioned separately.
-import_config "prod.secret.exs"
+if File.exists?("./config/prod.secret.exs"), do: import_config("prod.secret.exs")
 
 if File.exists?("./config/prod.exported_from_db.secret.exs"),
   do: import_config("prod.exported_from_db.secret.exs")

config/test.exs

@@ -7,6 +7,11 @@ config :pleroma, Pleroma.Web.Endpoint,
   url: [port: 4001],
   server: true
 
+config :pleroma, Pleroma.InstallerWeb.Endpoint,
+  http: [port: 4002],
+  url: [port: 4002],
+  server: true
+
 # Disable captha for tests
 config :pleroma, Pleroma.Captcha,
   # It should not be enabled for automatic tests
@@ -46,7 +51,7 @@ config :pleroma, Pleroma.Repo,
   username: "postgres",
   password: "postgres",
   database: "pleroma_test",
-  hostname: System.get_env("DB_HOST") || "localhost",
+  hostname: System.get_env("DB_HOST", "localhost"),
   pool: Ecto.Adapters.SQL.Sandbox,
   pool_size: 50
 
@@ -91,7 +96,9 @@ config :pleroma, Pleroma.ScheduledActivity,
   total_user_limit: 3,
   enabled: false
 
-config :pleroma, :rate_limit, %{}
+# Hack to drop default settings from `config.exs`, because keywords are deeply merged, so there is no other way to do it.
+config :pleroma, :rate_limit, nil
+config :pleroma, :rate_limit, []
 
 config :pleroma, :http_security, report_uri: "https://endpoint.com"
 
@@ -133,6 +140,10 @@ config :pleroma, :side_effects,
   ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock,
   logger: Pleroma.LoggerMock
 
+config :pleroma, :installer,
+  repo: Pleroma.Installer.InstallerRepoMock,
+  file: Pleroma.Installer.FileMock
+
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
 else

docs/administration/CLI_tasks/config.md

@@ -2,7 +2,7 @@
 
 {! backend/administration/CLI_tasks/general_cli_task_info.include !}
 
-## Transfer config from file to DB.
+## Transfer config from file to DB
 
 !!! note
     You need to add the following to your config before executing this command:
@@ -150,4 +150,24 @@ This forcibly removes all saved values in the database.
 
     ```sh
     mix pleroma.config [--force] reset
+
+## Rollback config version
+
+!!! note
+    You need to add the following to your config before executing this command:
+
+    ```elixir
+    config :pleroma, configurable_from_database: true
+    ```
+
+Rollback will restore last backup by default. If you want to restore older version use `-s` parameter.
+
+=== "OTP"
+    ```sh
+     ./bin/pleroma_ctl config rollback [-s 2]
+    ```
+
+=== "From Source"
+    ```sh
+    mix pleroma.config rollback [-s 2]
     ```

docs/development/API/admin_api.md

@@ -893,10 +893,10 @@ Status: 404
 
 **Only works when configuration from database is enabled.**
 
-- Params: none
-- Response:
-  - On failure:
-    - 400 Bad Request `"To use this endpoint you need to enable configuration from database."`
+* Params: none
+* Response:
+  * On failure:
+    * 400 Bad Request `"To use this endpoint you need to enable configuration from database."`
 
 ```json
 {}
@@ -906,9 +906,10 @@ Status: 404
 
 ### Returns the flag whether the pleroma should be restarted
 
-- Params: none
-- Response:
-  - `need_reboot` - boolean
+* Params: none
+* Response:
+  * `need_reboot` - boolean
+
 ```json
 {
   "need_reboot": false
@@ -917,17 +918,17 @@ Status: 404
 
 ## `GET /api/v1/pleroma/admin/config`
 
-### Get list of merged default settings with saved in database.
+### Get list of merged default settings with saved in database
 
 *If `need_reboot` is `true`, instance must be restarted, so reboot time settings can take effect.*
 
 **Only works when configuration from database is enabled.**
 
-- Params:
-  - `only_db`: true (*optional*, get only saved in database settings)
-- Response:
-  - On failure:
-    - 400 Bad Request `"To use this endpoint you need to enable configuration from database."`
+* Params:
+  * `only_db`: true (*optional*, get only saved in database settings)
+* Response:
+  * On failure:
+    * 400 Bad Request `"To use this endpoint you need to enable configuration from database."`
 
 ```json
 {
@@ -952,72 +953,71 @@ Status: 404
 
 Some modifications are necessary to save the config settings correctly:
 
-- strings which start with `Pleroma.`, `Phoenix.`, `Tesla.` or strings like `Oban`, `Ueberauth` will be converted to modules;
-```
-"Pleroma.Upload" -> Pleroma.Upload
-"Oban" -> Oban
-```
-- strings starting with `:` will be converted to atoms;
-```
-":pleroma" -> :pleroma
-```
-- objects with `tuple` key and array value will be converted to tuples;
-```
-{"tuple": ["string", "Pleroma.Upload", []]} -> {"string", Pleroma.Upload, []}
-```
-- arrays with *tuple objects* will be converted to keywords;
-```
-[{"tuple": [":key1", "value"]}, {"tuple": [":key2", "value"]}] -> [key1: "value", key2: "value"]
-```
+* strings which start with `Pleroma.`, `Phoenix.`, `Tesla.` or strings like `Oban`, `Ueberauth` will be converted to modules
+  * `"Pleroma.Upload"` -> `Pleroma.Upload`
+  * `"Oban"` -> `Oban`
+* strings starting with `:` will be converted to atoms
+  * `":pleroma"` -> `:pleroma`
+* objects with `tuple` key and array value will be converted to tuples
+  * `{"tuple": ["string", "Pleroma.Upload", []]}` -> `{"string", Pleroma.Upload, []}`
+* arrays with *tuple objects* will be converted to keywords
+  * `[{"tuple": [":key1", "value"]}, {"tuple": [":key2", "value"]}]` -> `[key1: "value", key2: "value"]`
 
-Most of the settings will be applied in `runtime`, this means that you don't need to restart the instance. But some settings are applied in `compile time` and require a reboot of the instance, such as:
-- all settings inside these keys:
-  - `:hackney_pools`
-  - `:connections_pool`
-  - `:pools`
-  - `:chat`
-- partially settings inside these keys:
-  - `:seconds_valid` in `Pleroma.Captcha`
-  - `:proxy_remote` in `Pleroma.Upload`
-  - `:upload_limit` in `:instance`
+Most of the settings will be applied in `runtime`, this means that changes will be applied immediately. But some settings are applied on `startup time` and will take effect after restart of the pleroma parts, such as:
 
-- Params:
-  - `configs` - array of config objects
-  - config object params:
-    - `group` - string (**required**)
-    - `key` - string (**required**)
-    - `value` - string, [], {} or {"tuple": []} (**required**)
-    - `delete` - true (*optional*, if setting must be deleted)
-    - `subkeys` - array of strings (*optional*, only works when `delete=true` parameter is passed, otherwise will be ignored)
+* all settings inside these keys
+  * `:chat`
+  * `Oban`
+  * `:rate_limit`
+  * `:streamer`
+  * `:pools`
+  * `:connections_pool`
+  * `:hackney_pools`
+  * `:gopher`
+  * `:eshhd`
+  * `:ex_aws`
+* partially settings inside these keys:
+  * `:seconds_valid` in `Pleroma.Captcha`
+  * `:proxy_remote` in `Pleroma.Upload`
+  * `:upload_limit` in `:instance`
+  * `:enabled` in `:fed_sockets`
 
-*When a value have several nested settings, you can delete only some nested settings by passing a parameter `subkeys`, without deleting all settings by key.*
-```
-[subkey: val1, subkey2: val2, subkey3: val3] \\ initial value
-{"group": ":pleroma", "key": "some_key", "delete": true, "subkeys": [":subkey", ":subkey3"]} \\ passing json for deletion
-[subkey2: val2] \\ value after deletion
-```
+* Params:
+  * `configs` - array of config objects
+  * config object params:
+    * `group` - string (**required**)
+    * `key` - string (**required**)
+    * `value` - string, [], {} or {"tuple": []} (**required**)
+    * `delete` - true (*optional*, if setting must be deleted)
+    * `subkeys` - array of strings (*optional*, only works when `delete=true` parameter is passed, otherwise will be ignored)
 
-*Most of the settings can be partially updated through merge old values with new values, except settings value of which is list or is not keyword.*
+#### Partial deletion
+
+Keys inside value can be partially deleted by passing `subkeys` parameter. If after partial deleting an empty list remains, then the entire setting will be deleted.
+
+Example:
 
-Example of setting without keyword in value:
 ```elixir
-config :tesla, :adapter, Tesla.Adapter.Hackney
+# initial value
+[subkey: :val1, subkey2: :val2, subkey3: :val3]
+```
+
+```json
+// config object for deletion
+{"group": ":pleroma", "key": "some_key", "delete": true, "subkeys": [":subkey", ":subkey3"]}
 ```
 
-List of settings which support only full update by key:
 ```elixir
-@full_key_update [
-    {:pleroma, :ecto_repos},
-    {:quack, :meta},
-    {:mime, :types},
-    {:cors_plug, [:max_age, :methods, :expose, :headers]},
-    {:auto_linker, :opts},
-    {:swarm, :node_blacklist},
-    {:logger, :backends}
-  ]
+# value after deletion
+[subkey2: :val2]
 ```
 
-List of settings which support only full update by subkey:
+#### Partial update
+
+Most settings can be partially updated: new values will be merged with existing ones.
+
+The following settings are exceptions and should be fully updated:
+
 ```elixir
 @full_subkey_update [
     {:pleroma, :assets, :mascots},
@@ -1028,23 +1028,25 @@ List of settings which support only full update by subkey:
   ]
 ```
 
-*Settings without explicit key must be sended in separate config object params.*
+#### Settings without explicit keys
+
+Settings without explicit key must be sended in one config object with null value as key.
+
 ```elixir
 config :quack,
   level: :debug,
-  meta: [:all],
-  ...
+  meta: [:all]
 ```
+
 ```json
 {
   "configs": [
-    {"group": ":quack", "key": ":level", "value": ":debug"},
-    {"group": ":quack", "key": ":meta", "value": [":all"]},
-    ...
+    {"group": ":quack", "key": null, "value": [{"tuple": [":level", ":debug"]}, {"tuple": [":meta", ":all"]}]}
   ]
 }
 ```
-- Request:
+
+* Request:
 
 ```json
 {
@@ -1075,9 +1077,10 @@ config :quack,
 }
 ```
 
-- Response:
-  - On failure:
-    - 400 Bad Request `"To use this endpoint you need to enable configuration from database."`
+* Response:
+  * On failure:
+    * 400 Bad Request `"To use this endpoint you need to enable configuration from database."`
+
 ```json
 {
   "configs": [
@@ -1091,9 +1094,10 @@ config :quack,
 }
 ```
 
-## ` GET /api/v1/pleroma/admin/config/descriptions`
+## `GET /api/v1/pleroma/admin/config/descriptions`
+
+### Get JSON with config descriptions
 
-### Get JSON with config descriptions.
 Loads json generated from `config/descriptions.exs`.
 
 - Params: none
@@ -1124,6 +1128,37 @@ Loads json generated from `config/descriptions.exs`.
 }]
 ```
 
+## `GET /api/v1/pleroma/admin/config/versions/rollback/:id`
+
+### Rollback config changes for a given version
+
+- Params:
+  - `id` - version id for rollback
+- Response:
+  - On success: `204`, empty response
+  - On failure:
+    - 400 Bad Request `"To use this endpoint you need to enable configuration from database."` or endpoint error
+    - 404 Not found
+
+## `GET /api/v1/pleroma/admin/config/versions`
+
+### Get list of config versions
+
+- Params: none
+- Response:
+
+```json
+{
+  "versions": [
+    {
+      "id": 1,
+      "current": true,
+      "inserted_at": "2020-04-21T15:11:46.000Z"
+     }
+  ]
+}
+```
+
 ## `GET /api/v1/pleroma/admin/moderation_log`
 
 ### Get moderation log

docs/development/authentication_authorization.md

@@ -16,6 +16,6 @@
 
 * With non-OAuth authentication ([HTTP Basic Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization) or HTTP header- or params-provided auth), OAuth scopes check is _not_ performed for any action (since password is provided during the auth, requester is able to obtain a token with full permissions anyways); auth plugs invoke `Pleroma.Helpers.AuthHelper.skip_oauth(conn)` in this case.
 
-## Auth-related configuration, OAuth consumer mode etc.
+### Auth-related configuration, OAuth consumer mode etc.
 
 See `Authentication` section of [the configuration cheatsheet](../configuration/cheatsheet.md#authentication).

docs/development/config_versioning.md

@@ -0,0 +1,40 @@
+# Config versioning
+
+Database configuration supports simple versioning. Every change (list of changes or only one change) through adminFE creates new version with backup from config table. It is possible to do rollback on N steps (1 by default). Rollback will recreate `config` table from backup.
+
+**IMPORTANT** Destructive operations with `Pleroma.ConfigDB` and `Pleroma.Config.Version` must be processed through `Pleroma.Config.Versioning` module for correct versioning work, especially migration changes.
+
+Example:
+
+* new config setting is added directly using `Pleroma.ConfigDB` module
+* user is doing rollback and setting is lost
+
+## Creating new version
+
+Creating new version is done with `Pleroma.Config.Versioning.new_version/1`, which accepts list of changes. Changes can include adding/updating/deleting operations in `config` table at the same time.
+
+Process of creating new version:
+
+* saving config changes in `config` table
+* saving new version with current configs
+  * `backup` - keyword with all configs from `config` table (binary)
+  * `current` - flag, which marks current version (boolean)
+
+## Version rollback
+
+Version control also supports a simple N steps back mechanism.
+
+Rollback process:
+
+* cleaning `config` table
+* splitting `backup` field into separate settings and inserting them into `config` table
+* removing subsequent versions
+
+## Config migrations
+
+Sometimes it becomes necessary to make changes to the configuration, which can be stored in the user's database. Config versioning makes this process more complicated, as we also must update this setting in versions backups.
+
+Versioning module contains two functions for migrations:
+
+* `Pleroma.Config.Versioning.migrate_namespace/2` - for simple renaming, e.g. group or key of the setting must be renamed.
+* `Pleroma.Config.Versioning.migrate_configs_and_versions/2` - abstract function for more complex migrations. Accepts two functions, the first one to make changes with configs, another to make changes with version backups.

installer/assets/css/app.css

@@ -0,0 +1,3 @@
+/* This file is for your main application css. */
+
+@import "./phoenix.css";

installer/assets/js/app.js

@@ -0,0 +1,59 @@
+// We need to import the CSS so that webpack will load it.
+// The MiniCssExtractPlugin is used to separate it out into
+// its own CSS file.
+import css from "../css/app.css"
+
+// webpack automatically bundles all modules in your
+// entry points. Those entry points can be configured
+// in "webpack.config.js".
+//
+// Import dependencies
+//
+import "phoenix_html"
+import "phoenix"
+import { Ajax } from "phoenix"
+
+// Import local files
+//
+// Local files can be imported directly using relative paths, for example:
+// import socket from "./socket"
+window.onload = function () {
+  let endpointUrlEl = document.getElementById('config_form_endpoint_url')
+  if (endpointUrlEl) {
+    endpointUrlEl.addEventListener('change', function (evt) {
+      let endpointUrl = new URL(endpointUrlEl.value);
+
+      let adminEmailEl = document.getElementById('config_form_instance_email');
+      if (adminEmailEl.value == '') {
+        adminEmailEl.value = 'admin@' + endpointUrl.hostname;
+      }
+
+      let adminUserEmailEl = document.getElementById('config_form_admin_email');
+      if (adminUserEmailEl.value == '') {
+        adminUserEmailEl.value = 'admin@' + endpointUrl.hostname;
+      }
+
+      let notifyEmailEl = document.getElementById('config_form_instance_notify_email');
+      if (notifyEmailEl.value == '') {
+        notifyEmailEl.value = 'no-reply@' + endpointUrl.hostname;
+      }
+    });
+  }
+
+  let migrations = document.getElementById('migrations')
+
+  if (migrations) {
+    Ajax.request("GET", "/run_migrations", "application/json", "", 20000, show_error, (resp) => {
+      if (resp == "ok") {
+        window.location = "/config";
+      } else {
+        show_error();
+      }
+    });
+  }
+}
+
+function show_error() {
+  let errorEl = document.getElementById('error');
+  errorEl.style.visibility = 'visible';
+}

installer/assets/package.json

@@ -0,0 +1,25 @@
+{
+  "repository": {},
+  "license": "MIT",
+  "scripts": {
+    "deploy": "webpack --mode production",
+    "watch": "webpack --mode development --watch"
+  },
+  "dependencies": {
+    "call-bind": "^1.0.0",
+    "phoenix": "file:../../deps/phoenix",
+    "phoenix_html": "file:../../deps/phoenix_html"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.0.0",
+    "@babel/preset-env": "^7.0.0",
+    "babel-loader": "^8.0.0",
+    "copy-webpack-plugin": "^5.1.1",
+    "css-loader": "^3.4.2",
+    "mini-css-extract-plugin": "^0.9.0",
+    "optimize-css-assets-webpack-plugin": "^5.0.1",
+    "terser-webpack-plugin": "^2.3.2",
+    "webpack": "4.41.5",
+    "webpack-cli": "^3.3.2"
+  }
+}

installer/assets/static/images/logo.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   version="1.1"
+   id="svg4485"
+   width="512"
+   height="512"
+   viewBox="0 0 512 512"
+   sodipodi:docname="logo.svg"
+   inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)">
+  <metadata
+     id="metadata4491">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs4489" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1274"
+     inkscape:window-height="1410"
+     id="namedview4487"
+     showgrid="false"
+     inkscape:zoom="1.2636719"
+     inkscape:cx="305.99333"
+     inkscape:cy="304.30809"
+     inkscape:window-x="1280"
+     inkscape:window-y="22"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="g4612"
+     inkscape:document-rotation="0" />
+  <g
+     id="g4612">
+    <g
+       id="g850"
+       transform="matrix(0.99659595,0,0,0.99659595,0.37313949,0.87143746)">
+      <path
+         style="opacity:1;fill:#fba457;fill-opacity:1;stroke:#009bff;stroke-width:0;stroke-linecap:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:0.175879"
+         d="m 194.75841,124.65165 a 20.449443,20.449443 0 0 0 -20.44944,20.44945 v 242.24725 h 65.28091 v -262.6967 z"
+         id="path4497" />
+      <path
+         style="fill:#fba457;fill-opacity:1;stroke:#000000;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+         d="M 272.6236,124.65165 V 256 h 45.61799 a 20.449443,20.449443 0 0 0 20.44944,-20.44945 v -110.8989 z"
+         id="path4516" />
+      <path
+         style="opacity:1;fill:#fba457;fill-opacity:1;stroke:#000000;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+         d="m 272.6236,322.06744 v 65.28091 h 45.61799 a 20.449443,20.449443 0 0 0 20.44944,-20.44945 v -44.83146 z"
+         id="path4516-5" />
+    </g>
+  </g>
+</svg>

installer/assets/webpack.config.js

@@ -0,0 +1,41 @@
+const path = require('path');
+const glob = require('glob');
+const MiniCssExtractPlugin = require('mini-css-extract-plugin');
+const TerserPlugin = require('terser-webpack-plugin');
+const OptimizeCSSAssetsPlugin = require('optimize-css-assets-webpack-plugin');
+const CopyWebpackPlugin = require('copy-webpack-plugin');
+
+module.exports = (env, options) => ({
+  optimization: {
+    minimizer: [
+      new TerserPlugin({ cache: true, parallel: true, sourceMap: false }),
+      new OptimizeCSSAssetsPlugin({})
+    ]
+  },
+  entry: {
+    './js/app.js': glob.sync('./vendor/**/*.js').concat(['./js/app.js'])
+  },
+  output: {
+    filename: 'app.js',
+    path: path.resolve(__dirname, '../../priv/static/installer/js')
+  },
+  module: {
+    rules: [
+      {
+        test: /\.js$/,
+        exclude: /node_modules/,
+        use: {
+          loader: 'babel-loader'
+        }
+      },
+      {
+        test: /\.css$/,
+        use: [MiniCssExtractPlugin.loader, 'css-loader']
+      }
+    ]
+  },
+  plugins: [
+    new MiniCssExtractPlugin({ filename: '../css/app.css' }),
+    new CopyWebpackPlugin([{ from: 'static/', to: '../' }])
+  ]
+});

installer/pleroma/installer/file.ex

@@ -0,0 +1,10 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Installer.File do
+  @callback write(Path.t(), iodata()) :: :ok | {:error, :file.posix()}
+  @callback write(Path.t(), iodata(), [atom()]) :: :ok | {:error, :file.posix()}
+
+  defdelegate write(path, content, modes \\ []), to: File
+end

installer/pleroma/installer/installer_repo.ex

@@ -0,0 +1,147 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Installer.InstallerRepo do
+  require Logger
+
+  alias Pleroma.Repo
+
+  @dynamic_repo :installer
+
+  @callback check_database(keyword()) :: :ok | {:error, term()}
+  @callback create_database(keyword()) :: :ok | {:error, term()}
+  @callback stop() :: :ok
+  @callback run_migrations(keyword(), [Path.t()]) :: :ok | {:error, :migrations_error}
+  @callback start_repo(keyword()) :: {:ok, pid()} | {:error, term()}
+
+  @spec dynamic_repo() :: atom()
+  def dynamic_repo, do: @dynamic_repo
+
+  @spec check_database(keyword()) :: :ok | {:error, term()}
+  def check_database(credentials) do
+    credentials = Keyword.put(credentials, :name, @dynamic_repo)
+
+    with {:ok, _} <- start_repo(credentials),
+         {:ok, _} <- check_connection(),
+         :ok <- check_extensions(credentials[:rum_enabled]) do
+      :ok
+    end
+  end
+
+  @spec create_database(keyword()) :: :ok | {:error, term()}
+  def create_database(credentials) do
+    credentials = Keyword.put(credentials, :name, @dynamic_repo)
+
+    maintenance_credentials = Keyword.merge(credentials, username: "postgres", password: "")
+
+    with :ok <- storage_up(maintenance_credentials),
+         {:ok, _} <- start_repo(maintenance_credentials),
+         {:ok, _} <- check_connection() do
+      queries = [
+        "CREATE USER #{credentials[:username]} WITH ENCRYPTED PASSWORD '#{credentials[:password]}';",
+        "ALTER DATABASE #{credentials[:database]} OWNER TO #{credentials[:username]};",
+        "CREATE EXTENSION IF NOT EXISTS citext;",
+        "CREATE EXTENSION IF NOT EXISTS pg_trgm;",
+        "CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";"
+      ]
+
+      queries =
+        if credentials[:rum_enabled] do
+          queries ++ ["CREATE EXTENSION IF NOT EXISTS rum;"]
+        else
+          queries
+        end
+
+      Enum.reduce_while(queries, :ok, fn query, acc ->
+        case Repo.query(query) do
+          {:ok, _} -> {:cont, acc}
+          error -> {:halt, error}
+        end
+      end)
+    end
+  end
+
+  defp storage_up(credentials) do
+    with {:error, _} = error <- Ecto.Adapters.Postgres.storage_up(credentials) do
+      Logger.error("Can't create repo due to error: #{inspect(error)}")
+      {:error, :create_repo}
+    end
+  end
+
+  @spec run_migrations(keyword(), [Path.t()]) ::
+          :ok | {:error, :migrations_error} | {:error, term()}
+  def run_migrations(credentials, paths) do
+    case start_repo(credentials) do
+      {:ok, _} ->
+        if Ecto.Migrator.run(Repo, paths, :up, all: true, dynamic_repo: @dynamic_repo) != [] do
+          :ok
+        else
+          Logger.error("Not all migrations were applied.")
+          {:error, :migrations_error}
+        end
+
+      error ->
+        Logger.error("Can't run migratios due to error: #{inspect(error)}")
+        error
+    end
+  end
+
+  @spec stop() :: :ok
+  def stop do
+    @dynamic_repo
+    |> Process.whereis()
+    |> case do
+      repo when is_pid(repo) -> Supervisor.stop(repo)
+      _ -> :ok
+    end
+  end
+
+  @spec start_repo(keyword()) :: {:ok, pid()} | {:error, term()}
+  def start_repo(credentials) when is_list(credentials) do
+    credentials = Keyword.put(credentials, :name, @dynamic_repo)
+
+    case Repo.start_link(credentials) do
+      {:ok, pid} = result ->
+        Repo.put_dynamic_repo(pid)
+        result
+
+      error ->
+        Logger.error("Can't start repo due to error: #{inspect(error)}")
+        {:error, :installer_repo_start}
+    end
+  end
+
+  defp check_connection do
+    with {:error, _} = error <- Repo.query("SELECT 1") do
+      Logger.error("Can't connect to database due to error #{inspect(error)}")
+      {:error, :query_execution}
+    end
+  end
+
+  defp check_extensions(rum_enabled?) do
+    default = ["citext", "pg_trgm", "uuid-ossp"]
+
+    required = if rum_enabled?, do: ["rum" | default], else: default
+
+    with {:ok, %{rows: extensions}} <- Repo.query("SELECT pg_available_extensions();") do
+      extensions = Enum.map(extensions, fn [{name, _, _}] -> name end)
+
+      not_installed =
+        Enum.reduce(required, [], fn ext, acc ->
+          if ext in extensions do
+            acc
+          else
+            [ext | acc]
+          end
+        end)
+
+      if not_installed == [] do
+        :ok
+      else
+        Logger.error("These extensions are not installed: #{Enum.join(not_installed, ",")}")
+        {:error, :extensions_not_installed}
+      end
+    end
+  end
+end

installer/pleroma/installer_web.ex

@@ -0,0 +1,47 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.InstallerWeb do
+  def controller do
+    quote do
+      use Phoenix.Controller, namespace: Pleroma.InstallerWeb
+
+      import Plug.Conn
+      alias Pleroma.InstallerWeb.Router.Helpers, as: Routes
+    end
+  end
+
+  def view do
+    quote do
+      use Phoenix.View,
+        root: "installer/pleroma/installer_web/templates",
+        namespace: Pleroma.InstallerWeb
+
+      import Phoenix.Controller,
+        only: [get_flash: 1, get_flash: 2, view_module: 1, view_template: 1]
+
+      use Phoenix.HTML
+
+      import Pleroma.InstallerWeb.ErrorHelpers
+      import Pleroma.Web.Gettext
+
+      alias Pleroma.InstallerWeb.Router.Helpers, as: Routes
+    end
+  end
+
+  def router do
+    quote do
+      use Phoenix.Router
+      import Plug.Conn
+      import Phoenix.Controller
+    end
+  end
+
+  @doc """
+  When used, dispatch to the appropriate controller/view/etc.
+  """
+  defmacro __using__(which) when is_atom(which) do
+    apply(__MODULE__, which, [])
+  end
+end

installer/pleroma/installer_web/controllers/setup_controller.ex

@@ -0,0 +1,151 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.InstallerWeb.SetupController do
+  use Pleroma.InstallerWeb, :controller
+
+  alias Pleroma.Config
+  alias Pleroma.InstallerWeb.Forms.ConfigForm
+  alias Pleroma.InstallerWeb.Forms.CredentialsForm
+
+  plug(:authenticate)
+
+  def index(conn, _) do
+    env = Config.get(:env)
+
+    render(conn, "index.html",
+      credentials:
+        CredentialsForm.changeset(%{
+          username: "pleroma",
+          password: "",
+          database: "pleroma_#{env}",
+          hostname: "localhost"
+        }),
+      error: nil
+    )
+  end
+
+  def save_credentials(conn, params) do
+    changeset = CredentialsForm.changeset(params["credentials_form"], generate_password: true)
+
+    case CredentialsForm.save_credentials(changeset) do
+      :ok ->
+        redirect(conn, to: Routes.setup_path(conn, :migrations))
+
+      {:error, %Ecto.Changeset{} = changeset} ->
+        render(conn, "index.html",
+          credentials: changeset,
+          error: nil
+        )
+
+      {:error, %DBConnection.ConnectionError{}, _} ->
+        render(conn, "index.html",
+          credentials: changeset,
+          error:
+            "Pleroma can't connect to the database with these credentials. Please check them and try one more time."
+        )
+
+      {:error, :create_repo, psql_path} ->
+        render(conn, "run_psql.html", psql_path: psql_path, error: nil)
+
+      {:error, error, _} ->
+        render(conn, "index.html",
+          credentials: changeset,
+          error: inspect(error)
+        )
+    end
+  end
+
+  def recheck(conn, _) do
+    case CredentialsForm.check_database_and_write_config() do
+      :ok ->
+        redirect(conn, to: Routes.setup_path(conn, :migrations))
+
+      {:error, %DBConnection.ConnectionError{}, _} ->
+        render(conn, "run_psql.html",
+          error:
+            "Pleroma can't connect to the database with these credentials. Please check them and try one more time."
+        )
+
+      {:error, :extensions_not_installed, _} ->
+        render(conn, "run_psql.html", error: "Some required extensions were not found.")
+
+      {:error, error, _} ->
+        render(conn, "run_psql.html", error: inspect(error))
+    end
+  end
+
+  def migrations(conn, _) do
+    render(conn, "migrations.html")
+  end
+
+  def run_migrations(conn, _) do
+    response =
+      case CredentialsForm.migrations() do
+        :ok -> "ok"
+        _ -> "Error occuried while migrations were run."
+      end
+
+    json(conn, response)
+  end
+
+  def config(conn, _) do
+    render(conn, "config.html",
+      config:
+        ConfigForm.changeset(%{
+          instance_static_dir: "instance/static",
+          endpoint_url_port: 443,
+          endpoint_http_ip: "127.0.0.1",
+          endpoint_http_port: 4000,
+          local_uploads_dir: "uploads"
+        }),
+      error: nil
+    )
+  end
+
+  def save_config(conn, params) do
+    changeset = ConfigForm.changeset(params["config_form"])
+
+    case ConfigForm.save(changeset) do
+      :ok ->
+        Config.delete(:installer_token)
+
+        if Config.get(:env) != :test do
+          if endpoint = Process.whereis(Pleroma.Web.Endpoint) do
+            Supervisor.stop(endpoint)
+          end
+
+          Pleroma.Application.stop_installer_and_start_pleroma()
+        end
+
+        redirect(conn, external: Pleroma.Web.Endpoint.url())
+
+      {:error, %Ecto.Changeset{} = changeset} ->
+        render(conn, "config.html", config: changeset, error: "Some values have incorrect values.")
+
+      {:error, :config_file_not_found} ->
+        render(conn, "config.html", config: changeset, error: "Something went wrong.")
+
+      {:error, error} ->
+        render(conn, "config.html", config: changeset, error: inspect(error))
+    end
+  end
+
+  defp authenticate(conn, _) do
+    token = Config.get(:installer_token)
+
+    cond do
+      token && get_session(conn, :token) == token ->
+        conn
+
+      token && conn.query_params["token"] == token ->
+        put_session(conn, :token, token)
+
+      true ->
+        conn
+        |> text("Token is invalid")
+        |> halt()
+    end
+  end
+end

installer/pleroma/installer_web/endpoint.ex

@@ -0,0 +1,34 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.InstallerWeb.Endpoint do
+  use Phoenix.Endpoint, otp_app: :pleroma
+
+  @session_options [
+    store: :cookie,
+    key: "_pleroma_installer_key",
+    signing_salt: "4aGH1qnr"
+  ]
+
+  plug(Plug.Static,
+    at: "/",
+    from: {:pleroma, "priv/static/installer"},
+    gzip: false,
+    only: ~w(css images js favicon.ico)
+  )
+
+  if code_reloading? do
+    plug(Phoenix.CodeReloader)
+  end
+
+  plug(Plug.Parsers,
+    parsers: [:urlencoded],
+    pass: ["*/*"],
+    json_decoder: Phoenix.json_library()
+  )
+
+  plug(Plug.MethodOverride)
+  plug(Plug.Session, @session_options)
+  plug(Pleroma.InstallerWeb.Router)
+end

installer/pleroma/installer_web/error_helpers.ex

@@ -0,0 +1,51 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.InstallerWeb.ErrorHelpers do
+  @moduledoc """
+  Conveniences for translating and building error messages.
+  """
+
+  use Phoenix.HTML
+
+  @doc """
+  Generates tag for inlined form input errors.
+  """
+  def error_tag(form, field) do
+    Enum.map(Keyword.get_values(form.errors, field), fn error ->
+      content_tag(:span, translate_error(error),
+        class: "help-block",
+        data: [phx_error_for: input_id(form, field)]
+      )
+    end)
+  end
+
+  @doc """
+  Translates an error message using gettext.
+  """
+  def translate_error({msg, opts}) do
+    # When using gettext, we typically pass the strings we want
+    # to translate as a static argument:
+    #
+    #     # Translate "is invalid" in the "errors" domain
+    #     dgettext("errors", "is invalid")
+    #
+    #     # Translate the number of files with plural rules
+    #     dngettext("errors", "1 file", "%{count} files", count)
+    #
+    # Because the error messages we show in our forms and APIs
+    # are defined inside Ecto, we need to translate them dynamically.
+    # This requires us to call the Gettext module passing our gettext
+    # backend as first argument.
+    #
+    # Note we use the "errors" domain, which means translations
+    # should be written to the errors.po file. The :count option is
+    # set by Ecto and indicates we should also apply plural rules.
+    if count = opts[:count] do
+      Gettext.dngettext(Pleroma.Web.Gettext, "errors", msg, msg, count, opts)
+    else
+      Gettext.dgettext(Pleroma.Web.Gettext, "errors", msg, opts)
+    end
+  end
+end

installer/pleroma/installer_web/forms/config_form.ex

@@ -0,0 +1,285 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.InstallerWeb.Forms.ConfigForm do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+
+  alias Pleroma.Config
+  alias Pleroma.User
+
+  @to_file [
+    :endpoint_url_scheme,
+    :endpoint_url_host,
+    :endpoint_url_port,
+    :endpoint_http_ip,
+    :endpoint_http_port,
+    :endpoint_secret_key_base,
+    :endpoint_signing_salt,
+    :joken_default_signer,
+    :configurable_from_database
+  ]
+
+  @primary_key false
+  @repo Config.get([:installer, :repo], Pleroma.Installer.InstallerRepo)
+  @admin_user_keys [:admin_email, :admin_nickname, :admin_password]
+
+  embedded_schema do
+    field(:instance_name)
+    field(:instance_email)
+    field(:instance_notify_email)
+    field(:instance_static_dir, :string)
+
+    field(:endpoint_url)
+    field(:endpoint_url_host)
+    field(:endpoint_url_port, :integer)
+    field(:endpoint_url_scheme, :string)
+    field(:endpoint_http_ip)
+    field(:endpoint_http_port, :integer)
+    field(:endpoint_secret_key_base)
+    field(:endpoint_signing_salt)
+
+    field(:local_uploads_dir)
+
+    field(:joken_default_signer)
+
+    field(:web_push_encryption_public_key)
+    field(:web_push_encryption_private_key)
+
+    field(:configurable_from_database, :boolean, default: true)
+    field(:indexable, :boolean, default: true)
+
+    field(:create_admin_user, :boolean, default: true)
+    field(:admin_nickname)
+    field(:admin_email)
+    field(:admin_password)
+  end
+
+  @spec changeset(map()) :: Ecto.Changeset.t()
+  def changeset(attrs \\ %{}) do
+    keys = [
+      :instance_name,
+      :instance_email,
+      :instance_notify_email,
+      :instance_static_dir,
+      :endpoint_url,
+      :endpoint_http_ip,
+      :endpoint_http_port,
+      :local_uploads_dir,
+      :configurable_from_database,
+      :indexable,
+      :create_admin_user
+    ]
+
+    %__MODULE__{}
+    |> cast(
+      attrs,
+      keys ++ @admin_user_keys
+    )
+    |> validate_change(:endpoint_url, fn :endpoint_url, url ->
+      case URI.parse(url) do
+        %{scheme: nil} -> [endpoint_url: "url must have scheme"]
+        %{host: nil} -> [endpoint_url: "url bad format"]
+        _ -> []
+      end
+    end)
+    |> set_url_fields()
+    |> add_endpoint_secret()
+    |> add_endpoint_signing_salt()
+    |> add_joken_default_signer()
+    |> add_web_push_keys()
+    |> validate_required(keys)
+    |> validate_format(:instance_email, User.email_regex())
+    |> validate_format(:instance_notify_email, User.email_regex())
+    |> maybe_validate_admin_user_fields()
+  end
+
+  defp set_url_fields(%{changes: %{endpoint_url: url}} = changeset) do
+    uri = URI.parse(url)
+
+    change(changeset,
+      endpoint_url_host: uri.host,
+      endpoint_url_port: uri.port,
+      endpoint_url_scheme: uri.scheme
+    )
+  end
+
+  defp set_url_fields(changeset), do: changeset
+
+  defp add_endpoint_secret(changeset) do
+    change(changeset, endpoint_secret_key_base: crypt(64))
+  end
+
+  defp add_endpoint_signing_salt(changeset) do
+    change(changeset, endpoint_signing_salt: crypt(8))
+  end
+
+  defp add_joken_default_signer(changeset) do
+    change(changeset, joken_default_signer: crypt(64))
+  end
+
+  defp crypt(bytes) do
+    bytes
+    |> :crypto.strong_rand_bytes()
+    |> Base.encode64()
+    |> binary_part(0, bytes)
+  end
+
+  defp add_web_push_keys(changeset) do
+    {web_push_public_key, web_push_private_key} = :crypto.generate_key(:ecdh, :prime256v1)
+
+    change(changeset,
+      web_push_encryption_public_key: Base.url_encode64(web_push_public_key, padding: false),
+      web_push_encryption_private_key: Base.url_encode64(web_push_private_key, padding: false)
+    )
+  end
+
+  defp maybe_validate_admin_user_fields(%{changes: %{create_admin_user: false}} = changeset) do
+    changeset
+  end
+
+  defp maybe_validate_admin_user_fields(changeset) do
+    changeset
+    |> validate_required(@admin_user_keys)
+    |> validate_format(:admin_email, User.email_regex())
+  end
+
+  @spec save(Ecto.Changeset.t()) ::
+          :ok
+          | {:error, Ecto.Changeset.t()}
+          | {:error, :config_file_not_found}
+          | {:error, :file.posix()}
+  def save(changeset) do
+    with {:ok, struct} <- apply_action(changeset, :insert) do
+      # we expect that config file was already created and contains database credentials,
+      # so if file doesn't exist we return error
+      config_path = Pleroma.Application.config_path()
+
+      if File.exists?(config_path) do
+        config =
+          struct
+          |> Map.from_struct()
+          |> Map.to_list()
+
+        with :ok <- do_save(config, config_path) do
+          if config[:create_admin_user] do
+            # in tests is always started
+            if !Process.whereis(Pleroma.Web.Endpoint) do
+              {:ok, _} = Pleroma.Web.Endpoint.start_link()
+            end
+
+            params = %{
+              nickname: config[:admin_nickname],
+              email: config[:admin_email],
+              password: config[:admin_password],
+              password_confirmation: config[:admin_password],
+              name: config[:admin_nickname],
+              bio: ""
+            }
+
+            changeset = User.register_changeset(%User{}, params, need_confirmation: false)
+
+            with {:ok, user} <- User.register(changeset) do
+              {:ok, _user} = User.admin_api_update(user, %{is_admin: true})
+            end
+          end
+
+          generate_robots_txt(config)
+        end
+      else
+        {:error, :config_file_not_found}
+      end
+    end
+  end
+
+  defp do_save(config, config_path) do
+    config
+    |> save_to_file(config_path)
+    |> maybe_save_to_database()
+  end
+
+  defp save_to_file(config, config_path) do
+    {keys, template} =
+      if config[:configurable_from_database] do
+        {@to_file, "installer/templates/config_part.eex"}
+      else
+        keys =
+          [
+            :local_uploads_dir,
+            :web_push_encryption_public_key,
+            :web_push_encryption_private_key,
+            :instance_name,
+            :instance_email,
+            :instance_notify_email,
+            :instance_static_dir
+          ] ++ @to_file
+
+        {keys, "installer/templates/config_full.eex"}
+      end
+
+    assigns = Keyword.take(config, keys)
+
+    content = EEx.eval_file(template, assigns)
+
+    file_mod = Config.get([:installer, :file], Pleroma.Installer.File)
+
+    with :ok <- file_mod.write(config_path, ["\n", content], [:append]) do
+      config
+    end
+  end
+
+  defp maybe_save_to_database(config) when is_list(config) do
+    if config[:configurable_from_database] do
+      web_push = [
+        subject: "mailto:" <> config[:instance_email],
+        public_key: config[:web_push_encryption_public_key],
+        private_key: config[:web_push_encryption_private_key]
+      ]
+
+      changes = [
+        %{
+          group: :pleroma,
+          key: :instance,
+          value:
+            Keyword.take(
+              config,
+              [:instance_name, :instance_email, :instance_notify_email, :instance_static_dir]
+            )
+        },
+        %{
+          group: :web_push_encryption,
+          key: :vapid_details,
+          value: web_push
+        },
+        %{
+          group: :pleroma,
+          key: Pleroma.Uploaders.Local,
+          value: [uploads: config[:local_uploads_dir]]
+        }
+      ]
+
+      config = Config.get(:credentials)
+
+      with {:ok, _} <- @repo.start_repo(config),
+           {:ok, _} <- Config.Versioning.new_version(changes) do
+        Config.delete(:credentials)
+      end
+    else
+      :ok
+    end
+  end
+
+  defp maybe_save_to_database(result), do: result
+
+  defp generate_robots_txt(config) do
+    templates_dir = Application.app_dir(:pleroma, "priv") <> "/templates"
+
+    Mix.Tasks.Pleroma.Instance.write_robots_txt(
+      config[:instance_static_dir],
+      config[:indexable],
+      templates_dir
+    )
+  end
+end

installer/pleroma/installer_web/forms/credentials_form.ex

@@ -0,0 +1,148 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.InstallerWeb.Forms.CredentialsForm do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+
+  require Logger
+
+  alias Pleroma.Config
+  alias Pleroma.Repo
+
+  @repo Config.get([:installer, :repo], Pleroma.Installer.InstallerRepo)
+
+  @primary_key false
+
+  embedded_schema do
+    field(:username, :string)
+    field(:password, :string, default: "")
+    field(:database, :string)
+    field(:hostname, :string)
+    field(:pool_size, :integer, default: 2)
+    field(:rum_enabled, :boolean, default: false)
+  end
+
+  @spec changeset(map()) :: Ecto.Changeset.t()
+  def changeset(attrs \\ %{}, opts \\ []) do
+    %__MODULE__{}
+    |> cast(attrs, [:username, :password, :database, :hostname, :rum_enabled])
+    |> maybe_add_password(opts)
+    |> validate_required([:username, :database, :hostname, :rum_enabled, :password])
+  end
+
+  defp maybe_add_password(%{changes: %{password: _}} = changeset, _), do: changeset
+
+  defp maybe_add_password(changeset, opts) do
+    if opts[:generate_password] do
+      generated = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
+      change(changeset, password: generated)
+    else
+      changeset
+    end
+  end
+
+  @spec save_credentials(Ecto.Changeset.t()) ::
+          :ok
+          | {:error, :psql_commands_execution, Path.t()}
+          | {:error, Ecto.Changeset.t()}
+          | {:error, term(), keyword()}
+  def save_credentials(changeset) do
+    with {:ok, struct} <- apply_action(changeset, :insert) do
+      struct
+      |> Map.from_struct()
+      |> Keyword.new()
+      |> check_database()
+      |> maybe_create_database()
+      |> write_config_file()
+    end
+  end
+
+  @spec check_database_and_write_config() :: :ok | {:error, term(), keyword()}
+  def check_database_and_write_config do
+    :credentials
+    |> Config.get()
+    |> check_database()
+    |> write_config_file()
+  end
+
+  defp check_database(credentials) when is_list(credentials) do
+    case @repo.check_database(credentials) do
+      :ok -> credentials
+      error -> Tuple.append(error, credentials)
+    end
+  end
+
+  defp maybe_create_database(credentials) when is_list(credentials), do: credentials
+
+  defp maybe_create_database({:error, _, credentials}) do
+    # we stop started repo in `check_database`
+    @repo.stop()
+
+    case @repo.create_database(credentials) do
+      :ok ->
+        @repo.stop()
+        check_database(credentials)
+
+      {:error, :create_repo} ->
+        # something went wrong with repo creation,
+        # we save psql file and let the user to run it manually
+        Config.put(:credentials, credentials)
+        psql_path = "/tmp/setup_db.psql"
+
+        Logger.warn("Writing the postgres script to #{psql_path}.")
+
+        psql =
+          EEx.eval_file(
+            "installer/templates/sample_psql.eex",
+            credentials
+          )
+
+        with :ok <- File.write(psql_path, psql) do
+          {:error, :create_repo, psql_path}
+        end
+
+      error ->
+        error
+    end
+  end
+
+  defp write_config_file(credentials) when is_list(credentials) do
+    config_path = Pleroma.Application.config_path()
+
+    config = EEx.eval_file("installer/templates/credentials.eex", credentials)
+
+    case File.write(config_path, config) do
+      :ok ->
+        updated_config = Keyword.merge(Repo.config(), credentials)
+
+        Config.put(Repo, updated_config)
+        Config.put([:database, :rum_enabled], credentials[:rum_enabled])
+        Config.put(:credentials, credentials)
+
+        @repo.stop()
+
+      error ->
+        error
+    end
+  end
+
+  defp write_config_file(error), do: error
+
+  @spec migrations() :: :ok | {:error, :migrations_error}
+  def migrations do
+    path = Ecto.Migrator.migrations_path(Repo)
+
+    paths =
+      if Config.get([:database, :rum_enabled]) do
+        [path, "priv/repo/optional_migrations/rum_indexing/"]
+      else
+        path
+      end
+
+    Config.get(:credentials)
+    |> @repo.run_migrations(paths)
+  end
+end

installer/pleroma/installer_web/router.ex

@@ -0,0 +1,31 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.InstallerWeb.Router do
+  use Pleroma.InstallerWeb, :router
+
+  pipeline :browser do
+    plug(:accepts, ["html"])
+    plug(:fetch_session)
+    plug(:fetch_flash)
+    plug(:protect_from_forgery)
+    plug(:put_secure_browser_headers)
+  end
+
+  scope "/", Pleroma.InstallerWeb do
+    pipe_through(:browser)
+
+    get("/", SetupController, :index)
+
+    post("/credentials", SetupController, :save_credentials)
+
+    get("/recheck", SetupController, :recheck)
+
+    get("/migrations", SetupController, :migrations)
+    get("/run_migrations", SetupController, :run_migrations)
+
+    get("/config", SetupController, :config)
+    post("/config", SetupController, :save_config)
+  end
+end

installer/pleroma/installer_web/templates/layout/app.html.eex

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8"/>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+    <title>Pleroma Installer</title>
+    <link rel=icon type=image/png href=<%= Routes.static_path(@conn, "/images/favicon.png") %>>
+    <link rel="stylesheet" href="<%= Routes.static_path(@conn, "/css/app.css") %>"/>
+  </head>
+  <body>
+    <header>
+      <div class="container" style="margin-top: 3rem;">
+        <div style="float: right; margin: -20px -20px 0 0;"><img width="80" src="<%= Routes.static_path(@conn, "/images/logo.svg") %>" alt="Pleroma Logo"/></div>
+        <h1>Pleroma Installer</h1>
+      </div>
+    </header>
+    <main role="main" class="container">
+      <p class="alert alert-info" role="alert"><%= get_flash(@conn, :info) %></p>
+      <p class="alert alert-danger" role="alert"><%= get_flash(@conn, :error) %></p>
+      <%= @inner_content %>
+    </main>
+    <script type="text/javascript" src="<%= Routes.static_path(@conn, "/js/app.js") %>"></script>
+  </body>
+</html>

installer/pleroma/installer_web/templates/setup/config.html.eex

@@ -0,0 +1,124 @@
+<section class="row">
+  <article class="column">
+    <%= if @error do %>
+      <p style="color: red;"> Error occuried: <%= @error %></p>
+      <a class="button" href="<%= Routes.setup_path(@conn, :index) %>">Try one more time</a>
+    <% end %>
+    <%= form_for @config, Routes.setup_path(@conn, :save_config), fn f -> %>
+      <div>
+        <h4>What is the name of your instance?</h4>
+        <%= text_input f, :instance_name, placeholder: "Pleroma/Soykaf"%>
+        <%= error_tag f, :instance_name %>
+      </div>
+      <div>
+        <h4>What directory should custom public files be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)?</h4>
+        <%= text_input f, :instance_static_dir, placeholder: "instance/static" %>
+        <%= error_tag f, :instance_static_dir %>
+      </div>
+      <div>
+        <h4>What directory should media uploads go in (when using the local uploader)?</h4>
+        <%= text_input f, :local_uploads_dir, placeholder: "uploads" %>
+        <%= error_tag f, :local_uploads_dir %>
+      </div>
+      <hr>
+
+      <div>
+        <div class="row">
+          <div class="column">
+            <label>URL</label>
+            <%= text_input f, :endpoint_url, placeholder: "https://pleroma.com" %>
+            <%= error_tag f, :endpoint_url %>
+          </div>
+        </div>
+      </div>
+      <hr>
+
+      <div>
+        <h4>Network Settings</h4>
+        <div class="row">
+          <div class="column">
+            <label>IP</label>
+            <%= text_input f, :endpoint_http_ip, placeholder: "127.0.0.1" %>
+            <%= error_tag f, :endpoint_http_ip %>
+          </div>
+          <div class="column">
+            <label>Port</label>
+            <%= text_input f, :endpoint_http_port, placeholder: 4000 %>
+            <%= error_tag f, :endpoint_http_port %>
+          </div>
+        </div>
+      </div>
+
+      <div>
+        <h4>Email</h4>
+        <div class="row">
+          <div class="column">
+            <label>Admin Email</label>
+            <%= email_input f, :instance_email, placeholder: "user@example.com" %>
+            <%= error_tag f, :instance_email %>
+          </div>
+          <div class="column">
+            <label>Email For Sending Notifications</label>
+            <%= email_input f, :instance_notify_email, placeholder: "user@example.com" %>
+            <%= error_tag f, :instance_notify_email %>
+          </div>
+        </div>
+      </div>
+      <hr>
+
+      <div>
+        <h4>Do you want to deny Search Engine bots from crawling the site?</h4>
+        <%= radio_button f, :indexable, true %>
+        <%= label f, :indexable, "No", class: "label-inline" %>
+        <%= radio_button f, :indexable, false, style: "margin-left: 1.5rem;" %>
+        <%= label f, :indexable, "Yes", class: "label-inline" %>
+      </div>
+      <hr>
+      <div>
+        <h4>Do you want to store the configuration in the database (allows controlling it from admin-fe)?</h4>
+        <%= radio_button f, :configurable_from_database, true %>
+        <%= label f, :configurable_from_database, "Yes", class: "label-inline" %>
+        <%= radio_button f, :configurable_from_database, false, style: "margin-left: 1.5rem;" %>
+        <%= label f, :configurable_from_database, "No", class: "label-inline" %>
+      </div>
+      <hr>
+      <div>
+        <h4>Admin user</h4>
+        <h4>Fill in the fields below, if you want to create admin user.</h4>
+        <%= radio_button f, :create_admin_user, true %>
+        <%= label f, :create_admin_user, "Yes", class: "label-inline" %>
+        <%= radio_button f, :create_admin_user, false, style: "margin-left: 1.5rem;" %>
+        <%= label f, :create_admin_user, "No", class: "label-inline" %>
+      </div>
+
+      <div>
+        <div class="row">
+          <div class="column">
+            <label>Nickname</label>
+            <%= text_input f, :admin_nickname, placeholder: "Lain" %>
+            <%= error_tag f, :admin_nickname %>
+          </div>
+          <div class="column">
+            <label>Email</label>
+            <%= email_input f, :admin_email, placeholder: "user@example.com" %>
+            <%= error_tag f, :admin_email %>
+          </div>
+        </div>
+      </div>
+
+      <div>
+        <div class="row">
+          <div class="column">
+            <label>Password</label>
+            <%= password_input f, :admin_password %>
+            <%= error_tag f, :admin_password %>
+          </div>
+        </div>
+      </div>
+
+      <hr>
+
+      <%= submit "Save Config" %>
+    <% end %>
+  </article>
+</section>

installer/pleroma/installer_web/templates/setup/index.html.eex

@@ -0,0 +1,48 @@
+<section class="row">
+  <article class="column">
+    <%= if @error do %>
+    <p style="color: red;"> Error occuried: <%= @error %></p>
+    <% end %>
+    <%= form_for @credentials, Routes.setup_path(@conn, :save_credentials), fn f -> %>
+    <div>
+      <h3>Database settings</h3>
+      <p>If you have already configured PostgreSQL, please fill in your credentials. If not, please fill in preferred credentials, so the installer can prepare the database.</p>
+      <div class="row">
+        <div class="column">
+          <%= label f, :hostname %>
+          <%= text_input f, :hostname %>
+          <%= error_tag f, :hostname %>
+        </div>
+        <div class="column">
+          <%= label f, :database %>
+          <%= text_input f, :database %>
+          <%= error_tag f, :database %>
+        </div>
+      </div>
+      <div class="row">
+        <div class="column">
+          <%= label f, :username %>
+          <%= text_input f, :username %>
+          <%= error_tag f, :username %>
+        </div>
+        <div class="column">
+          <%= label f, :password, "Password (if empty will be autogenerated)" %>
+          <%= text_input f, :password %>
+          <%= error_tag f, :password %>
+        </div>
+
+      </div>
+      <div>
+        <h4>Would you like to use RUM indices?</h4>
+        <%= radio_button f, :rum_enabled, true %>
+        <%= label f, :rum_enabled, "Yes", class: "label-inline" %>
+        <%= radio_button f, :rum_enabled, false, style: "margin-left: 1.5rem;" %>
+        <%= label f, :rum_enabled, "No", class: "label-inline" %>
+        <p>RUM extension must be <a href="https://github.com/postgrespro/rum#installation" target="_blank">installed</a></p>
+      </div>
+    </div>
+    <hr>
+    <%= submit "Save credentials" %>
+    <% end %>
+  </article>
+</section>

installer/pleroma/installer_web/templates/setup/migrations.html.eex

@@ -0,0 +1,8 @@
+<div class="container">
+  <div class="row">
+    <div class="column" id="migrations">
+      <p>The database is almost ready. Migrations are running.</p>
+      <p style="color: red; visibility: hidden;" id="error">Something went wrong. <a class="button" href="<%= Routes.setup_path(@conn, :index) %>">Start over</a></p>
+    </div>
+  </div>
+</div>

installer/pleroma/installer_web/templates/setup/run_psql.html.eex

@@ -0,0 +1,19 @@
+<div class="container">
+  <div class="row">
+    <div class="column">
+      <%= if @error do %>
+        <p style="color: red;"><%= @error %></p>
+        <p>Are you sure psql file was executed?</p>
+
+        <a class="button" href="<%= Routes.setup_path(@conn, :recheck) %>">Try one more time</a>
+        <a class="button" href="<%= Routes.setup_path(@conn, :index) %>">Start over</a>
+      <% else %>
+        <p>Run following command to setup PostgreSQL:</p>
+        <pre><code>
+        sudo -Hu postgres psql -f <%= @psql_path %>
+        </code></pre>
+        <a class="button" href="<%= Routes.setup_path(@conn, :recheck) %>">Next</a>
+      <% end %>
+    </div>
+  </div>
+</div>

installer/pleroma/installer_web/views/error_view.ex

@@ -0,0 +1,11 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.InstallerWeb.ErrorView do
+  use Pleroma.InstallerWeb, :view
+
+  def template_not_found(template, _assigns) do
+    Phoenix.Controller.status_message_from_template(template)
+  end
+end

installer/pleroma/installer_web/views/layout_view.ex

@@ -0,0 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.InstallerWeb.LayoutView do
+  use Pleroma.InstallerWeb, :view
+end

installer/pleroma/installer_web/views/setup_view.ex

@@ -0,0 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.InstallerWeb.SetupView do
+  use Pleroma.InstallerWeb, :view
+end

installer/templates/config_full.eex

@@ -0,0 +1,53 @@
+config :pleroma, Pleroma.Web.Endpoint,
+   url: [host: "<%= endpoint_url_host %>", scheme: "<%= endpoint_url_scheme %>", port: <%= endpoint_url_port %>],
+   http: [ip: {<%= String.replace(endpoint_http_ip, ".", ", ") %>}, port: <%= endpoint_http_port %>],
+   secret_key_base: "<%= endpoint_secret_key_base %>",
+   signing_salt: "<%= endpoint_signing_salt %>"
+
+config :pleroma, :instance,
+  name: "<%= instance_name %>",
+  email: "<%= instance_email %>",
+  notify_email: "<%= instance_notify_email %>",
+  limit: 5000,
+  registrations_open: true,
+  static_dir: "<%= instance_static_dir %>"
+
+config :pleroma, :media_proxy,
+  enabled: false,
+  redirect_on_failure: true
+  #base_url: "https://cache.pleroma.social"
+
+# Configure web push notifications
+config :web_push_encryption, :vapid_details,
+  subject: "mailto:<%= instance_email %>",
+  public_key: "<%= web_push_encryption_public_key %>",
+  private_key: "<%= web_push_encryption_private_key %>"
+
+config :pleroma, Pleroma.Uploaders.Local, uploads: "<%= local_uploads_dir %>"
+
+# Enable Strict-Transport-Security once SSL is working:
+# config :pleroma, :http_security,
+#   sts: true
+
+# Configure S3 support if desired.
+# The public S3 endpoint is different depending on region and provider,
+# consult your S3 provider's documentation for details on what to use.
+#
+# config :pleroma, Pleroma.Uploaders.S3,
+#   bucket: "some-bucket",
+#   public_endpoint: "https://s3.amazonaws.com"
+#
+# Configure S3 credentials:
+# config :ex_aws, :s3,
+#   access_key_id: "xxxxxxxxxxxxx",
+#   secret_access_key: "yyyyyyyyyyyy",
+#   region: "us-east-1",
+#   scheme: "https://"
+#
+# For using third-party S3 clones like wasabi, also do:
+# config :ex_aws, :s3,
+#   host: "s3.wasabisys.com"
+
+config :joken, default_signer: "<%= joken_default_signer %>"
+
+config :pleroma, configurable_from_database: <%= configurable_from_database %>

installer/templates/config_part.eex

@@ -0,0 +1,9 @@
+config :pleroma, Pleroma.Web.Endpoint,
+  url: [host: "<%= endpoint_url_host %>", scheme: "<%= endpoint_url_scheme %>", port: <%= endpoint_url_port %>],
+  http: [ip: {<%= String.replace(endpoint_http_ip, ".", ", ") %>}, port: <%= endpoint_http_port %>],
+  secret_key_base: "<%= endpoint_secret_key_base %>",
+  signing_salt: "<%= endpoint_signing_salt %>"
+
+config :pleroma, configurable_from_database: <%= configurable_from_database %>
+
+config :joken, default_signer: "<%= joken_default_signer %>"

installer/templates/credentials.eex

@@ -0,0 +1,20 @@
+# Pleroma instance configuration
+
+# NOTE: This file should not be committed to a repo or otherwise made public
+# without removing sensitive information.
+
+<%= if Code.ensure_loaded?(Config) or not Code.ensure_loaded?(Mix.Config) do
+  "import Config"
+else
+  "use Mix.Config"
+end %>
+
+config :pleroma, Pleroma.Repo,
+  adapter: Ecto.Adapters.Postgres,
+  username: "<%= username %>",
+  password: "<%= password %>",
+  database: "<%= database %>",
+  hostname: "<%= hostname %>",
+  pool_size: 10
+
+config :pleroma, :database, rum_enabled: <%= rum_enabled %>

installer/templates/sample_psql.eex

@@ -0,0 +1,11 @@
+CREATE USER <%= username %> WITH ENCRYPTED PASSWORD '<%= password %>';
+CREATE DATABASE <%= database %> OWNER <%= username %>;
+\c <%= database %>;
+CREATE EXTENSION IF NOT EXISTS citext;
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+<%= if rum_enabled do
+  "CREATE EXTENSION IF NOT EXISTS rum;"
+else
+""
+end %>

lib/mix/pleroma.ex

@@ -4,7 +4,6 @@
 
 defmodule Mix.Pleroma do
   @apps [
-    :restarter,
     :ecto,
     :ecto_sql,
     :postgrex,
@@ -16,12 +15,15 @@ defmodule Mix.Pleroma do
     :fast_html,
     :oban
   ]
+
   @cachex_children ["object", "user", "scrubber", "web_resp"]
+
   @doc "Common functions to be reused in mix tasks"
+  @spec start_pleroma() :: {:ok, pid()}
   def start_pleroma do
     Pleroma.Config.Holder.save_default()
-    Pleroma.Config.Oban.warn()
     Pleroma.Application.limiters_setup()
+    Pleroma.Config.DeprecationWarnings.check_oban_config()
     Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
 
     unless System.get_env("DEBUG") do
@@ -47,37 +49,28 @@ defmodule Mix.Pleroma do
       plugins: []
     ]
 
-    children =
-      [
-        Pleroma.Repo,
-        Pleroma.Emoji,
-        {Pleroma.Config.TransferTask, false},
-        Pleroma.Web.Endpoint,
-        {Oban, oban_config},
-        {Majic.Pool,
-         [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
-      ] ++
-        http_children(adapter)
+    children = [
+      Pleroma.Application.ConfigDependentDeps,
+      Pleroma.Repo,
+      Pleroma.Emoji,
+      Supervisor.child_spec({Task, &Pleroma.Application.Environment.load_from_db_and_update/0},
+        id: :update_env
+      ),
+      {Oban, oban_config},
+      {Majic.Pool,
+       [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]},
+      Pleroma.Web.Endpoint
+    ]
 
-    cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, []))
+    children = [Pleroma.Application.StartUpDependencies.adapter_module() | children]
+
+    cachex_children =
+      Enum.map(@cachex_children, &Pleroma.Application.StartUpDependencies.cachex_spec({&1, []}))
 
     Supervisor.start_link(children ++ cachex_children,
       strategy: :one_for_one,
       name: Pleroma.Supervisor
     )
-
-    if Pleroma.Config.get(:env) not in [:test, :benchmark] do
-      pleroma_rebooted?()
-    end
-  end
-
-  defp pleroma_rebooted? do
-    if Restarter.Pleroma.rebooted?() do
-      :ok
-    else
-      Process.sleep(10)
-      pleroma_rebooted?()
-    end
   end
 
   def load_pleroma do
@@ -129,11 +122,4 @@ defmodule Mix.Pleroma do
   def escape_sh_path(path) do
     ~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(')
   end
-
-  defp http_children(Tesla.Adapter.Gun) do
-    Pleroma.Gun.ConnectionPool.children() ++
-      [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
-  end
-
-  defp http_children(_), do: []
 end

lib/mix/tasks/pleroma/config.ex

@@ -14,10 +14,13 @@ defmodule Mix.Tasks.Pleroma.Config do
   @shortdoc "Manages the location of the config"
   @moduledoc File.read!("docs/administration/CLI_tasks/config.md")
 
-  def run(["migrate_to_db"]) do
+  def run(["migrate_to_db" | options]) do
     check_configdb(fn ->
       start_pleroma()
-      migrate_to_db()
+
+      {opts, _} = OptionParser.parse!(options, strict: [config: :string])
+
+      migrate_to_db(opts)
     end)
   end
 
@@ -39,15 +42,13 @@ defmodule Mix.Tasks.Pleroma.Config do
     check_configdb(fn ->
       start_pleroma()
 
-      header = config_header()
-
       settings =
         ConfigDB
         |> Repo.all()
         |> Enum.sort()
 
       unless settings == [] do
-        shell_info("#{header}")
+        shell_info("#{Pleroma.Config.Loader.config_header()}")
 
         Enum.each(settings, &dump(&1))
       else
@@ -73,9 +74,10 @@ defmodule Mix.Tasks.Pleroma.Config do
     check_configdb(fn ->
       start_pleroma()
 
-      group = maybe_atomize(group)
-
-      dump_group(group)
+      group
+      |> maybe_atomize()
+      |> ConfigDB.get_all_by_group()
+      |> Enum.each(&dump/1)
     end)
   end
 
@@ -100,7 +102,7 @@ defmodule Mix.Tasks.Pleroma.Config do
   def run(["reset", "--force"]) do
     check_configdb(fn ->
       start_pleroma()
-      truncatedb()
+      Pleroma.Config.Versioning.reset()
       shell_info("The ConfigDB settings have been removed from the database.")
     end)
   end
@@ -119,7 +121,7 @@ defmodule Mix.Tasks.Pleroma.Config do
       shell_error("\nTHIS CANNOT BE UNDONE!")
 
       if shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
-        truncatedb()
+        Pleroma.Config.Versioning.reset()
 
         shell_info("The ConfigDB settings have been removed from the database.")
       else
@@ -134,14 +136,12 @@ defmodule Mix.Tasks.Pleroma.Config do
     group = maybe_atomize(group)
     key = maybe_atomize(key)
 
-    with true <- key_exists?(group, key) do
+    with %ConfigDB{} = config <- ConfigDB.get_by_group_and_key(group, key) do
       shell_info("The following settings will be removed from ConfigDB:\n")
 
-      group
-      |> ConfigDB.get_by_group_and_key(key)
-      |> dump()
+      dump(config)
 
-      delete_key(group, key)
+      Pleroma.Config.Versioning.new_version(%{group: config.group, key: config.key, delete: true})
     else
       _ ->
         shell_error("No settings in ConfigDB for #{inspect(group)}, #{inspect(key)}. Aborting.")
@@ -153,12 +153,22 @@ defmodule Mix.Tasks.Pleroma.Config do
 
     group = maybe_atomize(group)
 
-    with true <- group_exists?(group) do
+    configs = ConfigDB.get_all_by_group(group)
+
+    if configs != [] do
       shell_info("The following settings will be removed from ConfigDB:\n")
-      dump_group(group)
-      delete_group(group)
+
+      Enum.each(configs, fn config ->
+        dump(config)
+
+        Pleroma.Config.Versioning.new_version(%{
+          group: config.group,
+          key: config.key,
+          delete: true
+        })
+      end)
     else
-      _ -> shell_error("No settings in ConfigDB for #{inspect(group)}. Aborting.")
+      shell_error("No settings in ConfigDB for #{inspect(group)}. Aborting.")
     end
   end
 
@@ -168,15 +178,17 @@ defmodule Mix.Tasks.Pleroma.Config do
     group = maybe_atomize(group)
     key = maybe_atomize(key)
 
-    with true <- key_exists?(group, key) do
+    with %ConfigDB{} = config <- ConfigDB.get_by_group_and_key(group, key) do
       shell_info("The following settings will be removed from ConfigDB:\n")
 
-      group
-      |> ConfigDB.get_by_group_and_key(key)
-      |> dump()
+      dump(config)
 
       if shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
-        delete_key(group, key)
+        Pleroma.Config.Versioning.new_version(%{
+          group: config.group,
+          key: config.key,
+          delete: true
+        })
       else
         shell_error("No changes made.")
       end
@@ -191,35 +203,67 @@ defmodule Mix.Tasks.Pleroma.Config do
 
     group = maybe_atomize(group)
 
-    with true <- group_exists?(group) do
+    configs = ConfigDB.get_all_by_group(group)
+
+    if configs != [] do
       shell_info("The following settings will be removed from ConfigDB:\n")
-      dump_group(group)
+      Enum.each(configs, &dump/1)
 
       if shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
-        delete_group(group)
+        Enum.each(configs, fn config ->
+          Pleroma.Config.Versioning.new_version(%{
+            group: config.group,
+            key: config.key,
+            delete: true
+          })
+        end)
       else
         shell_error("No changes made.")
       end
     else
-      _ -> shell_error("No settings in ConfigDB for #{inspect(group)}. Aborting.")
+      shell_error("No settings in ConfigDB for #{inspect(group)}. Aborting.")
     end
   end
 
-  @spec migrate_to_db(Path.t() | nil) :: any()
-  def migrate_to_db(file_path \\ nil) do
-    with :ok <- Pleroma.Config.DeprecationWarnings.warn() do
-      config_file =
-        if file_path do
-          file_path
-        else
-          if Pleroma.Config.get(:release) do
-            Pleroma.Config.get(:config_path)
-          else
-            "config/#{Pleroma.Config.get(:env)}.secret.exs"
-          end
-        end
+  def run(["rollback" | options]) do
+    check_configdb(fn ->
+      start_pleroma()
+      {opts, _} = OptionParser.parse!(options, strict: [steps: :integer], aliases: [s: :steps])
 
-      do_migrate_to_db(config_file)
+      do_rollback(opts)
+    end)
+  end
+
+  defp do_rollback(opts) do
+    steps = opts[:steps] || 1
+
+    case Pleroma.Config.Versioning.rollback(steps) do
+      {:ok, _} ->
+        shell_info("Success rollback")
+
+      {:error, :no_current_version} ->
+        shell_error("No version to rollback")
+
+      {:error, :rollback_not_possible} ->
+        shell_error("Rollback not possible. Incorrect steps value.")
+
+      {:error, _, _, _} ->
+        shell_error("Problem with backup. Rollback not possible.")
+
+      error ->
+        shell_error("error occuried: #{inspect(error)}")
+    end
+  end
+
+  defp migrate_to_db(opts) do
+    with :ok <- Pleroma.Config.DeprecationWarnings.warn() do
+      config_file = opts[:config] || Pleroma.Application.config_path()
+
+      if File.exists?(config_file) do
+        do_migrate_to_db(config_file)
+      else
+        shell_info("To migrate settings, you must define custom settings in #{config_file}.")
+      end
     else
       _ ->
         shell_error("Migration is not allowed until all deprecation warnings have been resolved.")
@@ -227,33 +271,9 @@ defmodule Mix.Tasks.Pleroma.Config do
   end
 
   defp do_migrate_to_db(config_file) do
-    if File.exists?(config_file) do
-      shell_info("Migrating settings from file: #{Path.expand(config_file)}")
-      truncatedb()
-
-      custom_config =
-        config_file
-        |> read_file()
-        |> elem(0)
-
-      custom_config
-      |> Keyword.keys()
-      |> Enum.each(&create(&1, custom_config))
-    else
-      shell_info("To migrate settings, you must define custom settings in #{config_file}.")
-    end
-  end
-
-  defp create(group, settings) do
-    group
-    |> Pleroma.Config.Loader.filter_group(settings)
-    |> Enum.each(fn {key, value} ->
-      {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value})
-
-      shell_info("Settings for key #{key} migrated.")
-    end)
-
-    shell_info("Settings for group #{inspect(group)} migrated.")
+    shell_info("Migrating settings from file: #{Path.expand(config_file)}")
+    {:ok, _} = Pleroma.Config.Versioning.migrate(config_file)
+    shell_info("Settings migrated.")
   end
 
   defp migrate_from_db(opts) do
@@ -270,12 +290,42 @@ defmodule Mix.Tasks.Pleroma.Config do
       |> Path.join("#{env}.exported_from_db.secret.exs")
 
     file = File.open!(config_path, [:write, :utf8])
+    IO.write(file, Pleroma.Config.Loader.config_header())
 
-    IO.write(file, config_header())
+    changes =
+      ConfigDB
+      |> Repo.all()
+      |> Enum.reduce([], fn %{group: group} = config, acc ->
+        group_str = inspect(group)
+        value = inspect(config.value, limit: :infinity)
 
-    ConfigDB
-    |> Repo.all()
-    |> Enum.each(&write_and_delete(&1, file, opts[:delete]))
+        msg =
+          if group in ConfigDB.groups_without_keys() do
+            IO.write(file, "config #{group_str}, #{value}\r\n\r\n")
+            "config #{group_str} was deleted."
+          else
+            key_str = inspect(config.key)
+            IO.write(file, "config #{group_str}, #{key_str}, #{value}\r\n\r\n")
+            "config #{group_str}, #{key_str} was deleted."
+          end
+
+        if opts[:delete] do
+          shell_info(msg)
+
+          change =
+            config
+            |> Map.take([:group, :key])
+            |> Map.put(:delete, true)
+
+          [change | acc]
+        else
+          acc
+        end
+      end)
+
+    if opts[:delete] and changes != [] do
+      Pleroma.Config.Versioning.new_version(changes)
+    end
 
     :ok = File.close(file)
     System.cmd("mix", ["format", config_path])
@@ -285,38 +335,6 @@ defmodule Mix.Tasks.Pleroma.Config do
     )
   end
 
-  if Code.ensure_loaded?(Config.Reader) do
-    defp config_header, do: "import Config\r\n\r\n"
-    defp read_file(config_file), do: Config.Reader.read_imports!(config_file)
-  else
-    defp config_header, do: "use Mix.Config\r\n\r\n"
-    defp read_file(config_file), do: Mix.Config.eval!(config_file)
-  end
-
-  defp write_and_delete(config, file, delete?) do
-    config
-    |> write(file)
-    |> delete(delete?)
-  end
-
-  defp write(config, file) do
-    value = inspect(config.value, limit: :infinity)
-
-    IO.write(file, "config #{inspect(config.group)}, #{inspect(config.key)}, #{value}\r\n\r\n")
-
-    config
-  end
-
-  defp delete(config, true) do
-    {:ok, _} = Repo.delete(config)
-
-    shell_info(
-      "config #{inspect(config.group)}, #{inspect(config.key)} was deleted from the ConfigDB."
-    )
-  end
-
-  defp delete(_config, _), do: :ok
-
   defp dump(%ConfigDB{} = config) do
     value = inspect(config.value, limit: :infinity)
 
@@ -325,31 +343,12 @@ defmodule Mix.Tasks.Pleroma.Config do
 
   defp dump(_), do: :noop
 
-  defp dump_group(group) when is_atom(group) do
-    group
-    |> ConfigDB.get_all_by_group()
-    |> Enum.each(&dump/1)
-  end
-
-  defp group_exists?(group) do
-    group
-    |> ConfigDB.get_all_by_group()
-    |> Enum.any?()
-  end
-
-  defp key_exists?(group, key) do
-    group
-    |> ConfigDB.get_by_group_and_key(key)
-    |> is_nil
-    |> Kernel.!()
-  end
-
   defp maybe_atomize(arg) when is_atom(arg), do: arg
 
   defp maybe_atomize(":" <> arg), do: maybe_atomize(arg)
 
   defp maybe_atomize(arg) when is_binary(arg) do
-    if ConfigDB.module_name?(arg) do
+    if Pleroma.Config.Converter.module_name?(arg) do
       String.to_existing_atom("Elixir." <> arg)
     else
       String.to_atom(arg)
@@ -366,23 +365,4 @@ defmodule Mix.Tasks.Pleroma.Config do
         )
     end
   end
-
-  defp delete_key(group, key) do
-    check_configdb(fn ->
-      ConfigDB.delete(%{group: group, key: key})
-    end)
-  end
-
-  defp delete_group(group) do
-    check_configdb(fn ->
-      group
-      |> ConfigDB.get_all_by_group()
-      |> Enum.each(&ConfigDB.delete/1)
-    end)
-  end
-
-  defp truncatedb do
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
-    Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
-  end
 end

lib/mix/tasks/pleroma/docs.ex

@@ -32,7 +32,7 @@ defmodule Mix.Tasks.Pleroma.Docs do
   defp do_run(implementation) do
     start_pleroma()
 
-    with descriptions <- Pleroma.Config.Loader.read("config/description.exs"),
+    with descriptions <- Pleroma.Config.Loader.read!("config/description.exs"),
          {:ok, file_path} <-
            Pleroma.Docs.Generator.process(
              implementation,

lib/mix/tasks/pleroma/instance.ex

@@ -86,9 +86,9 @@ defmodule Mix.Tasks.Pleroma.Instance do
         get_option(
           options,
           :indexable,
-          "Do you want search engines to index your site? (y/n)",
-          "y"
-        ) === "y"
+          "Do you want to deny Search Engine bots from crawling the site? (y/n)",
+          "n"
+        ) === "n"
 
       db_configurable? =
         get_option(
@@ -275,7 +275,8 @@ defmodule Mix.Tasks.Pleroma.Instance do
     end
   end
 
-  defp write_robots_txt(static_dir, indexable, template_dir) do
+  @spec write_robots_txt(Path.t(), boolean(), Path.t()) :: :ok | no_return()
+  def write_robots_txt(static_dir, indexable, template_dir) do
     robots_txt =
       EEx.eval_file(
         template_dir <> "/robots_txt.eex",
@@ -289,7 +290,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
       shell_info("Backing up existing robots.txt to #{robots_txt_path}.bak")
     end
 
-    File.write(robots_txt_path, robots_txt)
+    :ok = File.write(robots_txt_path, robots_txt)
     shell_info("Writing #{robots_txt_path}.")
   end
 

lib/pleroma/application.ex

@@ -5,8 +5,6 @@
 defmodule Pleroma.Application do
   use Application
 
-  import Cachex.Spec
-
   alias Pleroma.Config
 
   require Logger
@@ -15,12 +13,17 @@ defmodule Pleroma.Application do
   @version Mix.Project.config()[:version]
   @repository Mix.Project.config()[:source_url]
   @mix_env Mix.env()
+  @dynamic_supervisor Pleroma.Application.Supervisor
+
+  @type env() :: :test | :benchmark | :dev | :prod
 
   def name, do: @name
   def version, do: @version
   def named_version, do: @name <> " " <> @version
   def repository, do: @repository
+  def dynamic_supervisor, do: @dynamic_supervisor
 
+  @spec user_agent() :: String.t()
   def user_agent do
     if Process.whereis(Pleroma.Web.Endpoint) do
       case Config.get([:http, :user_agent], :default) do
@@ -37,9 +40,92 @@ defmodule Pleroma.Application do
     end
   end
 
-  # See http://elixir-lang.org/docs/stable/elixir/Application.html
-  # for more information on OTP Applications
+  @spec config_path() :: Path.t()
+  def config_path do
+    if Config.get(:release) do
+      Config.get(:config_path)
+    else
+      Config.get(:config_path_in_test) || "config/#{@mix_env}.secret.exs"
+    end
+  end
+
+  @doc """
+  Checks that config file exists and starts application, otherwise starts web UI for configuration.
+  Under main supervisor is started DynamicSupervisor, which later starts pleroma startup dependencies.
+  Pleroma start is splitted into three `phases`:
+    - running prestart requirements (runtime compilation, warnings, deprecations, monitoring, etc.)
+    - loading and updating environment (if database config is used and enabled)
+    - starting dependencies
+  """
+  @impl true
   def start(_type, _args) do
+    children = [
+      {DynamicSupervisor, strategy: :one_for_one, name: @dynamic_supervisor},
+      {Pleroma.Application.ConfigDependentDeps, [dynamic_supervisor: @dynamic_supervisor]}
+    ]
+
+    {:ok, main_supervisor} =
+      Supervisor.start_link(children, strategy: :one_for_one, name: Pleroma.Supervisor)
+
+    if @mix_env == :test or File.exists?(Pleroma.Application.config_path()) do
+      :ok = start_pleroma()
+    else
+      DynamicSupervisor.start_child(
+        @dynamic_supervisor,
+        Pleroma.InstallerWeb.Endpoint
+      )
+
+      token = Ecto.UUID.generate()
+
+      Pleroma.Config.put(:installer_token, token)
+
+      installer_port =
+        Pleroma.InstallerWeb.Endpoint
+        |> Pleroma.Config.get()
+        |> get_in([:http, :port])
+
+      ip =
+        with {:ok, ip} <- Pleroma.Helpers.ServerIPHelper.real_ip() do
+          ip
+        else
+          _ -> "IP not found"
+        end
+
+      Logger.warn("Access installer at http://#{ip}:#{installer_port}/?token=#{token}")
+    end
+
+    {:ok, main_supervisor}
+  end
+
+  defp start_pleroma do
+    {:ok, _} = DynamicSupervisor.start_child(@dynamic_supervisor, Pleroma.Repo)
+    run_prestart_requirements()
+
+    Pleroma.Application.Environment.load_from_db_and_update()
+
+    Pleroma.Application.StartUpDependencies.start_all(@mix_env)
+  end
+
+  @spec stop_installer_and_start_pleroma() :: {:ok, pid()}
+  def stop_installer_and_start_pleroma do
+    Pleroma.Application.config_path()
+    |> Pleroma.Application.Environment.update()
+
+    start_pleroma()
+
+    Task.start(fn ->
+      Process.sleep(100)
+
+      installer_endpoint = Process.whereis(Pleroma.InstallerWeb.Endpoint)
+
+      DynamicSupervisor.terminate_child(
+        @dynamic_supervisor,
+        installer_endpoint
+      )
+    end)
+  end
+
+  defp run_prestart_requirements do
     # Scrubbers are compiled at runtime and therefore will cause a conflict
     # every time the application is restarted, so we disable module
     # conflicts at runtime
@@ -47,72 +133,26 @@ defmodule Pleroma.Application do
     # Disable warnings_as_errors at runtime, it breaks Phoenix live reload
     # due to protocol consolidation warnings
     Code.compiler_options(warnings_as_errors: false)
-    Pleroma.Telemetry.Logger.attach()
-    Config.Holder.save_default()
+
+    # compilation in runtime
     Pleroma.HTML.compile_scrubbers()
-    Pleroma.Config.Oban.warn()
+    compile_custom_modules()
+    Pleroma.Docs.JSON.compile()
+
+    # telemetry and prometheus
+    Pleroma.Telemetry.Logger.attach()
+    setup_instrumenters()
+
+    Config.Holder.save_default()
+
     Config.DeprecationWarnings.warn()
     Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
-    Pleroma.ApplicationRequirements.verify!()
-    setup_instrumenters()
-    load_custom_modules()
-    Pleroma.Docs.JSON.compile()
+
     limiters_setup()
 
-    adapter = Application.get_env(:tesla, :adapter)
-
-    if adapter == Tesla.Adapter.Gun do
-      if version = Pleroma.OTPVersion.version() do
-        [major, minor] =
-          version
-          |> String.split(".")
-          |> Enum.map(&String.to_integer/1)
-          |> Enum.take(2)
-
-        if (major == 22 and minor < 2) or major < 22 do
-          raise "
-            !!!OTP VERSION WARNING!!!
-            You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. Please update your Erlang/OTP to at least 22.2.
-            "
-        end
-      else
-        raise "
-          !!!OTP VERSION WARNING!!!
-          To support correct handling of unordered certificates chains - OTP version must be > 22.2.
-          "
-      end
-    end
-
-    # Define workers and child supervisors to be supervised
-    children =
-      [
-        Pleroma.Repo,
-        Config.TransferTask,
-        Pleroma.Emoji,
-        Pleroma.Web.Plugs.RateLimiter.Supervisor
-      ] ++
-        cachex_children() ++
-        http_children(adapter, @mix_env) ++
-        [
-          Pleroma.Stats,
-          Pleroma.JobQueueMonitor,
-          {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
-          {Oban, Config.get(Oban)},
-          Pleroma.Web.Endpoint
-        ] ++
-        task_children(@mix_env) ++
-        dont_run_in_test(@mix_env) ++
-        chat_child(chat_enabled?()) ++
-        [Pleroma.Gopher.Server]
-
-    # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
-    # for other strategies and supported options
-    opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
-    result = Supervisor.start_link(children, opts)
-
     set_postgres_server_version()
 
-    result
+    Pleroma.Application.Requirements.verify!()
   end
 
   defp set_postgres_server_version do
@@ -132,7 +172,7 @@ defmodule Pleroma.Application do
     :persistent_term.put({Pleroma.Repo, :postgres_version}, version)
   end
 
-  def load_custom_modules do
+  defp compile_custom_modules do
     dir = Config.get([:modules, :runtime_dir])
 
     if dir && File.exists?(dir) do
@@ -177,128 +217,6 @@ defmodule Pleroma.Application do
     PrometheusPhx.setup()
   end
 
-  defp cachex_children do
-    [
-      build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
-      build_cachex("user", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
-      build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
-      build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
-      build_cachex("scrubber", limit: 2500),
-      build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
-      build_cachex("web_resp", limit: 2500),
-      build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
-      build_cachex("failed_proxy_url", limit: 2500),
-      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
-      build_cachex("chat_message_id_idempotency_key",
-        expiration: chat_message_id_idempotency_key_expiration(),
-        limit: 500_000
-      )
-    ]
-  end
-
-  defp emoji_packs_expiration,
-    do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60))
-
-  defp idempotency_expiration,
-    do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
-
-  defp chat_message_id_idempotency_key_expiration,
-    do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60))
-
-  defp seconds_valid_interval,
-    do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
-
-  @spec build_cachex(String.t(), keyword()) :: map()
-  def build_cachex(type, opts),
-    do: %{
-      id: String.to_atom("cachex_" <> type),
-      start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]},
-      type: :worker
-    }
-
-  defp chat_enabled?, do: Config.get([:chat, :enabled])
-
-  defp dont_run_in_test(env) when env in [:test, :benchmark], do: []
-
-  defp dont_run_in_test(_) do
-    [
-      {Registry,
-       [
-         name: Pleroma.Web.Streamer.registry(),
-         keys: :duplicate,
-         partitions: System.schedulers_online()
-       ]}
-    ] ++ background_migrators()
-  end
-
-  defp background_migrators do
-    [
-      Pleroma.Migrators.HashtagsTableMigrator
-    ]
-  end
-
-  defp chat_child(true) do
-    [
-      Pleroma.Web.ChatChannel.ChatChannelState,
-      {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
-    ]
-  end
-
-  defp chat_child(_), do: []
-
-  defp task_children(:test) do
-    [
-      %{
-        id: :web_push_init,
-        start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
-        restart: :temporary
-      }
-    ]
-  end
-
-  defp task_children(_) do
-    [
-      %{
-        id: :web_push_init,
-        start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
-        restart: :temporary
-      },
-      %{
-        id: :internal_fetch_init,
-        start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
-        restart: :temporary
-      }
-    ]
-  end
-
-  # start hackney and gun pools in tests
-  defp http_children(_, :test) do
-    http_children(Tesla.Adapter.Hackney, nil) ++ http_children(Tesla.Adapter.Gun, nil)
-  end
-
-  defp http_children(Tesla.Adapter.Hackney, _) do
-    pools = [:federation, :media]
-
-    pools =
-      if Config.get([Pleroma.Upload, :proxy_remote]) do
-        [:upload | pools]
-      else
-        pools
-      end
-
-    for pool <- pools do
-      options = Config.get([:hackney_pools, pool])
-      :hackney_pool.child_spec(pool, options)
-    end
-  end
-
-  defp http_children(Tesla.Adapter.Gun, _) do
-    Pleroma.Gun.ConnectionPool.children() ++
-      [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
-  end
-
-  defp http_children(_, _), do: []
-
   @spec limiters_setup() :: :ok
   def limiters_setup do
     config = Config.get(ConcurrentLimiter, [])

lib/pleroma/application/chat_supervisor.ex

@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Application.ChatSupervisor do
+  use Supervisor
+
+  def start_link(_) do
+    Supervisor.start_link(__MODULE__, :no_args)
+  end
+
+  def init(_) do
+    [
+      Pleroma.Web.ChatChannel.ChatChannelState,
+      {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
+    ]
+    |> Supervisor.init(strategy: :one_for_one)
+  end
+end

lib/pleroma/application/config_dependent_deps.ex

@@ -0,0 +1,244 @@
+# # Pleroma: A lightweight social networking server
+# # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# # SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Application.ConfigDependentDeps do
+  use GenServer
+
+  require Logger
+
+  @config_path_mods_relation [
+    {{:pleroma, :chat}, Pleroma.Application.ChatSupervisor},
+    {{:pleroma, Oban}, Oban},
+    {{:pleroma, :rate_limit}, Pleroma.Web.Plugs.RateLimiter.Supervisor},
+    {{:pleroma, :streamer}, Pleroma.Web.Streamer.registry()},
+    {{:pleroma, :pools}, Pleroma.Gun.GunSupervisor},
+    {{:pleroma, :connections_pool}, Pleroma.Gun.GunSupervisor},
+    {{:pleroma, :hackney_pools}, Pleroma.HTTP.HackneySupervisor},
+    {{:pleroma, :gopher}, Pleroma.Gopher.Server},
+    {{:pleroma, Pleroma.Captcha, [:seconds_valid]}, Pleroma.Web.Endpoint},
+    {{:pleroma, Pleroma.Upload, [:proxy_remote]},
+     Pleroma.Application.StartUpDependencies.adapter_module()},
+    {{:pleroma, :instance, [:upload_limit]}, Pleroma.Web.Endpoint},
+    {{:pleroma, :fed_sockets, [:enabled]}, Pleroma.Web.Endpoint},
+    {:eshhd, :eshhd},
+    {:ex_aws, :ex_aws}
+  ]
+
+  def start_link(opts) do
+    opts = Keyword.put_new(opts, :relations, @config_path_mods_relation)
+
+    GenServer.start_link(__MODULE__, opts, name: opts[:name] || __MODULE__)
+  end
+
+  @impl true
+  def init(opts) do
+    init_state = %{
+      dynamic_supervisor: opts[:dynamic_supervisor],
+      relations: opts[:relations],
+      reboot_paths: [],
+      pids: %{}
+    }
+
+    {:ok, init_state}
+  end
+
+  def start_dependency(module, server \\ __MODULE__) do
+    GenServer.call(server, {:start_dependency, module})
+  end
+
+  def need_reboot?(server \\ __MODULE__) do
+    GenServer.call(server, :need_reboot?)
+  end
+
+  def restart_dependencies(server \\ __MODULE__) do
+    GenServer.call(server, :restart_dependencies)
+  end
+
+  def clear_state(server \\ __MODULE__) do
+    GenServer.call(server, :clear_state)
+  end
+
+  def save_config_paths_for_restart(changes, server \\ __MODULE__) do
+    GenServer.call(server, {:save_config_paths, changes})
+  end
+
+  @impl true
+  def handle_call({:start_dependency, module}, _, state) do
+    {result, state} =
+      with {pid, state} when is_pid(pid) <- start_module(module, state) do
+        {{:ok, pid}, state}
+      else
+        error -> {error, state}
+      end
+
+    {:reply, result, state}
+  end
+
+  @impl true
+  def handle_call(:need_reboot?, _, state) do
+    {:reply, state[:reboot_paths] != [], state}
+  end
+
+  @impl true
+  def handle_call(:restart_dependencies, _, state) do
+    {paths, state} = Map.get_and_update(state, :reboot_paths, &{&1, []})
+    started_apps = Application.started_applications()
+
+    {result, state} =
+      Enum.reduce_while(paths, {:ok, state}, fn
+        path, {:ok, acc} when is_tuple(path) ->
+          case restart(path, acc, acc[:pids][path], with_terminate: true) do
+            {pid, state} when is_pid(pid) ->
+              {:cont, {:ok, state}}
+
+            :ignore ->
+              Logger.info("path #{inspect(path)} is ignored.")
+              {:cont, {:ok, acc}}
+
+            error ->
+              {:halt, {error, acc}}
+          end
+
+        app, {:ok, acc}
+        when is_atom(app) and app not in [:logger, :quack, :pleroma, :prometheus, :postgrex] ->
+          restart_app(app, started_apps)
+          {:cont, {:ok, acc}}
+      end)
+
+    {:reply, result, state}
+  end
+
+  @impl true
+  def handle_call(:clear_state, _, state) do
+    state =
+      state
+      |> Map.put(:reboot_paths, [])
+      |> Map.put(:pids, %{})
+
+    {:reply, :ok, state}
+  end
+
+  @impl true
+  def handle_call({:save_config_paths, changes}, _, state) do
+    paths =
+      Enum.reduce(changes, state[:reboot_paths], fn
+        %{group: group, key: key, value: value}, acc ->
+          with {path, _} <- find_relation(state[:relations], group, key, value) do
+            if path not in acc do
+              [path | acc]
+            else
+              acc
+            end
+          else
+            _ ->
+              acc
+          end
+      end)
+
+    {:reply, paths, put_in(state[:reboot_paths], paths)}
+  end
+
+  @impl true
+  def handle_info({:DOWN, _ref, :process, pid, _reason}, state) do
+    updated_state =
+      with {path, ^pid} <-
+             Enum.find(state[:pids], fn {_, registered_pid} -> registered_pid == pid end) do
+        {_new_pid, new_state} = restart(path, state, pid)
+        new_state
+      else
+        _ -> state
+      end
+
+    {:noreply, updated_state}
+  end
+
+  defp start_module(module, state) do
+    with {:ok, relations} <- find_relations(state[:relations], module) do
+      start_module(module, relations, state)
+    end
+  end
+
+  defp start_module(module, relations, state) do
+    spec =
+      module
+      |> Pleroma.Application.StartUpDependencies.spec()
+      |> Supervisor.child_spec(restart: :temporary)
+
+    with {:ok, pid} <-
+           DynamicSupervisor.start_child(
+             state[:dynamic_supervisor],
+             spec
+           ) do
+      pids = Map.new(relations, fn {path, _} -> {path, pid} end)
+      Process.monitor(pid)
+      {pid, put_in(state[:pids], Map.merge(state[:pids], pids))}
+    end
+  end
+
+  defp restart(path, state, pid, opts \\ [])
+
+  defp restart(path, state, nil, _) do
+    with {_, module} <- find_relation(state[:relations], path) do
+      start_module(module, state)
+    end
+  end
+
+  defp restart(path, state, pid, opts) when is_pid(pid) do
+    with {_, module} <- find_relation(state[:relations], path),
+         {:ok, relations} <- find_relations(state[:relations], module) do
+      if opts[:with_terminate] do
+        :ok = DynamicSupervisor.terminate_child(state[:dynamic_supervisor], pid)
+      end
+
+      paths_for_remove = Enum.map(relations, fn {path, _} -> path end)
+      state = put_in(state[:pids], Map.drop(state[:pids], paths_for_remove))
+
+      start_module(module, relations, state)
+    end
+  end
+
+  defp restart_app(app, started_applications) do
+    with {^app, _, _} <- List.keyfind(started_applications, app, 0) do
+      :ok = Application.stop(app)
+      :ok = Application.start(app)
+    else
+      nil ->
+        Logger.info("#{app} is not started.")
+
+      error ->
+        error
+        |> inspect()
+        |> Logger.error()
+    end
+  end
+
+  defp find_relations(relations, module) do
+    case Enum.filter(relations, fn {_, mod} -> mod == module end) do
+      [] ->
+        {:error, :relations_not_found}
+
+      relations ->
+        {:ok, relations}
+    end
+  end
+
+  defp find_relation(relations, group, key, value) do
+    Enum.find(relations, fn
+      {g, _} when is_atom(g) ->
+        g == group
+
+      {{g, k}, _} ->
+        g == group and k == key
+
+      {{g, k, subkeys}, _} ->
+        g == group and k == key and Enum.any?(Keyword.keys(value), &(&1 in subkeys))
+    end)
+  end
+
+  def find_relation(relations, path) do
+    with nil <- Enum.find(relations, fn {key, _} -> key == path end) do
+      {:error, :relation_not_found}
+    end
+  end
+end

lib/pleroma/application/environment.ex

@@ -0,0 +1,105 @@
+# # Pleroma: A lightweight social networking server
+# # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# # SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Application.Environment do
+  @moduledoc """
+  Overwrites environment config with settings from config file or database.
+  """
+
+  require Logger
+
+  @doc """
+  Method is called on pleroma start.
+  Config dependent parts don't require restart, because are not started yet.
+  But started apps need restart.
+  """
+  @spec load_from_db_and_update() :: :ok
+  def load_from_db_and_update do
+    Pleroma.ConfigDB.all()
+    |> update(restart_apps: true)
+  end
+
+  @spec update(Path.t()) :: :ok
+  def update(config_path) when is_binary(config_path) do
+    config_path
+    |> Pleroma.Config.Loader.read!()
+    |> Application.put_all_env()
+  end
+
+  @spec update([Pleroma.ConfigDB.t()], keyword()) :: :ok
+  def update(changes, opts \\ []) when is_list(changes) do
+    if Pleroma.Config.get(:configurable_from_database) do
+      defaults = Pleroma.Config.Holder.default_config()
+
+      changes
+      |> filter_logger()
+      |> prepare_logger_changes(defaults)
+      |> Enum.each(&configure_logger/1)
+
+      changes
+      |> Pleroma.ConfigDB.merge_changes_with_defaults(defaults)
+      |> Enum.each(&update_env(&1))
+
+      if opts[:restart_apps] do
+        # restart only apps on pleroma start
+        changes
+        |> Enum.filter(fn %{group: group} ->
+          group not in [:logger, :quack, :pleroma, :prometheus, :postgrex]
+        end)
+        |> Pleroma.Application.ConfigDependentDeps.save_config_paths_for_restart()
+
+        Pleroma.Application.ConfigDependentDeps.restart_dependencies()
+      else
+        Pleroma.Application.ConfigDependentDeps.save_config_paths_for_restart(changes)
+      end
+    end
+
+    :ok
+  end
+
+  defp filter_logger(changes) do
+    Enum.filter(changes, fn %{group: group} -> group in [:logger, :quack] end)
+  end
+
+  defp prepare_logger_changes(changes, defaults) do
+    Enum.map(changes, fn %{group: group} = change ->
+      {change, Pleroma.ConfigDB.merge_change_value_with_default(change, defaults[group])}
+    end)
+  end
+
+  defp configure_logger({%{group: :quack}, merged_value}) do
+    Logger.configure_backend(Quack.Logger, merged_value)
+  end
+
+  defp configure_logger({%{group: :logger} = change, merged_value}) do
+    if change.value[:backends] do
+      Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1)
+
+      Enum.each(merged_value[:backends], &Logger.add_backend/1)
+    end
+
+    if change.value[:console] do
+      console = merged_value[:console]
+      console = put_in(console[:format], console[:format] <> "\n")
+
+      Logger.configure_backend(:console, console)
+    end
+
+    if change.value[:ex_syslogger] do
+      Logger.configure_backend({ExSyslogger, :ex_syslogger}, merged_value[:ex_syslogger])
+    end
+
+    Logger.configure(merged_value)
+  end
+
+  defp update_env(%{group: group, key: key, value: nil}), do: Application.delete_env(group, key)
+
+  defp update_env(%{group: group, value: config} = change) do
+    if group in Pleroma.ConfigDB.groups_without_keys() do
+      Application.put_all_env([{group, config}])
+    else
+      Application.put_env(group, change.key, config)
+    end
+  end
+end

lib/pleroma/application/requirements.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.ApplicationRequirements do
+defmodule Pleroma.Application.Requirements do
   @moduledoc """
   The module represents the collection of validations to runs before start server.
   """
@@ -18,6 +18,8 @@ defmodule Pleroma.ApplicationRequirements do
 
   @spec verify!() :: :ok | VerifyError.t()
   def verify! do
+    adapter = Application.get_env(:tesla, :adapter)
+
     :ok
     |> check_system_commands!()
     |> check_confirmation_accounts!()
@@ -25,11 +27,12 @@ defmodule Pleroma.ApplicationRequirements do
     |> check_welcome_message_config!()
     |> check_rum!()
     |> check_repo_pool_size!()
-    |> handle_result()
+    |> check_otp_version!(adapter)
+    |> handle_result!()
   end
 
-  defp handle_result(:ok), do: :ok
-  defp handle_result({:error, message}), do: raise(VerifyError, message: message)
+  defp handle_result!(:ok), do: :ok
+  defp handle_result!({:error, message}), do: raise(VerifyError, message: message)
 
   defp check_welcome_message_config!(:ok) do
     if Pleroma.Config.get([:welcome, :email, :enabled], false) and
@@ -160,9 +163,9 @@ defmodule Pleroma.ApplicationRequirements do
 
   defp check_system_commands!(:ok) do
     filter_commands_statuses = [
-      check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"),
-      check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"),
-      check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify")
+      check_filter!(Pleroma.Upload.Filters.Exiftool, "exiftool"),
+      check_filter!(Pleroma.Upload.Filters.Mogrify, "mogrify"),
+      check_filter!(Pleroma.Upload.Filters.Mogrifun, "mogrify")
     ]
 
     preview_proxy_commands_status =
@@ -213,7 +216,7 @@ defmodule Pleroma.ApplicationRequirements do
 
   defp check_repo_pool_size!(result), do: result
 
-  defp check_filter(filter, command_required) do
+  defp check_filter!(filter, command_required) do
     filters = Config.get([Pleroma.Upload, :filters])
 
     if filter in filters and not Pleroma.Utils.command_available?(command_required) do
@@ -227,4 +230,32 @@ defmodule Pleroma.ApplicationRequirements do
       true
     end
   end
+
+  defp check_otp_version!(:ok, Tesla.Adapter.Gun) do
+    if version = Pleroma.OTPVersion.version() do
+      [major, minor] =
+        version
+        |> String.split(".")
+        |> Enum.map(&String.to_integer/1)
+        |> Enum.take(2)
+
+      if (major == 22 and minor < 2) or major < 22 do
+        Logger.error("
+            !!!OTP VERSION ERROR!!!
+            You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. Please update your Erlang/OTP to at least 22.2.
+            ")
+        {:error, "OTP version error"}
+      else
+        :ok
+      end
+    else
+      Logger.error("
+          !!!OTP VERSION ERROR!!!
+          To support correct handling of unordered certificates chains - OTP version must be > 22.2.
+          ")
+      {:error, "OTP version error"}
+    end
+  end
+
+  defp check_otp_version!(result, _), do: result
 end

lib/pleroma/application/start_up_dependencies.ex

@@ -0,0 +1,182 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Application.StartUpDependencies do
+  alias Pleroma.Config
+  alias Pleroma.Web.Endpoint
+
+  require Cachex.Spec
+  require Logger
+
+  @type config_path() :: {atom(), atom()} | {atom(), atom(), [atom()]}
+  @type relation() :: {config_path(), module()}
+
+  @spec start_all(Pleroma.Application.env()) ::
+          :ok | {:error, {:already_started, pid()} | :max_children | term()}
+  def start_all(env) do
+    with :ok <- start_common_deps(env),
+         :ok <- start_config_dependent_deps(env) do
+      :ok
+    end
+  end
+
+  @spec adapter_module() :: module()
+  def adapter_module do
+    if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do
+      Pleroma.Gun.GunSupervisor
+    else
+      Pleroma.HTTP.HackneySupervisor
+    end
+  end
+
+  @spec spec(module()) :: module() | {module(), keyword()}
+  def spec(Oban), do: {Oban, Config.get(Oban)}
+
+  def spec(Pleroma.Web.StreamerRegistry) do
+    {Registry,
+     [
+       name: Pleroma.Web.Streamer.registry(),
+       keys: :duplicate,
+       partitions: System.schedulers_online()
+     ]}
+  end
+
+  def spec(child), do: child
+
+  @spec cachex_spec({String.t(), keyword()}) :: :supervisor.child_spec()
+  def cachex_spec({type, opts}) do
+    %{
+      id: String.to_atom("cachex_" <> type),
+      start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]},
+      type: :worker
+    }
+  end
+
+  defp start_common_deps(env) do
+    fun = fn child ->
+      DynamicSupervisor.start_child(Pleroma.Application.dynamic_supervisor(), spec(child))
+    end
+
+    [
+      Pleroma.Emoji,
+      Pleroma.Stats,
+      Pleroma.JobQueueMonitor,
+      {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
+      %{
+        id: :web_push_init,
+        start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
+        restart: :temporary
+      }
+    ]
+    |> add_cachex_deps()
+    |> maybe_add_init_internal_fetch_actor_task(env)
+    |> maybe_add_background_migrator(env)
+    |> start_while(fun)
+  end
+
+  defp start_config_dependent_deps(env) do
+    fun = fn child -> Pleroma.Application.ConfigDependentDeps.start_dependency(child) end
+
+    [
+      Pleroma.Web.Plugs.RateLimiter.Supervisor,
+      Oban,
+      Endpoint,
+      Pleroma.Gopher.Server
+    ]
+    |> add_http_children(env)
+    |> maybe_add(:streamer, env)
+    |> maybe_add_chat_child()
+    |> start_while(fun)
+  end
+
+  defp start_while(deps, fun) do
+    Enum.reduce_while(deps, :ok, fn child, acc ->
+      case fun.(child) do
+        {:ok, _} ->
+          {:cont, acc}
+
+        # consider this behavior is normal
+        :ignore ->
+          Logger.info("#{inspect(child)} is ignored.")
+          {:cont, acc}
+
+        error ->
+          Logger.error("Child #{inspect(child)} can't be started. #{inspect(error)}")
+          {:halt, error}
+      end
+    end)
+  end
+
+  @spec cachex_deps() :: [tuple()]
+  def cachex_deps do
+    captcha_clean_up_interval =
+      [Pleroma.Captcha, :seconds_valid]
+      |> Config.get!()
+      |> :timer.seconds()
+
+    [
+      {"used_captcha", expiration: Cachex.Spec.expiration(interval: captcha_clean_up_interval)},
+      {"user", expiration: cachex_expiration(25_000, 1000), limit: 2500},
+      {"object", expiration: cachex_expiration(25_000, 1000), limit: 2500},
+      {"rich_media",
+       expiration: Cachex.Spec.expiration(default: :timer.minutes(120)), limit: 5000},
+      {"scrubber", limit: 2500},
+      {"idempotency", expiration: cachex_expiration(21_600, 60), limit: 2500},
+      {"web_resp", limit: 2500},
+      {"emoji_packs", expiration: cachex_expiration(300, 60), limit: 10},
+      {"failed_proxy_url", limit: 2500},
+      {"banned_urls",
+       expiration: Cachex.Spec.expiration(default: :timer.hours(24 * 30)), limit: 5_000},
+      {"chat_message_id_idempotency_key",
+       expiration: cachex_expiration(:timer.minutes(2), :timer.seconds(60)), limit: 500_000}
+    ]
+  end
+
+  defp add_cachex_deps(application_deps) do
+    cachex_deps()
+    |> Enum.reduce(application_deps, fn cachex_init_args, acc ->
+      [cachex_spec(cachex_init_args) | acc]
+    end)
+  end
+
+  defp cachex_expiration(default, interval) do
+    Cachex.Spec.expiration(default: :timer.seconds(default), interval: :timer.seconds(interval))
+  end
+
+  defp maybe_add_init_internal_fetch_actor_task(children, :test), do: children
+
+  defp maybe_add_init_internal_fetch_actor_task(children, _) do
+    [
+      %{
+        id: :internal_fetch_init,
+        start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
+        restart: :temporary
+      }
+      | children
+    ]
+  end
+
+  defp maybe_add_background_migrator(children, env) when env in [:test, :benchmark], do: children
+
+  defp maybe_add_background_migrator(children, _) do
+    [Pleroma.Migrators.HashtagsTableMigrator | children]
+  end
+
+  defp maybe_add(children, _, env) when env in [:test, :benchmark], do: children
+  defp maybe_add(children, :streamer, _), do: [Pleroma.Web.Streamer.registry() | children]
+
+  defp add_http_children(children, :test) do
+    [Pleroma.HTTP.HackneySupervisor, Pleroma.Gun.GunSupervisor | children]
+  end
+
+  defp add_http_children(children, _), do: [adapter_module() | children]
+
+  defp maybe_add_chat_child(children) do
+    if Config.get([:chat, :enabled]) do
+      [Pleroma.Application.ChatSupervisor | children]
+    else
+      children
+    end
+  end
+end

lib/pleroma/config/converter.ex

@@ -0,0 +1,195 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Config.Converter do
+  @moduledoc """
+  Converts json structures with strings into elixir structures and types and vice versa.
+  """
+  @spec to_elixir_types(boolean() | String.t() | map() | list()) :: term()
+  def to_elixir_types(%{"tuple" => [":args", args]}) when is_list(args) do
+    arguments =
+      Enum.map(args, fn arg ->
+        if String.contains?(arg, ["{", "}"]) do
+          {elem, []} = Code.eval_string(arg)
+          elem
+        else
+          to_elixir_types(arg)
+        end
+      end)
+
+    {:args, arguments}
+  end
+
+  def to_elixir_types(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}) do
+    {:proxy_url, {string_to_elixir_types!(type), parse_host(host), port}}
+  end
+
+  def to_elixir_types(%{"tuple" => [":partial_chain", entity]}) do
+    {partial_chain, []} =
+      entity
+      |> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "")
+      |> Code.eval_string()
+
+    {:partial_chain, partial_chain}
+  end
+
+  def to_elixir_types(%{"tuple" => entity}) do
+    Enum.reduce(entity, {}, &Tuple.append(&2, to_elixir_types(&1)))
+  end
+
+  def to_elixir_types(entity) when is_map(entity) do
+    Map.new(entity, fn {k, v} -> {to_elixir_types(k), to_elixir_types(v)} end)
+  end
+
+  def to_elixir_types(entity) when is_list(entity) do
+    Enum.map(entity, &to_elixir_types/1)
+  end
+
+  def to_elixir_types(entity) when is_binary(entity) do
+    entity
+    |> String.trim()
+    |> string_to_elixir_types!()
+  end
+
+  def to_elixir_types(entity), do: entity
+
+  defp parse_host("localhost"), do: :localhost
+
+  defp parse_host(host) do
+    charlist = to_charlist(host)
+
+    case :inet.parse_address(charlist) do
+      {:error, :einval} ->
+        charlist
+
+      {:ok, ip} ->
+        ip
+    end
+  end
+
+  @spec string_to_elixir_types!(String.t()) ::
+          atom() | Regex.t() | module() | String.t() | no_return()
+  def string_to_elixir_types!("~r" <> _pattern = regex) do
+    pattern =
+      ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u
+
+    delimiters = ["/", "|", "\"", "'", {"(", ")"}, {"[", "]"}, {"{", "}"}, {"<", ">"}]
+
+    with %{"modifier" => modifier, "pattern" => pattern, "delimiter" => regex_delimiter} <-
+           Regex.named_captures(pattern, regex),
+         {:ok, {leading, closing}} <- find_valid_delimiter(delimiters, pattern, regex_delimiter),
+         {result, _} <- Code.eval_string("~r#{leading}#{pattern}#{closing}#{modifier}") do
+      result
+    end
+  end
+
+  def string_to_elixir_types!(":" <> atom), do: String.to_atom(atom)
+
+  def string_to_elixir_types!(value) do
+    if module_name?(value) do
+      String.to_existing_atom("Elixir." <> value)
+    else
+      value
+    end
+  end
+
+  defp find_valid_delimiter([], _string, _) do
+    raise(ArgumentError, message: "valid delimiter for Regex expression not found")
+  end
+
+  defp find_valid_delimiter([{leading, closing} = delimiter | others], pattern, regex_delimiter)
+       when is_tuple(delimiter) do
+    if String.contains?(pattern, closing) do
+      find_valid_delimiter(others, pattern, regex_delimiter)
+    else
+      {:ok, {leading, closing}}
+    end
+  end
+
+  defp find_valid_delimiter([delimiter | others], pattern, regex_delimiter) do
+    if String.contains?(pattern, delimiter) do
+      find_valid_delimiter(others, pattern, regex_delimiter)
+    else
+      {:ok, {delimiter, delimiter}}
+    end
+  end
+
+  @spec module_name?(String.t()) :: boolean()
+  def module_name?(string) do
+    Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or
+      string in ["Oban", "Ueberauth", "ExSyslogger"]
+  end
+
+  @spec to_json_types(term()) :: map() | list() | boolean() | String.t() | integer()
+  def to_json_types(entity) when is_list(entity) do
+    Enum.map(entity, &to_json_types/1)
+  end
+
+  def to_json_types(%Regex{} = entity), do: inspect(entity)
+
+  def to_json_types(entity) when is_map(entity) do
+    Map.new(entity, fn {k, v} -> {to_json_types(k), to_json_types(v)} end)
+  end
+
+  def to_json_types({:args, args}) when is_list(args) do
+    arguments =
+      Enum.map(args, fn
+        arg when is_tuple(arg) -> inspect(arg)
+        arg -> to_json_types(arg)
+      end)
+
+    %{"tuple" => [":args", arguments]}
+  end
+
+  def to_json_types({:proxy_url, {type, :localhost, port}}) do
+    %{"tuple" => [":proxy_url", %{"tuple" => [to_json_types(type), "localhost", port]}]}
+  end
+
+  def to_json_types({:proxy_url, {type, host, port}}) when is_tuple(host) do
+    ip =
+      host
+      |> :inet_parse.ntoa()
+      |> to_string()
+
+    %{
+      "tuple" => [
+        ":proxy_url",
+        %{"tuple" => [to_json_types(type), ip, port]}
+      ]
+    }
+  end
+
+  def to_json_types({:proxy_url, {type, host, port}}) do
+    %{
+      "tuple" => [
+        ":proxy_url",
+        %{"tuple" => [to_json_types(type), to_string(host), port]}
+      ]
+    }
+  end
+
+  def to_json_types({:partial_chain, entity}),
+    do: %{"tuple" => [":partial_chain", inspect(entity)]}
+
+  def to_json_types(entity) when is_tuple(entity) do
+    value =
+      entity
+      |> Tuple.to_list()
+      |> to_json_types()
+
+    %{"tuple" => value}
+  end
+
+  def to_json_types(entity) when is_binary(entity), do: entity
+
+  def to_json_types(entity) when is_boolean(entity) or is_number(entity) or is_nil(entity) do
+    entity
+  end
+
+  def to_json_types(entity) when entity in [:"tlsv1.1", :"tlsv1.2", :"tlsv1.3"] do
+    ":#{entity}"
+  end
+
+  def to_json_types(entity) when is_atom(entity), do: inspect(entity)
+end

lib/pleroma/config/deprecation_warnings.ex

@@ -41,7 +41,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
          :ok <- check_gun_pool_options(),
          :ok <- check_activity_expiration_config(),
          :ok <- check_remote_ip_plug_name(),
-         :ok <- check_uploders_s3_public_endpoint() do
+         :ok <- check_uploders_s3_public_endpoint(),
+         :ok <- check_oban_config() do
       :ok
     else
       _ ->
@@ -79,7 +80,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
     move_namespace_and_warn(@mrf_config_map, warning_preface)
   end
 
-  @spec move_namespace_and_warn([config_map()], String.t()) :: :ok | nil
+  @spec move_namespace_and_warn([config_map()], String.t()) :: :ok | :error
   def move_namespace_and_warn(config_map, warning_preface) do
     warning =
       Enum.reduce(config_map, "", fn
@@ -102,7 +103,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
     end
   end
 
-  @spec check_media_proxy_whitelist_config() :: :ok | nil
+  @spec check_media_proxy_whitelist_config() :: :ok | :error
   def check_media_proxy_whitelist_config do
     whitelist = Config.get([:media_proxy, :whitelist])
 
@@ -163,7 +164,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
     end
   end
 
-  @spec check_activity_expiration_config() :: :ok | nil
+  @spec check_activity_expiration_config() :: :ok | :error
   def check_activity_expiration_config do
     warning_preface = """
     !!!DEPRECATION WARNING!!!
@@ -215,4 +216,41 @@ defmodule Pleroma.Config.DeprecationWarnings do
       :ok
     end
   end
+
+  @spec check_oban_config() :: :ok | :error
+  def check_oban_config do
+    oban_config = Config.get(Oban)
+
+    {crontab, changed?} =
+      [
+        Pleroma.Workers.Cron.StatsWorker,
+        Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker,
+        Pleroma.Workers.Cron.ClearOauthTokenWorker
+      ]
+      |> Enum.reduce({oban_config[:crontab], false}, fn removed_worker, {acc, changed?} ->
+        with acc when is_list(acc) <- acc,
+             setting when is_tuple(setting) <-
+               Enum.find(acc, fn {_, worker} -> worker == removed_worker end) do
+          """
+          !!!OBAN CONFIG WARNING!!!
+          You are using old workers in Oban crontab settings, which were removed.
+          Please, remove setting from crontab in your config file (prod.secret.exs): #{
+            inspect(setting)
+          }
+          """
+          |> Logger.warn()
+
+          {List.delete(acc, setting), true}
+        else
+          _ -> {acc, changed?}
+        end
+      end)
+
+    if changed? do
+      Config.put(Oban, Keyword.put(oban_config, :crontab, crontab))
+      :error
+    else
+      :ok
+    end
+  end
 end

lib/pleroma/config/loader.ex

@@ -3,57 +3,74 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config.Loader do
+  @reject_groups [
+    :postgrex,
+    :tesla,
+    :phoenix,
+    :tzdata,
+    :http_signatures,
+    :web_push_encryption,
+    :floki,
+    :pbkdf2_elixir
+  ]
+
   @reject_keys [
     Pleroma.Repo,
     Pleroma.Web.Endpoint,
+    Pleroma.InstallerWeb.Endpoint,
     :env,
     :configurable_from_database,
     :database,
-    :swarm
-  ]
-
-  @reject_groups [
-    :postgrex,
-    :tesla
+    :ecto_repos,
+    Pleroma.Gun,
+    Pleroma.ReverseProxy.Client,
+    Pleroma.Web.Auth.Authenticator
   ]
 
   if Code.ensure_loaded?(Config.Reader) do
     @reader Config.Reader
-
-    def read(path), do: @reader.read!(path)
+    @config_header "import Config\r\n\r\n"
   else
     # support for Elixir less than 1.9
     @reader Mix.Config
-    def read(path) do
-      path
-      |> @reader.eval!()
-      |> elem(0)
-    end
+    @config_header "use Mix.Config\r\n\r\n"
   end
 
-  @spec read(Path.t()) :: keyword()
+  @spec read!(Path.t()) :: keyword()
+  def read!(path), do: @reader.read!(path)
 
   @spec merge(keyword(), keyword()) :: keyword()
   def merge(c1, c2), do: @reader.merge(c1, c2)
 
+  @spec config_header() :: String.t()
+  def config_header, do: @config_header
+
   @spec default_config() :: keyword()
   def default_config do
-    "config/config.exs"
-    |> read()
-    |> filter()
+    config =
+      "config/config.exs"
+      |> read!()
+      |> filter()
+
+    logger_config =
+      :logger
+      |> Application.get_all_env()
+      |> Enum.filter(fn {key, _} -> key in [:backends, :console, :ex_syslogger] end)
+
+    merge(config, logger: logger_config)
   end
 
-  defp filter(configs) do
-    configs
-    |> Keyword.keys()
-    |> Enum.reduce([], &Keyword.put(&2, &1, filter_group(&1, configs)))
-  end
+  @spec filter(keyword()) :: keyword()
+  def filter(configs) do
+    Enum.reduce(configs, [], fn
+      {group, _settings}, group_acc when group in @reject_groups ->
+        group_acc
 
-  @spec filter_group(atom(), keyword()) :: keyword()
-  def filter_group(group, configs) do
-    Enum.reject(configs[group], fn {key, _v} ->
-      key in @reject_keys or group in @reject_groups or
-        (group == :phoenix and key == :serve_endpoints)
+      {group, settings}, group_acc ->
+        Enum.reduce(settings, group_acc, fn
+          {key, _value}, acc when key in @reject_keys -> acc
+          setting, acc -> Keyword.update(acc, group, [setting], &Keyword.merge(&1, [setting]))
+        end)
     end)
   end
 end

lib/pleroma/config/oban.ex

@@ -1,38 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Config.Oban do
-  require Logger
-
-  def warn do
-    oban_config = Pleroma.Config.get(Oban)
-
-    crontab =
-      [
-        Pleroma.Workers.Cron.StatsWorker,
-        Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker,
-        Pleroma.Workers.Cron.ClearOauthTokenWorker
-      ]
-      |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc ->
-        with acc when is_list(acc) <- acc,
-             setting when is_tuple(setting) <-
-               Enum.find(acc, fn {_, worker} -> worker == removed_worker end) do
-          """
-          !!!OBAN CONFIG WARNING!!!
-          You are using old workers in Oban crontab settings, which were removed.
-          Please, remove setting from crontab in your config file (prod.secret.exs): #{
-            inspect(setting)
-          }
-          """
-          |> Logger.warn()
-
-          List.delete(acc, setting)
-        else
-          _ -> acc
-        end
-      end)
-
-    Pleroma.Config.put(Oban, Keyword.put(oban_config, :crontab, crontab))
-  end
-end

lib/pleroma/config/transfer_task.ex

@@ -1,201 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Config.TransferTask do
-  use Task
-
-  alias Pleroma.Config
-  alias Pleroma.ConfigDB
-  alias Pleroma.Repo
-
-  require Logger
-
-  @type env() :: :test | :benchmark | :dev | :prod
-
-  @reboot_time_keys [
-    {:pleroma, :hackney_pools},
-    {:pleroma, :chat},
-    {:pleroma, Oban},
-    {:pleroma, :rate_limit},
-    {:pleroma, :markup},
-    {:pleroma, :streamer},
-    {:pleroma, :pools},
-    {:pleroma, :connections_pool}
-  ]
-
-  @reboot_time_subkeys [
-    {:pleroma, Pleroma.Captcha, [:seconds_valid]},
-    {:pleroma, Pleroma.Upload, [:proxy_remote]},
-    {:pleroma, :instance, [:upload_limit]},
-    {:pleroma, :gopher, [:enabled]}
-  ]
-
-  def start_link(restart_pleroma? \\ true) do
-    load_and_update_env([], restart_pleroma?)
-    if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
-    :ignore
-  end
-
-  @spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok
-  def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do
-    with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do
-      # We need to restart applications for loaded settings take effect
-
-      {logger, other} =
-        (Repo.all(ConfigDB) ++ deleted_settings)
-        |> Enum.map(&merge_with_default/1)
-        |> Enum.split_with(fn {group, _, _, _} -> group in [:logger, :quack] end)
-
-      logger
-      |> Enum.sort()
-      |> Enum.each(&configure/1)
-
-      started_applications = Application.started_applications()
-
-      # TODO: some problem with prometheus after restart!
-      reject = [nil, :prometheus, :postgrex]
-
-      reject =
-        if restart_pleroma? do
-          reject
-        else
-          [:pleroma | reject]
-        end
-
-      other
-      |> Enum.map(&update/1)
-      |> Enum.uniq()
-      |> Enum.reject(&(&1 in reject))
-      |> maybe_set_pleroma_last()
-      |> Enum.each(&restart(started_applications, &1, Config.get(:env)))
-
-      :ok
-    else
-      {:configurable, false} -> Restarter.Pleroma.rebooted()
-    end
-  end
-
-  defp maybe_set_pleroma_last(apps) do
-    # to be ensured that pleroma will be restarted last
-    if :pleroma in apps do
-      apps
-      |> List.delete(:pleroma)
-      |> List.insert_at(-1, :pleroma)
-    else
-      Restarter.Pleroma.rebooted()
-      apps
-    end
-  end
-
-  defp merge_with_default(%{group: group, key: key, value: value} = setting) do
-    default = Config.Holder.default_config(group, key)
-
-    merged =
-      cond do
-        Ecto.get_meta(setting, :state) == :deleted -> default
-        can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value)
-        true -> value
-      end
-
-    {group, key, value, merged}
-  end
-
-  # change logger configuration in runtime, without restart
-  defp configure({:quack, key, _, merged}) do
-    Logger.configure_backend(Quack.Logger, [{key, merged}])
-    :ok = update_env(:quack, key, merged)
-  end
-
-  defp configure({_, :backends, _, merged}) do
-    # removing current backends
-    Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1)
-
-    Enum.each(merged, &Logger.add_backend/1)
-
-    :ok = update_env(:logger, :backends, merged)
-  end
-
-  defp configure({_, key, _, merged}) when key in [:console, :ex_syslogger] do
-    merged =
-      if key == :console do
-        put_in(merged[:format], merged[:format] <> "\n")
-      else
-        merged
-      end
-
-    backend =
-      if key == :ex_syslogger,
-        do: {ExSyslogger, :ex_syslogger},
-        else: key
-
-    Logger.configure_backend(backend, merged)
-    :ok = update_env(:logger, key, merged)
-  end
-
-  defp configure({_, key, _, merged}) do
-    Logger.configure([{key, merged}])
-    :ok = update_env(:logger, key, merged)
-  end
-
-  defp update({group, key, value, merged}) do
-    try do
-      :ok = update_env(group, key, merged)
-
-      if group != :pleroma or pleroma_need_restart?(group, key, value), do: group
-    rescue
-      error ->
-        error_msg =
-          "updating env causes error, group: #{inspect(group)}, key: #{inspect(key)}, value: #{
-            inspect(value)
-          } error: #{inspect(error)}"
-
-        Logger.warn(error_msg)
-
-        nil
-    end
-  end
-
-  defp update_env(group, key, nil), do: Application.delete_env(group, key)
-  defp update_env(group, key, value), do: Application.put_env(group, key, value)
-
-  @spec pleroma_need_restart?(atom(), atom(), any()) :: boolean()
-  def pleroma_need_restart?(group, key, value) do
-    group_and_key_need_reboot?(group, key) or group_and_subkey_need_reboot?(group, key, value)
-  end
-
-  defp group_and_key_need_reboot?(group, key) do
-    Enum.any?(@reboot_time_keys, fn {g, k} -> g == group and k == key end)
-  end
-
-  defp group_and_subkey_need_reboot?(group, key, value) do
-    Keyword.keyword?(value) and
-      Enum.any?(@reboot_time_subkeys, fn {g, k, subkeys} ->
-        g == group and k == key and
-          Enum.any?(Keyword.keys(value), &(&1 in subkeys))
-      end)
-  end
-
-  defp restart(_, :pleroma, env), do: Restarter.Pleroma.restart_after_boot(env)
-
-  defp restart(started_applications, app, _) do
-    with {^app, _, _} <- List.keyfind(started_applications, app, 0),
-         :ok <- Application.stop(app) do
-      :ok = Application.start(app)
-    else
-      nil ->
-        Logger.warn("#{app} is not started.")
-
-      error ->
-        error
-        |> inspect()
-        |> Logger.warn()
-    end
-  end
-
-  defp can_be_merged?(val1, val2) when is_list(val1) and is_list(val2) do
-    Keyword.keyword?(val1) and Keyword.keyword?(val2)
-  end
-
-  defp can_be_merged?(_val1, _val2), do: false
-end

lib/pleroma/config/version.ex

@@ -0,0 +1,25 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Config.Version do
+  @moduledoc """
+  IMPORTANT!!!
+  Before modifying records in the database directly, please read "Config versioning" in `docs/development/config_versioning.md`.
+  """
+
+  use Ecto.Schema
+
+  import Ecto.Query, only: [from: 2]
+
+  schema "config_versions" do
+    field(:backup, Pleroma.EctoType.Config.BinaryValue)
+    field(:current, :boolean, default: true)
+
+    timestamps()
+  end
+
+  def all do
+    from(v in __MODULE__, order_by: [desc: v.id]) |> Pleroma.Repo.all()
+  end
+end

lib/pleroma/config/versioning.ex

@@ -0,0 +1,292 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Config.Versioning do
+  @moduledoc """
+  Module that manages versions of database configs.
+  """
+
+  import Ecto.Query, only: [from: 2]
+
+  alias Ecto.Multi
+  alias Pleroma.Config.Version
+  alias Pleroma.ConfigDB
+  alias Pleroma.Repo
+
+  @type change :: %{
+          optional(:delete) => boolean(),
+          optional(:value) => any(),
+          group: atom(),
+          key: atom() | nil
+        }
+
+  @doc """
+  Creates new config version:
+    - convert changes to elixir types
+    - splits changes by type and processes them in `config` table
+    - sets all pointers to false
+    - gets all rows from `config` table and inserts them as keyword in `backup` field
+  """
+  @spec new_version([change()] | change()) ::
+          {:ok, map()} | {:error, :no_changes} | {:error, atom() | tuple(), any(), any()}
+  def new_version([]), do: {:error, :empty_changes}
+  def new_version(change) when is_map(change), do: new_version([change])
+
+  def new_version(changes) when is_list(changes) do
+    changes
+    |> Enum.reduce(Multi.new(), fn
+      %{delete: true} = deletion, acc ->
+        Multi.run(acc, {:delete_or_update, deletion[:group], deletion[:key]}, fn _, _ ->
+          ConfigDB.delete_or_update(deletion)
+        end)
+
+      operation, acc ->
+        {name, fun} =
+          if Keyword.keyword?(operation[:value]) or
+               (operation[:group] == :pleroma and
+                  operation[:key] in ConfigDB.pleroma_not_keyword_values()) do
+            {:insert_or_update,
+             fn _, _ ->
+               ConfigDB.update_or_create(operation)
+             end}
+          else
+            {:error,
+             fn _, _ ->
+               {:error, {:value_must_be_keyword, operation}}
+             end}
+          end
+
+        Multi.run(acc, {name, operation[:group], operation[:key]}, fun)
+    end)
+    |> set_current_flag_false_for_all_versions()
+    |> insert_new_version()
+    |> Repo.transaction()
+  end
+
+  def new_version(_), do: {:error, :bad_format}
+
+  defp set_current_flag_false_for_all_versions(multi) do
+    Multi.update_all(multi, :update_all_versions, Version, set: [current: false])
+  end
+
+  defp insert_new_version(multi) do
+    Multi.run(multi, :insert_version, fn repo, _ ->
+      %Version{
+        backup: ConfigDB.all_as_keyword()
+      }
+      |> repo.insert()
+    end)
+  end
+
+  @doc """
+  Rollbacks config version by N steps:
+    - checks possibility for rollback
+    - truncates config table and restarts pk
+    - inserts config settings from backup
+    - sets all pointers to false
+    - sets current pointer to true for rollback version
+    - deletes versions after current
+  """
+  @spec rollback(pos_integer()) ::
+          {:ok, map()}
+          | {:error, atom() | tuple(), any(), any()}
+          | {:error, :steps_format}
+          | {:error, :no_current_version}
+          | {:error, :rollback_not_possible}
+  def rollback(steps \\ 1)
+
+  def rollback(steps) when is_integer(steps) and steps > 0 do
+    with version_id when is_integer(version_id) <- get_current_version_id(),
+         %Version{} = version <- get_version_by_steps(steps) do
+      do_rollback(version)
+    end
+  end
+
+  def rollback(_), do: {:error, :steps_format}
+
+  @doc """
+  Same as `rollback/1`, but rollbacks for a given version id.
+  """
+  @spec rollback_by_id(pos_integer()) ::
+          {:ok, map()}
+          | {:error, atom() | tuple(), any(), any()}
+          | {:error, :not_found}
+          | {:error, :version_is_already_current}
+  def rollback_by_id(id) when is_integer(id) do
+    with %Version{current: false} = version <- get_version_by_id(id) do
+      do_rollback(version)
+    else
+      %Version{current: true} -> {:error, :version_is_already_current}
+      error -> error
+    end
+  end
+
+  defp get_current_version_id do
+    query = from(v in Version, where: v.current == true)
+
+    with nil <- Repo.aggregate(query, :max, :id) do
+      {:error, :no_current_version}
+    end
+  end
+
+  defp get_version_by_id(id) do
+    with nil <- Repo.get(Version, id) do
+      {:error, :not_found}
+    end
+  end
+
+  defp get_version_by_steps(steps) do
+    query = from(v in Version, order_by: [desc: v.id], limit: 1, offset: ^steps)
+
+    with nil <- Repo.one(query) do
+      {:error, :rollback_not_possible}
+    end
+  end
+
+  defp do_rollback(version) do
+    multi =
+      truncate_config_table()
+      |> reset_pk_in_config_table()
+
+    version.backup
+    |> ConfigDB.from_keyword_to_maps()
+    |> add_insert_commands(multi)
+    |> set_current_flag_false_for_all_versions()
+    |> Multi.update(:move_current_pointer, Ecto.Changeset.change(version, current: true))
+    |> Multi.delete_all(
+      :delete_next_versions,
+      from(v in Version, where: v.id > ^version.id)
+    )
+    |> Repo.transaction()
+  end
+
+  defp truncate_config_table(multi \\ Multi.new()) do
+    Multi.run(multi, :truncate_config_table, fn repo, _ ->
+      repo.query("TRUNCATE config;")
+    end)
+  end
+
+  defp reset_pk_in_config_table(multi) do
+    Multi.run(multi, :reset_pk, fn repo, _ ->
+      repo.query("ALTER SEQUENCE config_id_seq RESTART;")
+    end)
+  end
+
+  defp add_insert_commands(changes, multi) do
+    Enum.reduce(changes, multi, fn change, acc ->
+      Multi.run(acc, {:insert, change[:group], change[:key]}, fn _, _ ->
+        ConfigDB.update_or_create(change)
+      end)
+    end)
+  end
+
+  @doc """
+  Resets config table and creates new empty version.
+  """
+  @spec reset() :: {:ok, map()} | {:error, atom() | tuple(), any(), any()}
+  def reset do
+    truncate_config_table()
+    |> reset_pk_in_config_table()
+    |> set_current_flag_false_for_all_versions()
+    |> insert_new_version()
+    |> Repo.transaction()
+  end
+
+  @doc """
+  Migrates settings from config file into database:
+    - truncates config table and restarts pk
+    - inserts settings from config file
+    - sets all pointers to false
+    - gets all rows from `config` table and inserts them as keyword in `backup` field
+  """
+  @spec migrate(Path.t()) :: {:ok, map()} | {:error, atom() | tuple(), any(), any()}
+  def migrate(config_path) do
+    multi =
+      truncate_config_table()
+      |> reset_pk_in_config_table()
+
+    config_path
+    |> Pleroma.Config.Loader.read!()
+    |> Pleroma.Config.Loader.filter()
+    |> ConfigDB.from_keyword_to_maps()
+    |> add_insert_commands(multi)
+    |> set_current_flag_false_for_all_versions()
+    |> insert_new_version()
+    |> Repo.transaction()
+  end
+
+  @doc """
+  Common function to migrate old config namespace to the new one keeping the old value.
+  """
+  @spec migrate_namespace({atom(), atom()}, {atom(), atom()}) ::
+          {:ok, map()} | {:error, atom() | tuple(), any(), any()}
+  def migrate_namespace({o_group, o_key}, {n_group, n_key}) do
+    config = ConfigDB.get_by_params(%{group: o_group, key: o_key})
+
+    configs_changes_fun =
+      if config do
+        fn ->
+          config
+          |> Ecto.Changeset.change(group: n_group, key: n_key)
+          |> Repo.update()
+        end
+      else
+        fn -> {:ok, nil} end
+      end
+
+    versions_changes_fun = fn %{backup: backup} = version ->
+      with {value, rest} when not is_nil(value) <- pop_in(backup[o_group][o_key]) do
+        rest =
+          if rest[o_group] == [] do
+            Keyword.delete(rest, o_group)
+          else
+            rest
+          end
+
+        updated_backup =
+          if Keyword.has_key?(rest, n_group) do
+            put_in(rest[n_group][n_key], value)
+          else
+            Keyword.put(rest, n_group, [{n_key, value}])
+          end
+
+        version
+        |> Ecto.Changeset.change(backup: updated_backup)
+        |> Repo.update()
+      else
+        _ -> {:ok, nil}
+      end
+    end
+
+    migrate_configs_and_versions(configs_changes_fun, versions_changes_fun)
+  end
+
+  @doc """
+  Abstract function for config migrations to keep changes in config table and changes in versions backups in transaction.
+  Accepts two functions:
+    - first function makes changes to the configs
+    - second function makes changes to the backups in versions
+  """
+  @spec migrate_configs_and_versions(function(), function()) ::
+          {:ok, map()} | {:error, atom() | tuple(), any(), any()}
+  def migrate_configs_and_versions(configs_changes_fun, version_change_fun)
+      when is_function(configs_changes_fun, 0) and
+             is_function(version_change_fun, 1) do
+    versions = Repo.all(Version)
+
+    multi =
+      Multi.new()
+      |> Multi.run(:configs_changes, fn _, _ ->
+        configs_changes_fun.()
+      end)
+
+    versions
+    |> Enum.reduce(multi, fn version, acc ->
+      Multi.run(acc, {:version_change, version.id}, fn _, _ ->
+        version_change_fun.(version)
+      end)
+    end)
+    |> Repo.transaction()
+  end
+end

lib/pleroma/config_db.ex

@@ -6,8 +6,7 @@ defmodule Pleroma.ConfigDB do
   use Ecto.Schema
 
   import Ecto.Changeset
-  import Ecto.Query, only: [select: 3, from: 2]
-  import Pleroma.Web.Gettext
+  import Ecto.Query, only: [from: 2]
 
   alias __MODULE__
   alias Pleroma.Repo
@@ -22,6 +21,10 @@ defmodule Pleroma.ConfigDB do
     {:pleroma, :mrf_keyword, :replace}
   ]
 
+  @groups_without_keys [:quack, :mime, :cors_plug, :esshd, :ex_aws, :joken, :logger, :swoosh]
+
+  @pleroma_not_keyword_values [Pleroma.Web.Auth.Authenticator, :admin_token]
+
   schema "config" do
     field(:key, Pleroma.EctoType.Config.Atom)
     field(:group, Pleroma.EctoType.Config.Atom)
@@ -31,13 +34,35 @@ defmodule Pleroma.ConfigDB do
     timestamps()
   end
 
-  @spec get_all_as_keyword() :: keyword()
-  def get_all_as_keyword do
-    ConfigDB
-    |> select([c], {c.group, c.key, c.value})
-    |> Repo.all()
-    |> Enum.reduce([], fn {group, key, value}, acc ->
-      Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}]))
+  @spec all() :: [t()]
+  def all, do: Repo.all(ConfigDB)
+
+  @spec all_with_db() :: [t()]
+  def all_with_db do
+    all()
+    |> Enum.map(fn
+      %{group: :pleroma, key: key} = change when key in @pleroma_not_keyword_values ->
+        %{change | db: [change.key]}
+
+      %{value: value} = change ->
+        %{change | db: Keyword.keys(value)}
+    end)
+  end
+
+  @spec all_as_keyword() :: keyword()
+  def all_as_keyword do
+    all()
+    |> as_keyword()
+  end
+
+  @spec as_keyword([t()]) :: keyword()
+  def as_keyword(changes) do
+    Enum.reduce(changes, [], fn
+      %{group: group, key: nil, value: value}, acc ->
+        Keyword.update(acc, group, value, &Keyword.merge(&1, value))
+
+      %{group: group, key: key, value: value}, acc ->
+        Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}]))
     end)
   end
 
@@ -52,14 +77,22 @@ defmodule Pleroma.ConfigDB do
   end
 
   @spec get_by_params(map()) :: ConfigDB.t() | nil
-  def get_by_params(%{group: _, key: _} = params), do: Repo.get_by(ConfigDB, params)
+  def get_by_params(%{group: group, key: key} = params)
+      when not is_nil(key) and not is_nil(group) do
+    Repo.get_by(ConfigDB, params)
+  end
+
+  def get_by_params(%{group: group}) do
+    from(c in ConfigDB, where: c.group == ^group and is_nil(c.key)) |> Repo.one()
+  end
 
   @spec changeset(ConfigDB.t(), map()) :: Changeset.t()
   def changeset(config, params \\ %{}) do
     config
     |> cast(params, [:key, :group, :value])
-    |> validate_required([:key, :group, :value])
+    |> validate_required([:group, :value])
     |> unique_constraint(:key, name: :config_group_key_index)
+    |> unique_constraint(:key, name: :config_group__key_is_null_index)
   end
 
   defp create(params) do
@@ -74,53 +107,104 @@ defmodule Pleroma.ConfigDB do
     |> Repo.update()
   end
 
-  @spec get_db_keys(keyword(), any()) :: [String.t()]
-  def get_db_keys(value, key) do
-    keys =
-      if Keyword.keyword?(value) do
-        Keyword.keys(value)
-      else
-        [key]
-      end
+  @doc """
+  IMPORTANT!!!
+  Before modifying records in the database directly, please read "Config versioning" in `docs/development/config_versioning.md`.
+  """
+  @spec update_or_create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()}
+  def update_or_create(params) do
+    search_opts = Map.take(params, [:group, :key])
 
-    Enum.map(keys, &to_json_types(&1))
+    with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts) do
+      new_value = merge_group(config.group, config.key, config.value, params[:value])
+
+      update(config, %{value: new_value})
+    else
+      nil ->
+        create(params)
+    end
   end
 
-  @spec merge_group(atom(), atom(), keyword(), keyword()) :: keyword()
-  def merge_group(group, key, old_value, new_value) do
+  @doc """
+  IMPORTANT!!!
+  Before modifying records in the database directly, please read "Config versioning" in `docs/development/config_versioning.md`.
+  """
+  @spec delete(ConfigDB.t() | map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()}
+  def delete(%ConfigDB{} = config), do: Repo.delete(config)
+
+  @doc """
+  IMPORTANT!!!
+  Before modifying records in the database directly, please read "Config versioning" in `docs/development/config_versioning.md`.
+  """
+  @spec delete_or_update(map()) :: {:ok, t()} | {:ok, nil} | {:error, Changeset.t()}
+  def delete_or_update(%{group: _, key: key} = params) when not is_nil(key) do
+    search_opts = Map.take(params, [:group, :key])
+
+    with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts) do
+      do_delete_or_update(config, params[:subkeys])
+    else
+      _ -> {:ok, nil}
+    end
+  end
+
+  def delete_or_update(%{group: group}) do
+    query = from(c in ConfigDB, where: c.group == ^group)
+
+    with {num, _} <- Repo.delete_all(query) do
+      {:ok, num}
+    end
+  end
+
+  defp do_delete_or_update(%ConfigDB{} = config, subkeys)
+       when is_list(subkeys) and subkeys != [] do
+    new_value = Keyword.drop(config.value, subkeys)
+
+    if new_value == [] do
+      delete(config)
+    else
+      update(config, %{value: new_value})
+    end
+  end
+
+  defp do_delete_or_update(%ConfigDB{} = config, _), do: delete(config)
+
+  defp merge_group(group, key, old_value, new_value)
+       when is_list(old_value) and is_list(new_value) do
     new_keys = to_mapset(new_value)
 
     intersect_keys = old_value |> to_mapset() |> MapSet.intersection(new_keys) |> MapSet.to_list()
 
-    merged_value = ConfigDB.merge(old_value, new_value)
+    merged_value = deep_merge(old_value, new_value)
 
     @full_subkey_update
-    |> Enum.map(fn
-      {g, k, subkey} when g == group and k == key ->
-        if subkey in intersect_keys, do: subkey, else: []
+    |> Enum.reduce([], fn
+      {g, k, subkey}, acc when g == group and k == key ->
+        if subkey in intersect_keys do
+          [subkey | acc]
+        else
+          acc
+        end
 
-      _ ->
-        []
+      _, acc ->
+        acc
     end)
-    |> List.flatten()
     |> Enum.reduce(merged_value, &Keyword.put(&2, &1, new_value[&1]))
   end
 
-  defp to_mapset(keyword) do
+  defp merge_group(_group, _key, _old_value, new_value) when is_list(new_value), do: new_value
+
+  defp merge_group(:pleroma, key, _old_value, new_value)
+       when key in @pleroma_not_keyword_values do
+    new_value
+  end
+
+  defp to_mapset(keyword) when is_list(keyword) do
     keyword
     |> Keyword.keys()
     |> MapSet.new()
   end
 
-  @spec sub_key_full_update?(atom(), atom(), [Keyword.key()]) :: boolean()
-  def sub_key_full_update?(group, key, subkeys) do
-    Enum.any?(@full_subkey_update, fn {g, k, subkey} ->
-      g == group and k == key and subkey in subkeys
-    end)
-  end
-
-  @spec merge(keyword(), keyword()) :: keyword()
-  def merge(config1, config2) when is_list(config1) and is_list(config2) do
+  defp deep_merge(config1, config2) when is_list(config1) and is_list(config2) do
     Keyword.merge(config1, config2, fn _, app1, app2 ->
       if Keyword.keyword?(app1) and Keyword.keyword?(app2) do
         Keyword.merge(app1, app2, &deep_merge/3)
@@ -138,255 +222,99 @@ defmodule Pleroma.ConfigDB do
     end
   end
 
-  @spec update_or_create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()}
-  def update_or_create(params) do
-    params = Map.put(params, :value, to_elixir_types(params[:value]))
-    search_opts = Map.take(params, [:group, :key])
+  @spec reduce_defaults_and_merge_with_changes([t()], keyword()) :: {[t()], keyword()}
+  def reduce_defaults_and_merge_with_changes(changes, defaults) do
+    Enum.reduce(changes, {[], defaults}, &reduce_default_and_merge_with_change/2)
+  end
 
-    with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts),
-         {_, true, config} <- {:partial_update, can_be_partially_updated?(config), config},
-         {_, true, config} <-
-           {:can_be_merged, is_list(params[:value]) and is_list(config.value), config} do
-      new_value = merge_group(config.group, config.key, config.value, params[:value])
-      update(config, %{value: new_value})
+  defp reduce_default_and_merge_with_change(%{group: group} = change, {acc, defaults})
+       when group in @groups_without_keys do
+    {default, remaining_defaults} = Keyword.pop(defaults, group)
+
+    change = merge_change_with_default(change, default)
+    {[change | acc], remaining_defaults}
+  end
+
+  defp reduce_default_and_merge_with_change(%{group: group, key: key} = change, {acc, defaults}) do
+    if defaults[group] do
+      {default, remaining_group_defaults} = Keyword.pop(defaults[group], key)
+
+      remaining_defaults =
+        if remaining_group_defaults == [] do
+          Keyword.delete(defaults, group)
+        else
+          Keyword.put(defaults, group, remaining_group_defaults)
+        end
+
+      change = merge_change_with_default(change, default)
+
+      {[change | acc], remaining_defaults}
     else
-      {reason, false, config} when reason in [:partial_update, :can_be_merged] ->
-        update(config, params)
-
-      nil ->
-        create(params)
+      {[change | acc], defaults}
     end
   end
 
-  defp can_be_partially_updated?(%ConfigDB{} = config), do: not only_full_update?(config)
+  @spec from_keyword_to_structs(keyword(), [] | [t()]) :: [t()]
+  def from_keyword_to_structs(keyword, initial_acc \\ []) do
+    Enum.reduce(keyword, initial_acc, &reduce_to_structs/2)
+  end
 
-  defp only_full_update?(%ConfigDB{group: group, key: key}) do
-    full_key_update = [
-      {:pleroma, :ecto_repos},
-      {:quack, :meta},
-      {:mime, :types},
-      {:cors_plug, [:max_age, :methods, :expose, :headers]},
-      {:swarm, :node_blacklist},
-      {:logger, :backends}
-    ]
+  defp reduce_to_structs({group, config}, group_acc) when group in @groups_without_keys do
+    [struct(%ConfigDB{}, to_map(group, config)) | group_acc]
+  end
 
-    Enum.any?(full_key_update, fn
-      {s_group, s_key} ->
-        group == s_group and ((is_list(s_key) and key in s_key) or key == s_key)
+  defp reduce_to_structs({group, config}, group_acc) do
+    Enum.reduce(config, group_acc, fn {key, value}, acc ->
+      [struct(%ConfigDB{}, to_map(group, key, value)) | acc]
     end)
   end
 
-  @spec delete(ConfigDB.t() | map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()}
-  def delete(%ConfigDB{} = config), do: Repo.delete(config)
+  @spec from_keyword_to_maps(keyword(), [] | [map()]) :: [map()]
+  def from_keyword_to_maps(keyword, initial_acc \\ []) do
+    Enum.reduce(keyword, initial_acc, &reduce_to_maps/2)
+  end
 
-  def delete(params) do
-    search_opts = Map.delete(params, :subkeys)
+  defp reduce_to_maps({group, config}, group_acc) when group in @groups_without_keys do
+    [to_map(group, config) | group_acc]
+  end
 
-    with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts),
-         {config, sub_keys} when is_list(sub_keys) <- {config, params[:subkeys]},
-         keys <- Enum.map(sub_keys, &string_to_elixir_types(&1)),
-         {_, config, new_value} when new_value != [] <-
-           {:partial_remove, config, Keyword.drop(config.value, keys)} do
-      update(config, %{value: new_value})
+  defp reduce_to_maps({group, config}, group_acc) do
+    Enum.reduce(config, group_acc, fn {key, value}, acc ->
+      [to_map(group, key, value) | acc]
+    end)
+  end
+
+  defp to_map(group, config), do: %{group: group, value: config}
+
+  defp to_map(group, key, value), do: %{group: group, key: key, value: value}
+
+  @spec merge_changes_with_defaults([t()], keyword()) :: [t()]
+  def merge_changes_with_defaults(changes, defaults) when is_list(changes) do
+    Enum.map(changes, fn
+      %{group: group} = change when group in @groups_without_keys ->
+        merge_change_with_default(change, defaults[group])
+
+      %{group: group, key: key} = change ->
+        merge_change_with_default(change, defaults[group][key])
+    end)
+  end
+
+  defp merge_change_with_default(change, default) do
+    %{change | value: merge_change_value_with_default(change, default)}
+  end
+
+  @spec merge_change_value_with_default(t(), keyword()) :: keyword()
+  def merge_change_value_with_default(change, default) do
+    if Ecto.get_meta(change, :state) == :deleted do
+      default
     else
-      {:partial_remove, config, []} ->
-        Repo.delete(config)
-
-      {config, nil} ->
-        Repo.delete(config)
-
-      nil ->
-        err =
-          dgettext("errors", "Config with params %{params} not found", params: inspect(params))
-
-        {:error, err}
+      merge_group(change.group, change.key, default, change.value)
     end
   end
 
-  @spec to_json_types(term()) :: map() | list() | boolean() | String.t()
-  def to_json_types(entity) when is_list(entity) do
-    Enum.map(entity, &to_json_types/1)
-  end
+  @spec groups_without_keys() :: [atom()]
+  def groups_without_keys, do: @groups_without_keys
 
-  def to_json_types(%Regex{} = entity), do: inspect(entity)
-
-  def to_json_types(entity) when is_map(entity) do
-    Map.new(entity, fn {k, v} -> {to_json_types(k), to_json_types(v)} end)
-  end
-
-  def to_json_types({:args, args}) when is_list(args) do
-    arguments =
-      Enum.map(args, fn
-        arg when is_tuple(arg) -> inspect(arg)
-        arg -> to_json_types(arg)
-      end)
-
-    %{"tuple" => [":args", arguments]}
-  end
-
-  def to_json_types({:proxy_url, {type, :localhost, port}}) do
-    %{"tuple" => [":proxy_url", %{"tuple" => [to_json_types(type), "localhost", port]}]}
-  end
-
-  def to_json_types({:proxy_url, {type, host, port}}) when is_tuple(host) do
-    ip =
-      host
-      |> :inet_parse.ntoa()
-      |> to_string()
-
-    %{
-      "tuple" => [
-        ":proxy_url",
-        %{"tuple" => [to_json_types(type), ip, port]}
-      ]
-    }
-  end
-
-  def to_json_types({:proxy_url, {type, host, port}}) do
-    %{
-      "tuple" => [
-        ":proxy_url",
-        %{"tuple" => [to_json_types(type), to_string(host), port]}
-      ]
-    }
-  end
-
-  def to_json_types({:partial_chain, entity}),
-    do: %{"tuple" => [":partial_chain", inspect(entity)]}
-
-  def to_json_types(entity) when is_tuple(entity) do
-    value =
-      entity
-      |> Tuple.to_list()
-      |> to_json_types()
-
-    %{"tuple" => value}
-  end
-
-  def to_json_types(entity) when is_binary(entity), do: entity
-
-  def to_json_types(entity) when is_boolean(entity) or is_number(entity) or is_nil(entity) do
-    entity
-  end
-
-  def to_json_types(entity) when entity in [:"tlsv1.1", :"tlsv1.2", :"tlsv1.3"] do
-    ":#{entity}"
-  end
-
-  def to_json_types(entity) when is_atom(entity), do: inspect(entity)
-
-  @spec to_elixir_types(boolean() | String.t() | map() | list()) :: term()
-  def to_elixir_types(%{"tuple" => [":args", args]}) when is_list(args) do
-    arguments =
-      Enum.map(args, fn arg ->
-        if String.contains?(arg, ["{", "}"]) do
-          {elem, []} = Code.eval_string(arg)
-          elem
-        else
-          to_elixir_types(arg)
-        end
-      end)
-
-    {:args, arguments}
-  end
-
-  def to_elixir_types(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}) do
-    {:proxy_url, {string_to_elixir_types(type), parse_host(host), port}}
-  end
-
-  def to_elixir_types(%{"tuple" => [":partial_chain", entity]}) do
-    {partial_chain, []} =
-      entity
-      |> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "")
-      |> Code.eval_string()
-
-    {:partial_chain, partial_chain}
-  end
-
-  def to_elixir_types(%{"tuple" => entity}) do
-    Enum.reduce(entity, {}, &Tuple.append(&2, to_elixir_types(&1)))
-  end
-
-  def to_elixir_types(entity) when is_map(entity) do
-    Map.new(entity, fn {k, v} -> {to_elixir_types(k), to_elixir_types(v)} end)
-  end
-
-  def to_elixir_types(entity) when is_list(entity) do
-    Enum.map(entity, &to_elixir_types/1)
-  end
-
-  def to_elixir_types(entity) when is_binary(entity) do
-    entity
-    |> String.trim()
-    |> string_to_elixir_types()
-  end
-
-  def to_elixir_types(entity), do: entity
-
-  @spec string_to_elixir_types(String.t()) ::
-          atom() | Regex.t() | module() | String.t() | no_return()
-  def string_to_elixir_types("~r" <> _pattern = regex) do
-    pattern =
-      ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u
-
-    delimiters = ["/", "|", "\"", "'", {"(", ")"}, {"[", "]"}, {"{", "}"}, {"<", ">"}]
-
-    with %{"modifier" => modifier, "pattern" => pattern, "delimiter" => regex_delimiter} <-
-           Regex.named_captures(pattern, regex),
-         {:ok, {leading, closing}} <- find_valid_delimiter(delimiters, pattern, regex_delimiter),
-         {result, _} <- Code.eval_string("~r#{leading}#{pattern}#{closing}#{modifier}") do
-      result
-    end
-  end
-
-  def string_to_elixir_types(":" <> atom), do: String.to_atom(atom)
-
-  def string_to_elixir_types(value) do
-    if module_name?(value) do
-      String.to_existing_atom("Elixir." <> value)
-    else
-      value
-    end
-  end
-
-  defp parse_host("localhost"), do: :localhost
-
-  defp parse_host(host) do
-    charlist = to_charlist(host)
-
-    case :inet.parse_address(charlist) do
-      {:error, :einval} ->
-        charlist
-
-      {:ok, ip} ->
-        ip
-    end
-  end
-
-  defp find_valid_delimiter([], _string, _) do
-    raise(ArgumentError, message: "valid delimiter for Regex expression not found")
-  end
-
-  defp find_valid_delimiter([{leading, closing} = delimiter | others], pattern, regex_delimiter)
-       when is_tuple(delimiter) do
-    if String.contains?(pattern, closing) do
-      find_valid_delimiter(others, pattern, regex_delimiter)
-    else
-      {:ok, {leading, closing}}
-    end
-  end
-
-  defp find_valid_delimiter([delimiter | others], pattern, regex_delimiter) do
-    if String.contains?(pattern, delimiter) do
-      find_valid_delimiter(others, pattern, regex_delimiter)
-    else
-      {:ok, {delimiter, delimiter}}
-    end
-  end
-
-  @spec module_name?(String.t()) :: boolean()
-  def module_name?(string) do
-    Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or
-      string in ["Oban", "Ueberauth", "ExSyslogger"]
-  end
+  @spec pleroma_not_keyword_values() :: [atom()]
+  def pleroma_not_keyword_values, do: @pleroma_not_keyword_values
 end

lib/pleroma/docs/json.ex

@@ -5,7 +5,7 @@
 defmodule Pleroma.Docs.JSON do
   @behaviour Pleroma.Docs.Generator
   @external_resource "config/description.exs"
-  @raw_config Pleroma.Config.Loader.read("config/description.exs")
+  @raw_config Pleroma.Config.Loader.read!("config/description.exs")
   @raw_descriptions @raw_config[:pleroma][:config_description]
   @term __MODULE__.Compiled
 

lib/pleroma/ecto_type/config/atom.ex

@@ -12,13 +12,13 @@ defmodule Pleroma.EctoType.Config.Atom do
   end
 
   def cast(key) when is_binary(key) do
-    {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)}
+    {:ok, Pleroma.Config.Converter.string_to_elixir_types!(key)}
   end
 
   def cast(_), do: :error
 
   def load(key) do
-    {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)}
+    {:ok, Pleroma.Config.Converter.string_to_elixir_types!(key)}
   end
 
   def dump(key) when is_atom(key), do: {:ok, inspect(key)}

lib/pleroma/ecto_type/config/binary_value.ex

@@ -15,6 +15,10 @@ defmodule Pleroma.EctoType.Config.BinaryValue do
     end
   end
 
+  def cast(value) when is_map(value) or is_list(value) do
+    {:ok, Pleroma.Config.Converter.to_elixir_types(value)}
+  end
+
   def cast(value), do: {:ok, value}
 
   def load(value) when is_binary(value) do

lib/pleroma/emails/user_email.ex

@@ -8,12 +8,11 @@ defmodule Pleroma.Emails.UserEmail do
   use Phoenix.Swoosh, view: Pleroma.Web.EmailView, layout: {Pleroma.Web.LayoutView, :email}
 
   alias Pleroma.Config
+  alias Pleroma.Helpers.ConfigHelper
   alias Pleroma.User
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.Router
 
-  import Pleroma.Config.Helpers, only: [instance_name: 0, sender: 0]
-
   defp recipient(email, nil), do: email
   defp recipient(email, name), do: {name, email}
   defp recipient(%User{} = user), do: recipient(user.email, user.name)
@@ -22,25 +21,25 @@ defmodule Pleroma.Emails.UserEmail do
   def welcome(user, opts \\ %{}) do
     new()
     |> to(recipient(user))
-    |> from(Map.get(opts, :sender, sender()))
-    |> subject(Map.get(opts, :subject, "Welcome to #{instance_name()}!"))
-    |> html_body(Map.get(opts, :html, "Welcome to #{instance_name()}!"))
-    |> text_body(Map.get(opts, :text, "Welcome to #{instance_name()}!"))
+    |> from(Map.get(opts, :sender, ConfigHelper.sender()))
+    |> subject(Map.get(opts, :subject, "Welcome to #{ConfigHelper.instance_name()}!"))
+    |> html_body(Map.get(opts, :html, "Welcome to #{ConfigHelper.instance_name()}!"))
+    |> text_body(Map.get(opts, :text, "Welcome to #{ConfigHelper.instance_name()}!"))
   end
 
   def password_reset_email(user, token) when is_binary(token) do
     password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token)
 
     html_body = """
-    <h3>Reset your password at #{instance_name()}</h3>
-    <p>Someone has requested password change for your account at #{instance_name()}.</p>
+    <h3>Reset your password at #{ConfigHelper.instance_name()}</h3>
+    <p>Someone has requested password change for your account at #{ConfigHelper.instance_name()}.</p>
     <p>If it was you, visit the following link to proceed: <a href="#{password_reset_url}">reset password</a>.</p>
     <p>If it was someone else, nothing to worry about: your data is secure and your password has not been changed.</p>
     """
 
     new()
     |> to(recipient(user))
-    |> from(sender())
+    |> from(ConfigHelper.sender())
     |> subject("Password reset")
     |> html_body(html_body)
   end
@@ -59,15 +58,15 @@ defmodule Pleroma.Emails.UserEmail do
       )
 
     html_body = """
-    <h3>You are invited to #{instance_name()}</h3>
-    <p>#{user.name} invites you to join #{instance_name()}, an instance of Pleroma federated social networking platform.</p>
+    <h3>You are invited to #{ConfigHelper.instance_name()}</h3>
+    <p>#{user.name} invites you to join #{ConfigHelper.instance_name()}, an instance of Pleroma federated social networking platform.</p>
     <p>Click the following link to register: <a href="#{registration_url}">accept invitation</a>.</p>
     """
 
     new()
     |> to(recipient(to_email, to_name))
-    |> from(sender())
-    |> subject("Invitation to #{instance_name()}")
+    |> from(ConfigHelper.sender())
+    |> subject("Invitation to #{ConfigHelper.instance_name()}")
     |> html_body(html_body)
   end
 
@@ -81,27 +80,27 @@ defmodule Pleroma.Emails.UserEmail do
       )
 
     html_body = """
-    <h3>Thank you for registering on #{instance_name()}</h3>
+    <h3>Thank you for registering on  #{ConfigHelper.instance_name()}!</h3>
     <p>Email confirmation is required to activate the account.</p>
     <p>Please click the following link to <a href="#{confirmation_url}">activate your account</a>.</p>
     """
 
     new()
     |> to(recipient(user))
-    |> from(sender())
-    |> subject("#{instance_name()} account confirmation")
+    |> from(ConfigHelper.sender())
+    |> subject("#{ConfigHelper.instance_name()} account confirmation")
     |> html_body(html_body)
   end
 
   def approval_pending_email(user) do
     html_body = """
     <h3>Awaiting Approval</h3>
-    <p>Your account at #{instance_name()} is being reviewed by staff. You will receive another email once your account is approved.</p>
+    <p>Your account at #{ConfigHelper.instance_name()} is being reviewed by staff. You will receive another email once your account is approved.</p>
     """
 
     new()
     |> to(recipient(user))
-    |> from(sender())
+    |> from(ConfigHelper.sender())
     |> subject("Your account is awaiting approval")
     |> html_body(html_body)
   end
@@ -109,14 +108,14 @@ defmodule Pleroma.Emails.UserEmail do
   def successful_registration_email(user) do
     html_body = """
     <h3>Hello @#{user.nickname},</h3>
-    <p>Your account at #{instance_name()} has been registered successfully.</p>
+    <p>Your account at #{ConfigHelper.instance_name()} has been registered successfully.</p>
     <p>No further action is required to activate your account.</p>
     """
 
     new()
     |> to(recipient(user))
-    |> from(sender())
-    |> subject("Account registered on #{instance_name()}")
+    |> from(ConfigHelper.sender())
+    |> subject("Account registered on #{ConfigHelper.instance_name()}")
     |> html_body(html_body)
   end
 
@@ -168,7 +167,7 @@ defmodule Pleroma.Emails.UserEmail do
       logo = Config.get([__MODULE__, :logo])
 
       html_data = %{
-        instance: instance_name(),
+        instance: ConfigHelper.instance_name(),
         user: user,
         mentions: mentions,
         followers: followers,
@@ -185,8 +184,8 @@ defmodule Pleroma.Emails.UserEmail do
 
       new()
       |> to(recipient(user))
-      |> from(sender())
-      |> subject("Your digest from #{instance_name()}")
+      |> from(ConfigHelper.sender())
+      |> subject("Your digest from #{ConfigHelper.instance_name()}")
       |> put_layout(false)
       |> render_body("digest.html", html_data)
       |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.svg", type: :inline))
@@ -238,7 +237,7 @@ defmodule Pleroma.Emails.UserEmail do
 
     new()
     |> to(recipient(user))
-    |> from(sender())
+    |> from(ConfigHelper.sender())
     |> subject("Your account archive is ready")
     |> html_body(html_body)
   end

lib/pleroma/gopher/server.ex

@@ -12,13 +12,14 @@ defmodule Pleroma.Gopher.Server do
     port = Keyword.get(config, :port, 1234)
 
     if Keyword.get(config, :enabled, false) do
-      GenServer.start_link(__MODULE__, [ip, port], [])
+      GenServer.start_link(__MODULE__, [ip, port])
     else
       Logger.info("Gopher server disabled")
       :ignore
     end
   end
 
+  @impl true
   def init([ip, port]) do
     Logger.info("Starting gopher server on #{port}")
 
@@ -31,8 +32,14 @@ defmodule Pleroma.Gopher.Server do
       []
     )
 
+    Process.flag(:trap_exit, true)
     {:ok, %{ip: ip, port: port}}
   end
+
+  @impl true
+  def terminate(_reason, _state) do
+    :ranch.stop_listener(:gopher)
+  end
 end
 
 defmodule Pleroma.Gopher.Server.ProtocolHandler do

lib/pleroma/gun/gun_supervisor.ex

@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun.GunSupervisor do
+  use Supervisor
+
+  def start_link(_) do
+    Supervisor.start_link(__MODULE__, :no_args)
+  end
+
+  def init(_) do
+    children =
+      Pleroma.Gun.ConnectionPool.children() ++
+        [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
+
+    Supervisor.init(children, strategy: :one_for_one)
+  end
+end

lib/pleroma/helpers/config_helper.ex

@@ -2,16 +2,20 @@
 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Config.Helpers do
+defmodule Pleroma.Helpers.ConfigHelper do
   alias Pleroma.Config
 
+  require Logger
+
+  @spec instance_name() :: String.t() | nil
   def instance_name, do: Config.get([:instance, :name])
 
+  @spec sender() :: {String.t() | nil, String.t() | nil}
+  def sender do
+    {instance_name(), instance_notify_email()}
+  end
+
   defp instance_notify_email do
     Config.get([:instance, :notify_email]) || Config.get([:instance, :email])
   end
-
-  def sender do
-    {instance_name(), instance_notify_email()}
-  end
 end

lib/pleroma/helpers/server_ip_helper.ex

@@ -0,0 +1,63 @@
+defmodule Pleroma.Helpers.ServerIPHelper do
+  @moduledoc """
+  Module tries to get server real ip address from system or makes request to the remote server.
+  """
+
+  # Taken from https://ipinfo.io/bogon
+  @bogon_ranges [
+                  "0.0.0.0/8",
+                  "10.0.0.0/8",
+                  "100.64.0.0/10",
+                  "127.0.0.0/8",
+                  "127.0.53.53/32",
+                  "169.254.0.0/16",
+                  "172.16.0.0/12",
+                  "192.0.0.0/24",
+                  "192.0.2.0/24",
+                  "192.168.0.0/16",
+                  "198.18.0.0/15",
+                  "198.51.100.0/24",
+                  "203.0.113.0/24",
+                  "224.0.0.0/4",
+                  "240.0.0.0/4",
+                  "255.255.255.255/32"
+                ]
+                |> Enum.map(&InetCidr.parse/1)
+
+  @spec real_ip() :: {:ok, String.t()} | {:error, term()}
+  def real_ip do
+    if Pleroma.Config.get(:env) == :prod do
+      from_system() || from_remote_server()
+    else
+      {:ok, "127.0.0.1"}
+    end
+  end
+
+  defp from_system do
+    with {:ok, interfaces} <- :inet.getifaddrs(),
+         {_name, addresses} <-
+           Enum.find(interfaces, fn {_name, addresses} ->
+             addr = Keyword.get(addresses, :addr)
+
+             Enum.all?([:up, :broadcast, :running], &(&1 in addresses[:flags])) and
+               not Enum.any?(@bogon_ranges, &InetCidr.contains?(&1, addr))
+           end) do
+      ip =
+        addresses
+        |> Keyword.get(:addr)
+        |> :inet.ntoa()
+        |> to_string()
+
+      {:ok, ip}
+    else
+      _ -> nil
+    end
+  end
+
+  defp from_remote_server do
+    with {:ok, %{body: body}} <- Pleroma.HTTP.get("https://api.myip.com") do
+      %{"ip" => ip} = Jason.decode!(body)
+      {:ok, ip}
+    end
+  end
+end

lib/pleroma/http/hackney_supervisor.ex

@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.HackneySupervisor do
+  use Supervisor
+
+  def start_link(_) do
+    Supervisor.start_link(__MODULE__, :no_arg)
+  end
+
+  def init(_) do
+    pools = [:federation, :media]
+
+    pools =
+      if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
+        [:upload | pools]
+      else
+        pools
+      end
+
+    children =
+      for pool <- pools do
+        options = Pleroma.Config.get([:hackney_pools, pool])
+        :hackney_pool.child_spec(pool, options)
+      end
+
+    Supervisor.init(children, strategy: :one_for_one)
+  end
+end

lib/pleroma/user.ex

@@ -2483,4 +2483,7 @@ defmodule Pleroma.User do
     |> where([u], u.local == true)
     |> Repo.aggregate(:count)
   end
+
+  @spec email_regex() :: Regex.t()
+  def email_regex, do: @email_regex
 end

lib/pleroma/user/welcome_email.ex

@@ -9,10 +9,9 @@ defmodule Pleroma.User.WelcomeEmail do
 
   alias Pleroma.Config
   alias Pleroma.Emails
+  alias Pleroma.Helpers.ConfigHelper
   alias Pleroma.User
 
-  import Pleroma.Config.Helpers, only: [instance_name: 0]
-
   @spec enabled?() :: boolean()
   def enabled?, do: Config.get([:welcome, :email, :enabled], false)
 
@@ -24,7 +23,7 @@ defmodule Pleroma.User.WelcomeEmail do
   end
 
   defp email_options(user) do
-    bindings = [user: user, instance_name: instance_name()]
+    bindings = [user: user, instance_name: ConfigHelper.instance_name()]
 
     %{}
     |> add_sender(Config.get([:welcome, :email, :sender], nil))
@@ -45,7 +44,7 @@ defmodule Pleroma.User.WelcomeEmail do
   end
 
   defp add_sender(opts, sender) when is_binary(sender) do
-    add_sender(opts, {instance_name(), sender})
+    add_sender(opts, {ConfigHelper.instance_name(), sender})
   end
 
   defp add_sender(opts, _), do: opts

lib/pleroma/web/admin_api/controllers/admin_api_controller.ex

@@ -392,14 +392,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
   def restart(conn, _params) do
     with :ok <- configurable_from_database() do
-      Restarter.Pleroma.restart(Config.get(:env), 50)
+      Task.start(Pleroma.Application.ConfigDependentDeps, :restart_dependencies, [])
 
       json(conn, %{})
     end
   end
 
   def need_reboot(conn, _params) do
-    json(conn, %{need_reboot: Restarter.Pleroma.need_reboot?()})
+    json(conn, %{need_reboot: Pleroma.Application.ConfigDependentDeps.need_reboot?()})
   end
 
   defp configurable_from_database do

lib/pleroma/web/admin_api/controllers/config_controller.ex

@@ -5,19 +5,24 @@
 defmodule Pleroma.Web.AdminAPI.ConfigController do
   use Pleroma.Web, :controller
 
+  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
+
+  alias Pleroma.Application
   alias Pleroma.Config
   alias Pleroma.ConfigDB
   alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
-  plug(OAuthScopesPlug, %{scopes: ["admin:write"]} when action == :update)
+  plug(OAuthScopesPlug, %{scopes: ["admin:write"]} when action in [:update, :rollback])
 
   plug(
     OAuthScopesPlug,
     %{scopes: ["admin:read"]}
-    when action in [:show, :descriptions]
+    when action in [:show, :descriptions, :versions]
   )
 
+  plug(:check_possibility_configuration_from_database when action != :descriptions)
+
   action_fallback(Pleroma.Web.AdminAPI.FallbackController)
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.ConfigOperation
@@ -29,100 +34,111 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do
   end
 
   def show(conn, %{only_db: true}) do
-    with :ok <- configurable_from_database() do
-      configs = Pleroma.Repo.all(ConfigDB)
+    configs = ConfigDB.all_with_db()
 
-      render(conn, "index.json", %{
-        configs: configs,
-        need_reboot: Restarter.Pleroma.need_reboot?()
-      })
-    end
+    render(conn, "index.json", %{
+      configs: configs,
+      need_reboot: Application.ConfigDependentDeps.need_reboot?()
+    })
   end
 
   def show(conn, _params) do
-    with :ok <- configurable_from_database() do
-      configs = ConfigDB.get_all_as_keyword()
+    defaults = Config.Holder.default_config()
+    changes = ConfigDB.all_with_db()
 
-      merged =
-        Config.Holder.default_config()
-        |> ConfigDB.merge(configs)
-        |> Enum.map(fn {group, values} ->
-          Enum.map(values, fn {key, value} ->
-            db =
-              if configs[group][key] do
-                ConfigDB.get_db_keys(configs[group][key], key)
-              end
+    {changes_values_merged_with_defaults, remaining_defaults} =
+      ConfigDB.reduce_defaults_and_merge_with_changes(changes, defaults)
 
-            db_value = configs[group][key]
+    changes_merged_with_defaults =
+      ConfigDB.from_keyword_to_structs(remaining_defaults, changes_values_merged_with_defaults)
 
-            merged_value =
-              if not is_nil(db_value) and Keyword.keyword?(db_value) and
-                   ConfigDB.sub_key_full_update?(group, key, Keyword.keys(db_value)) do
-                ConfigDB.merge_group(group, key, value, db_value)
-              else
-                value
-              end
-
-            %ConfigDB{
-              group: group,
-              key: key,
-              value: merged_value
-            }
-            |> Pleroma.Maps.put_if_present(:db, db)
-          end)
-        end)
-        |> List.flatten()
-
-      render(conn, "index.json", %{
-        configs: merged,
-        need_reboot: Restarter.Pleroma.need_reboot?()
-      })
-    end
+    render(conn, "index.json", %{
+      configs: changes_merged_with_defaults,
+      need_reboot: Application.ConfigDependentDeps.need_reboot?()
+    })
   end
 
   def update(%{body_params: %{configs: configs}} = conn, _) do
-    with :ok <- configurable_from_database() do
-      results =
-        configs
-        |> Enum.filter(&whitelisted_config?/1)
-        |> Enum.map(fn
-          %{group: group, key: key, delete: true} = params ->
-            ConfigDB.delete(%{group: group, key: key, subkeys: params[:subkeys]})
+    result =
+      configs
+      |> Enum.filter(&whitelisted_config?/1)
+      |> Enum.map(&Config.Converter.to_elixir_types/1)
+      |> Config.Versioning.new_version()
 
-          %{group: group, key: key, value: value} ->
-            ConfigDB.update_or_create(%{group: group, key: key, value: value})
-        end)
-        |> Enum.reject(fn {result, _} -> result == :error end)
+    case result do
+      {:ok, changes} ->
+        inserts_and_deletions =
+          changes
+          |> Enum.reduce([], fn
+            {{operation, _, _}, %ConfigDB{} = change}, acc
+            when operation in [:insert_or_update, :delete_or_update] ->
+              if Ecto.get_meta(change, :state) == :deleted do
+                [change | acc]
+              else
+                if change.group == :pleroma and
+                     change.key in ConfigDB.pleroma_not_keyword_values() do
+                  [%{change | db: [change.key]} | acc]
+                else
+                  [%{change | db: Keyword.keys(change.value)} | acc]
+                end
+              end
 
-      {deleted, updated} =
-        results
-        |> Enum.map(fn {:ok, %{key: key, value: value} = config} ->
-          Map.put(config, :db, ConfigDB.get_db_keys(value, key))
-        end)
-        |> Enum.split_with(&(Ecto.get_meta(&1, :state) == :deleted))
+            _, acc ->
+              acc
+          end)
 
-      Config.TransferTask.load_and_update_env(deleted, false)
+        Application.Environment.update(inserts_and_deletions)
 
-      if not Restarter.Pleroma.need_reboot?() do
-        changed_reboot_settings? =
-          (updated ++ deleted)
-          |> Enum.any?(&Config.TransferTask.pleroma_need_restart?(&1.group, &1.key, &1.value))
+        render(conn, "index.json", %{
+          configs: Enum.reject(inserts_and_deletions, &(Ecto.get_meta(&1, :state) == :deleted)),
+          need_reboot: Application.ConfigDependentDeps.need_reboot?()
+        })
 
-        if changed_reboot_settings?, do: Restarter.Pleroma.need_reboot()
-      end
+      {:error, error} ->
+        {:error, "Updating config failed: #{inspect(error)}"}
 
-      render(conn, "index.json", %{
-        configs: updated,
-        need_reboot: Restarter.Pleroma.need_reboot?()
-      })
+      {:error, _, {error, operation}, _} ->
+        {:error,
+         "Updating config failed: #{inspect(error)}, group: #{operation[:group]}, key: #{
+           operation[:key]
+         }, value: #{inspect(operation[:value])}"}
     end
   end
 
-  defp configurable_from_database do
+  def rollback(conn, %{id: id}) do
+    case Config.Versioning.rollback_by_id(id) do
+      {:ok, _} ->
+        json_response(conn, :no_content, "")
+
+      {:error, :not_found} ->
+        {:error, :not_found}
+
+      {:error, error} ->
+        {:error, "Rollback is not possible: #{inspect(error)}"}
+
+      {:error, _, {error, operation}, _} ->
+        {:error,
+         "Rollback is not possible, backup restore error: #{inspect(error)}, operation error: #{
+           inspect(operation)
+         }"}
+    end
+  end
+
+  def versions(conn, _) do
+    versions = Pleroma.Config.Version.all()
+
+    render(conn, "index.json", %{versions: versions})
+  end
+
+  defp check_possibility_configuration_from_database(conn, _) do
     if Config.get(:configurable_from_database) do
-      :ok
+      conn
     else
-      {:error, "You must enable configurable_from_database in your config file."}
+      Pleroma.Web.AdminAPI.FallbackController.call(
+        conn,
+        {:error, "You must enable configurable_from_database in your config file."}
+      )
+      |> halt()
     end
   end
 

lib/pleroma/web/admin_api/views/config_view.ex

@@ -5,8 +5,6 @@
 defmodule Pleroma.Web.AdminAPI.ConfigView do
   use Pleroma.Web, :view
 
-  alias Pleroma.ConfigDB
-
   def render("index.json", %{configs: configs} = params) do
     %{
       configs: render_many(configs, __MODULE__, "show.json", as: :config),
@@ -14,17 +12,23 @@ defmodule Pleroma.Web.AdminAPI.ConfigView do
     }
   end
 
-  def render("show.json", %{config: config}) do
-    map = %{
-      key: ConfigDB.to_json_types(config.key),
-      group: ConfigDB.to_json_types(config.group),
-      value: ConfigDB.to_json_types(config.value)
+  def render("index.json", %{versions: versions}) do
+    %{
+      versions: render_many(versions, __MODULE__, "show.json", as: :version)
     }
+  end
 
-    if config.db != [] do
-      Map.put(map, :db, config.db)
-    else
-      map
-    end
+  def render("show.json", %{config: config}) do
+    config
+    |> Map.take([:group, :key, :value, :db])
+    |> Map.new(fn
+      {k, v} -> {k, Pleroma.Config.Converter.to_json_types(v)}
+    end)
+  end
+
+  def render("show.json", %{version: version}) do
+    version
+    |> Map.take([:id, :current])
+    |> Map.put(:inserted_at, Pleroma.Web.CommonAPI.Utils.to_masto_date(version.inserted_at))
   end
 end

lib/pleroma/web/api_spec/operations/admin/config_operation.ex

@@ -53,7 +53,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ConfigOperation do
                 type: :object,
                 properties: %{
                   group: %Schema{type: :string},
-                  key: %Schema{type: :string},
+                  key: %Schema{type: :string, nullable: true},
                   value: any(),
                   delete: %Schema{type: :boolean},
                   subkeys: %Schema{type: :array, items: %Schema{type: :string}}
@@ -107,6 +107,56 @@ defmodule Pleroma.Web.ApiSpec.Admin.ConfigOperation do
     }
   end
 
+  def rollback_operation do
+    %Operation{
+      tags: ["Admin", "Config"],
+      summary: "Rollback config changes.",
+      operationId: "AdminAPI.ConfigController.rollback",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [
+        Operation.parameter(:id, :path, %Schema{type: :integer}, "Version id to rollback",
+          required: true
+        )
+        | admin_api_params()
+      ],
+      responses: %{
+        204 => no_content_response(),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def versions_operation do
+    %Operation{
+      tags: ["Admin", "Config"],
+      summary: "Get list with config versions.",
+      operationId: "AdminAPI.ConfigController.versions",
+      security: [%{"oAuth" => ["read"]}],
+      parameters: admin_api_params(),
+      responses: %{
+        200 =>
+          Operation.response("Config Version", "application/json", %Schema{
+            type: :object,
+            properties: %{
+              versions: %Schema{
+                type: :array,
+                items: %Schema{
+                  type: :object,
+                  properties: %{
+                    id: %Schema{type: :integer},
+                    current: %Schema{type: :boolean},
+                    inserted_at: %Schema{type: :string, format: :"date-time"}
+                  }
+                }
+              }
+            }
+          }),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
   defp any do
     %Schema{
       oneOf: [
@@ -129,7 +179,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ConfigOperation do
             type: :object,
             properties: %{
               group: %Schema{type: :string},
-              key: %Schema{type: :string},
+              key: %Schema{type: :string, nullable: true},
               value: any()
             }
           }

lib/pleroma/web/rich_media/helpers.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright _ 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Helpers do

lib/pleroma/web/router.ex

@@ -232,6 +232,8 @@ defmodule Pleroma.Web.Router do
     get("/config", ConfigController, :show)
     post("/config", ConfigController, :update)
     get("/config/descriptions", ConfigController, :descriptions)
+    get("/config/versions", ConfigController, :versions)
+    get("/config/versions/rollback/:id", ConfigController, :rollback)
     get("/need_reboot", AdminAPIController, :need_reboot)
     get("/restart", AdminAPIController, :restart)
 

mix.exs

@@ -77,9 +77,8 @@ defmodule Pleroma.Mixfile do
         :logger,
         :runtime_tools,
         :comeonin,
-        :quack,
-        :fast_sanitize,
-        :ssl
+        :ssl,
+        :fast_sanitize
       ],
       included_applications: [:ex_syslogger]
     ]
@@ -87,8 +86,8 @@ defmodule Pleroma.Mixfile do
 
   # Specifies which paths to compile per environment.
   defp elixirc_paths(:benchmark), do: ["lib", "benchmarks"]
-  defp elixirc_paths(:test), do: ["lib", "test/support"]
-  defp elixirc_paths(_), do: ["lib"]
+  defp elixirc_paths(:test), do: ["lib", "test/support", "installer/pleroma"]
+  defp elixirc_paths(_), do: ["lib", "installer/pleroma"]
 
   defp warnings_as_errors(:prod), do: false
   defp warnings_as_errors(_), do: true
@@ -192,7 +191,6 @@ defmodule Pleroma.Mixfile do
       {:captcha,
        git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git",
        ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"},
-      {:restarter, path: "./restarter"},
       {:majic,
        git: "https://git.pleroma.social/pleroma/elixir-libraries/majic.git",
        ref: "289cda1b6d0d70ccb2ba508a2b0bd24638db2880"},

mix.lock

@@ -98,7 +98,7 @@
   "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},
-  "postgrex": {:hex, :postgrex, "0.15.7", "724410acd48abac529d0faa6c2a379fb8ae2088e31247687b16cacc0e0883372", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "88310c010ff047cecd73d5ceca1d99205e4b1ab1b9abfdab7e00f5c9d20ef8f9"},
+  "postgrex": {:hex, :postgrex, "0.15.8", "f5e782bbe5e8fa178d5e3cd1999c857dc48eda95f0a4d7f7bd92a50e84a0d491", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "698fbfacea34c4cf22c8281abeb5cf68d99628d541874f085520ab3b53d356fe"},
   "pot": {:hex, :pot, "0.11.0", "61bad869a94534739dd4614a25a619bc5c47b9970e9a0ea5bef4628036fc7a16", [:rebar3], [], "hexpm", "57ee6ee6bdeb639661ffafb9acefe3c8f966e45394de6a766813bb9e1be4e54b"},
   "prometheus": {:hex, :prometheus, "4.6.0", "20510f381db1ccab818b4cf2fac5fa6ab5cc91bc364a154399901c001465f46f", [:mix, :rebar3], [], "hexpm", "4905fd2992f8038eccd7aa0cd22f40637ed618c0bed1f75c05aacec15b7545de"},
   "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.3", "3dd4da1812b8e0dbee81ea58bb3b62ed7588f2eae0c9e97e434c46807ff82311", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "8d66289f77f913b37eda81fd287340c17e61a447549deb28efc254532b2bed82"},

priv/repo/migrations/20170912114248_add_context_index.exs

@@ -1,6 +1,7 @@
 defmodule Pleroma.Repo.Migrations.AddContextIndex do
   use Ecto.Migration
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def change do
     create(

priv/repo/migrations/20170916090107_add_fts_index_to_activities.exs

@@ -1,6 +1,7 @@
 defmodule Pleroma.Repo.Migrations.AddFTSIndexToActivities do
   use Ecto.Migration
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def change do
     create(

priv/repo/migrations/20170917120416_add_tag_index.exs

@@ -2,6 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddTagIndex do
   use Ecto.Migration
 
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def change do
     create(

priv/repo/migrations/20171023155035_add_second_object_index_to_activty.exs

@@ -2,6 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddSecondObjectIndexToActivty do
   use Ecto.Migration
 
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def change do
     drop_if_exists(

priv/repo/migrations/20171024121413_add_object_actor_index.exs

@@ -2,6 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddObjectActorIndex do
   use Ecto.Migration
 
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def change do
     create(

priv/repo/migrations/20171109091239_add_actor_to_activity.exs

@@ -2,6 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddActorToActivity do
   use Ecto.Migration
 
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def up do
     alter table(:activities) do

priv/repo/migrations/20171109141309_add_sort_index_to_activities.exs

@@ -1,6 +1,7 @@
 defmodule Pleroma.Repo.Migrations.AddSortIndexToActivities do
   use Ecto.Migration
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def change do
     create(index(:activities, ["id desc nulls last"], concurrently: true))

priv/repo/migrations/20180325172351_add_follower_address_index_to_users.exs

@@ -2,6 +2,8 @@ defmodule Pleroma.Repo.Migrations.AddFollowerAddressIndexToUsers do
   use Ecto.Migration
 
   @disable_ddl_transaction true
+  @disable_migration_lock true
+
   def change do
     create(index(:users, [:follower_address], concurrently: true))
     create(index(:users, [:following], concurrently: true, using: :gin))

priv/repo/migrations/20180513104714_modify_activity_index.exs

@@ -1,6 +1,7 @@
 defmodule Pleroma.Repo.Migrations.ModifyActivityIndex do
   use Ecto.Migration
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def change do
     create(index(:activities, ["id desc nulls last", "local"], concurrently: true))

priv/repo/migrations/20180606173637_create_apid_host_extraction_index.exs

@@ -1,6 +1,7 @@
 defmodule Pleroma.Repo.Migrations.CreateApidHostExtractionIndex do
   use Ecto.Migration
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def change do
     create(

priv/repo/migrations/20180617221540_create_activities_in_reply_to_index.exs

@@ -1,6 +1,7 @@
 defmodule Pleroma.Repo.Migrations.CreateActivitiesInReplyToIndex do
   use Ecto.Migration
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def change do
     create(

priv/repo/migrations/20190109152453_add_visibility_function.exs

@@ -1,6 +1,7 @@
 defmodule Pleroma.Repo.Migrations.AddVisibilityFunction do
   use Ecto.Migration
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def up do
     definition = """

priv/repo/migrations/20190127151220_add_activities_likes_index.exs

@@ -1,6 +1,7 @@
 defmodule Pleroma.Repo.Migrations.AddActivitiesLikesIndex do
   use Ecto.Migration
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def change do
     create(

priv/repo/migrations/20190204200237_add_correct_dm_index.exs

@@ -1,6 +1,7 @@
 defmodule Pleroma.Repo.Migrations.AddCorrectDMIndex do
   use Ecto.Migration
   @disable_ddl_transaction true
+  @disable_migration_lock true
 
   def up do
     drop_if_exists(

priv/repo/migrations/20190405160700_add_index_on_subscribers.exs

@@ -2,6 +2,8 @@ defmodule Pleroma.Repo.Migrations.AddIndexOnSubscribers do
   use Ecto.Migration
 
   @disable_ddl_transaction true
+  @disable_migration_lock true
+
   def change do
     create(
       index(:users, ["(info->'subscribers')"],

priv/repo/migrations/20190710125051_add_following_address_index_to_user.exs

@@ -2,6 +2,8 @@ defmodule Pleroma.Repo.Migrations.AddFollowingAddressIndexToUser do
   use Ecto.Migration
 
   @disable_ddl_transaction true
+  @disable_migration_lock true
+
   def change do
     create(index(:users, [:following_address], concurrently: true))
   end

priv/repo/migrations/20191118084500_data_migration_populate_user_relationships.exs

@@ -1,7 +1,6 @@
 defmodule Pleroma.Repo.Migrations.DataMigrationPopulateUserRelationships do
   use Ecto.Migration
 
-  alias Ecto.Adapters.SQL
   alias Pleroma.Repo
 
   require Logger
@@ -25,7 +24,7 @@ defmodule Pleroma.Repo.Migrations.DataMigrationPopulateUserRelationships do
     Logger.info("Processing users.#{field}...")
 
     {:ok, %{rows: field_rows}} =
-      SQL.query(Repo, "SELECT id, #{field} FROM users WHERE #{field} != '{}'")
+      Repo.query("SELECT id, #{field} FROM users WHERE #{field} != '{}'")
 
     target_ap_ids =
       Enum.flat_map(
@@ -36,7 +35,7 @@ defmodule Pleroma.Repo.Migrations.DataMigrationPopulateUserRelationships do
 
     # Selecting ids of all targets at once in order to reduce the number of SELECT queries
     {:ok, %{rows: target_ap_id_id}} =
-      SQL.query(Repo, "SELECT ap_id, id FROM users WHERE ap_id = ANY($1)", [target_ap_ids])
+      Repo.query("SELECT ap_id, id FROM users WHERE ap_id = ANY($1)", [target_ap_ids])
 
     target_id_by_ap_id = Enum.into(target_ap_id_id, %{}, fn [k, v] -> {k, v} end)
 

priv/repo/migrations/20200716195806_autolinker_to_linkify.exs

@@ -15,7 +15,7 @@ defmodule Pleroma.Repo.Migrations.AutolinkerToLinkify do
 
   defp move_config(%{} = old, %{} = new) do
     {:ok, _} = ConfigDB.update_or_create(new)
-    {:ok, _} = ConfigDB.delete(old)
+    {:ok, _} = ConfigDB.delete_or_update(old)
     :ok
   end
 

priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs

@@ -4,7 +4,7 @@ defmodule Pleroma.Repo.Migrations.MoveActivityExpirationsToOban do
   import Ecto.Query, only: [from: 2]
 
   def change do
-    Pleroma.Config.Oban.warn()
+    Pleroma.Config.DeprecationWarnings.check_oban_config()
 
     Application.ensure_all_started(:oban)
 

priv/repo/migrations/20200907092050_move_tokens_expiration_into_oban.exs

@@ -4,7 +4,7 @@ defmodule Pleroma.Repo.Migrations.MoveTokensExpirationIntoOban do
   import Ecto.Query, only: [from: 2]
 
   def change do
-    Pleroma.Config.Oban.warn()
+    Pleroma.Config.DeprecationWarnings.check_oban_config()
 
     Application.ensure_all_started(:oban)
 

priv/repo/migrations/20201007131955_add_config_version.exs

@@ -0,0 +1,14 @@
+defmodule Pleroma.Repo.Migrations.AddConfigVersion do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists table(:config_versions) do
+      add(:backup, :binary)
+      add(:current, :boolean)
+
+      timestamps()
+    end
+
+    create_if_not_exists(unique_index(:config_versions, [:current], where: "current = true"))
+  end
+end