added verify RUM settings before start app

This commit is contained in:
Maksim Pechnikov 2020-06-22 17:27:49 +03:00
parent 4cb7b1ebc6
commit b0a40fc2e4
6 changed files with 189 additions and 93 deletions

View file

@ -39,7 +39,7 @@ defmodule Pleroma.Application do
Pleroma.HTML.compile_scrubbers() Pleroma.HTML.compile_scrubbers()
Config.DeprecationWarnings.warn() Config.DeprecationWarnings.warn()
Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
Pleroma.Repo.check_migrations_applied!() Pleroma.ApplicationRequirements.verify!()
setup_instrumenters() setup_instrumenters()
load_custom_modules() load_custom_modules()

View file

@ -0,0 +1,103 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ApplicationRequirements do
@moduledoc """
The module represents the collection of validations to runs before start server.
"""
defmodule VerifyError, do: defexception([:message])
import Ecto.Query
require Logger
@spec verify!() :: :ok | VerifyError.t()
def verify! do
:ok
|> check_migrations_applied!()
|> check_rum!()
|> handle_result()
end
defp handle_result(:ok), do: :ok
defp handle_result({:error, message}), do: raise(VerifyError, message: message)
defp check_migrations_applied!(:ok) do
unless Pleroma.Config.get(
[:i_am_aware_this_may_cause_data_loss, :disable_migration_check],
false
) do
{_, res, _} =
Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
down_migrations =
Ecto.Migrator.migrations(repo)
|> Enum.reject(fn
{:up, _, _} -> true
{:down, _, _} -> false
end)
if length(down_migrations) > 0 do
down_migrations_text =
Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end)
Logger.error(
"The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
)
{:error, "Unapplied Migrations detected"}
else
:ok
end
end)
res
else
:ok
end
end
defp check_migrations_applied!(result), do: result
defp check_rum!(:ok) do
{_, res, _} =
Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
migrate =
from(o in "columns",
where: o.table_name == "objects",
where: o.column_name == "fts_content"
)
|> repo.exists?(prefix: "information_schema")
setting = Pleroma.Config.get([:database, :rum_enabled], false)
do_check_rum!(setting, migrate)
end)
res
end
defp check_rum!(result), do: result
defp do_check_rum!(setting, migrate) do
case {setting, migrate} do
{true, false} ->
Logger.error(
"Use `RUM` index is enabled, but were not applied migrations for it.\nIf you want to start Pleroma anyway, set\nconfig :pleroma, :database, rum_enabled: false\nOtherwise apply the following migrations:\n`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`"
)
{:error, "Unapplied RUM Migrations detected"}
{false, true} ->
Logger.error(
"Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\nIf you want to use `RUM`, set\nconfig :pleroma, :database, rum_enabled: true\nOtherwise roll `RUM` migrations back.\n`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`"
)
{:error, "RUM Migrations detected"}
_ ->
:ok
end
end
end

View file

@ -11,9 +11,7 @@ defmodule Pleroma.Repo do
import Ecto.Query import Ecto.Query
require Logger require Logger
defmodule Instrumenter do defmodule Instrumenter, do: use(Prometheus.EctoInstrumenter)
use Prometheus.EctoInstrumenter
end
@doc """ @doc """
Dynamically loads the repository url from the Dynamically loads the repository url from the
@ -51,35 +49,6 @@ defmodule Pleroma.Repo do
end end
end end
def check_migrations_applied!() do
unless Pleroma.Config.get(
[:i_am_aware_this_may_cause_data_loss, :disable_migration_check],
false
) do
Ecto.Migrator.with_repo(__MODULE__, fn repo ->
down_migrations =
Ecto.Migrator.migrations(repo)
|> Enum.reject(fn
{:up, _, _} -> true
{:down, _, _} -> false
end)
if length(down_migrations) > 0 do
down_migrations_text =
Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end)
Logger.error(
"The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
)
raise Pleroma.Repo.UnappliedMigrationsError
end
end)
else
:ok
end
end
def chunk_stream(query, chunk_size) do def chunk_stream(query, chunk_size) do
# We don't actually need start and end funcitons of resource streaming, # We don't actually need start and end funcitons of resource streaming,
# but it seems to be the only way to not fetch records one-by-one and # but it seems to be the only way to not fetch records one-by-one and
@ -107,7 +76,3 @@ defmodule Pleroma.Repo do
) )
end end
end end
defmodule Pleroma.Repo.UnappliedMigrationsError do
defexception message: "Unapplied Migrations detected"
end

View file

@ -2,29 +2,24 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
use Ecto.Migration use Ecto.Migration
def up do def up do
if Pleroma.Config.get([:database, :rum_enabled]) do execute("create extension if not exists rum")
execute("create extension if not exists rum") drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) alter table(:objects) do
alter table(:objects) do add(:fts_content, :tsvector)
add(:fts_content, :tsvector)
end
execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$
begin
new.fts_content := to_tsvector('english', new.data->>'content');
return new;
end
$$ LANGUAGE plpgsql")
execute("create index if not exists objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');")
execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects
FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
execute("UPDATE objects SET updated_at = NOW()")
else
raise Ecto.MigrationError,
message: "Migration is not allowed. You can change this behavior by setting `database/rum_enabled` to true."
end end
execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$
begin
new.fts_content := to_tsvector('english', new.data->>'content');
return new;
end
$$ LANGUAGE plpgsql")
execute("create index if not exists objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');")
execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects
FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
execute("UPDATE objects SET updated_at = NOW()")
end end
def down do def down do

View file

@ -0,0 +1,67 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.RepoTest do
use Pleroma.DataCase
import ExUnit.CaptureLog
import Mock
describe "check_rum!" do
setup_with_mocks([
{Ecto.Migrator, [],
[
with_repo: fn repo, fun -> passthrough([repo, fun]) end,
migrations: fn Pleroma.Repo -> [] end
]}
]) do
:ok
end
setup do: clear_config([:database, :rum_enabled])
test "raises if rum is enabled and detects unapplied rum migrations" do
Pleroma.Config.put([:database, :rum_enabled], true)
assert_raise Pleroma.ApplicationRequirements.VerifyError,
"Unapplied RUM Migrations detected",
fn ->
capture_log(&Pleroma.ApplicationRequirements.verify!/0)
end
end
end
describe "check_migrations_applied!" do
setup_with_mocks([
{Ecto.Migrator, [],
[
with_repo: fn repo, fun -> passthrough([repo, fun]) end,
migrations: fn Pleroma.Repo ->
[
{:up, 20_191_128_153_944, "fix_missing_following_count"},
{:up, 20_191_203_043_610, "create_report_notes"},
{:down, 20_191_220_174_645, "add_scopes_to_pleroma_feo_auth_records"}
]
end
]}
]) do
:ok
end
setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check])
test "raises if it detects unapplied migrations" do
assert_raise Pleroma.ApplicationRequirements.VerifyError,
"Unapplied Migrations detected",
fn ->
capture_log(&Pleroma.ApplicationRequirements.verify!/0)
end
end
test "doesn't do anything if disabled" do
Pleroma.Config.put([:i_am_aware_this_may_cause_data_loss, :disable_migration_check], true)
assert :ok == Pleroma.ApplicationRequirements.verify!()
end
end
end

View file

@ -4,9 +4,7 @@
defmodule Pleroma.RepoTest do defmodule Pleroma.RepoTest do
use Pleroma.DataCase use Pleroma.DataCase
import ExUnit.CaptureLog
import Pleroma.Factory import Pleroma.Factory
import Mock
alias Pleroma.User alias Pleroma.User
@ -49,36 +47,4 @@ defmodule Pleroma.RepoTest do
assert Repo.get_assoc(token, :user) == {:error, :not_found} assert Repo.get_assoc(token, :user) == {:error, :not_found}
end end
end end
describe "check_migrations_applied!" do
setup_with_mocks([
{Ecto.Migrator, [],
[
with_repo: fn repo, fun -> passthrough([repo, fun]) end,
migrations: fn Pleroma.Repo ->
[
{:up, 20_191_128_153_944, "fix_missing_following_count"},
{:up, 20_191_203_043_610, "create_report_notes"},
{:down, 20_191_220_174_645, "add_scopes_to_pleroma_feo_auth_records"}
]
end
]}
]) do
:ok
end
setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check])
test "raises if it detects unapplied migrations" do
assert_raise Pleroma.Repo.UnappliedMigrationsError, fn ->
capture_log(&Repo.check_migrations_applied!/0)
end
end
test "doesn't do anything if disabled" do
Pleroma.Config.put([:i_am_aware_this_may_cause_data_loss, :disable_migration_check], true)
assert :ok == Repo.check_migrations_applied!()
end
end
end end