diff --git a/Dockerfile b/Dockerfile index 6ceef20..b5b8f67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,13 +12,13 @@ # - https://pkgs.org/ - resource for finding needed packages # - Ex: hexpm/elixir:1.12.0-erlang-24.0.1-debian-bullseye-20210902-slim # -ARG BUILDER_IMAGE="hexpm/elixir:1.12.0-erlang-24.0.1-debian-bullseye-20210902-slim" +ARG BUILDER_IMAGE="hexpm/elixir:1.14.0-erlang-24.0.1-debian-bullseye-20210902-slim" ARG RUNNER_IMAGE="debian:bullseye-20210902-slim" FROM ${BUILDER_IMAGE} as builder # install build dependencies -RUN apt-get update -y && apt-get install -y build-essential git \ +RUN apt-get update -y && apt-get install -y build-essential git curl ffmpeg \ && apt-get clean && rm -f /var/lib/apt/lists/*_* # prepare build dir @@ -30,6 +30,7 @@ RUN mix local.hex --force && \ # set build ENV ENV MIX_ENV="prod" +ENV BUMBLEBEE_CACHE_DIR="/app/.bumblebee" # install mix dependencies COPY mix.exs mix.lock ./ @@ -57,6 +58,7 @@ COPY assets assets RUN mix assets.deploy RUN mix compile +RUN mix run -e 'LiveBeats.Application.load_serving()' --no-start # Changes to config/runtime.exs don't require recompiling the code COPY config/runtime.exs config/ @@ -68,7 +70,7 @@ RUN mix release # the compiled release and other runtime necessities FROM ${RUNNER_IMAGE} -RUN apt-get update -y && apt-get install -y libstdc++6 openssl libncurses5 locales \ +RUN apt-get update -y && apt-get install -y libstdc++6 openssl libncurses5 locales curl ffmpeg \ && apt-get clean && rm -f /var/lib/apt/lists/*_* # Set the locale @@ -80,9 +82,11 @@ ENV LC_ALL en_US.UTF-8 WORKDIR "/app" RUN chown nobody /app +ENV BUMBLEBEE_CACHE_DIR="/app/.bumblebee" # Only copy the final release from the build stage COPY --from=builder --chown=nobody:root /app/_build/prod/rel/live_beats ./ +COPY --from=builder --chown=nobody:root /app/.bumblebee/ ./.bumblebee USER nobody diff --git a/config/config.exs b/config/config.exs index e60bde9..a0d2a9f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -34,7 +34,7 @@ config :esbuild, # Configure tailwind (the version is required) config :tailwind, - version: "3.1.8", + version: "3.2.7", default: [ args: ~w( --config=tailwind.config.js diff --git a/config/dev.exs b/config/dev.exs index 7401360..0686acb 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -2,7 +2,7 @@ import Config config :live_beats, :files, uploads_dir: Path.expand("../priv/uploads", __DIR__), - host: [scheme: "http", host: "localhost", port: 4000], + host: [scheme: "http", host: "localhost", port: 4001], server_ip: "127.0.0.1", hostname: "localhost", transport_opts: [] diff --git a/fly.toml b/fly.toml index 4f9205d..23116c5 100644 --- a/fly.toml +++ b/fly.toml @@ -1,45 +1,46 @@ -app = "livebeats" +# fly.toml app configuration file generated for livebeats on 2023-05-19T22:42:40-04:00 +# +# See https://fly.io/docs/reference/configuration/ for information about how to use this file. +# +app = "livebeats" +primary_region = "ord" kill_signal = "SIGTERM" -kill_timeout = 5 -processes = [] +kill_timeout = "5s" + +[experimental] + auto_rollback = true [deploy] release_command = "/app/bin/migrate" [env] + BUMBLEBEE_CACHE_DIR = "/app/.bumblebee" PHX_HOST = "livebeats.fly.dev" [mounts] source="data" destination="/app/uploads" -[experimental] - allowed_public_ports = [] - auto_rollback = true - [[services]] - http_checks = [] + protocol = "tcp" internal_port = 4000 processes = ["app"] - protocol = "tcp" - script_checks = [] + [[services.ports]] + port = 80 + handlers = ["http"] + + [[services.ports]] + port = 443 + handlers = ["tls", "http"] [services.concurrency] + type = "connections" hard_limit = 2500 soft_limit = 2000 - type = "connections" - - [[services.ports]] - handlers = ["http"] - port = 80 - - [[services.ports]] - handlers = ["tls", "http"] - port = 443 [[services.tcp_checks]] - grace_period = "20s" # allow some time for startup interval = "15s" + timeout = "2s" + grace_period = "20s" restart_limit = 0 - timeout = "2s" \ No newline at end of file diff --git a/lib/live_beats/application.ex b/lib/live_beats/application.ex index 7cac791..f5cd977 100644 --- a/lib/live_beats/application.ex +++ b/lib/live_beats/application.ex @@ -5,32 +5,46 @@ defmodule LiveBeats.Application do use Application + def load_serving do + {:ok, whisper} = Bumblebee.load_model({:hf, "openai/whisper-tiny"}) + {:ok, featurizer} = Bumblebee.load_featurizer({:hf, "openai/whisper-tiny"}) + {:ok, tokenizer} = Bumblebee.load_tokenizer({:hf, "openai/whisper-tiny"}) + + Bumblebee.Audio.speech_to_text(whisper, featurizer, tokenizer, + compile: [batch_size: 1], + max_new_tokens: 100, + defn_options: [compiler: EXLA] + ) + end + @impl true def start(_type, _args) do LiveBeats.MediaLibrary.attach() topologies = Application.get_env(:libcluster, :topologies) || [] - children = [ - {Cluster.Supervisor, [topologies, [name: LiveBeats.ClusterSupervisor]]}, - {Task.Supervisor, name: LiveBeats.TaskSupervisor}, - # Start the Ecto repository - LiveBeats.Repo, - LiveBeats.ReplicaRepo, - # Start the Telemetry supervisor - LiveBeatsWeb.Telemetry, - # Start the PubSub system - {Phoenix.PubSub, name: LiveBeats.PubSub}, - # start presence - LiveBeatsWeb.Presence, - {Finch, name: LiveBeats.Finch}, - # Start the Endpoint (http/https) - LiveBeatsWeb.Endpoint, - # Expire songs every six hours - {LiveBeats.SongsCleaner, interval: {3600 * 6, :second}} - - # Start a worker by calling: LiveBeats.Worker.start_link(arg) - # {LiveBeats.Worker, arg} - ] + children = + [ + {Nx.Serving, name: WhisperServing, serving: load_serving()}, + {Cluster.Supervisor, [topologies, [name: LiveBeats.ClusterSupervisor]]}, + {Task.Supervisor, name: LiveBeats.TaskSupervisor}, + {Task.Supervisor, name: Fly.Machine.TaskSupervisor}, + # Start the Ecto repository + LiveBeats.Repo, + LiveBeats.ReplicaRepo, + # Start the Telemetry supervisor + LiveBeatsWeb.Telemetry, + # Start the PubSub system + {Phoenix.PubSub, name: LiveBeats.PubSub}, + # start presence + LiveBeatsWeb.Presence, + {Finch, name: LiveBeats.Finch}, + # Start the Endpoint (http/https) + LiveBeatsWeb.Endpoint, + # Expire songs every six hours + {LiveBeats.SongsCleaner, interval: {3600 * 6, :second}} + # Start a worker by calling: LiveBeats.Worker.start_link(arg) + # {LiveBeats.Worker, arg} + ] # See https://hexdocs.pm/elixir/Supervisor.html # for other strategies and supported options diff --git a/lib/live_beats/audio.ex b/lib/live_beats/audio.ex new file mode 100644 index 0000000..f4340b5 --- /dev/null +++ b/lib/live_beats/audio.ex @@ -0,0 +1,17 @@ +defmodule LiveBeats.Audio do + def speech_to_text(path, chunk_time, func) do + {:ok, stat} = LiveBeats.MP3Stat.parse(path) + + 0..stat.duration//chunk_time + |> Task.async_stream( + fn ss -> + args = ~w(-ac 1 -ar 16k -f f32le -ss #{ss} -t #{chunk_time} -v quiet -) + {data, 0} = System.cmd("ffmpeg", ["-i", path] ++ args) + {ss, Nx.Serving.batched_run(WhisperServing, Nx.from_binary(data, :f32))} + end, + max_concurrency: 2, + timeout: :infinity + ) + |> Enum.map(fn {:ok, {ss, %{results: [%{text: text}]}}} -> func.(ss, text) end) + end +end diff --git a/lib/live_beats/media_library.ex b/lib/live_beats/media_library.ex index ac9b959..e5b6f94 100644 --- a/lib/live_beats/media_library.ex +++ b/lib/live_beats/media_library.ex @@ -52,13 +52,7 @@ defmodule LiveBeats.MediaLibrary do user.id == song.user_id end - def play_song(%Song{id: id}) do - play_song(id) - end - - def play_song(id) do - song = get_song!(id) - + def play_song(%Song{} = song) do played_at = cond do playing?(song) -> @@ -97,6 +91,12 @@ defmodule LiveBeats.MediaLibrary do new_song end + def play_song(id) do + id + |> get_song!() + |> play_song() + end + def pause_song(%Song{} = song) do now = DateTime.truncate(DateTime.utc_now(), :second) set = [status: :paused, paused_at: now] @@ -211,6 +211,7 @@ defmodule LiveBeats.MediaLibrary do |> Enum.filter(&match?({{:song, _ref}, _}, &1)) |> Enum.map(fn {{:song, ref}, song} -> consume_file.(ref, fn tmp_path -> store_mp3(song, tmp_path) end) + async_transcribe(song, user) {ref, song} end) @@ -231,6 +232,22 @@ defmodule LiveBeats.MediaLibrary do end end + defp async_transcribe(%Song{} = song, %Accounts.User{} = user) do + Task.Supervisor.start_child(LiveBeats.TaskSupervisor, fn -> + segments = + LiveBeats.Audio.speech_to_text(song.mp3_filepath, 20, fn ss, text -> + segment = %Song.TranscriptSegment{ss: ss, text: text} + broadcast!(user.id, {segment, song.id}) + + segment + end) + + Repo.update_all(from(s in Song, where: s.id == ^song.id), + set: [transcript_segments: segments] + ) + end) + end + defp broadcast_imported(%Accounts.User{} = user, songs) do songs = Enum.map(songs, fn {_ref, song} -> song end) broadcast!(user.id, %Events.SongsImported{user_id: user.id, songs: songs}) diff --git a/lib/live_beats/media_library/song.ex b/lib/live_beats/media_library/song.ex index e445212..43c1513 100644 --- a/lib/live_beats/media_library/song.ex +++ b/lib/live_beats/media_library/song.ex @@ -25,6 +25,11 @@ defmodule LiveBeats.MediaLibrary.Song do belongs_to :user, Accounts.User belongs_to :genre, LiveBeats.MediaLibrary.Genre + embeds_many :transcript_segments, TranscriptSegment do + field :ss, :integer + field :text, :string + end + timestamps() end diff --git a/lib/live_beats_web/controllers/oauth_callback_controller.ex b/lib/live_beats_web/controllers/oauth_callback_controller.ex index d4ebc54..bc45b2d 100644 --- a/lib/live_beats_web/controllers/oauth_callback_controller.ex +++ b/lib/live_beats_web/controllers/oauth_callback_controller.ex @@ -15,7 +15,7 @@ defmodule LiveBeatsWeb.OAuthCallbackController do |> LiveBeatsWeb.UserAuth.log_in_user(user) else {:error, %Ecto.Changeset{} = changeset} -> - Logger.debug("failed GitHub insert #{inspect(changeset.errors)}") + Logger.info("failed GitHub insert #{inspect(changeset.errors)}") conn |> put_flash( @@ -25,7 +25,7 @@ defmodule LiveBeatsWeb.OAuthCallbackController do |> redirect(to: "/") {:error, reason} -> - Logger.debug("failed GitHub exchange #{inspect(reason)}") + Logger.info("failed GitHub exchange #{inspect(reason)}") conn |> put_flash(:error, "We were unable to contact GitHub. Please try again later") diff --git a/lib/live_beats_web/live/profile_live.ex b/lib/live_beats_web/live/profile_live.ex index 4fe7cee..bc89302 100644 --- a/lib/live_beats_web/live/profile_live.ex +++ b/lib/live_beats_web/live/profile_live.ex @@ -60,6 +60,13 @@ defmodule LiveBeatsWeb.ProfileLive do total_count={@presences_count} /> +