Browse Source

worker settings

master
Inhji Y. 2 years ago
parent
commit
3cbfe602fc
  1. 29
      assets/css/app.scss
  2. 8
      lib/dagon/listens/albums.ex
  3. 8
      lib/dagon/listens/listens.ex
  4. 28
      lib/dagon/listens/rate_limit.ex
  5. 136
      lib/dagon/listens/workers/discogs_worker.ex
  6. 99
      lib/dagon/listens/workers/listenbrainz_worker.ex
  7. 14
      lib/dagon/listens/workers/worker.ex
  8. 2
      lib/dagon_web/controllers/album_controller.ex
  9. 44
      lib/dagon_web/controllers/worker_controller.ex
  10. 2
      lib/dagon_web/router.ex
  11. 9
      lib/dagon_web/templates/album/index.html.eex
  12. 31
      lib/dagon_web/templates/album/show.html.eex
  13. 1
      lib/dagon_web/templates/artist/show.html.eex
  14. 2
      lib/dagon_web/templates/layout/app.html.eex
  15. 9
      lib/dagon_web/templates/listen/show.html.eex
  16. 14
      lib/dagon_web/templates/page/index.html.eex
  17. 4
      lib/dagon_web/templates/worker/edit.html.eex
  18. 11
      lib/dagon_web/templates/worker/form.html.eex
  19. 53
      lib/dagon_web/templates/worker/index.html.eex
  20. 3
      lib/dagon_web/views/worker_view.ex

29
assets/css/app.scss

@ -1,4 +1,20 @@
@import "~bulma/bulma.sass";
@charset "utf-8";
// Import a Google Font
@import url('https://fonts.googleapis.com/css?family=Nunito:400,700');
$family-sans-serif: "Nunito", sans-serif;
$dimensions: 16 24 32 48 64 96 128 196;
$spacing-default: 1.5rem;
@import "~bulma/sass/utilities/_all";
@import "~bulma/sass/base/_all";
@import "~bulma/sass/elements/_all";
@import "~bulma/sass/components/_all";
@import "~bulma/sass/form/_all";
@import "~bulma/sass/grid/_all";
@import "~bulma/sass/layout/_all";
.media.is-dense {
margin-top: 0;
@ -6,10 +22,13 @@
border: 0;
}
.has-padding-bottom {
padding-bottom: 1rem;
.has-padding {
padding: $spacing-default;
}
.has-margin-bottom {
margin-bottom: 1rem;
.has-padding-bottom {
padding-bottom: $spacing-default;
}
.has-margin-bottom { margin-bottom: $spacing-default; }
.has-margin-top { margin-top: $spacing-default; }

8
lib/dagon/listens/albums.ex

@ -19,8 +19,9 @@ defmodule Dagon.Listens.Albums do
"""
def list_albums do
Album
|> order_by(:name)
|> order_by([:image, :name])
|> Repo.all()
|> Repo.preload(:artist)
end
def list_albums_without_cover do
@ -53,7 +54,10 @@ defmodule Dagon.Listens.Albums do
** (Ecto.NoResultsError)
"""
def get_album!(id), do: Repo.get!(Album, id)
def get_album!(id),
do:
Repo.get!(Album, id)
|> Repo.preload([:artist])
@doc """
Creates a album.

8
lib/dagon/listens/listens.ex

@ -1,4 +1,6 @@
defmodule Dagon.Listens.Listens do
@preloads [:artist, :album, :track]
@moduledoc """
The Listens.Listens context.
"""
@ -21,7 +23,7 @@ defmodule Dagon.Listens.Listens do
Listen
|> order_by(desc: :listened_at)
|> Repo.all()
|> Repo.preload([:artist, :album, :track])
|> Repo.preload(@preloads)
end
@doc """
@ -38,7 +40,7 @@ defmodule Dagon.Listens.Listens do
|> order_by(desc: :listened_at)
|> limit(^limit)
|> Repo.all()
|> Repo.preload([:artist, :album, :track])
|> Repo.preload(@preloads)
end
def listens_per_month_by_artist(artist) do
@ -64,7 +66,7 @@ defmodule Dagon.Listens.Listens do
** (Ecto.NoResultsError)
"""
def get_listen!(id), do: Repo.get!(Listen, id)
def get_listen!(id), do: Repo.get!(Listen, id) |> Repo.preload(@preloads)
def get_oldest_listen() do
Listen

28
lib/dagon/listens/rate_limit.ex

@ -0,0 +1,28 @@
defmodule Dagon.Listens.RateLimit do
@id_listenbrainz "LB"
@id_discogs "DC"
def calculate(headers, @id_listenbrainz) do
keyword_list = convert_to_keyword_list(headers)
%{
total: Keyword.get(keyword_list, :"X-RateLimit-Limit"),
remaining: Keyword.get(keyword_list, :"X-RateLimit-Remaining")
}
end
def calculate(headers, @id_discogs) do
keyword_list = convert_to_keyword_list(headers)
%{
total: Keyword.get(keyword_list, :"X-Discogs-Ratelimit"),
remaining: Keyword.get(keyword_list, :"X-Discogs-Ratelimit-Remaining")
}
end
defp convert_to_keyword_list(list) do
Enum.map(list, fn {name, value} ->
{String.to_atom(name), value}
end)
end
end

136
lib/dagon/listens/workers/discogs_worker.ex

@ -4,7 +4,7 @@ defmodule Dagon.Listens.Workers.DiscogsWorker do
alias Dagon.Listens.Albums
@fetch_interval 1 * 60 * 1_000
@fetch_interval 2 * 60 * 1_000
@base_url "https://api.discogs.com/database/search"
@token "kRIDCYTMRucJojWzQKlXlDAnDlQSgmXboMEZiUBT"
@invalid_discogs_id -1
@ -18,69 +18,119 @@ defmodule Dagon.Listens.Workers.DiscogsWorker do
GenServer.cast(__MODULE__, :update)
end
def init(state \\ %{}) do
schedule_fetch(10_000)
def update_fetch_interval(interval_in_seconds) do
GenServer.cast(__MODULE__, {:update_fetch_interval, interval_in_seconds})
end
def init(_state) do
state = %{
rate_limit: %{
total: -1,
remaining: -1
},
fetch_interval: @fetch_interval,
updated_at: DateTime.utc_now()
}
schedule_fetch(state, 10_000)
{:ok, state}
end
def get_state() do
GenServer.call(__MODULE__, :get_state)
end
def handle_call(:get_state, _from, state) do
{:reply, state, state}
end
def handle_cast({:update_fetch_interval, interval_in_seconds}, state) do
Logger.info("Updating fetch_interval to #{interval_in_seconds}s")
new_state = Map.replace!(state, :fetch_interval, interval_in_seconds * 1000)
{:noreply, new_state}
end
def handle_info(:fetch, state) do
new_state = do_fetch(state)
{:noreply, new_state}
end
def schedule_fetch(wait_time \\ @fetch_interval) do
def handle_info({:ssl_closed, _}, state) do
Logger.error("Temporary TLS error")
{:noreply, state}
end
def schedule_fetch(state) do
Logger.info("Scheduling Discogs Worker..")
Process.send_after(self(), :fetch, state.fetch_interval)
end
def schedule_fetch(_state, wait_time) do
Logger.info("Scheduling Discogs Worker..")
Process.send_after(self(), :fetch, wait_time)
end
def do_fetch(state) do
albums = Albums.list_albums_without_cover()
Enum.each(albums, &fetch_album_cover/1)
# todo
state =
Enum.reduce(albums, state, fn album, state ->
fetch_album_cover(state, album)
end)
schedule_fetch()
schedule_fetch(state)
state
end
def fetch_album_cover(album) do
def fetch_album_cover(state, album) do
url = get_url(album)
case HTTPoison.get!(url, [{"User-Agent", "Dagon/0.1.0"}]) do
%HTTPoison.Response{body: body} ->
data =
body
|> Jason.decode!(keys: :atoms)
|> Map.get(:results)
|> List.first()
if not is_nil(data) do
discogs_id = data.id
image = data.cover_image
Logger.info("Saving cover image for #{album.name}")
Logger.debug("Remote filename: #{image}")
Albums.update_album(
album,
%{
image: image,
discogs_id: discogs_id
}
)
else
Logger.warn("No data found for #{album.name}")
Albums.update_album(
album,
%{
discogs_id: @invalid_discogs_id
}
)
end
_ ->
nil
state =
case HTTPoison.get!(url, [{"User-Agent", "Dagon/0.1.0"}]) do
%HTTPoison.Response{body: body, headers: headers} ->
handle_fetch_response(body, album)
rate_limit = Dagon.Listens.RateLimit.calculate(headers, "DC")
Map.put(state, :rate_limit, rate_limit)
_ ->
state
end
state = Map.put(state, :updated_at, DateTime.utc_now())
state
end
def handle_fetch_response(body, album) do
data =
body
|> Jason.decode!(keys: :atoms)
|> Map.get(:results)
|> List.first()
if not is_nil(data) do
discogs_id = data.id
image = data.cover_image
Logger.info("Saving cover image for '#{album.name}'.")
Logger.debug("Remote filename: #{image}")
Albums.update_album(
album,
%{
image: image,
discogs_id: discogs_id
}
)
else
Logger.warn("No data found for '#{album.name}'. Marking album cover as invalid.")
Albums.update_album(
album,
%{
discogs_id: @invalid_discogs_id
}
)
end
end

99
lib/dagon/listens/workers/listenbrainz_worker.ex

@ -22,8 +22,25 @@ defmodule Dagon.Listens.Workers.ListenbrainzWorker do
GenServer.cast(__MODULE__, :update)
end
def init(state \\ %{}) do
schedule_fetch(10_000)
def update_fetch_interval(interval_in_seconds) do
GenServer.cast(__MODULE__, {:update_fetch_interval, interval_in_seconds})
end
def get_state() do
GenServer.call(__MODULE__, :get_state)
end
def init(_state) do
state = %{
rate_limit: %{
total: -1,
remaining: -1
},
fetch_interval: @fetch_interval,
updated_at: DateTime.utc_now()
}
schedule_fetch(state, 10_000)
{:ok, state}
end
@ -32,7 +49,27 @@ defmodule Dagon.Listens.Workers.ListenbrainzWorker do
{:noreply, new_state}
end
def schedule_fetch(wait_time \\ @fetch_interval) do
def handle_info({:ssl_closed, _}, state) do
Logger.error("Temporary TLS error")
{:noreply, state}
end
def handle_cast({:update_fetch_interval, interval_in_seconds}, state) do
Logger.info("Updating fetch_interval to #{interval_in_seconds}s")
new_state = Map.replace!(state, :fetch_interval, interval_in_seconds * 1000)
{:noreply, new_state}
end
def handle_call(:get_state, _from, state) do
{:reply, state, state}
end
def schedule_fetch(state) do
Logger.info("Scheduling Listenbrainz Worker..")
Process.send_after(self(), :fetch, state.fetch_interval)
end
def schedule_fetch(_state, wait_time) do
Logger.info("Scheduling Listenbrainz Worker..")
Process.send_after(self(), :fetch, wait_time)
end
@ -45,27 +82,36 @@ defmodule Dagon.Listens.Workers.ListenbrainzWorker do
Logger.info("Fetching new Listens for Timestamp #{last_ts}:")
Logger.info(url)
listens =
case HTTPoison.get!(url) do
%HTTPoison.Response{body: body} ->
body
|> Jason.decode!(keys: :atoms)
|> Map.get(:payload)
|> Map.get(:listens)
response = HTTPoison.get!(url)
state =
case response do
%HTTPoison.Response{body: body, headers: headers} ->
handle_fetch_response(body)
rate_limit = Dagon.Listens.RateLimit.calculate(headers, "LB")
Map.put(state, :rate_limit, rate_limit)
_ ->
nil
state
end
listens
schedule_fetch(state)
state = Map.replace!(state, :updated_at, DateTime.utc_now())
state
end
def handle_fetch_response(body) do
body
|> Jason.decode!(keys: :atoms)
|> Map.get(:payload)
|> Map.get(:listens)
|> prepare_listens()
|> Enum.filter(fn l -> !is_nil(l) end)
|> Enum.each(fn changeset ->
Repo.insert(changeset, log: false)
end)
schedule_fetch()
state
end
def last_listen_timestamp do
@ -102,13 +148,17 @@ defmodule Dagon.Listens.Workers.ListenbrainzWorker do
})
else
{:error, reason} ->
Logger.error(inspect(reason))
Logger.error(reason)
nil
{:warn, reason} ->
Logger.warn(reason)
nil
end
end
def maybe_create_artist(nil, _) do
{:error, "Artist name was nil, skipping."}
{:warn, "Artist name was nil, skipping."}
end
def maybe_create_artist(name, messybrainz_id) do
@ -120,7 +170,7 @@ defmodule Dagon.Listens.Workers.ListenbrainzWorker do
%Artist{}
|> Artist.changeset(%{
name: name,
slug: Slugger.slugify_downcase(name),
slug: slugify(name),
msid: messybrainz_id
})
|> Repo.insert!(log: false)
@ -132,8 +182,15 @@ defmodule Dagon.Listens.Workers.ListenbrainzWorker do
{:ok, artist}
end
def slugify(string) do
case Slugger.slugify_downcase(string) do
"" -> Ecto.UUID.generate()
slug -> slug
end
end
def maybe_create_album(nil, _, _) do
{:error, "Album name was nil, skipping."}
{:warn, "Album name was nil, skipping."}
end
def maybe_create_album(name, messybrainz_id, artist) do
@ -145,7 +202,7 @@ defmodule Dagon.Listens.Workers.ListenbrainzWorker do
%Album{}
|> Album.changeset(%{
name: name,
slug: Slugger.slugify_downcase(name),
slug: slugify(name),
msid: messybrainz_id,
artist_id: artist.id
})
@ -162,7 +219,7 @@ defmodule Dagon.Listens.Workers.ListenbrainzWorker do
track =
Track
|> Repo.get_by([name: name, artist_id: artist.id, album_id: album.id], log: false)
|> Repo.preload([:album, :artist])
|> Repo.preload([:album, :artist], log: false)
track =
case track do

14
lib/dagon/listens/workers/worker.ex

@ -0,0 +1,14 @@
defmodule Dagon.Listens.Workers.Worker do
import Ecto.Changeset
@types %{
id: :string,
fetch_interval: :integer
}
def change(params \\ %{}) do
{%{}, @types}
|> cast(params, Map.keys(@types))
|> validate_required([:id, :fetch_interval])
end
end

2
lib/dagon_web/controllers/album_controller.ex

@ -5,7 +5,7 @@ defmodule DagonWeb.AlbumController do
alias Dagon.Listens.Albums.Album
def index(conn, _params) do
albums = Albums.list_albums_with_cover()
albums = Albums.list_albums()
render(conn, "index.html", albums: albums)
end

44
lib/dagon_web/controllers/worker_controller.ex

@ -0,0 +1,44 @@
defmodule DagonWeb.WorkerController do
use DagonWeb, :controller
alias Dagon.Listens.Workers
def index(conn, _params) do
render(conn, "index.html",
state: %{
listenbrainz: get_worker_state("listenbrainz"),
discogs: get_worker_state("discogs")
}
)
end
def edit(conn, %{"id" => id}) do
state = get_worker_state(id)
changeset = Workers.Worker.change(%{id: id, fetch_interval: div(state.fetch_interval, 1000)})
render(conn, "edit.html", id: id, changeset: changeset)
end
def update(conn, %{"id" => id, "worker" => worker_params}) do
changeset = Workers.Worker.change(worker_params)
case changeset.valid? do
true ->
settings = Ecto.Changeset.apply_changes(changeset)
update_fetch_interval(id, settings.fetch_interval)
false ->
render(conn, "edit.html", id: id, changeset: changeset)
end
redirect(conn, to: Routes.worker_path(conn, :index))
end
defp get_worker_state("listenbrainz"), do: Workers.ListenbrainzWorker.get_state()
defp get_worker_state("discogs"), do: Workers.DiscogsWorker.get_state()
defp update_fetch_interval("listenbrainz", interval),
do: Workers.ListenbrainzWorker.update_fetch_interval(interval)
defp update_fetch_interval("discogs", interval),
do: Workers.DiscogsWorker.update_fetch_interval(interval)
end

2
lib/dagon_web/router.ex

@ -22,6 +22,8 @@ defmodule DagonWeb.Router do
resources "/albums", AlbumController
resources "/tracks", TrackController
resources "/listens", ListenController
resources "/workers", WorkerController, only: [:index, :edit, :update]
end
# Other scopes may use custom stacks.

9
lib/dagon_web/templates/album/index.html.eex

@ -3,9 +3,12 @@
<div class="columns is-multiline">
<%= for album <- @albums do %>
<div class="column is-1">
<figure class="image is-96x96">
<img src="<%= Dagon.Listens.Albums.Uploader.url({album.image, album}, :thumb) %>">
</figure>
<a href="<%= Routes.album_path(@conn, :show, album) %>">
<figure class="image is-96x96">
<img src="<%= Dagon.Listens.Albums.Uploader.url({album.image, album}, :thumb) %>">
</figure>
</a>
<%= album.name %>
</div>
<% end %>
</div>

31
lib/dagon_web/templates/album/show.html.eex

@ -1,23 +1,14 @@
<h1>Show Album</h1>
<ul>
<li>
<strong>Name:</strong>
<%= @album.name %>
</li>
<li>
<strong>Slug:</strong>
<%= @album.slug %>
</li>
<li>
<strong>Mbid:</strong>
<%= @album.mbid %>
</li>
</ul>
<div class="media">
<div class="media-left">
<figure class="image is-196x196">
<img src="<%= Dagon.Listens.Albums.Uploader.url({@album.image, @album}, :large) %>">
</figure>
</div>
<div class="media-content">
<h1 class="title"><%= @album.name %></h1>
<h2 class="subtitle"><%= @album.artist.name %></h2>
</div>
</div>
<span><%= link "Edit", to: Routes.album_path(@conn, :edit, @album) %></span>
<span><%= link "Back", to: Routes.album_path(@conn, :index) %></span>

1
lib/dagon_web/templates/artist/show.html.eex

@ -2,6 +2,7 @@
<div class="listen-chart" style="width: 200px; height: 50px">
<div id="dailychart"
data-dailychart-close="0"
data-dailychart-values='<%= Enum.join(@per_month.listens, ",") %>'
data-dailychart-length="<%= Enum.count(@per_month.listens) %>"></div>
<small class="has-text-grey is-pulled-left"><%= Timex.format!(@per_month.oldest.listened_at, "{YYYY}") %></small>

2
lib/dagon_web/templates/layout/app.html.eex

@ -45,7 +45,7 @@
</nav>
</section>
</header>
<main role="main" class="container">
<main role="main" class="container has-padding-bottom">
<p class="alert alert-info" role="alert"><%= get_flash(@conn, :info) %></p>
<p class="alert alert-danger" role="alert"><%= get_flash(@conn, :error) %></p>
<%= render @view_module, @view_template, assigns %>

9
lib/dagon_web/templates/listen/show.html.eex

@ -1,8 +1 @@
<h1>Show Listen</h1>
<ul>
</ul>
<span><%= link "Edit", to: Routes.listen_path(@conn, :edit, @listen) %></span>
<span><%= link "Back", to: Routes.listen_path(@conn, :index) %></span>
<%= render "listen.html", conn: @conn, listen: @listen %>

14
lib/dagon_web/templates/page/index.html.eex

@ -22,19 +22,5 @@
</div>
</div>
</article>
<article class="column">
<div class="card">
<div class="card-header">
<h3 class="card-header-title">
Popular Tracks
</h3>
</div>
<div class="card-content">
lol
</div>
</div>
</article>
</section>

4
lib/dagon_web/templates/worker/edit.html.eex

@ -0,0 +1,4 @@
<h1 class="title">Edit Worker</h1>
<%= render "form.html", Map.put(assigns, :action, Routes.worker_path(@conn, :update, @id)) %>

11
lib/dagon_web/templates/worker/form.html.eex

@ -0,0 +1,11 @@
<%= form_for @changeset, @action, [as: :worker, class: "form", method: "PUT"], fn f -> %>
<%= hidden_input f, :id %>
<%= label f, :fetch_interval, class: "label" %>
<%= text_input f, :fetch_interval, class: "input" %>
<%= error_tag f, :fetch_interval %>
<div>
<%= submit "Save", class: "button" %>
</div>
<% end %>

53
lib/dagon_web/templates/worker/index.html.eex

@ -0,0 +1,53 @@
<h1 class="title">Worker Settings</h1>
<div class="columns">
<div class="column">
<div class="card">
<div class="card-header has-background-primary">
<h3 class="card-header-title has-text-light">
Discogs Worker
</h3>
</div>
<div class="card-content">
<p>
<%= if not is_nil(@state.discogs) do %>
<div>Ratelimit Total: <%= @state.discogs.rate_limit.total %></div>
<div>Ratelimit Remaining: <%= @state.discogs.rate_limit.remaining %></div>
<div>Updated At: <%= Timex.from_now(@state.discogs.updated_at) %></div>
<div>Fetch Interval: <%= @state.discogs.fetch_interval / 1000 %> seconds</div>
<% end %>
</p>
</div>
<footer class="card-footer">
<a href="<%= Routes.worker_path(@conn, :edit, :discogs) %>" class="card-footer-item">Edit Settings</a>
<a href="#" class="card-footer-item">Run Now</a>
</footer>
</div>
</div>
<div class="column">
<div class="card">
<div class="card-header has-background-primary">
<h3 class="card-header-title has-text-light">
Listenbrainz Worker
</h3>
</div>
<div class="card-content">
<p>
<%= if not is_nil(@state.listenbrainz) do %>
<div>Ratelimit Total: <%= @state.listenbrainz.rate_limit.total %></div>
<div>Ratelimit Remaining: <%= @state.listenbrainz.rate_limit.remaining %></div>
<div>Updated At: <%= Timex.from_now(@state.listenbrainz.updated_at) %></div>
<div>Fetch Interval: <%= @state.listenbrainz.fetch_interval / 1000 %> seconds</div>
<% end %>
</p>
</div>
<footer class="card-footer">
<a href="<%= Routes.worker_path(@conn, :edit, :listenbrainz) %>" class="card-footer-item">Edit Settings</a>
<a href="#" class="card-footer-item">Run Now</a>
</footer>
</div>
</div>
</div>

3
lib/dagon_web/views/worker_view.ex

@ -0,0 +1,3 @@
defmodule DagonWeb.WorkerView do
use DagonWeb, :view
end
Loading…
Cancel
Save