Browse Source

ref: move scraper functionality to http, move most workers to tomie_web

master
Inhji Y. 10 months ago
parent
commit
3fa50a934f
  1. 7
      apps/bookmarks/mix.exs
  2. 13
      apps/http/lib/application.ex
  3. 30
      apps/http/lib/html.ex
  4. 51
      apps/http/lib/weather.ex
  5. 7
      apps/http/mix.exs
  6. 2
      apps/tomie_web/lib/tomie_web/handlers/micropub_handler.ex
  7. 10
      apps/tomie_web/lib/tomie_web/workers/bookmark_worker.ex
  8. 12
      apps/tomie_web/lib/tomie_web/workers/weather_worker.ex
  9. 3
      apps/tomie_web/mix.exs
  10. 4
      config/config.exs
  11. 1
      mix.exs

7
apps/bookmarks/mix.exs

@ -20,23 +20,20 @@ defmodule Bookmarks.MixProject do
def application do
[
mod: {Bookmarks.Application, []},
extra_applications: [:logger, :tesla]
extra_applications: [:logger]
]
end
# Run "mix help deps" to learn about dependencies.
defp deps do
[
{:oban, "~> 1.2"},
{:earmark, "~> 1.4"},
{:ecto_sql, "~> 3.1"},
{:phoenix_pubsub, "~> 2.0"},
{:mock, "~> 0.3.0", only: :test},
{:excoveralls, "~> 0.13.0", only: :test},
{:db, in_umbrella: true},
{:tags, in_umbrella: true},
{:scraper, in_umbrella: true},
{:webmentions, "~> 0.4.2"}
{:http, in_umbrella: true},
]
end
end

13
apps/http/lib/application.ex

@ -0,0 +1,13 @@
defmodule Http.Application do
use Application
def start(_type, _args) do
import Supervisor.Spec, warn: false
children = [
worker(Cachex, [:weather, []], id: :weather)
]
Supervisor.start_link(children, strategy: :one_for_one, name: Http.Supervisor)
end
end

30
apps/http/lib/html.ex

@ -0,0 +1,30 @@
defmodule Http.Html do
def get_html(url) do
case Http.get(url, [], follow_redirect: true) do
{:ok, %{body: body, status_code: 200}} ->
{:ok, body}
{:ok, status_code: status_code} ->
{:error, status_code}
{:error, error} ->
{:error, error}
end
end
def get_title!(html) do
{:ok, document} = Floki.parse_document(html)
document
|> Floki.find("head title")
|> Floki.text()
end
def parse(html) do
title = get_title!(html)
og = OpenGraphExtended.parse(html)
result = Map.merge(%{title: title}, og, fn _k, v1, v2 -> v2 || v1 end)
{:ok, result}
end
end

51
apps/http/lib/weather.ex

@ -0,0 +1,51 @@
defmodule Http.Weather do
@base_url "http://api.openweathermap.org/data/2.5/onecall"
@opts "units=metric,exclude=minutely,daily"
def get_weather() do
[api_key: api_key, latitude: lat, longitude: lon, city_id: _] =
Application.fetch_env!(:http, :weather)
url = "#{@base_url}?lat=#{lat}&lon=#{lon}&appid=#{api_key}&#{@opts}"
with %{body: body, status_code: 200} <- Http.get!(url),
{:ok, json} <- Jason.decode!(body, keys: :atoms) do
json
else
error -> error
end
end
def save_weather(%{
current: current,
daily: daily,
hourly: _hourly,
lat: _lat,
lon: _lon,
timezone: _timezone,
timezone_offset: _timezone_offset
}) do
daily_data = Enum.map(daily, &map_daily/1)
Cachex.put(:weather, :daily, daily_data)
Cachex.put(:weather, :now, %{
pretty: List.first(current.weather),
temp: current.temp,
clouds: current.clouds,
timestamp: current.dt,
humidity: current.humidity
})
end
defp map_daily(day) do
date =
day.dt
|> DateTime.from_unix!()
|> DateTime.to_date()
day
|> Map.put(:date, date)
|> Map.put(:pretty, List.first(day.weather))
end
end

7
apps/http/mix.exs

@ -18,6 +18,7 @@ defmodule Http.MixProject do
# Run "mix help compile.app" to learn about applications.
def application do
[
mod: {Http.Application, []},
extra_applications: [:logger]
]
end
@ -25,7 +26,11 @@ defmodule Http.MixProject do
# Run "mix help deps" to learn about dependencies.
defp deps do
[
{:httpoison, "~> 1.6"}
{:cachex, "~> 3.2"},
{:floki, "~> 0.26.0"},
{:httpoison, "~> 1.6"},
{:oban, "~> 1.2"},
{:open_graph_extended, git: "https://git.inhji.de/inhji/open_graph.git"}
]
end
end

2
apps/tomie_web/lib/tomie_web/handlers/micropub_handler.ex

@ -33,7 +33,7 @@ defmodule TomieWeb.MicropubHandler do
case Bookmarks.create_bookmark(attrs) do
{:ok, bookmark} ->
Bookmarks.Worker.new(%{
TomieWeb.BookmarkWorker.new(%{
id: bookmark.id,
url: Routes.home_url(TomieWeb.Endpoint, :bookmark, bookmark)
})

10
apps/bookmarks/lib/bookmarks/worker.ex → apps/tomie_web/lib/tomie_web/workers/bookmark_worker.ex

@ -1,5 +1,5 @@
defmodule Bookmarks.Worker do
use Oban.Worker,
defmodule TomieWeb.BookmarkWorker do
use Oban.Worker,
queue: :bookmarks,
max_attempts: 5
@ -18,8 +18,8 @@ defmodule Bookmarks.Worker do
end
def scrape_bookmark_url(%Bookmark{source: source} = bookmark) do
with {:ok, html} <- Scraper.Html.get_html(source),
{:ok, result} <- Scraper.Html.parse(html),
with {:ok, html} <- Http.Html.get_html(source),
{:ok, result} <- Http.Html.parse(html),
{:ok, updated_bookmark} <- Bookmarks.update_bookmark(bookmark, %{title: result.title}) do
{:ok, updated_bookmark} =
Tags.list_tags_with_rules()
@ -48,4 +48,4 @@ defmodule Bookmarks.Worker do
{:error, message} -> {:error, message}
end
end
end
end

12
apps/tomie_web/lib/tomie_web/workers/weather_worker.ex

@ -0,0 +1,12 @@
defmodule TomieWeb.WeatherWorker do
use Oban.Worker,
queue: :scraper,
max_attempts: 1,
unique: [period: 60, fields: [:worker]]
@impl Oban.Worker
def perform(_args, _job) do
Http.Weather.get_weather()
|> Http.Weather.save_weather()
end
end

3
apps/tomie_web/mix.exs

@ -25,7 +25,7 @@ defmodule TomieWeb.MixProject do
def application do
[
mod: {TomieWeb.Application, []},
extra_applications: [:logger, :runtime_tools, :uuid, :mnesia]
extra_applications: [:logger, :runtime_tools, :uuid, :mnesia, :tesla]
]
end
@ -47,6 +47,7 @@ defmodule TomieWeb.MixProject do
{:indie, in_umbrella: true},
{:jason, "~> 1.0"},
{:listens, in_umbrella: true},
{:oban, "~> 1.2.0"},
{:phoenix, "~> 1.5.0"},
{:phoenix_active_link, "~> 0.3.0"},
{:phoenix_ecto, "~> 4.0"},

4
config/config.exs

@ -19,7 +19,7 @@ config :tomie,
config :listens,
discogs_token: "kRIDCYTMRucJojWzQKlXlDAnDlQSgmXboMEZiUBT"
config :scraper, :weather,
config :http, :weather,
api_key: "5e8d6569a828fd7860acf87bf1113a9e",
latitude: 50.583832,
longitude: 8.677890,
@ -53,7 +53,7 @@ config :tomie, Oban,
{"* * * * *", Listens.Workers.DiscogsAlbum, args: %{action: "search_id"}},
{"* * * * *", Listens.Workers.DiscogsAlbum, args: %{action: "fetch_cover"}},
{"* * * * *", Listens.Workers.DiscogsAlbum, args: %{action: "fetch_genres"}},
{"*/5 * * * *", Scraper.Workers.Weather, args: %{}}
{"*/5 * * * *", TomieWeb.WeatherWorker, args: %{}}
]
config :tomie_web,

1
mix.exs

@ -17,7 +17,6 @@ defmodule Tomie.Umbrella.MixProject do
http: :permanent,
indie: :permanent,
listens: :permanent,
scraper: :permanent,
tags: :permanent,
tomie: :permanent,
tomie_web: :permanent,

Loading…
Cancel
Save