Browse Source

feat: add http app, consolidate http requests

master
Inhji Y. 11 months ago
parent
commit
e03c9ab011
  1. 4
      apps/http/.formatter.exs
  2. 24
      apps/http/.gitignore
  3. 21
      apps/http/README.md
  4. 16
      apps/http/lib/http.ex
  5. 31
      apps/http/mix.exs
  6. 8
      apps/http/test/http_test.exs
  7. 1
      apps/http/test/test_helper.exs
  8. 18
      apps/listens/lib/listens/workers/discogs_album.ex
  9. 3
      apps/listens/lib/listens/workers/discogs_artist.ex
  10. 3
      apps/listens/lib/listens/workers/listenbrainz.ex
  11. 1
      apps/listens/mix.exs
  12. 10
      apps/scraper/lib/scraper.ex
  13. 1
      apps/scraper/mix.exs
  14. 5
      config/config.exs
  15. 3
      mix.exs

4
apps/http/.formatter.exs

@ -0,0 +1,4 @@
# Used by "mix format"
[
inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
]

24
apps/http/.gitignore

@ -0,0 +1,24 @@
# The directory Mix will write compiled artifacts to.
/_build/
# If you run "mix test --cover", coverage assets end up here.
/cover/
# The directory Mix downloads your dependencies sources to.
/deps/
# Where third-party dependencies like ExDoc output generated docs.
/doc/
# Ignore .fetch files in case you like to edit your project deps locally.
/.fetch
# If the VM crashes, it generates a dump, let's ignore it too.
erl_crash.dump
# Also ignore archive artifacts (built via "mix archive.build").
*.ez
# Ignore package tarball (built via "mix hex.build").
http-*.tar

21
apps/http/README.md

@ -0,0 +1,21 @@
# Http
**TODO: Add description**
## Installation
If [available in Hex](https://hex.pm/docs/publish), the package can be installed
by adding `http` to your list of dependencies in `mix.exs`:
```elixir
def deps do
[
{:http, "~> 0.1.0"}
]
end
```
Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc)
and published on [HexDocs](https://hexdocs.pm). Once published, the docs can
be found at [https://hexdocs.pm/http](https://hexdocs.pm/http).

16
apps/http/lib/http.ex

@ -0,0 +1,16 @@
defmodule Http do
@moduledoc """
Documentation for `Http`.
"""
use HTTPoison.Base
@headers [
{"User-Agent", "Tomie/0.x +https://dev.inhji.de"}
]
@options [ssl: [{:versions, [:"tlsv1.2", :"tlsv1.1", :tlsv1]}]]
def process_request_headers(headers), do: headers ++ @headers
def process_request_options(options), do: options ++ @options
end

31
apps/http/mix.exs

@ -0,0 +1,31 @@
defmodule Http.MixProject do
use Mix.Project
def project do
[
app: :http,
version: "0.1.0",
build_path: "../../_build",
config_path: "../../config/config.exs",
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.10",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications.
def application do
[
extra_applications: [:logger]
]
end
# Run "mix help deps" to learn about dependencies.
defp deps do
[
{:httpoison, "~> 1.6"}
]
end
end

8
apps/http/test/http_test.exs

@ -0,0 +1,8 @@
defmodule HttpTest do
use ExUnit.Case
doctest Http
test "greets the world" do
assert Http.hello() == :world
end
end

1
apps/http/test/test_helper.exs

@ -0,0 +1 @@
ExUnit.start()

18
apps/listens/lib/listens/workers/discogs_album.ex

@ -19,7 +19,7 @@ defmodule Listens.Workers.DiscogsAlbum do
@impl Oban.Worker
def perform(_args, _job) do
albums = Albums.list_albums_without_cover(log: false)
Enum.each(albums, &fetch_album_cover/1)
Enum.each(albums, &search_album_cover/1)
Listens.Cache.try_put(@cache, "updated_at", DateTime.utc_now())
@ -30,13 +30,12 @@ defmodule Listens.Workers.DiscogsAlbum do
:ok
end
def fetch_album_cover(album) do
url = get_url(album)
opts = Application.fetch_env!(:httpoison, :options)
def search_album_cover(album) do
url = get_search_url(album)
case HTTPoison.get!(url, [], opts) do
case Http.get!(url) do
%HTTPoison.Response{body: body, headers: headers} ->
handle_fetch_response(body, album)
handle_search_response(body, album)
rate_limit = Listens.RateLimit.calculate(headers, :discogs)
Listens.Cache.try_put(@cache, @rate_limit, rate_limit)
@ -45,7 +44,7 @@ defmodule Listens.Workers.DiscogsAlbum do
end
end
def handle_fetch_response(body, album) do
def handle_search_response(body, album) do
with decoded <- Jason.decode!(body, keys: :atoms),
results <- Map.get(decoded, :results),
false <- is_nil(results),
@ -75,10 +74,13 @@ defmodule Listens.Workers.DiscogsAlbum do
end
end
def get_url(album) do
def get_search_url(album) do
album_name = URI.encode(album.name)
artist_name = URI.encode(album.artist.name)
"#{@base_url}?token=#{@token}&release_title=#{album_name}&artist=#{artist_name}"
end
def get_fetch_url(%{discogs_id: discogs_id}) do
end
end

3
apps/listens/lib/listens/workers/discogs_artist.ex

@ -33,9 +33,8 @@ defmodule Listens.Workers.DiscogsArtist do
def fetch_artist_image(artist) do
url = get_url(artist)
opts = Application.fetch_env!(:httpoison, :options)
case HTTPoison.get!(url, [], opts) do
case Http.get!(url) do
%HTTPoison.Response{body: body, headers: headers} ->
handle_fetch_response(body, artist)
rate_limit = Listens.RateLimit.calculate(headers, :discogs)

3
apps/listens/lib/listens/workers/listenbrainz.ex

@ -21,12 +21,11 @@ defmodule Listens.Workers.Listenbrainz do
count = Listens.Cache.try_get(@cache, @fetch_listen_count, @default_listen_count)
last_ts = Handler.last_listen_timestamp()
url = "#{@base_url}/#{user}/listens?min_ts=#{last_ts}&count=#{count}"
opts = Application.fetch_env!(:httpoison, :options)
Logger.info("Fetching new Listens for Timestamp #{last_ts}:")
Logger.info(url)
case HTTPoison.get!(url, [], opts) do
case Http.get!(url) do
%HTTPoison.Response{body: body, headers: headers} ->
newest_timestamp =
body

1
apps/listens/mix.exs

@ -31,6 +31,7 @@ defmodule Listens.MixProject do
{:waffle_ecto, "~> 0.0.8"},
{:phoenix_pubsub, "~> 2.0"},
{:ecto_sql, "~> 3.1"},
{:http, in_umbrella: true},
{:db, in_umbrella: true}
]
end

10
apps/scraper/lib/scraper.ex

@ -3,16 +3,8 @@ defmodule Scraper do
Documentation for `Scraper`.
"""
def request_headers() do
user_agent = Application.get_env(:tomie, :user_agent)
[
{"User-Agent", user_agent}
]
end
def get_html(url) do
case HTTPoison.get(url, request_headers(), follow_redirect: true) do
case Http.get(url, [], follow_redirect: true) do
{:ok, %{body: body, status_code: 200}} ->
{:ok, body}

1
apps/scraper/mix.exs

@ -29,6 +29,7 @@ defmodule Scraper.MixProject do
{:open_graph_extended, git: "https://git.inhji.de/inhji/open_graph.git"},
{:httpoison, "~> 1.6"},
{:excoveralls, "~> 0.12.3", only: :test},
{:http, in_umbrella: true},
{:floki, "~> 0.26.0"}
]
end

5
config/config.exs

@ -14,7 +14,7 @@ config :db,
ecto_repos: [Db.Repo]
config :tomie,
user_agent: "Tomie/0.x (https://inhji.de)"
user_agent: "Tomie/0.x +https://dev.inhji.de"
config :listens,
discogs_token: "kRIDCYTMRucJojWzQKlXlDAnDlQSgmXboMEZiUBT"
@ -62,9 +62,6 @@ config :chartkick, json_serializer: Jason
config :earmark,
render_options: [code_class_prefix: "language-"]
config :httpoison,
options: [ssl: [{:versions, [:"tlsv1.2", :"tlsv1.1", :tlsv1]}]]
config :waffle,
storage: Waffle.Storage.Local,
storage_dir_prefix: "uploads"

3
mix.exs

@ -25,7 +25,8 @@ defmodule Tomie.Umbrella.MixProject do
bookmarks: :permanent,
scraper: :permanent,
tags: :permanent,
listens: :permanent
listens: :permanent,
http: :permanent
]
]
]

Loading…
Cancel
Save