From 5b36e71581bb5583a842e429f71061bbdf05b675 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Mon, 3 May 2021 14:52:37 +0200 Subject: [PATCH] Fix rich media parsers Signed-off-by: Thomas Citharel --- .../__snapshots__/CommentTree.spec.ts.snap | 8 +++---- lib/service/rich_media/parser.ex | 10 ++++----- lib/service/rich_media/parsers/fallback.ex | 12 ++++++++-- .../rich_media/parsers/meta_tags_parser.ex | 22 ++++++++++++++----- .../rich_media/parsers/oembed_parser.ex | 4 +++- 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/js/tests/unit/specs/components/Comment/__snapshots__/CommentTree.spec.ts.snap b/js/tests/unit/specs/components/Comment/__snapshots__/CommentTree.spec.ts.snap index 6754048a5..f00005715 100644 --- a/js/tests/unit/specs/components/Comment/__snapshots__/CommentTree.spec.ts.snap +++ b/js/tests/unit/specs/components/Comment/__snapshots__/CommentTree.spec.ts.snap @@ -3,25 +3,25 @@ exports[`CommentTree renders a comment tree 1`] = `
- +
No comments yet
- +
`; exports[`CommentTree renders a comment tree 2`] = `
- +
No comments yet
- +
`; diff --git a/lib/service/rich_media/parser.ex b/lib/service/rich_media/parser.ex index 879741482..6ffcb2d5f 100644 --- a/lib/service/rich_media/parser.ex +++ b/lib/service/rich_media/parser.ex @@ -74,12 +74,11 @@ defmodule Mobilizon.Service.RichMedia.Parser do {:is_html, _response_headers, true} <- {:is_html, response_headers, is_html(response_headers)} do body - |> parse_html() |> maybe_parse() |> Map.put(:url, url) |> maybe_add_favicon() |> clean_parsed_data() - |> check_parsed_data() + |> check_parsed_data(body) |> check_remote_picture_path() else {:is_html, response_headers, false} -> @@ -193,8 +192,7 @@ defmodule Mobilizon.Service.RichMedia.Parser do end end - defp parse_html(html), do: Floki.parse_document!(html) - + @spec maybe_parse(String.t()) :: {:halt, map()} | {:cont, map()} defp maybe_parse(html) do Enum.reduce_while(parsers(), %{}, fn parser, acc -> case parser.parse(html, acc) do @@ -207,7 +205,9 @@ defmodule Mobilizon.Service.RichMedia.Parser do end) end - defp check_parsed_data(%{title: title} = data) + defp check_parsed_data(data, html, first_run \\ true) + + defp check_parsed_data(%{title: title} = data, _html, _first_run) when is_binary(title) and byte_size(title) > 0 do data end diff --git a/lib/service/rich_media/parsers/fallback.ex b/lib/service/rich_media/parsers/fallback.ex index 5c2d8d573..2a92113fe 100644 --- a/lib/service/rich_media/parsers/fallback.ex +++ b/lib/service/rich_media/parsers/fallback.ex @@ -29,11 +29,19 @@ defmodule Mobilizon.Service.RichMedia.Parsers.Fallback do end defp get_page(html, :title) do - html |> Floki.find("html head title") |> List.first() |> Floki.text() |> String.trim() + html + |> Floki.parse_document!() + |> Floki.find("html title") + |> List.first() + |> Floki.text() + |> String.trim() end defp get_page(html, :description) do - case html |> Floki.find("html head meta[name='description']") |> List.first() do + case html + |> Floki.parse_document!() + |> Floki.find("html meta[name='description']") + |> List.first() do nil -> "" elem -> elem |> Floki.attribute("content") |> List.first() |> String.trim() end diff --git a/lib/service/rich_media/parsers/meta_tags_parser.ex b/lib/service/rich_media/parsers/meta_tags_parser.ex index 316e444c5..768baffa2 100644 --- a/lib/service/rich_media/parsers/meta_tags_parser.ex +++ b/lib/service/rich_media/parsers/meta_tags_parser.ex @@ -36,7 +36,7 @@ defmodule Mobilizon.Service.RichMedia.Parsers.MetaTagsParser do end defp get_elements(html, key_name, prefix) do - html |> Floki.find("meta[#{to_string(key_name)}^='#{prefix}:']") + html |> Floki.parse_document!() |> Floki.find("meta[#{to_string(key_name)}^='#{prefix}:']") end defp normalize_attributes(html_node, prefix, key_name, value_name, allowed_attributes) do @@ -83,14 +83,26 @@ defmodule Mobilizon.Service.RichMedia.Parsers.MetaTagsParser do defp maybe_put_description(meta, _), do: meta + @spec get_page_title(String.t()) :: String.t() defp get_page_title(html) do - html |> Floki.find("html head title") |> List.first() |> Floki.text() + with {:ok, document} <- Floki.parse_document(html), + elem when not is_nil(elem) <- document |> Floki.find("html head title") |> List.first(), + title when is_binary(title) <- Floki.text(elem) do + title + else + _ -> "" + end end + @spec get_page_description(String.t()) :: String.t() defp get_page_description(html) do - case html |> Floki.find("html head meta[name='description']") |> List.first() do - nil -> "" - elem -> Floki.attribute(elem, "content") + with {:ok, document} <- Floki.parse_document(html), + elem when not is_nil(elem) <- + document |> Floki.find("html head meta[name='description']") |> List.first(), + description when is_binary(description) <- Floki.attribute(elem, "content") do + description + else + _ -> "" end end end diff --git a/lib/service/rich_media/parsers/oembed_parser.ex b/lib/service/rich_media/parsers/oembed_parser.ex index ce450d0d9..567277472 100644 --- a/lib/service/rich_media/parsers/oembed_parser.ex +++ b/lib/service/rich_media/parsers/oembed_parser.ex @@ -32,7 +32,9 @@ defmodule Mobilizon.Service.RichMedia.Parsers.OEmbed do end defp get_discovery_data(html) do - html |> Floki.find("link[type='application/json+oembed']") + with {:ok, document} <- Floki.parse_document(html) do + Floki.find(document, "link[type='application/json+oembed']") + end end defp get_oembed_url(nodes) do