From 5b36e71581bb5583a842e429f71061bbdf05b675 Mon Sep 17 00:00:00 2001
From: Thomas Citharel <tcit@tcit.fr>
Date: Mon, 3 May 2021 14:52:37 +0200
Subject: [PATCH] Fix rich media parsers
Signed-off-by: Thomas Citharel <tcit@tcit.fr>
---
.../__snapshots__/CommentTree.spec.ts.snap | 8 +++----
lib/service/rich_media/parser.ex | 10 ++++-----
lib/service/rich_media/parsers/fallback.ex | 12 ++++++++--
.../rich_media/parsers/meta_tags_parser.ex | 22 ++++++++++++++-----
.../rich_media/parsers/oembed_parser.ex | 4 +++-
5 files changed, 39 insertions(+), 17 deletions(-)
diff --git a/js/tests/unit/specs/components/Comment/__snapshots__/CommentTree.spec.ts.snap b/js/tests/unit/specs/components/Comment/__snapshots__/CommentTree.spec.ts.snap
index 6754048a5..f00005715 100644
--- a/js/tests/unit/specs/components/Comment/__snapshots__/CommentTree.spec.ts.snap
+++ b/js/tests/unit/specs/components/Comment/__snapshots__/CommentTree.spec.ts.snap
@@ -3,25 +3,25 @@
exports[`CommentTree renders a comment tree 1`] = `
<div>
<!---->
- <transition-stub name="comment-empty-list" mode="out-in">
+ <transition-group-stub name="comment-empty-list" mode="out-in">
<transition-group-stub tag="ul" name="comment-list" class="comment-list">
<comment-stub comment="[object Object]" event="[object Object]" class="root-comment"></comment-stub>
<comment-stub comment="[object Object]" event="[object Object]" class="root-comment"></comment-stub>
</transition-group-stub>
<div class="no-comments"><span>No comments yet</span></div>
- </transition-stub>
+ </transition-group-stub>
</div>
`;
exports[`CommentTree renders a comment tree 2`] = `
<div>
<!---->
- <transition-stub name="comment-empty-list" mode="out-in">
+ <transition-group-stub name="comment-empty-list" mode="out-in">
<transition-group-stub tag="ul" name="comment-list" class="comment-list">
<comment-stub comment="[object Object]" event="[object Object]" class="root-comment"></comment-stub>
<comment-stub comment="[object Object]" event="[object Object]" class="root-comment"></comment-stub>
</transition-group-stub>
<div class="no-comments"><span>No comments yet</span></div>
- </transition-stub>
+ </transition-group-stub>
</div>
`;
diff --git a/lib/service/rich_media/parser.ex b/lib/service/rich_media/parser.ex
index 879741482..6ffcb2d5f 100644
--- a/lib/service/rich_media/parser.ex
+++ b/lib/service/rich_media/parser.ex
@@ -74,12 +74,11 @@ defmodule Mobilizon.Service.RichMedia.Parser do
{:is_html, _response_headers, true} <-
{:is_html, response_headers, is_html(response_headers)} do
body
- |> parse_html()
|> maybe_parse()
|> Map.put(:url, url)
|> maybe_add_favicon()
|> clean_parsed_data()
- |> check_parsed_data()
+ |> check_parsed_data(body)
|> check_remote_picture_path()
else
{:is_html, response_headers, false} ->
@@ -193,8 +192,7 @@ defmodule Mobilizon.Service.RichMedia.Parser do
end
end
- defp parse_html(html), do: Floki.parse_document!(html)
-
+ @spec maybe_parse(String.t()) :: {:halt, map()} | {:cont, map()}
defp maybe_parse(html) do
Enum.reduce_while(parsers(), %{}, fn parser, acc ->
case parser.parse(html, acc) do
@@ -207,7 +205,9 @@ defmodule Mobilizon.Service.RichMedia.Parser do
end)
end
- defp check_parsed_data(%{title: title} = data)
+ defp check_parsed_data(data, html, first_run \\ true)
+
+ defp check_parsed_data(%{title: title} = data, _html, _first_run)
when is_binary(title) and byte_size(title) > 0 do
data
end
diff --git a/lib/service/rich_media/parsers/fallback.ex b/lib/service/rich_media/parsers/fallback.ex
index 5c2d8d573..2a92113fe 100644
--- a/lib/service/rich_media/parsers/fallback.ex
+++ b/lib/service/rich_media/parsers/fallback.ex
@@ -29,11 +29,19 @@ defmodule Mobilizon.Service.RichMedia.Parsers.Fallback do
end
defp get_page(html, :title) do
- html |> Floki.find("html head title") |> List.first() |> Floki.text() |> String.trim()
+ html
+ |> Floki.parse_document!()
+ |> Floki.find("html title")
+ |> List.first()
+ |> Floki.text()
+ |> String.trim()
end
defp get_page(html, :description) do
- case html |> Floki.find("html head meta[name='description']") |> List.first() do
+ case html
+ |> Floki.parse_document!()
+ |> Floki.find("html meta[name='description']")
+ |> List.first() do
nil -> ""
elem -> elem |> Floki.attribute("content") |> List.first() |> String.trim()
end
diff --git a/lib/service/rich_media/parsers/meta_tags_parser.ex b/lib/service/rich_media/parsers/meta_tags_parser.ex
index 316e444c5..768baffa2 100644
--- a/lib/service/rich_media/parsers/meta_tags_parser.ex
+++ b/lib/service/rich_media/parsers/meta_tags_parser.ex
@@ -36,7 +36,7 @@ defmodule Mobilizon.Service.RichMedia.Parsers.MetaTagsParser do
end
defp get_elements(html, key_name, prefix) do
- html |> Floki.find("meta[#{to_string(key_name)}^='#{prefix}:']")
+ html |> Floki.parse_document!() |> Floki.find("meta[#{to_string(key_name)}^='#{prefix}:']")
end
defp normalize_attributes(html_node, prefix, key_name, value_name, allowed_attributes) do
@@ -83,14 +83,26 @@ defmodule Mobilizon.Service.RichMedia.Parsers.MetaTagsParser do
defp maybe_put_description(meta, _), do: meta
+ @spec get_page_title(String.t()) :: String.t()
defp get_page_title(html) do
- html |> Floki.find("html head title") |> List.first() |> Floki.text()
+ with {:ok, document} <- Floki.parse_document(html),
+ elem when not is_nil(elem) <- document |> Floki.find("html head title") |> List.first(),
+ title when is_binary(title) <- Floki.text(elem) do
+ title
+ else
+ _ -> ""
+ end
end
+ @spec get_page_description(String.t()) :: String.t()
defp get_page_description(html) do
- case html |> Floki.find("html head meta[name='description']") |> List.first() do
- nil -> ""
- elem -> Floki.attribute(elem, "content")
+ with {:ok, document} <- Floki.parse_document(html),
+ elem when not is_nil(elem) <-
+ document |> Floki.find("html head meta[name='description']") |> List.first(),
+ description when is_binary(description) <- Floki.attribute(elem, "content") do
+ description
+ else
+ _ -> ""
end
end
end
diff --git a/lib/service/rich_media/parsers/oembed_parser.ex b/lib/service/rich_media/parsers/oembed_parser.ex
index ce450d0d9..567277472 100644
--- a/lib/service/rich_media/parsers/oembed_parser.ex
+++ b/lib/service/rich_media/parsers/oembed_parser.ex
@@ -32,7 +32,9 @@ defmodule Mobilizon.Service.RichMedia.Parsers.OEmbed do
end
defp get_discovery_data(html) do
- html |> Floki.find("link[type='application/json+oembed']")
+ with {:ok, document} <- Floki.parse_document(html) do
+ Floki.find(document, "link[type='application/json+oembed']")
+ end
end
defp get_oembed_url(nodes) do