Detect and convert html body in the correct charset before parsing it

Signed-off-by: Thomas Citharel <tcit@tcit.fr>

config/config.exs

@@ -185,6 +185,12 @@ config :phoenix, :filter_parameters, ["password", "token"]
 config :absinthe, schema: Mobilizon.GraphQL.Schema
 config :absinthe, Absinthe.Logger, filter_variables: ["token", "password", "secret"]
 
+config :codepagex, :encodings, [
+  :ascii,
+  ~r[iso8859]i,
+  :"VENDORS/MICSFT/WINDOWS/CP1252"
+]
+
 config :mobilizon, Mobilizon.Web.Gettext, split_module_by: [:locale, :domain]
 
 config :ex_cldr,

lib/service/rich_media/parser.ex

@@ -74,6 +74,7 @@ defmodule Mobilizon.Service.RichMedia.Parser do
            {:is_html, _response_headers, true} <-
              {:is_html, response_headers, is_html(response_headers)} do
         body
+        |> convert_utf8(response_headers)
         |> maybe_parse()
         |> Map.put(:url, url)
         |> maybe_add_favicon()
@@ -317,4 +318,78 @@ defmodule Mobilizon.Service.RichMedia.Parser do
   defp default_user_agent(_url) do
     Config.instance_user_agent()
   end
+
+  defp convert_utf8(body, headers) do
+    headers
+    |> get_header("Content-Type")
+    |> handle_charset(body)
+  end
+
+  defp handle_charset(nil, body) do
+    case detect_charset_from_meta(body) do
+      "" -> body
+      nil -> body
+      charset -> convert_body(body, charset)
+    end
+  end
+
+  defp handle_charset(content_type, body) do
+    case charset_from_content_type(content_type) do
+      nil -> handle_charset(nil, body)
+      charset -> convert_body(body, charset)
+    end
+  end
+
+  defp charset_from_content_type(content_type) do
+    with [_, params] <- :binary.split(content_type, ";"),
+         %{"charset" => charset} <- Utils.params(params) do
+      charset
+    else
+      _ -> nil
+    end
+  end
+
+  defp detect_charset_from_meta(body) do
+    Logger.debug("Trying to detect charset from meta")
+
+    document = Floki.parse_document!(body)
+
+    case document
+         |> Floki.find("meta[http-equiv=\"content-type\"]")
+         |> List.first() do
+      nil ->
+        case document
+             |> Floki.find("meta[http-equiv=\"Content-Type\"]")
+             |> List.first() do
+          nil -> nil
+          meta -> content_type_from_meta(meta)
+        end
+
+      meta ->
+        content_type_from_meta(meta)
+    end
+  end
+
+  defp content_type_from_meta(meta) do
+    Logger.debug("Finding content-type into <meta> element")
+
+    meta
+    |> Floki.attribute("content")
+    |> List.first()
+    |> String.trim()
+    |> charset_from_content_type()
+  end
+
+  defp convert_body(body, "utf-8"), do: body
+
+  defp convert_body(body, charset) do
+    Logger.debug("Converting body from #{charset}")
+    Codepagex.to_string!(body, fix_charset(charset))
+  end
+
+  defp fix_charset("windows-1252"), do: :"VENDORS/MICSFT/WINDOWS/CP1252"
+
+  defp fix_charset(charset) do
+    String.replace(charset, "-", "_")
+  end
 end

mix.exs

@@ -203,6 +203,7 @@ defmodule Mobilizon.Mixfile do
       {:export, "~> 0.1.0"},
       {:tz_world, "~> 1.0"},
       {:tzdata, "~> 1.1"},
+      {:codepagex, "~> 0.1.6"},
       # Dev and test dependencies
       {:phoenix_live_reload, "~> 1.2", only: [:dev, :e2e]},
       {:ex_machina, "~> 2.3", only: [:dev, :test]},

mix.lock

@@ -11,6 +11,7 @@
   "cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"},
   "certifi": {:hex, :certifi, "2.8.0", "d4fb0a6bb20b7c9c3643e22507e42f356ac090a1dcea9ab99e27e0376d695eba", [:rebar3], [], "hexpm", "6ac7efc1c6f8600b08d625292d4bbf584e14847ce1b6b5c44d983d273e1097ea"},
   "cldr_utils": {:hex, :cldr_utils, "2.17.0", "05453797e5b89f936c54c5602ac881e46b1ba4423a803c27a414466f4b598c94", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.5", [hex: :certifi, repo: "hexpm", optional: true]}, {:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "6077ddaaa155f27755638225617bdc00c004f39b3c9355b688e52a3fc98d57e8"},
+  "codepagex": {:hex, :codepagex, "0.1.6", "49110d09a25ee336a983281a48ef883da4c6190481e0b063afe2db481af6117e", [:mix], [], "hexpm", "1521461097dde281edf084062f525a4edc6a5e49f4fd1f5ec41c9c4955d5bd59"},
   "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
   "comeonin": {:hex, :comeonin, "5.3.2", "5c2f893d05c56ae3f5e24c1b983c2d5dfb88c6d979c9287a76a7feb1e1d8d646", [:mix], [], "hexpm", "d0993402844c49539aeadb3fe46a3c9bd190f1ecf86b6f9ebd71957534c95f04"},
   "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"},