Implement search engine & service in backend

Signed-off-by: Thomas Citharel <tcit@tcit.fr>

config/dev.exs

@@ -71,4 +71,5 @@ config :mobilizon, Mobilizon.Storage.Repo,
   database: System.get_env("MOBILIZON_DATABASE_DBNAME") || "mobilizon_dev",
   hostname: System.get_env("MOBILIZON_DATABASE_HOST") || "localhost",
   port: System.get_env("MOBILIZON_DATABASE_PORT") || "5432",
-  pool_size: 10
+  pool_size: 10,
+  show_sensitive_data_on_connection_error: true

lib/mobilizon/events/events.ex

@@ -12,6 +12,7 @@ defmodule Mobilizon.Events do
 
   alias Mobilizon.Actors.Actor
   alias Mobilizon.Addresses.Address
+  alias Mobilizon.Service.Search
 
   alias Mobilizon.Events.{
     Comment,
@@ -246,6 +247,7 @@ defmodule Mobilizon.Events do
              role: :creator,
              event_id: event.id
            }) do
+      Search.insert_search_event(event)
       {:ok, event}
     else
       # We don't create a creator participant if the event is a draft
@@ -259,20 +261,11 @@ defmodule Mobilizon.Events do
     with {:ok, %Event{} = event} <-
            %Event{}
            |> Event.changeset(attrs)
+           |> Ecto.Changeset.put_assoc(:tags, Map.get(attrs, "tags", []))
            |> Repo.insert(),
          %Event{} = event <-
-           Repo.preload(event, [:tags, :organizer_actor, :physical_address, :picture]),
+           Repo.preload(event, [:tags, :organizer_actor, :physical_address, :picture]) do
-         {:has_tags, true, _} <- {:has_tags, Map.has_key?(attrs, "tags"), event} do
+      {:ok, event}
-      event
-      |> Ecto.Changeset.change()
-      |> Ecto.Changeset.put_assoc(:tags, attrs["tags"])
-      |> Repo.update()
-    else
-      {:has_tags, false, event} ->
-        {:ok, event}
-
-      error ->
-        error
     end
   end
 
@@ -306,6 +299,8 @@ defmodule Mobilizon.Events do
           changes
         )
 
+        Search.update_search_event(new_event)
+
         {:ok, new_event}
       end
     end
@@ -418,11 +413,14 @@ defmodule Mobilizon.Events do
   @doc """
   Builds a page struct for events by their name.
   """
-  @spec build_events_by_name(String.t(), integer | nil, integer | nil) :: Page.t()
+  @spec build_events_for_search(String.t(), integer | nil, integer | nil) :: Page.t()
-  def build_events_by_name(name, page \\ nil, limit \\ nil) do
+  def build_events_for_search(name, page \\ nil, limit \\ nil)
+  def build_events_for_search("", _page, _limit), do: %Page{total: 0, elements: []}
+
+  def build_events_for_search(name, page, limit) do
     name
-    |> String.trim()
+    |> normalize_search_string()
-    |> events_by_name_query()
+    |> events_for_search_query()
     |> Page.build_page(page, limit)
   end
 
@@ -1203,18 +1201,51 @@ defmodule Mobilizon.Events do
     )
   end
 
-  @spec events_by_name_query(String.t()) :: Ecto.Query.t()
+  defmacro matching_event_ids_and_ranks(search_string) do
-  defp events_by_name_query(name) do
+    quote do
-    from(
+      fragment(
-      e in Event,
+        """
-      where:
+        SELECT event_search.id AS id,
-        e.visibility == ^:public and
+        ts_rank(
-          fragment("f_unaccent(?) %> f_unaccent(?)", e.title, ^name),
+          event_search.document, plainto_tsquery(unaccent(?))
-      order_by: fragment("word_similarity(?, ?) desc", e.title, ^name),
+        ) AS rank
-      preload: [:organizer_actor]
+        FROM event_search
+        WHERE event_search.document @@ plainto_tsquery(unaccent(?))
+        OR event_search.title ILIKE ?
+        """,
+        ^unquote(search_string),
+        ^unquote(search_string),
+        ^"%#{unquote(search_string)}%"
+      )
+    end
+  end
+
+  @spec events_for_search_query(String.t()) :: Ecto.Query.t()
+  defp events_for_search_query(search_string) do
+    Event
+    |> where([e], e.visibility in ^@public_visibility)
+    |> do_event_for_search_query(search_string)
+  end
+
+  @spec do_event_for_search_query(Ecto.Query.t(), String.t()) :: Ecto.Query.t()
+  defp do_event_for_search_query(query, search_string) do
+    from(event in query,
+      join: id_and_rank in matching_event_ids_and_ranks(search_string),
+      on: id_and_rank.id == event.id,
+      order_by: [desc: id_and_rank.rank]
     )
   end
 
+  @spec normalize_search_string(String.t()) :: String.t()
+  defp normalize_search_string(search_string) do
+    search_string
+    |> String.downcase()
+    |> String.replace(~r/\n/, " ")
+    |> String.replace(~r/\t/, " ")
+    |> String.replace(~r/\s{2,}/, " ")
+    |> String.trim()
+  end
+
   @spec events_by_tags_query([integer], integer) :: Ecto.Query.t()
   def events_by_tags_query(tags_ids, limit) do
     from(

lib/mobilizon_web/api/search.ex

@@ -68,7 +68,7 @@ defmodule MobilizonWeb.API.Search do
         end
 
       true ->
-        {:ok, Events.build_events_by_name(search, page, limit)}
+        {:ok, Events.build_events_for_search(search, page, limit)}
     end
   end
 

lib/service/search.ex

@@ -0,0 +1,59 @@
+defmodule Mobilizon.Service.Search do
+  @moduledoc """
+  Module to handle search service
+  """
+
+  alias Mobilizon.Events.Event
+  alias Mobilizon.Storage.Repo
+  alias Ecto.Adapters.SQL
+
+  def insert_search_event(%Event{} = event) do
+    SQL.query(
+      Repo,
+      """
+        INSERT INTO event_search(id, title, document) VALUES ($1, $2, (
+          SELECT
+            setweight(to_tsvector(unaccent($2)), 'A') ||
+            setweight(to_tsvector(unaccent(coalesce($4, ' '))), 'B') ||
+            setweight(to_tsvector(unaccent($3)), 'C')
+          )
+        );
+      """,
+      [
+        event.id,
+        event.title,
+        HtmlSanitizeEx.strip_tags(event.description),
+        get_tags_string(event)
+      ]
+    )
+  end
+
+  def update_search_event(%Event{} = event) do
+    SQL.query(
+      Repo,
+      """
+        UPDATE event_search
+          SET document =
+            (SELECT
+              setweight(to_tsvector(unaccent($2)), 'A') ||
+              setweight(to_tsvector(unaccent(coalesce($4, ' '))), 'B') ||
+              setweight(to_tsvector(unaccent($3)), 'C')
+            ),
+          title = $2
+          WHERE id = $1;
+      """,
+      [
+        event.id,
+        event.title,
+        HtmlSanitizeEx.strip_tags(event.description),
+        get_tags_string(event)
+      ]
+    )
+  end
+
+  defp get_tags_string(%Event{tags: tags}) do
+    tags
+    |> Enum.map(& &1.title)
+    |> Enum.join(" ")
+  end
+end

mix.exs

@@ -99,6 +99,7 @@ defmodule Mobilizon.Mixfile do
       {:html_sanitize_ex, "~> 1.3.0"},
       {:ex_cldr_dates_times, "~> 2.0"},
       {:ex_optimizer, "~> 0.1"},
+      {:progress_bar, "~> 2.0"},
       # Dev and test dependencies
       {:phoenix_live_reload, "~> 1.2", only: [:dev, :e2e]},
       {:ex_machina, "~> 2.3", only: [:dev, :test]},

mix.lock

@@ -97,6 +97,7 @@
   "poison": {:hex, :poison, "4.0.1", "bcb755a16fac91cad79bfe9fc3585bb07b9331e50cfe3420a24bcc2d735709ae", [:mix], [], "hexpm"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"},
   "postgrex": {:hex, :postgrex, "0.15.1", "23ce3417de70f4c0e9e7419ad85bdabcc6860a6925fe2c6f3b1b5b1e8e47bf2f", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
+  "progress_bar": {:hex, :progress_bar, "2.0.0", "447285f533b4b8717881fdb7160c7360c2f2ab57276f8904ce6d40482857e573", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm"},
   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
   "rdf": {:hex, :rdf, "0.6.2", "1b85e37c135e232febeebda6b04ac4aba5f5e2bb1c3a2a6665ed4ccec19ade70", [:mix], [{:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm"},
   "rsa_ex": {:hex, :rsa_ex, "0.4.0", "e28dd7dc5236e156df434af0e4aa822384c8866c928e17b785d4edb7c253b558", [:mix], [], "hexpm"},

priv/repo/migrations/20191022083446_add_eager_materialized_view_for_searching_events.exs

@@ -0,0 +1,65 @@
+defmodule Mobilizon.Storage.Repo.Migrations.AddEagerMaterializedViewForSearchingEvents do
+  use Ecto.Migration
+  import Ecto.Query
+
+  alias Mobilizon.Storage.Repo
+  alias Mobilizon.Service.Search
+  alias Mobilizon.Events.Event
+
+  require Logger
+
+  def up do
+    create table(:event_search, primary_key: false) do
+      add(:id, references(:events, on_delete: :delete_all, on_update: :update_all),
+        primary_key: true
+      )
+
+      add(:title, :text, null: false)
+      add(:document, :tsvector)
+    end
+
+    # to support full-text searches
+    create(index("event_search", [:document], using: :gin))
+
+    # to support substring title matches with ILIKE
+    execute(
+      "CREATE INDEX event_search_title_trgm_index ON event_search USING gin (title gin_trgm_ops)"
+    )
+
+    # to support updating CONCURRENTLY
+    create(unique_index("event_search", [:id]))
+
+    flush()
+
+    events =
+      Event
+      |> preload([e], :tags)
+      |> Repo.all()
+
+    nb_events = length(events)
+
+    IO.puts("\nStarting setting up search for #{nb_events} events, this can take a while…\n")
+    insert_search_event(events, nb_events)
+  end
+
+  defp insert_search_event([%Event{url: url} = event | events], nb_events) do
+    with {:ok, _} <- Search.insert_search_event(event) do
+      Logger.debug("Added event #{url} to the search")
+    else
+      {:error, res} ->
+        Logger.error("Error while adding event #{url} to the search: #{inspect(res)}")
+    end
+
+    ProgressBar.render(nb_events - length(events), nb_events)
+
+    insert_search_event(events, nb_events)
+  end
+
+  defp insert_search_event([], nb_events) do
+    IO.puts("\nFinished setting up search for #{nb_events} events!\n")
+  end
+
+  def down do
+    drop(table(:event_search))
+  end
+end

test/mobilizon/events/events_test.exs

@@ -7,6 +7,7 @@ defmodule Mobilizon.EventsTest do
   alias Mobilizon.Events
   alias Mobilizon.Events.{Comment, Event, Participant, Session, Tag, TagRelation, Track}
   alias Mobilizon.Storage.Page
+  alias Mobilizon.Service.Search
 
   @event_valid_attrs %{
     begins_on: "2010-04-17 14:00:00Z",
@@ -22,6 +23,7 @@ defmodule Mobilizon.EventsTest do
     setup do
       actor = insert(:actor)
       event = insert(:event, organizer_actor: actor, visibility: :public)
+      Search.insert_search_event(event)
       {:ok, actor: actor, event: event}
     end
 
@@ -55,22 +57,31 @@ defmodule Mobilizon.EventsTest do
       assert Events.get_event_with_preload!(event.id).participants == []
     end
 
-    test "build_events_by_name/1 returns events for a given name", %{
+    test "build_events_for_search/1 returns events for a given name", %{
       event: %Event{title: title} = event
     } do
-      assert title == hd(Events.build_events_by_name(event.title).elements).title
+      assert title == hd(Events.build_events_for_search(event.title).elements).title
 
       %Event{} = event2 = insert(:event, title: "Special event")
+      Search.insert_search_event(event2)
 
       assert event2.title ==
-               Events.build_events_by_name("Special").elements |> hd() |> Map.get(:title)
+               Events.build_events_for_search("Special").elements |> hd() |> Map.get(:title)
 
       assert event2.title ==
-               Events.build_events_by_name("  Special  ").elements
+               Events.build_events_for_search("  Spécïal  ").elements
                |> hd()
                |> Map.get(:title)
 
-      assert %Page{elements: [], total: 0} == Events.build_events_by_name("")
+      tag1 = insert(:tag, title: "coucou")
+      tag2 = insert(:tag, title: "hola")
+      %Event{} = event3 = insert(:event, title: "Nothing like it", tags: [tag1, tag2])
+      Search.insert_search_event(event3)
+
+      assert event3.title ==
+               Events.build_events_for_search("hola").elements |> hd() |> Map.get(:title)
+
+      assert %Page{elements: [], total: 0} == Events.build_events_for_search("")
     end
 
     test "find_close_events/3 returns events in the area" do

test/mobilizon_web/api/search_test.exs

@@ -46,13 +46,13 @@ defmodule MobilizonWeb.API.SearchTest do
 
   test "search events" do
     with_mock Events,
-      build_events_by_name: fn "toto", 1, 10 ->
+      build_events_for_search: fn "toto", 1, 10 ->
         %Page{total: 1, elements: [%Event{title: "super toto event"}]}
       end do
       assert {:ok, %{total: 1, elements: [%Event{title: "super toto event"}]}} =
                Search.search_events("toto", 1, 10)
 
-      assert_called(Events.build_events_by_name("toto", 1, 10))
+      assert_called(Events.build_events_for_search("toto", 1, 10))
     end
   end
 end

test/mobilizon_web/resolvers/search_resolver_test.exs

@@ -2,6 +2,7 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
   use MobilizonWeb.ConnCase
   alias MobilizonWeb.AbsintheHelpers
   import Mobilizon.Factory
+  alias Mobilizon.Service.Search
 
   setup %{conn: conn} do
     user = insert(:user)
@@ -16,6 +17,7 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
     insert(:actor, user: user, preferred_username: "test_person")
     insert(:actor, type: :Group, preferred_username: "test_group")
     event = insert(:event, title: "test_event")
+    Search.insert_search_event(event)
 
     query = """
     {
@@ -48,7 +50,8 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
   } do
     actor = insert(:actor, user: user, preferred_username: "test_person")
     insert(:actor, type: :Group, preferred_username: "test_group")
-    insert(:event, title: "test_event")
+    event = insert(:event, title: "test_event")
+    Search.insert_search_event(event)
 
     query = """
     {
@@ -80,7 +83,8 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
   } do
     insert(:actor, user: user, preferred_username: "test_person")
     group = insert(:actor, type: :Group, preferred_username: "test_group")
-    insert(:event, title: "test_event")
+    event = insert(:event, title: "test_event")
+    Search.insert_search_event(event)
 
     query = """
     {
@@ -111,9 +115,12 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
     user: user
   } do
     insert(:actor, user: user, preferred_username: "person", name: "I like pineapples")
-    insert(:event, title: "Pineapple fashion week")
+    event1 = insert(:event, title: "Pineapple fashion week")
-    insert(:event, title: "I love pineAPPLE")
+    event2 = insert(:event, title: "I love pineAPPLE")
-    insert(:event, title: "Hello")
+    event3 = insert(:event, title: "Hello")
+    Search.insert_search_event(event1)
+    Search.insert_search_event(event2)
+    Search.insert_search_event(event3)
 
     query = """
     {
@@ -140,8 +147,8 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
 
     assert json_response(res, 200)["data"]["search_events"]["elements"]
            |> Enum.map(& &1["title"]) == [
-             "I love pineAPPLE",
+             "Pineapple fashion week",
-             "Pineapple fashion week"
+             "I love pineAPPLE"
            ]
   end
 
@@ -151,9 +158,12 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
   } do
     actor = insert(:actor, user: user, preferred_username: "person", name: "I like pineapples")
     insert(:actor, preferred_username: "group", type: :Group, name: "pineapple group")
-    insert(:event, title: "Pineapple fashion week")
+    event1 = insert(:event, title: "Pineapple fashion week")
-    insert(:event, title: "I love pineAPPLE")
+    event2 = insert(:event, title: "I love pineAPPLE")
-    insert(:event, title: "Hello")
+    event3 = insert(:event, title: "Hello")
+    Search.insert_search_event(event1)
+    Search.insert_search_event(event2)
+    Search.insert_search_event(event3)
 
     query = """
     {
@@ -188,6 +198,7 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
     insert(:actor, user: user, preferred_username: "person", name: "Torréfaction du Kafé")
     insert(:actor, type: :Group, preferred_username: "group", name: "Kafé group")
     event = insert(:event, title: "Tour du monde des Kafés")
+    Search.insert_search_event(event)
 
     # Elaborate query
     query = """
@@ -218,7 +229,8 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
   } do
     insert(:actor, user: user, preferred_username: "person", name: "Torréfaction du Kafé")
     group = insert(:actor, type: :Group, preferred_username: "group", name: "Kafé group")
-    insert(:event, title: "Tour du monde des Kafés")
+    event = insert(:event, title: "Tour du monde des Kafés")
+    Search.insert_search_event(event)
 
     # Elaborate query
     query = """