Refactor adding tags to an event

Also refactor extracting tags from content, now uses Pleroma's Formatter

Signed-off-by: Thomas Citharel <tcit@tcit.fr>

config/config.exs

@@ -94,6 +94,18 @@ config :geolix,
     }
   ]
 
+config :auto_linker,
+  opts: [
+    scheme: true,
+    extra: true,
+    # TODO: Set to :no_scheme when it works properly
+    validate_tld: true,
+    class: false,
+    strip_prefix: false,
+    new_window: false,
+    rel: false
+  ]
+
 config :phoenix, :format_encoders, json: Jason, "activity-json": Jason
 config :phoenix, :json_library, Jason
 

js/src/App.vue

@@ -72,6 +72,7 @@ export default class App extends Vue {
   @import "~bulma/sass/components/modal.sass";
   @import "~bulma/sass/components/navbar.sass";
   @import "~bulma/sass/components/pagination.sass";
+  @import "~bulma/sass/components/dropdown.sass";
   @import "~bulma/sass/elements/box.sass";
   @import "~bulma/sass/elements/button.sass";
   @import "~bulma/sass/elements/container.sass";
@@ -91,9 +92,11 @@ export default class App extends Vue {
   @import "~buefy/src/scss/components/datepicker";
   @import "~buefy/src/scss/components/notices";
   @import "~buefy/src/scss/components/dropdown";
+  @import "~buefy/src/scss/components/autocomplete";
   @import "~buefy/src/scss/components/form";
   @import "~buefy/src/scss/components/modal";
   @import "~buefy/src/scss/components/tag";
+  @import "~buefy/src/scss/components/taginput";
   @import "~buefy/src/scss/components/upload";
 
 .router-enter-active,

js/src/components/Event/TagInput.vue

@@ -0,0 +1,54 @@
+<template>
+    <b-field label="Enter some tags">
+        <b-taginput
+                v-model="tags"
+                :data="filteredTags"
+                autocomplete
+                :allow-new="true"
+                :field="path"
+                icon="label"
+                placeholder="Add a tag"
+                @typing="getFilteredTags">
+        </b-taginput>
+    </b-field>
+</template>
+<script lang="ts">
+import { Component, Prop, Vue, Watch } from 'vue-property-decorator';
+import { get } from 'lodash';
+import { ITag } from '@/types/tag.model';
+@Component
+export default class TagInput extends Vue {
+
+  @Prop({ required: false, default: () => [] }) data!: object[];
+  @Prop({ required: true, default: 'value' }) path!: string;
+  @Prop({ required: true }) value!: string;
+
+  filteredTags: object[] = [];
+  tags: object[] = [];
+
+  getFilteredTags(text) {
+    this.filteredTags = this.data.filter((option) => {
+      return get(option, this.path)
+                .toString()
+                .toLowerCase()
+                .indexOf(text.toLowerCase()) >= 0;
+    });
+  }
+
+  @Watch('tags')
+  onTagsChanged (tags) {
+    const tagEntities = tags.map((tag) => {
+      if (TagInput.isTag(tag)) {
+        return tag;
+      }
+      return { title: tag, slug: tag } as ITag;
+    });
+    console.log('tags changed', tagEntities);
+    this.$emit('input', tagEntities);
+  }
+
+  static isTag(x: any): x is ITag {
+    return x.slug !== undefined;
+  }
+}
+</script>

js/src/graphql/event.ts

@@ -143,7 +143,8 @@ export const CREATE_EVENT = gql`
   $organizerActorId: ID!,
   $category: String!,
   $beginsOn: DateTime!,
-  $picture: PictureInput!
+  $picture: PictureInput,
+  $tags: [String]
   ) {
     createEvent(
       title: $title,
@@ -151,7 +152,8 @@ export const CREATE_EVENT = gql`
       beginsOn: $beginsOn,
       organizerActorId: $organizerActorId,
       category: $category,
-      picture: $picture
+      picture: $picture,
+      tags: $tags
     ) {
       id,
       uuid,
@@ -203,8 +205,10 @@ export const LEAVE_EVENT = gql`
 export const DELETE_EVENT = gql`
   mutation DeleteEvent($id: Int!, $actorId: Int!) {
     deleteEvent(
-      id: $id,
+      eventId: $id,
       actorId: $actorId
-    )
+    ) {
+        id
+    }
   }
 `;

js/src/graphql/tags.ts

@@ -0,0 +1,16 @@
+import gql from 'graphql-tag';
+
+export const TAGS = gql`
+query {
+  tags {
+    id,
+    related {
+        id,
+        slug,
+        title
+    }
+    slug,
+    title
+  }
+}
+`;

js/src/types/event.model.ts

@@ -75,6 +75,8 @@ export interface IEvent {
   onlineAddress?: string;
   phoneAddress?: string;
   physicalAddress?: IAddress;
+
+  tags: ITag[];
 }
 
 
@@ -99,4 +101,5 @@ export class EventModel implements IEvent {
   onlineAddress: string = '';
   phoneAddress: string = '';
   picture: IAbstractPicture|null = null;
+  tags: ITag[] = [];
 }

js/src/views/Event/Create.vue

@@ -6,10 +6,14 @@
     <div v-if="$apollo.loading">Loading...</div>
     <div class="columns is-centered" v-else>
       <form class="column is-two-thirds-desktop" @submit="createEvent">
+        <picture-upload v-model="pictureFile" />
+
         <b-field :label="$gettext('Title')">
           <b-input aria-required="true" required v-model="event.title" maxlength="64" />
         </b-field>
 
+        <tag-input v-model="event.tags" :data="tags" path="title" />
+
         <date-time-picker v-model="event.beginsOn" :label="$gettext('Starts on…')" :step="15"/>
         <date-time-picker v-model="event.endsOn" :label="$gettext('Ends on…')" :step="15" />
 
@@ -28,8 +32,6 @@
           </b-select>
         </b-field>
 
-        <picture-upload v-model="pictureFile" />
-
         <button class="button is-primary">
           <translate>Create my event</translate>
         </button>
@@ -52,13 +54,19 @@ import { IPerson, Person } from '@/types/actor';
 import PictureUpload from '@/components/PictureUpload.vue';
 import Editor from '@/components/Editor.vue';
 import DateTimePicker from '@/components/Event/DateTimePicker.vue';
+import TagInput from '@/components/Event/TagInput.vue';
+import { TAGS } from '@/graphql/tags';
+import { ITag } from '@/types/tag.model';
 
 @Component({
-  components: { DateTimePicker, PictureUpload, Editor },
+  components: { TagInput, DateTimePicker, PictureUpload, Editor },
   apollo: {
     loggedPerson: {
       query: LOGGED_PERSON,
     },
+    tags: {
+      query: TAGS,
+    },
   },
 })
 export default class CreateEvent extends Vue {
@@ -123,11 +131,12 @@ export default class CreateEvent extends Vue {
      * Transform general variables
      */
     let pictureObj = {};
-    let obj = {
+    const obj = {
       organizerActorId: this.loggedPerson.id,
       beginsOn: this.event.beginsOn.toISOString(),
+      tags: this.event.tags.map((tag: ITag) => tag.title),
     };
-    let res = Object.assign({}, this.event, obj);
+    const res = Object.assign({}, this.event, obj);
 
     /**
      * Transform picture files

lib/mobilizon/events/event.ex

@@ -86,7 +86,6 @@ defmodule Mobilizon.Events.Event do
       :uuid,
       :picture_id
     ])
-    |> cast_assoc(:tags)
     |> cast_assoc(:physical_address)
     |> validate_required([
       :title,

lib/mobilizon/events/events.ex

@@ -367,7 +367,7 @@ defmodule Mobilizon.Events do
 
   """
   def create_event(attrs \\ %{}) do
-    with {:ok, %Event{} = event} <- %Event{} |> Event.changeset(attrs) |> Repo.insert(),
+    with %Event{} = event <- do_create_event(attrs),
          {:ok, %Participant{} = _participant} <-
            %Participant{}
            |> Participant.changeset(%{
@@ -376,7 +376,24 @@ defmodule Mobilizon.Events do
              event_id: event.id
            })
            |> Repo.insert() do
-      {:ok, Repo.preload(event, [:organizer_actor])}
+      {:ok, event}
+    end
+  end
+
+  defp do_create_event(attrs) do
+    with {:ok, %Event{} = event} <- %Event{} |> Event.changeset(attrs) |> Repo.insert(),
+         %Event{} = event <- event |> Repo.preload([:tags, :organizer_actor]),
+         {:has_tags, true, _} <- {:has_tags, Map.has_key?(attrs, "tags"), event} do
+      event
+      |> Ecto.Changeset.change()
+      |> Ecto.Changeset.put_assoc(:tags, attrs["tags"])
+      |> Repo.update()
+    else
+      {:has_tags, false, event} ->
+        event
+
+      error ->
+        error
     end
   end
 
@@ -491,6 +508,22 @@ defmodule Mobilizon.Events do
   """
   def get_tag!(id), do: Repo.get!(Tag, id)
 
+  def get_tag(id), do: Repo.get(Tag, id)
+
+  @doc """
+  Get an existing tag or create one
+  """
+  @spec get_or_create_tag(String.t()) :: {:ok, Tag.t()} | {:error, any()}
+  def get_or_create_tag(title) do
+    case Repo.get_by(Tag, title: title) do
+      %Tag{} = tag ->
+        {:ok, tag}
+
+      nil ->
+        create_tag(%{"title" => title})
+    end
+  end
+
   @doc """
   Creates a tag.
 

lib/mobilizon_web/api/comments.ex

@@ -6,10 +6,9 @@ defmodule MobilizonWeb.API.Comments do
   alias Mobilizon.Actors
   alias Mobilizon.Actors.Actor
   alias Mobilizon.Events.Comment
-  alias Mobilizon.Service.Formatter
   alias Mobilizon.Service.ActivityPub
   alias Mobilizon.Service.ActivityPub.Utils, as: ActivityPubUtils
-  import MobilizonWeb.API.Utils
+  alias MobilizonWeb.API.Utils
 
   @doc """
   Create a comment
@@ -20,23 +19,14 @@ defmodule MobilizonWeb.API.Comments do
   def create_comment(
         from_username,
         status,
-        visibility \\ "public",
+        visibility \\ :public,
         in_reply_to_comment_URL \\ nil
       ) do
     with {:local_actor, %Actor{url: url} = actor} <-
            {:local_actor, Actors.get_local_actor_by_name(from_username)},
-         status <- String.trim(status),
-         mentions <- Formatter.parse_mentions(status),
          in_reply_to_comment <- get_in_reply_to_comment(in_reply_to_comment_URL),
-         {to, cc} <- to_for_actor_and_mentions(actor, mentions, in_reply_to_comment, visibility),
+         {content_html, tags, to, cc} <-
-         tags <- Formatter.parse_tags(status),
+           Utils.prepare_content(actor, status, visibility, [], in_reply_to_comment),
-         content_html <-
-           make_content_html(
-             status,
-             mentions,
-             tags,
-             "text/plain"
-           ),
          comment <-
            ActivityPubUtils.make_comment_data(
              url,

lib/mobilizon_web/api/events.ex

@@ -4,10 +4,9 @@ defmodule MobilizonWeb.API.Events do
   """
   alias Mobilizon.Actors
   alias Mobilizon.Actors.Actor
-  alias Mobilizon.Service.Formatter
   alias Mobilizon.Service.ActivityPub
   alias Mobilizon.Service.ActivityPub.Utils, as: ActivityPubUtils
-  import MobilizonWeb.API.Utils
+  alias MobilizonWeb.API.Utils
 
   @doc """
   Create an event
@@ -19,24 +18,19 @@ defmodule MobilizonWeb.API.Events do
           description: description,
           organizer_actor_id: organizer_actor_id,
           begins_on: begins_on,
-          category: category
+          category: category,
+          tags: tags
         } = args
       ) do
+    require Logger
+
     with %Actor{url: url} = actor <-
            Actors.get_local_actor_with_everything(organizer_actor_id),
          title <- String.trim(title),
-         mentions <- Formatter.parse_mentions(description),
          visibility <- Map.get(args, :visibility, :public),
-         {to, cc} <- to_for_actor_and_mentions(actor, mentions, nil, Atom.to_string(visibility)),
-         tags <- Formatter.parse_tags(description),
          picture <- Map.get(args, :picture, nil),
-         content_html <-
+         {content_html, tags, to, cc} <-
-           make_content_html(
+           Utils.prepare_content(actor, description, visibility, tags, nil),
-             description,
-             mentions,
-             tags,
-             "text/plain"
-           ),
          event <-
            ActivityPubUtils.make_event_data(
              url,

lib/mobilizon_web/api/groups.ex

@@ -4,10 +4,9 @@ defmodule MobilizonWeb.API.Groups do
   """
   alias Mobilizon.Actors
   alias Mobilizon.Actors.Actor
-  alias Mobilizon.Service.Formatter
   alias Mobilizon.Service.ActivityPub
   alias Mobilizon.Service.ActivityPub.Utils, as: ActivityPubUtils
-  import MobilizonWeb.API.Utils
+  alias MobilizonWeb.API.Utils
 
   @doc """
   Create a group
@@ -24,17 +23,9 @@ defmodule MobilizonWeb.API.Groups do
            {:bad_actor, Actors.get_local_actor_by_name(admin_actor_username)},
          {:existing_group, nil} <- {:existing_group, Actors.get_group_by_title(title)},
          title <- String.trim(title),
-         mentions <- Formatter.parse_mentions(description),
+         visibility <- Map.get(args, :visibility, :public),
-         visibility <- Map.get(args, :visibility, "public"),
+         {content_html, tags, to, cc} <-
-         {to, cc} <- to_for_actor_and_mentions(actor, mentions, nil, visibility),
+           Utils.prepare_content(actor, description, visibility, [], nil),
-         tags <- Formatter.parse_tags(description),
-         content_html <-
-           make_content_html(
-             description,
-             mentions,
-             tags,
-             "text/plain"
-           ),
          group <-
            ActivityPubUtils.make_group_data(
              url,

lib/mobilizon_web/api/utils.ex

@@ -12,11 +12,9 @@ defmodule MobilizonWeb.API.Utils do
     * `to` : the mentionned actors, the eventual actor we're replying to and the public
     * `cc` : the actor's followers
   """
-  @spec to_for_actor_and_mentions(Actor.t(), list(), map(), String.t()) :: {list(), list()}
+  @spec get_to_and_cc(Actor.t(), list(), map(), String.t()) :: {list(), list()}
-  def to_for_actor_and_mentions(%Actor{} = actor, mentions, inReplyTo, "public") do
+  def get_to_and_cc(%Actor{} = actor, mentions, inReplyTo, :public) do
-    mentioned_actors = Enum.map(mentions, fn {_, %{url: url}} -> url end)
+    to = ["https://www.w3.org/ns/activitystreams#Public" | mentions]
-
-    to = ["https://www.w3.org/ns/activitystreams#Public" | mentioned_actors]
     cc = [actor.followers_url]
 
     if inReplyTo do
@@ -33,11 +31,9 @@ defmodule MobilizonWeb.API.Utils do
     * `to` : the mentionned actors, actor's followers and the eventual actor we're replying to
     * `cc` : public
   """
-  @spec to_for_actor_and_mentions(Actor.t(), list(), map(), String.t()) :: {list(), list()}
+  @spec get_to_and_cc(Actor.t(), list(), map(), String.t()) :: {list(), list()}
-  def to_for_actor_and_mentions(%Actor{} = actor, mentions, inReplyTo, "unlisted") do
+  def get_to_and_cc(%Actor{} = actor, mentions, inReplyTo, :unlisted) do
-    mentioned_actors = Enum.map(mentions, fn {_, %{url: url}} -> url end)
+    to = [actor.followers_url | mentions]
-
-    to = [actor.followers_url | mentioned_actors]
     cc = ["https://www.w3.org/ns/activitystreams#Public"]
 
     if inReplyTo do
@@ -54,9 +50,9 @@ defmodule MobilizonWeb.API.Utils do
     * `to` : the mentionned actors, actor's followers and the eventual actor we're replying to
     * `cc` : none
   """
-  @spec to_for_actor_and_mentions(Actor.t(), list(), map(), String.t()) :: {list(), list()}
+  @spec get_to_and_cc(Actor.t(), list(), map(), String.t()) :: {list(), list()}
-  def to_for_actor_and_mentions(%Actor{} = actor, mentions, inReplyTo, "private") do
+  def get_to_and_cc(%Actor{} = actor, mentions, inReplyTo, :private) do
-    {to, cc} = to_for_actor_and_mentions(actor, mentions, inReplyTo, "direct")
+    {to, cc} = get_to_and_cc(actor, mentions, inReplyTo, :direct)
     {[actor.followers_url | to], cc}
   end
 
@@ -67,58 +63,61 @@ defmodule MobilizonWeb.API.Utils do
     * `to` : the mentionned actors and the eventual actor we're replying to
     * `cc` : none
   """
-  @spec to_for_actor_and_mentions(Actor.t(), list(), map(), String.t()) :: {list(), list()}
+  @spec get_to_and_cc(Actor.t(), list(), map(), String.t()) :: {list(), list()}
-  def to_for_actor_and_mentions(_actor, mentions, inReplyTo, "direct") do
+  def get_to_and_cc(_actor, mentions, inReplyTo, :direct) do
-    mentioned_actors = Enum.map(mentions, fn {_, %{url: url}} -> url end)
-
     if inReplyTo do
-      {Enum.uniq([inReplyTo.actor | mentioned_actors]), []}
+      {Enum.uniq([inReplyTo.actor | mentions]), []}
     else
-      {mentioned_actors, []}
+      {mentions, []}
     end
   end
 
+  def get_to_and_cc(_user, mentions, _inReplyTo, {:list, _}), do: {mentions, []}
+
+  #  def get_addressed_users(_, to) when is_list(to) do
+  #    Actors.get(to)
+  #  end
+
+  def get_addressed_users(mentioned_users, _), do: mentioned_users
+
   @doc """
   Creates HTML content from text and mentions
   """
-  @spec make_content_html(String.t(), list(), list(), String.t()) :: String.t()
+  @spec make_content_html(String.t(), list(), String.t()) :: String.t()
   def make_content_html(
-        status,
+        text,
-        mentions,
+        additional_tags,
-        tags,
         content_type
-      ),
+      ) do
-      do: format_input(status, mentions, tags, content_type)
+    with {text, mentions, tags} <- format_input(text, content_type, []) do
+      {text, mentions, additional_tags ++ Enum.map(tags, fn {_, tag} -> tag end)}
+    end
+  end
 
-  def format_input(text, mentions, tags, "text/plain") do
+  def format_input(text, "text/plain", options) do
     text
     |> Formatter.html_escape("text/plain")
-    |> String.replace(~r/\r?\n/, "<br>")
+    |> Formatter.linkify(options)
-    |> (&{[], &1}).()
+    |> (fn {text, mentions, tags} ->
-    |> Formatter.add_links()
+          {String.replace(text, ~r/\r?\n/, "<br>"), mentions, tags}
-    |> Formatter.add_actor_links(mentions)
+        end).()
-    |> Formatter.add_hashtag_links(tags)
-    |> Formatter.finalize()
   end
 
-  def format_input(text, mentions, _tags, "text/html") do
+  def format_input(text, "text/html", options) do
     text
     |> Formatter.html_escape("text/html")
-    |> String.replace(~r/\r?\n/, "<br>")
+    |> Formatter.linkify(options)
-    |> (&{[], &1}).()
-    |> Formatter.add_actor_links(mentions)
-    |> Formatter.finalize()
   end
 
-  # def format_input(text, mentions, tags, "text/markdown") do
+  #  @doc """
+  #  Formatting text to markdown.
+  #  """
+  #  def format_input(text, "text/markdown", options) do
   #    text
+  #    |> Formatter.mentions_escape(options)
   #    |> Earmark.as_html!()
+  #    |> Formatter.linkify(options)
   #    |> Formatter.html_escape("text/html")
-  #   |> String.replace(~r/\r?\n/, "")
-  #   |> (&{[], &1}).()
-  #   |> Formatter.add_actor_links(mentions)
-  #   |> Formatter.add_hashtag_links(tags)
-  #   |> Formatter.finalize()
   #  end
 
   def make_report_content_html(nil), do: {:ok, {nil, [], []}}
@@ -132,4 +131,19 @@ defmodule MobilizonWeb.API.Utils do
       {:error, "Comment must be up to #{max_size} characters"}
     end
   end
+
+  def prepare_content(actor, content, visibility, tags, in_reply_to) do
+    with content <- String.trim(content),
+         {content_html, mentions, tags} <-
+           make_content_html(
+             content,
+             tags,
+             "text/plain"
+           ),
+         mentioned_users <- for({_, mentioned_user} <- mentions, do: mentioned_user.url),
+         addressed_users <- get_addressed_users(mentioned_users, nil),
+         {to, cc} <- get_to_and_cc(actor, addressed_users, in_reply_to, visibility) do
+      {content_html, tags, to, cc}
+    end
+  end
 end

lib/mobilizon_web/resolvers/tag.ex

@@ -2,7 +2,7 @@ defmodule MobilizonWeb.Resolvers.Tag do
   @moduledoc """
   Handles the tag-related GraphQL calls
   """
-  require Logger
+  alias Mobilizon.Events
   alias Mobilizon.Events.Event
   alias Mobilizon.Events.Tag
 
@@ -19,6 +19,15 @@ defmodule MobilizonWeb.Resolvers.Tag do
     {:ok, Mobilizon.Events.list_tags_for_event(id)}
   end
 
+  @doc """
+  Retrieve the list of tags for an event
+  """
+  def list_tags_for_event(%{url: url}, _args, _resolution) do
+    with %Event{id: event_id} <- Events.get_event_by_url(url) do
+      {:ok, Mobilizon.Events.list_tags_for_event(event_id)}
+    end
+  end
+
   #  @doc """
   #  Retrieve the list of related tags for a given tag ID
   #  """

lib/mobilizon_web/schema/event.ex

@@ -117,6 +117,11 @@ defmodule MobilizonWeb.Schema.EventType do
       arg(:public, :boolean)
       arg(:visibility, :event_visibility, default_value: :private)
 
+      arg(:tags, list_of(:string),
+        default_value: [],
+        description: "The list of tags associated to the event"
+      )
+
       arg(:picture, :picture_input,
         description:
           "The picture for the event, either as an object or directly the ID of an existing Picture"

lib/service/activity_pub/converters/event.ex

@@ -10,6 +10,8 @@ defmodule Mobilizon.Service.ActivityPub.Converters.Event do
   alias Mobilizon.Actors.Actor
   alias Mobilizon.Events.Event, as: EventModel
   alias Mobilizon.Service.ActivityPub.Converter
+  alias Mobilizon.Events
+  alias Mobilizon.Events.Tag
 
   @behaviour Converter
 
@@ -19,7 +21,8 @@ defmodule Mobilizon.Service.ActivityPub.Converters.Event do
   @impl Converter
   @spec as_to_model_data(map()) :: map()
   def as_to_model_data(object) do
-    with {:ok, %Actor{id: actor_id}} <- Actors.get_actor_by_url(object["actor"]) do
+    with {:ok, %Actor{id: actor_id}} <- Actors.get_actor_by_url(object["actor"]),
+         tags <- fetch_tags(object["tag"]) do
       picture_id =
         with true <- Map.has_key?(object, "attachment"),
              %Picture{id: picture_id} <-
@@ -43,11 +46,24 @@ defmodule Mobilizon.Service.ActivityPub.Converters.Event do
         "begins_on" => object["begins_on"],
         "category" => object["category"],
         "url" => object["id"],
-        "uuid" => object["uuid"]
+        "uuid" => object["uuid"],
+        "tags" => tags
       }
     end
   end
 
+  defp fetch_tags(tags) do
+    Enum.reduce(tags, [], fn tag, acc ->
+      case Events.get_or_create_tag(tag) do
+        {:ok, %Tag{} = tag} ->
+          acc ++ [tag]
+
+        _ ->
+          acc
+      end
+    end)
+  end
+
   @doc """
   Convert an event struct to an ActivityStream representation
   """

lib/service/activity_pub/utils.ex

@@ -296,7 +296,7 @@ defmodule Mobilizon.Service.ActivityPub.Utils do
       "actor" => actor,
       "id" => Routes.page_url(Endpoint, :event, uuid),
       "uuid" => uuid,
-      "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq()
+      "tag" => tags |> Enum.uniq()
     }
 
     if is_nil(picture), do: res, else: Map.put(res, "attachment", [make_picture_data(picture)])
@@ -328,7 +328,7 @@ defmodule Mobilizon.Service.ActivityPub.Utils do
       "actor" => actor,
       "id" => Routes.page_url(Endpoint, :comment, uuid),
       "uuid" => uuid,
-      "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq()
+      "tag" => tags |> Enum.uniq()
     }
 
     if inReplyTo do

lib/service/formatter/formatter.ex

@@ -1,5 +1,5 @@
 # Portions of this file are derived from Pleroma:
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social>
 # SPDX-License-Identifier: AGPL-3.0-only
 # Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/formatter.ex
 
@@ -10,68 +10,86 @@ defmodule Mobilizon.Service.Formatter do
   alias Mobilizon.Actors.Actor
   alias Mobilizon.Actors
 
-  @tag_regex ~r/\#\w+/u
+  @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
-  def parse_tags(text, data \\ %{}) do
+  @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
-    Regex.scan(@tag_regex, text)
+
-    |> Enum.map(fn ["#" <> tag = full_tag] -> {full_tag, String.downcase(tag)} end)
+  @auto_linker_config hashtag: true,
-    |> (fn map ->
+                      hashtag_handler: &Mobilizon.Service.Formatter.hashtag_handler/4,
-          if data["sensitive"] in [true, "True", "true", "1"],
+                      mention: true,
-            do: [{"#nsfw", "nsfw"}] ++ map,
+                      mention_handler: &Mobilizon.Service.Formatter.mention_handler/4
-            else: map
+
-        end).()
+  def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do
+    case Mobilizon.Actors.get_actor_by_name(nickname) do
+      %Actor{} ->
+        # escape markdown characters with `\\`
+        # (we don't want something like @user__name to be parsed by markdown)
+        String.replace(mention, @markdown_characters_regex, "\\\\\\1")
+
+      _ ->
+        buffer
+    end
   end
 
-  def parse_mentions(text) do
+  def mention_handler("@" <> nickname, buffer, _opts, acc) do
-    # Modified from https://www.w3.org/TR/html5/forms.html#valid-e-mail-address
+    case Actors.get_actor_by_name(nickname) do
-    regex =
+      %Actor{id: id, url: url, preferred_username: preferred_username} = actor ->
-      ~r/@[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]*@?[a-zA-Z0-9_-](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/u
+        link =
+          "<span class='h-card'><a data-user='#{id}' class='u-url mention' href='#{url}'>@<span>#{
+            preferred_username
+          }</span></a></span>"
 
-    Regex.scan(regex, text)
+        {link, %{acc | mentions: MapSet.put(acc.mentions, {"@" <> nickname, actor})}}
-    |> List.flatten()
+
-    |> Enum.uniq()
+      _ ->
-    |> Enum.map(fn "@" <> match = full_match ->
+        {buffer, acc}
-      {full_match, Actors.get_actor_by_name(match)}
+    end
-    end)
-    |> Enum.filter(fn {_match, user} -> user end)
   end
 
-  # def emojify(text) do
+  def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do
-  #   emojify(text, Emoji.get_all())
+    tag = String.downcase(tag)
-  # end
+    url = "#{MobilizonWeb.Endpoint.url()}/tag/#{tag}"
+    link = "<a class='hashtag' data-tag='#{tag}' href='#{url}' rel='tag'>#{tag_text}</a>"
 
-  # def emojify(text, nil), do: text
+    {link, %{acc | tags: MapSet.put(acc.tags, {tag_text, tag})}}
+  end
 
-  # def emojify(text, emoji) do
+  @doc """
-  #   Enum.reduce(emoji, text, fn {emoji, file}, text ->
+  Parses a text and replace plain text links with HTML. Returns a tuple with a result text, mentions, and hashtags.
-  #     emoji = HTML.strip_tags(emoji)
-  #     file = HTML.strip_tags(file)
 
-  #     String.replace(
+  If the 'safe_mention' option is given, only consecutive mentions at the start the post are actually mentioned.
-  #       text,
+  """
-  #       ":#{emoji}:",
+  @spec linkify(String.t(), keyword()) ::
-  #       "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
+          {String.t(), [{String.t(), Actor.t()}], [{String.t(), String.t()}]}
-  #         MediaProxy.url(file)
+  def linkify(text, options \\ []) do
-  #       }' />"
+    options = options ++ @auto_linker_config
-  #     )
-  #     |> HTML.filter_tags()
-  #   end)
-  # end
 
-  # def get_emoji(text) when is_binary(text) do
+    acc = %{mentions: MapSet.new(), tags: MapSet.new()}
-  #   Enum.filter(Emoji.get_all(), fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end)
+    {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options)
-  # end
 
-  # def get_emoji(_), do: []
+    {text, MapSet.to_list(mentions), MapSet.to_list(tags)}
+  end
 
-  @link_regex ~r/[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+/ui
+  @doc """
+  Escapes a special characters in mention names.
+  """
+  def mentions_escape(text, options \\ []) do
+    options =
+      Keyword.merge(options,
+        mention: true,
+        url: false,
+        mention_handler: &escape_mention_handler/4
+      )
 
-  @uri_schemes Application.get_env(:mobilizon, :uri_schemes, [])
+    AutoLinker.link(text, options)
-  @valid_schemes Keyword.get(@uri_schemes, :valid_schemes, [])
+  end
 
-  # # TODO: make it use something other than @link_regex
+  def html_escape({text, mentions, hashtags}, type) do
-  # def html_escape(text, "text/html") do
+    {html_escape(text, type), mentions, hashtags}
+  end
+
+  def html_escape(_text, "text/html") do
     #    HTML.filter_tags(text)
-  # end
+  end
 
   def html_escape(text, "text/plain") do
     Regex.split(@link_regex, text, include_captures: true)
@@ -82,84 +100,15 @@ defmodule Mobilizon.Service.Formatter do
     |> Enum.join("")
   end
 
-  @doc "changes scheme:... urls to html links"
+  def truncate(text, max_length \\ 200, omission \\ "...") do
-  def add_links({subs, text}) do
+    # Remove trailing whitespace
-    links =
+    text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}")
+
+    if String.length(text) < max_length do
       text
-      |> String.split([" ", "\t", "<br>"])
+    else
-      |> Enum.filter(fn word -> String.starts_with?(word, @valid_schemes) end)
+      length_with_omission = max_length - String.length(omission)
-      |> Enum.filter(fn word -> Regex.match?(@link_regex, word) end)
+      String.slice(text, 0, length_with_omission) <> omission
-      |> Enum.map(fn url -> {Ecto.UUID.generate(), url} end)
-      |> Enum.sort_by(fn {_, url} -> -String.length(url) end)
-
-    uuid_text =
-      links
-      |> Enum.reduce(text, fn {uuid, url}, acc -> String.replace(acc, url, uuid) end)
-
-    subs =
-      subs ++
-        Enum.map(links, fn {uuid, url} ->
-          {uuid, "<a href=\"#{url}\">#{url}</a>"}
-        end)
-
-    {subs, uuid_text}
     end
-
-  @doc "Adds the links to mentioned actors"
-  def add_actor_links({subs, text}, mentions) do
-    mentions =
-      mentions
-      |> Enum.sort_by(fn {name, _} -> -String.length(name) end)
-      |> Enum.map(fn {name, actor} -> {name, actor, Ecto.UUID.generate()} end)
-
-    uuid_text =
-      mentions
-      |> Enum.reduce(text, fn {match, _actor, uuid}, text ->
-        String.replace(text, match, uuid)
-      end)
-
-    subs =
-      subs ++
-        Enum.map(mentions, fn {match, %Actor{id: id, url: url}, uuid} ->
-          short_match = String.split(match, "@") |> tl() |> hd()
-
-          {uuid,
-           "<span><a data-user='#{id}' class='mention' href='#{url}'>@<span>#{short_match}</span></a></span>"}
-        end)
-
-    {subs, uuid_text}
-  end
-
-  @doc "Adds the hashtag links"
-  def add_hashtag_links({subs, text}, tags) do
-    tags =
-      tags
-      |> Enum.sort_by(fn {name, _} -> -String.length(name) end)
-      |> Enum.map(fn {name, short} -> {name, short, Ecto.UUID.generate()} end)
-
-    uuid_text =
-      tags
-      |> Enum.reduce(text, fn {match, _short, uuid}, text ->
-        String.replace(text, match, uuid)
-      end)
-
-    subs =
-      subs ++
-        Enum.map(tags, fn {tag_text, tag, uuid} ->
-          url =
-            "<a data-tag='#{tag}' href='#{MobilizonWeb.Endpoint.url()}/tag/#{tag}' rel='tag'>#{
-              tag_text
-            }</a>"
-
-          {uuid, url}
-        end)
-
-    {subs, uuid_text}
-  end
-
-  def finalize({subs, text}) do
-    Enum.reduce(subs, text, fn {uuid, replacement}, result_text ->
-      String.replace(result_text, uuid, replacement)
-    end)
   end
 end

mix.exs

@@ -90,6 +90,9 @@ defmodule Mobilizon.Mixfile do
       {:earmark, "~> 1.3.1"},
       {:geohax, "~> 0.3.0"},
       {:mogrify, "~> 0.7.2"},
+      {:auto_linker,
+       git: "https://git.pleroma.social/pleroma/auto_linker.git",
+       ref: "95e8188490e97505c56636c1379ffdf036c1fdde"},
       # Dev and test dependencies
       {:phoenix_live_reload, "~> 1.2", only: :dev},
       {:ex_machina, "~> 2.3", only: [:dev, :test]},

mix.lock

@@ -7,6 +7,7 @@
   "arc_ecto": {:git, "https://github.com/tcitworld/arc_ecto.git", "e0d8db119c564744404cff68157417e2a83941af", []},
   "argon2_elixir": {:hex, :argon2_elixir, "2.0.5", "0073a87d755c7e63fc4f9d08b1d1646585b93f144cecde126e15061b24240b20", [:make, :mix], [{:comeonin, "~> 5.1", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.5", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm"},
   "atomex": {:hex, :atomex, "0.3.0", "19b5d1a2aef8706dbd307385f7d5d9f6f273869226d317492c396c7bacf26402", [:mix], [{:xml_builder, "~> 2.0.0", [hex: :xml_builder, repo: "hexpm", optional: false]}], "hexpm"},
+  "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "95e8188490e97505c56636c1379ffdf036c1fdde", [ref: "95e8188490e97505c56636c1379ffdf036c1fdde"]},
   "bamboo": {:hex, :bamboo, "1.2.0", "8aebd24f7c606c32d0163c398004a11608ca1028182a169b2e527793bfab7561", [:mix], [{:hackney, ">= 1.13.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
   "bamboo_smtp": {:hex, :bamboo_smtp, "1.7.0", "f0d213e18ced1f08b551a72221e9b8cfbf23d592b684e9aa1ef5250f4943ef9b", [:mix], [{:bamboo, "~> 1.2", [hex: :bamboo, repo: "hexpm", optional: false]}, {:gen_smtp, "~> 0.14.0", [hex: :gen_smtp, repo: "hexpm", optional: false]}], "hexpm"},
   "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"},

test/mobilizon/service/formatter/formatter_test.exs

@@ -0,0 +1,205 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Service.FormatterTest do
+  alias Mobilizon.Service.Formatter
+  use Mobilizon.DataCase
+
+  import Mobilizon.Factory
+
+  describe ".add_hashtag_links" do
+    test "turns hashtags into links" do
+      text = "I love #cofe and #2hu"
+
+      expected_text =
+        "I love <a class='hashtag' data-tag='cofe' href='http://mobilizon.test/tag/cofe' rel='tag'>#cofe</a> and <a class='hashtag' data-tag='2hu' href='http://mobilizon.test/tag/2hu' rel='tag'>#2hu</a>"
+
+      assert {^expected_text, [], _tags} = Formatter.linkify(text)
+    end
+
+    test "does not turn html characters to tags" do
+      text = "#fact_3: pleroma does what mastodon't"
+
+      expected_text =
+        "<a class='hashtag' data-tag='fact_3' href='http://mobilizon.test/tag/fact_3' rel='tag'>#fact_3</a>: pleroma does what mastodon't"
+
+      assert {^expected_text, [], _tags} = Formatter.linkify(text)
+    end
+  end
+
+  describe ".add_links" do
+    test "turning urls into links" do
+      text = "Hey, check out https://www.youtube.com/watch?v=8Zg1-TufF%20zY?x=1&y=2#blabla ."
+
+      expected =
+        "Hey, check out <a href=\"https://www.youtube.com/watch?v=8Zg1-TufF%20zY?x=1&y=2#blabla\">https://www.youtube.com/watch?v=8Zg1-TufF%20zY?x=1&y=2#blabla</a> ."
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+
+      text = "https://mastodon.social/@lambadalambda"
+
+      expected =
+        "<a href=\"https://mastodon.social/@lambadalambda\">https://mastodon.social/@lambadalambda</a>"
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+
+      text = "https://mastodon.social:4000/@lambadalambda"
+
+      expected =
+        "<a href=\"https://mastodon.social:4000/@lambadalambda\">https://mastodon.social:4000/@lambadalambda</a>"
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+
+      text = "@lambadalambda"
+      expected = "@lambadalambda"
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+
+      text = "http://www.cs.vu.nl/~ast/intel/"
+      expected = "<a href=\"http://www.cs.vu.nl/~ast/intel/\">http://www.cs.vu.nl/~ast/intel/</a>"
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+
+      text = "https://forum.zdoom.org/viewtopic.php?f=44&t=57087"
+
+      expected =
+        "<a href=\"https://forum.zdoom.org/viewtopic.php?f=44&t=57087\">https://forum.zdoom.org/viewtopic.php?f=44&t=57087</a>"
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+
+      text = "https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul"
+
+      expected =
+        "<a href=\"https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul\">https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul</a>"
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+
+      text = "https://www.google.co.jp/search?q=Nasim+Aghdam"
+
+      expected =
+        "<a href=\"https://www.google.co.jp/search?q=Nasim+Aghdam\">https://www.google.co.jp/search?q=Nasim+Aghdam</a>"
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+
+      text = "https://en.wikipedia.org/wiki/Duff's_device"
+
+      expected =
+        "<a href=\"https://en.wikipedia.org/wiki/Duff's_device\">https://en.wikipedia.org/wiki/Duff's_device</a>"
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+
+      text = "https://pleroma.com https://pleroma.com/sucks"
+
+      expected =
+        "<a href=\"https://pleroma.com\">https://pleroma.com</a> <a href=\"https://pleroma.com/sucks\">https://pleroma.com/sucks</a>"
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+
+      text = "xmpp:contact@hacktivis.me"
+
+      expected = "<a href=\"xmpp:contact@hacktivis.me\">xmpp:contact@hacktivis.me</a>"
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+
+      text =
+        "magnet:?xt=urn:btih:7ec9d298e91d6e4394d1379caf073c77ff3e3136&tr=udp%3A%2F%2Fopentor.org%3A2710&tr=udp%3A%2F%2Ftracker.blackunicorn.xyz%3A6969&tr=udp%3A%2F%2Ftracker.ccc.de%3A80&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.openbittorrent.com%3A80&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com"
+
+      expected = "<a href=\"#{text}\">#{text}</a>"
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+    end
+  end
+
+  describe "add_user_links" do
+    test "gives a replacement for user links, using local nicknames in user links text" do
+      text = "@gsimg According to @archa_eme_, that is @daggsy. Also hello @archaeme@archae.me"
+      gsimg = insert(:actor, preferred_username: "gsimg")
+
+      archaeme =
+        insert(:actor, preferred_username: "archa_eme_", url: "https://archeme/@archa_eme_")
+
+      archaeme_remote = insert(:actor, preferred_username: "archaeme", domain: "archae.me")
+
+      {text, mentions, []} = Formatter.linkify(text)
+
+      assert length(mentions) == 3
+
+      expected_text =
+        "<span class='h-card'><a data-user='#{gsimg.id}' class='u-url mention' href='#{gsimg.url}'>@<span>gsimg</span></a></span> According to <span class='h-card'><a data-user='#{
+          archaeme.id
+        }' class='u-url mention' href='#{"https://archeme/@archa_eme_"}'>@<span>archa_eme_</span></a></span>, that is @daggsy. Also hello <span class='h-card'><a data-user='#{
+          archaeme_remote.id
+        }' class='u-url mention' href='#{archaeme_remote.url}'>@<span>archaeme</span></a></span>"
+
+      assert expected_text == text
+    end
+
+    test "gives a replacement for single-character local nicknames" do
+      text = "@o hi"
+      o = insert(:actor, preferred_username: "o")
+
+      {text, mentions, []} = Formatter.linkify(text)
+
+      assert length(mentions) == 1
+
+      expected_text =
+        "<span class='h-card'><a data-user='#{o.id}' class='u-url mention' href='#{o.url}'>@<span>o</span></a></span> hi"
+
+      assert expected_text == text
+    end
+
+    test "does not give a replacement for single-character local nicknames who don't exist" do
+      text = "@a hi"
+
+      expected_text = "@a hi"
+      assert {^expected_text, [] = _mentions, [] = _tags} = Formatter.linkify(text)
+    end
+  end
+
+  describe ".parse_tags" do
+    test "parses tags in the text" do
+      text = "Here's a #Test. Maybe these are #working or not. What about #漢字? And #は｡"
+
+      expected_tags = [
+        {"#Test", "test"},
+        {"#working", "working"},
+        {"#は", "は"},
+        {"#漢字", "漢字"}
+      ]
+
+      assert {_text, [], ^expected_tags} = Formatter.linkify(text)
+    end
+  end
+
+  test "it can parse mentions and return the relevant users" do
+    text =
+      "@@gsimg According to @archaeme, that is @daggsy. Also hello @archaeme@archae.me and @o and @@@jimm"
+
+    o = insert(:actor, preferred_username: "o")
+    jimm = insert(:actor, preferred_username: "jimm")
+    gsimg = insert(:actor, preferred_username: "gsimg")
+    archaeme = insert(:actor, preferred_username: "archaeme")
+    archaeme_remote = insert(:actor, preferred_username: "archaeme", domain: "archae.me")
+
+    expected_mentions = [
+      {"@archaeme", archaeme.id},
+      {"@archaeme@archae.me", archaeme_remote.id},
+      {"@gsimg", gsimg.id},
+      {"@jimm", jimm.id},
+      {"@o", o.id}
+    ]
+
+    {_text, mentions, []} = Formatter.linkify(text)
+
+    assert expected_mentions ==
+             Enum.map(mentions, fn {username, actor} -> {username, actor.id} end)
+  end
+
+  test "it escapes HTML in plain text" do
+    text = "hello & world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1"
+    expected = "hello &amp; world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1"
+
+    assert Formatter.html_escape(text, "text/plain") == expected
+  end
+end

test/mobilizon_web/resolvers/event_resolver_test.exs

@@ -84,6 +84,44 @@ defmodule MobilizonWeb.Resolvers.EventResolverTest do
       assert json_response(res, 200)["data"]["createEvent"]["title"] == "come to my event"
     end
 
+    test "create_event/3 creates an event with tags", %{conn: conn, actor: actor, user: user} do
+      mutation = """
+          mutation {
+              createEvent(
+                  title: "my event is referenced",
+                  description: "with tags!",
+                  begins_on: "#{
+        DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_iso8601()
+      }",
+                  organizer_actor_id: "#{actor.id}",
+                  category: "birthday",
+                  tags: ["nicolas", "birthday", "bad tag"]
+              ) {
+                title,
+                uuid,
+                tags {
+                  title,
+                  slug
+                }
+              }
+            }
+      """
+
+      res =
+        conn
+        |> auth_conn(user)
+        |> post("/api", AbsintheHelpers.mutation_skeleton(mutation))
+
+      assert json_response(res, 200)["errors"] == nil
+      assert json_response(res, 200)["data"]["createEvent"]["title"] == "my event is referenced"
+
+      assert json_response(res, 200)["data"]["createEvent"]["tags"] == [
+               %{"slug" => "nicolas", "title" => "nicolas"},
+               %{"slug" => "birthday", "title" => "birthday"},
+               %{"slug" => "bad-tag", "title" => "bad tag"}
+             ]
+    end
+
     test "create_event/3 creates an event with an attached picture", %{
       conn: conn,
       actor: actor,