Merge branch 'clean-old-activity' into 'master'

Clean old activity

See merge request framasoft/mobilizon!836
This commit is contained in:
Thomas Citharel 2021-03-02 14:28:28 +00:00
commit 800078f0d9
10 changed files with 193 additions and 7 deletions

View file

@ -32,6 +32,8 @@ config :mobilizon, :instance,
orphan_upload_grace_period_hours: 48, orphan_upload_grace_period_hours: 48,
remove_unconfirmed_users: true, remove_unconfirmed_users: true,
unconfirmed_user_grace_period_hours: 48, unconfirmed_user_grace_period_hours: 48,
activity_expire_days: 365,
activity_keep_number: 100,
email_from: "noreply@localhost", email_from: "noreply@localhost",
email_reply_to: "noreply@localhost" email_reply_to: "noreply@localhost"
@ -269,7 +271,8 @@ config :mobilizon, Oban,
{"17 * * * *", Mobilizon.Service.Workers.RefreshGroups, queue: :background}, {"17 * * * *", Mobilizon.Service.Workers.RefreshGroups, queue: :background},
# To be activated in Mobilizon 1.2 # To be activated in Mobilizon 1.2
# {"@hourly", Mobilizon.Service.Workers.CleanOrphanMediaWorker, queue: :background}, # {"@hourly", Mobilizon.Service.Workers.CleanOrphanMediaWorker, queue: :background},
{"@hourly", Mobilizon.Service.Workers.CleanUnconfirmedUsersWorker, queue: :background} {"@hourly", Mobilizon.Service.Workers.CleanUnconfirmedUsersWorker, queue: :background},
{"@daily", Mobilizon.Service.Workers.CleanOldActivityWorker, queue: :background}
]}, ]},
{Oban.Plugins.Pruner, max_age: 300} {Oban.Plugins.Pruner, max_age: 300}
] ]

View file

@ -100,6 +100,7 @@ defmodule Mobilizon.Federation.ActivityPub.Refresher do
def refresh_all_external_groups do def refresh_all_external_groups do
Repo.transaction(fn -> Repo.transaction(fn ->
Actors.list_external_groups_for_stream() Actors.list_external_groups_for_stream()
|> Stream.filter(&Actors.needs_update?/1)
|> Stream.map(&refresh_profile/1) |> Stream.map(&refresh_profile/1)
|> Stream.run() |> Stream.run()
end) end)

View file

@ -19,7 +19,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Activity do
with {:actor, %Actor{id: actor_id} = _actor} <- {:actor, Users.get_actor_for_user(user)}, with {:actor, %Actor{id: actor_id} = _actor} <- {:actor, Users.get_actor_for_user(user)},
{:member, true} <- {:member, Actors.is_member?(actor_id, group_id) or is_moderator(role)} do {:member, true} <- {:member, Actors.is_member?(actor_id, group_id) or is_moderator(role)} do
%Page{total: total, elements: elements} = %Page{total: total, elements: elements} =
Activities.list_activities_for_group(group_id, actor_id, [], page, limit) Activities.list_group_activities_for_member(group_id, actor_id, [], page, limit)
elements = elements =
Enum.map(elements, fn %Activity{} = activity -> Enum.map(elements, fn %Activity{} = activity ->

View file

@ -72,13 +72,14 @@ defmodule Mobilizon.Activities do
Repo.all(Activity) Repo.all(Activity)
end end
@spec list_activities_for_group( @spec list_group_activities_for_member(
integer() | String.t(),
integer() | String.t(), integer() | String.t(),
Keyword.t(), Keyword.t(),
integer() | nil, integer() | nil,
integer() | nil integer() | nil
) :: Page.t() ) :: Page.t()
def list_activities_for_group( def list_group_activities_for_member(
group_id, group_id,
actor_asking_id, actor_asking_id,
filters \\ [], filters \\ [],
@ -97,6 +98,26 @@ defmodule Mobilizon.Activities do
|> Page.build_page(page, limit) |> Page.build_page(page, limit)
end end
@spec list_group_activities(
integer() | String.t(),
Keyword.t(),
integer() | nil,
integer() | nil
) :: Page.t()
def list_group_activities(
group_id,
filters \\ [],
page \\ nil,
limit \\ nil
) do
Activity
|> where([a], a.group_id == ^group_id)
|> filter_object_type(Keyword.get(filters, :type))
|> order_by(desc: :inserted_at)
|> preload([:author, :group])
|> Page.build_page(page, limit)
end
@doc """ @doc """
Gets a single activity. Gets a single activity.

View file

@ -0,0 +1,64 @@
defmodule Mobilizon.Service.CleanOldActivity do
@moduledoc """
Service to clean old activities
"""
alias Mobilizon.Activities.Activity
alias Mobilizon.Actors.Actor
alias Mobilizon.Config
alias Mobilizon.Storage.Repo
import Ecto.Query
@doc """
Clean old activities
Remove activities that are older than a certain period
Options:
* `grace_period` how old in hours can the media be before it's taken into account for deletion
* `dry_run` just return the media that would have been deleted, don't actually delete it
"""
@spec clean(Keyword.t()) :: {:ok, list(Media.t())} | {:error, String.t()}
def clean(opts \\ []) do
{query, nb_actors} = find_activities(opts)
if Keyword.get(opts, :dry_run, false) do
nb_activities = Repo.aggregate(query, :count)
{:ok, actors: nb_actors, activities: nb_activities}
else
{nb_activities, _} = Repo.delete_all(query)
{:ok, actors: nb_actors, activities: nb_activities}
end
end
@spec find_activities(Keyword.t()) :: {Ecto.Query.t(), list()}
defp find_activities(opts) do
grace_period =
Keyword.get(opts, :grace_period, Config.get([:instance, :activity_expire_days], 365))
expiration_date = DateTime.add(DateTime.utc_now(), grace_period * -3600)
activities_to_keep =
Keyword.get(
opts,
:activity_keep_number,
Config.get([:instance, :activity_keep_number], 100)
)
actor_ids =
Actor
|> where(type: :Group)
|> join(:inner, [ac], a in Activity, on: a.group_id == ac.id)
|> group_by([ac], ac.id)
|> having([_ac, a], count(a.id) > ^activities_to_keep)
|> select([ac], ac.id)
|> Repo.all()
query =
Activity
|> where([a], a.inserted_at < ^expiration_date)
|> where([a], a.group_id in ^actor_ids)
{query, length(actor_ids)}
end
end

View file

@ -0,0 +1,13 @@
defmodule Mobilizon.Service.Workers.CleanOldActivityWorker do
@moduledoc """
Worker to clean old activity
"""
use Oban.Worker, queue: "background"
alias Mobilizon.Service.CleanOldActivity
@impl Oban.Worker
def perform(%Job{}) do
CleanOldActivity.clean()
end
end

View file

@ -0,0 +1,84 @@
defmodule Mobilizon.Service.CleanOldActivityTest do
use Mobilizon.DataCase
import Mobilizon.Factory
alias Mobilizon.{Activities, Config}
alias Mobilizon.Service.CleanOldActivity
@activity_inserted_at_1 DateTime.from_iso8601("2019-01-02T10:33:39.207493Z") |> elem(1)
@activity_inserted_at_2 DateTime.from_iso8601("2021-03-02T10:33:39.207493Z") |> elem(1)
setup do
group1 = insert(:group)
group2 = insert(:group)
Config.clear_config_cache()
{:ok, group1: group1, group2: group2}
end
describe "clean old activities" do
test "with default settings", %{group1: group1, group2: group2} do
create_activities(group1, group2)
Config.put([:instance, :activity_expire_days], 100)
Config.put([:instance, :activity_keep_number], 5)
assert Activities.list_group_activities(group1.id).total == 10
assert Activities.list_group_activities(group2.id).total == 5
assert {:ok, actors: 1, activities: 5} == CleanOldActivity.clean(dry_run: true)
assert Activities.list_group_activities(group1.id).total == 10
assert Activities.list_group_activities(group2.id).total == 5
assert {:ok, actors: 1, activities: 5} == CleanOldActivity.clean()
assert Activities.list_group_activities(group1.id).total == 5
assert Activities.list_group_activities(group2.id).total == 5
assert {:ok, actors: 0, activities: 0} == CleanOldActivity.clean()
assert Activities.list_group_activities(group1.id).total == 5
assert Activities.list_group_activities(group2.id).total == 5
Config.put([:instance, :activity_expire_days], 365)
Config.put([:instance, :activity_keep_number], 100)
end
test "with custom settings", %{group1: group1, group2: group2} do
create_activities(group1, group2)
assert Activities.list_group_activities(group1.id).total == 10
assert Activities.list_group_activities(group2.id).total == 5
assert {:ok, actors: 1, activities: 5} ==
CleanOldActivity.clean(grace_period: 100, activity_keep_number: 5, dry_run: true)
assert Activities.list_group_activities(group1.id).total == 10
assert Activities.list_group_activities(group2.id).total == 5
assert {:ok, actors: 1, activities: 5} ==
CleanOldActivity.clean(grace_period: 100, activity_keep_number: 5)
assert Activities.list_group_activities(group1.id).total == 5
assert Activities.list_group_activities(group2.id).total == 5
assert {:ok, actors: 0, activities: 0} ==
CleanOldActivity.clean(grace_period: 100, activity_keep_number: 5)
assert Activities.list_group_activities(group1.id).total == 5
assert Activities.list_group_activities(group2.id).total == 5
end
end
defp create_activities(group1, group2) do
Enum.each(1..5, fn _ ->
insert(:mobilizon_activity, group: group1, inserted_at: @activity_inserted_at_1)
end)
Enum.each(1..5, fn _ ->
insert(:mobilizon_activity, group: group1, inserted_at: @activity_inserted_at_2)
end)
Enum.each(1..5, fn _ ->
insert(:mobilizon_activity, group: group2, inserted_at: @activity_inserted_at_2)
end)
end
end

View file

@ -1,4 +1,4 @@
defmodule Mix.Tasks.Mobilizon.Media.CleanOrphanTest do defmodule Mobilizon.Service.CleanOrphanMediaTest do
use Mobilizon.DataCase use Mobilizon.DataCase
import Mobilizon.Factory import Mobilizon.Factory

View file

@ -1,4 +1,4 @@
defmodule Mix.Tasks.Mobilizon.User.CleanUnconfirmedUsersTest do defmodule Mobilizon.Service.CleanUnconfirmedUsersTest do
use Mobilizon.DataCase use Mobilizon.DataCase
import Mobilizon.Factory import Mobilizon.Factory

View file

@ -418,7 +418,7 @@ defmodule Mobilizon.Factory do
%Mobilizon.Activities.Activity{ %Mobilizon.Activities.Activity{
type: :event, type: :event,
subject: :event_created, subject: :event_created,
subject_params: %{event: event}, subject_params: %{event_title: event.title},
author: actor, author: actor,
group: group, group: group,
object_type: :event, object_type: :event,