From cb4e28f405735e9e4c5438a93320c2584cd386f3 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 18 Jan 2023 16:50:50 +0100 Subject: [PATCH] Add `tootctl domains purge` options to select subdomains and keep domain blocks (#22063) * Add --include-subdomains option to tootctl domains purge * Add support for '*.' subdomain wildcard patterns in `tootctl domains purge` * Fix custom emojis deletion not following subdomain and URI options * Change `tootctl domains purge` to not purge domain blocks unless --purge-domain-blocks is passed * Refactor `tootctl domains purge` * Add feedback on deleted domain blocks --- lib/mastodon/domains_cli.rb | 75 +++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index 77364ffbb..81ee53c18 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -18,6 +18,8 @@ module Mastodon option :dry_run, type: :boolean option :limited_federation_mode, type: :boolean option :by_uri, type: :boolean + option :include_subdomains, type: :boolean + option :purge_domain_blocks, type: :boolean desc 'purge [DOMAIN...]', 'Remove accounts from a DOMAIN without a trace' long_desc <<-LONG_DESC Remove all accounts from a given DOMAIN without leaving behind any @@ -33,40 +35,75 @@ module Mastodon that has the handle `foo@bar.com` but whose profile is at the URL `https://mastodon-bar.com/users/foo`, would be purged by either `tootctl domains purge bar.com` or `tootctl domains purge --by-uri mastodon-bar.com`. + + When the --include-subdomains option is given, not only DOMAIN is deleted, but all + subdomains as well. Note that this may be considerably slower. + + When the --purge-domain-blocks option is given, also purge matching domain blocks. LONG_DESC def purge(*domains) - dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + domains = domains.map { |domain| TagManager.instance.normalize_domain(domain) } + account_scope = Account.none + domain_block_scope = DomainBlock.none + emoji_scope = CustomEmoji.none - scope = begin - if options[:limited_federation_mode] - Account.remote.where.not(domain: DomainAllow.pluck(:domain)) - elsif !domains.empty? - if options[:by_uri] - domains.map { |domain| Account.remote.where(Account.arel_table[:uri].matches("https://#{domain}/%", false, true)) }.reduce(:or) - else - Account.remote.where(domain: domains) - end + # Sanity check on command arguments + if options[:limited_federation_mode] && !domains.empty? + say('DOMAIN parameter not supported with --limited-federation-mode', :red) + exit(1) + elsif domains.empty? && !options[:limited_federation_mode] + say('No domain(s) given', :red) + exit(1) + end + + # Build scopes from command arguments + if options[:limited_federation_mode] + account_scope = Account.remote.where.not(domain: DomainAllow.select(:domain)) + emoji_scope = CustomEmoji.remote.where.not(domain: DomainAllow.select(:domain)) + else + # Handle wildcard subdomains + subdomain_patterns = domains.filter_map { |domain| "%.#{Account.sanitize_sql_like(domain[2..])}" if domain.start_with?('*.') } + domains = domains.filter { |domain| !domain.start_with?('*.') } + # Handle --include-subdomains + subdomain_patterns += domains.map { |domain| "%.#{Account.sanitize_sql_like(domain)}" } if options[:include_subdomains] + uri_patterns = (domains.map { |domain| Account.sanitize_sql_like(domain) } + subdomain_patterns).map { |pattern| "https://#{pattern}/%" } + + if options[:purge_domain_blocks] + domain_block_scope = DomainBlock.where(domain: domains) + domain_block_scope = domain_block_scope.or(DomainBlock.where(DomainBlock.arel_table[:domain].matches_any(subdomain_patterns))) unless subdomain_patterns.empty? + end + + if options[:by_uri] + account_scope = Account.remote.where(Account.arel_table[:uri].matches_any(uri_patterns, false, true)) + emoji_scope = CustomEmoji.remote.where(CustomEmoji.arel_table[:uri].matches_any(uri_patterns, false, true)) else - say('No domain(s) given', :red) - exit(1) + account_scope = Account.remote.where(domain: domains) + account_scope = account_scope.or(Account.remote.where(Account.arel_table[:domain].matches_any(subdomain_patterns))) unless subdomain_patterns.empty? + emoji_scope = CustomEmoji.where(domain: domains) + emoji_scope = emoji_scope.or(CustomEmoji.remote.where(CustomEmoji.arel_table[:uri].matches_any(subdomain_patterns))) unless subdomain_patterns.empty? end end - processed, = parallelize_with_progress(scope) do |account| + # Actually perform the deletions + processed, = parallelize_with_progress(account_scope) do |account| DeleteAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless options[:dry_run] end - DomainBlock.where(domain: domains).destroy_all unless options[:dry_run] - say("Removed #{processed} accounts#{dry_run}", :green) - custom_emojis = CustomEmoji.where(domain: domains) - custom_emojis_count = custom_emojis.count - custom_emojis.destroy_all unless options[:dry_run] + if options[:purge_domain_blocks] + domain_block_count = domain_block_scope.count + domain_block_scope.in_batches.destroy_all unless options[:dry_run] + say("Removed #{domain_block_count} domain blocks#{dry_run}", :green) + end + + custom_emojis_count = emoji_scope.count + emoji_scope.in_batches.destroy_all unless options[:dry_run] Instance.refresh unless options[:dry_run] - say("Removed #{custom_emojis_count} custom emojis", :green) + say("Removed #{custom_emojis_count} custom emojis#{dry_run}", :green) end option :concurrency, type: :numeric, default: 50, aliases: [:c]