diff --git a/lib/federation/activity_pub/fetcher.ex b/lib/federation/activity_pub/fetcher.ex
index 4c4324fee..af519ab59 100644
--- a/lib/federation/activity_pub/fetcher.ex
+++ b/lib/federation/activity_pub/fetcher.ex
@@ -22,7 +22,9 @@ defmodule Mobilizon.Federation.ActivityPub.Fetcher do
           | {:error,
              :invalid_url | :http_gone | :http_error | :http_not_found | :content_not_json}
   def fetch(url, options \\ []) do
-    on_behalf_of = Keyword.get(options, :on_behalf_of, Relay.get_actor())
+    Logger.debug("Fetching #{url} with AP Fetcher")
+    on_behalf_of = Keyword.get(options, :on_behalf_of, actor_relay())
+    Logger.debug("Fetching on behalf of #{inspect(on_behalf_of.url)}")
     date = Signature.generate_date_header()
 
     headers =
@@ -32,29 +34,8 @@ defmodule Mobilizon.Federation.ActivityPub.Fetcher do
 
     client = ActivityPubClient.client(headers: headers)
 
-    if address_valid?(url) do
-      case ActivityPubClient.get(client, url) do
-        {:ok, %Tesla.Env{body: data, status: code}} when code in 200..299 and is_map(data) ->
-          {:ok, data}
-
-        {:ok, %Tesla.Env{status: 410}} ->
-          Logger.debug("Resource at #{url} is 410 Gone")
-          {:error, :http_gone}
-
-        {:ok, %Tesla.Env{status: 404}} ->
-          Logger.debug("Resource at #{url} is 404 Gone")
-          {:error, :http_not_found}
-
-        {:ok, %Tesla.Env{body: data}} when is_binary(data) ->
-          {:error, :content_not_json}
-
-        {:ok, %Tesla.Env{} = res} ->
-          Logger.debug("Resource returned bad HTTP code #{inspect(res)}")
-          {:error, :http_error}
-
-        {:error, err} ->
-          {:error, err}
-      end
+    if address_valid?(url) and url != on_behalf_of.url do
+      do_fetch(url, client)
     else
       {:error, :invalid_url}
     end
@@ -169,4 +150,38 @@ defmodule Mobilizon.Federation.ActivityPub.Fetcher do
     %URI{host: host, scheme: scheme} = URI.parse(address)
     is_valid_string(host) and is_valid_string(scheme)
   end
+
+  defp actor_relay do
+    Relay.get_actor()
+  end
+
+  @spec do_fetch(String.t(), Tesla.Client.t()) ::
+          {:ok, map()}
+          | {:error,
+             :invalid_url | :http_gone | :http_error | :http_not_found | :content_not_json}
+  defp do_fetch(url, client) do
+    case ActivityPubClient.get(client, url) do
+      {:ok, %Tesla.Env{body: data, status: code}} when code in 200..299 and is_map(data) ->
+        Logger.debug("Found the following from ActivityPubClient fetch: #{inspect(data)}")
+        {:ok, data}
+
+      {:ok, %Tesla.Env{status: 410}} ->
+        Logger.debug("Resource at #{url} is 410 Gone")
+        {:error, :http_gone}
+
+      {:ok, %Tesla.Env{status: 404}} ->
+        Logger.debug("Resource at #{url} is 404 Gone")
+        {:error, :http_not_found}
+
+      {:ok, %Tesla.Env{body: data}} when is_binary(data) ->
+        {:error, :content_not_json}
+
+      {:ok, %Tesla.Env{} = res} ->
+        Logger.debug("Resource returned bad HTTP code #{inspect(res)}")
+        {:error, :http_error}
+
+      {:error, err} ->
+        {:error, err}
+    end
+  end
 end
diff --git a/lib/federation/http_signatures/signature.ex b/lib/federation/http_signatures/signature.ex
index d60598ebf..6812cf0d1 100644
--- a/lib/federation/http_signatures/signature.ex
+++ b/lib/federation/http_signatures/signature.ex
@@ -12,6 +12,7 @@ defmodule Mobilizon.Federation.HTTPSignatures.Signature do
 
   alias Mobilizon.Actors.Actor
   alias Mobilizon.Federation.ActivityPub.Actor, as: ActivityPubActor
+  alias Mobilizon.Federation.ActivityPub.Relay
 
   require Logger
 
@@ -94,13 +95,19 @@ defmodule Mobilizon.Federation.HTTPSignatures.Signature do
     %{"keyId" => kid} = HTTPSignatures.signature_for_conn(conn)
     actor_url = key_id_to_actor_url(kid)
     Logger.debug("Refetching public key for #{actor_url}")
+    relay = Relay.get_actor()
 
-    # In this specific case we don't sign object fetches because
-    # this would cause infinite recursion when servers both need
-    # to fetch each other's keys
-    with {:ok, %Actor{} = actor} <-
-           ActivityPubActor.make_actor_from_url(actor_url, ignore_sign_object_fetches: true) do
-      get_actor_public_key(actor)
+    if actor_url == relay.url do
+      # Special case if ever it's our own actor fetching ourselves
+      get_actor_public_key(relay)
+    else
+      # In this specific case we don't sign object fetches because
+      # this would cause infinite recursion when servers both need
+      # to fetch each other's keys
+      with {:ok, %Actor{} = actor} <-
+             ActivityPubActor.make_actor_from_url(actor_url, ignore_sign_object_fetches: true) do
+        get_actor_public_key(actor)
+      end
     end
   end