diff --git a/.gitignore b/.gitignore
index dec9281ff..571bd8abf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,7 @@ priv/errors/*
cover/
site/
test/fixtures/image_tmp.jpg
+test/fixtures/DSCN0010_tmp.jpg
test/uploads/
uploads/*
release/
diff --git a/lib/mobilizon/config.ex b/lib/mobilizon/config.ex
index 6f7f6315f..87ad8ce5c 100644
--- a/lib/mobilizon/config.ex
+++ b/lib/mobilizon/config.ex
@@ -276,17 +276,19 @@ defmodule Mobilizon.Config do
end
end
- @spec put([module | atom], any) :: any
def put([key], value), do: put(key, value)
def put([parent_key | keys], value) do
- parent = put_in(Application.get_env(:mobilizon, parent_key), keys, value)
+ parent =
+ Application.get_env(:mobilizon, parent_key, [])
+ |> put_in(keys, value)
Application.put_env(:mobilizon, parent_key, parent)
end
- @spec put(module | atom, any) :: any
- def put(key, value), do: Application.put_env(:mobilizon, key, value)
+ def put(key, value) do
+ Application.put_env(:mobilizon, key, value)
+ end
@spec to_boolean(boolean | String.t()) :: boolean
defp to_boolean(boolean), do: "true" == String.downcase("#{boolean}")
diff --git a/lib/mobilizon/utils.ex b/lib/mobilizon/utils.ex
new file mode 100644
index 000000000..400edf4cb
--- /dev/null
+++ b/lib/mobilizon/utils.ex
@@ -0,0 +1,25 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Utils do
+ @moduledoc """
+ Module that provide generic utils functions for Mobilizon
+ """
+
+ @doc """
+ POSIX-compliant check if command is available in the system
+
+ ## Examples
+ iex> command_available?("git")
+ true
+ iex> command_available?("wrongcmd")
+ false
+
+ """
+ @spec command_available?(String.t()) :: boolean()
+ def command_available?(command) do
+ match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"]))
+ end
+end
diff --git a/lib/web/upload/filter/anonymize_filename.ex b/lib/web/upload/filter/anonymize_filename.ex
index 33074ba24..093479448 100644
--- a/lib/web/upload/filter/anonymize_filename.ex
+++ b/lib/web/upload/filter/anonymize_filename.ex
@@ -13,11 +13,20 @@ defmodule Mobilizon.Web.Upload.Filter.AnonymizeFilename do
@behaviour Mobilizon.Web.Upload.Filter
alias Mobilizon.Config
+ alias Mobilizon.Web.Upload
- def filter(upload) do
- extension = List.last(String.split(upload.name, "."))
- name = Config.get([__MODULE__, :text], random(extension))
- {:ok, %Mobilizon.Web.Upload{upload | name: name}}
+ def filter(%Upload{name: name} = upload) do
+ extension = List.last(String.split(name, "."))
+ name = predefined_name(extension) || random(extension)
+ {:ok, :filtered, %Upload{upload | name: name}}
+ end
+
+ def filter(_), do: {:ok, :noop}
+
+ @spec predefined_name(String.t()) :: String.t() | nil
+ defp predefined_name(extension) do
+ with name when not is_nil(name) <- Config.get([__MODULE__, :text]),
+ do: String.replace(name, "{extension}", extension)
end
defp random(extension) do
diff --git a/lib/web/upload/filter/dedupe.ex b/lib/web/upload/filter/dedupe.ex
index d1f48d757..1e7731a5d 100644
--- a/lib/web/upload/filter/dedupe.ex
+++ b/lib/web/upload/filter/dedupe.ex
@@ -10,11 +10,13 @@ defmodule Mobilizon.Web.Upload.Filter.Dedupe do
@behaviour Mobilizon.Web.Upload.Filter
alias Mobilizon.Web.Upload
- def filter(%Upload{name: name} = upload) do
+ def filter(%Upload{name: name, tempfile: tempfile} = upload) do
extension = name |> String.split(".") |> List.last()
- shasum = :crypto.hash(:sha256, File.read!(upload.tempfile)) |> Base.encode16(case: :lower)
+ shasum = :crypto.hash(:sha256, File.read!(tempfile)) |> Base.encode16(case: :lower)
filename = shasum <> "." <> extension
- {:ok, %Upload{upload | id: shasum, path: filename}}
+ {:ok, :filtered, %Upload{upload | id: shasum, path: filename}}
end
+
+ def filter(_), do: {:ok, :noop}
end
diff --git a/lib/web/upload/filter/exiftool.ex b/lib/web/upload/filter/exiftool.ex
new file mode 100644
index 000000000..fd7391f27
--- /dev/null
+++ b/lib/web/upload/filter/exiftool.ex
@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.Filter.Exiftool do
+ @moduledoc """
+ Strips GPS related EXIF tags and overwrites the file in place.
+ Also strips or replaces filesystem metadata e.g., timestamps.
+ """
+ alias Mobilizon.Web.Upload
+
+ @behaviour Mobilizon.Web.Upload.Filter
+
+ @spec filter(Upload.t()) :: {:ok, any()} | {:error, String.t()}
+
+ # webp is not compatible with exiftool at this time
+ def filter(%Upload{content_type: "image/webp"}), do: {:ok, :noop}
+
+ def filter(%Upload{tempfile: file, content_type: "image" <> _}) do
+ case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
+ {_response, 0} -> {:ok, :filtered}
+ {error, 1} -> {:error, error}
+ end
+ rescue
+ _e in ErlangError ->
+ {:error, "exiftool command not found"}
+ end
+
+ def filter(_), do: {:ok, :noop}
+end
diff --git a/lib/web/upload/filter/filter.ex b/lib/web/upload/filter/filter.ex
index 322acdf47..be85909a8 100644
--- a/lib/web/upload/filter/filter.ex
+++ b/lib/web/upload/filter/filter.ex
@@ -17,7 +17,10 @@ defmodule Mobilizon.Web.Upload.Filter do
require Logger
@callback filter(Mobilizon.Web.Upload.t()) ::
- :ok | {:ok, Mobilizon.Web.Upload.t()} | {:error, any()}
+ {:ok, :filtered}
+ | {:ok, :noop}
+ | {:ok, :filtered, Mobilizon.Web.Upload.t()}
+ | {:error, any()}
@spec filter([module()], Mobilizon.Web.Upload.t()) ::
{:ok, Mobilizon.Web.Upload.t()} | {:error, any()}
@@ -28,10 +31,13 @@ defmodule Mobilizon.Web.Upload.Filter do
def filter([filter | rest], upload) do
case filter.filter(upload) do
- :ok ->
+ {:ok, :filtered} ->
filter(rest, upload)
- {:ok, upload} ->
+ {:ok, :filtered, upload} ->
+ filter(rest, upload)
+
+ {:ok, :noop} ->
filter(rest, upload)
error ->
diff --git a/lib/web/upload/filter/mogrify.ex b/lib/web/upload/filter/mogrify.ex
index fbf9e491d..f10c49728 100644
--- a/lib/web/upload/filter/mogrify.ex
+++ b/lib/web/upload/filter/mogrify.ex
@@ -15,19 +15,24 @@ defmodule Mobilizon.Web.Upload.Filter.Mogrify do
@type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
@type conversions :: conversion() | [conversion()]
+ @spec filter(Mobilizon.Web.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
def filter(%Mobilizon.Web.Upload{tempfile: file, content_type: "image" <> _}) do
- filters = Config.get!([__MODULE__, :args])
+ do_filter(file, Config.get!([__MODULE__, :args]))
+ {:ok, :filtered}
+ rescue
+ _e in ErlangError ->
+ {:error, "mogrify command not found"}
+ end
+ def filter(_), do: {:ok, :noop}
+
+ def do_filter(file, filters) do
file
|> Mogrify.open()
|> mogrify_filter(filters)
|> Mogrify.save(in_place: true)
-
- :ok
end
- def filter(_), do: :ok
-
defp mogrify_filter(mogrify, nil), do: mogrify
defp mogrify_filter(mogrify, [filter | rest]) do
diff --git a/lib/web/upload/filter/optimize.ex b/lib/web/upload/filter/optimize.ex
index 0162f6c11..3c3e71222 100644
--- a/lib/web/upload/filter/optimize.ex
+++ b/lib/web/upload/filter/optimize.ex
@@ -21,7 +21,7 @@ defmodule Mobilizon.Web.Upload.Filter.Optimize do
case ExOptimizer.optimize(file, deps: optimizers) do
{:ok, _res} ->
- :ok
+ {:ok, :filtered}
{:error, err} ->
require Logger
@@ -30,12 +30,12 @@ defmodule Mobilizon.Web.Upload.Filter.Optimize do
"Unable to optimize file #{file}. The return from the process was #{inspect(err)}"
)
- :ok
+ {:ok, :noop}
err ->
- err
+ {:error, err}
end
end
- def filter(_), do: :ok
+ def filter(_), do: {:ok, :noop}
end
diff --git a/lib/web/upload/uploader/local.ex b/lib/web/upload/uploader/local.ex
index 5304820f4..d153ee3d6 100644
--- a/lib/web/upload/uploader/local.ex
+++ b/lib/web/upload/uploader/local.ex
@@ -12,10 +12,12 @@ defmodule Mobilizon.Web.Upload.Uploader.Local do
alias Mobilizon.Config
+ @impl true
def get_file(_) do
{:ok, {:static_dir, upload_path()}}
end
+ @impl true
def put_file(upload) do
{path, file} = local_path(upload.path)
result_file = Path.join(path, file)
@@ -27,6 +29,7 @@ defmodule Mobilizon.Web.Upload.Uploader.Local do
:ok
end
+ @impl true
def remove_file(path) do
with {path, file} <- local_path(path),
full_path <- Path.join(path, file),
diff --git a/test/fixtures/DSCN0010.jpg b/test/fixtures/DSCN0010.jpg
new file mode 100644
index 000000000..4a2c1552b
Binary files /dev/null and b/test/fixtures/DSCN0010.jpg differ
diff --git a/test/web/upload/filter/anonymize_filename_test.exs b/test/web/upload/filter/anonymize_filename_test.exs
new file mode 100644
index 000000000..cdc8ccb11
--- /dev/null
+++ b/test/web/upload/filter/anonymize_filename_test.exs
@@ -0,0 +1,44 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.Filter.AnonymizeFilenameTest do
+ use Mobilizon.DataCase
+ use Mobilizon.Tests.Helpers
+
+ alias Mobilizon.Config
+ alias Mobilizon.Web.Upload
+
+ setup do
+ File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+
+ upload_file = %Upload{
+ name: "an… image.jpg",
+ content_type: "image/jpeg",
+ path: Path.absname("test/fixtures/image_tmp.jpg")
+ }
+
+ %{upload_file: upload_file}
+ end
+
+ setup do: clear_config([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text])
+
+ test "it replaces filename on pre-defined text", %{upload_file: upload_file} do
+ Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.png")
+ {:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
+ assert name == "custom-file.png"
+ end
+
+ test "it replaces filename on pre-defined text expression", %{upload_file: upload_file} do
+ Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.{extension}")
+ {:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
+ assert name == "custom-file.jpg"
+ end
+
+ test "it replaces filename on random text", %{upload_file: upload_file} do
+ {:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
+ assert <<_::bytes-size(14)>> <> ".jpg" = name
+ refute name == "an… image.jpg"
+ end
+end
diff --git a/test/web/upload/filter/dedupe_test.exs b/test/web/upload/filter/dedupe_test.exs
new file mode 100644
index 000000000..14fbe7bb3
--- /dev/null
+++ b/test/web/upload/filter/dedupe_test.exs
@@ -0,0 +1,33 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.Filter.DedupeTest do
+ use Mobilizon.DataCase
+
+ alias Mobilizon.Web.Upload
+ alias Mobilizon.Web.Upload.Filter.Dedupe
+
+ @shasum "590523d60d3831ec92d05cdd871078409d5780903910efec5cd35ab1b0f19d11"
+
+ test "adds shasum" do
+ File.cp!(
+ "test/fixtures/image.jpg",
+ "test/fixtures/image_tmp.jpg"
+ )
+
+ upload = %Upload{
+ name: "an… image.jpg",
+ content_type: "image/jpeg",
+ path: Path.absname("test/fixtures/image_tmp.jpg"),
+ tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+ }
+
+ assert {
+ :ok,
+ :filtered,
+ %Upload{id: @shasum, path: @shasum <> ".jpg"}
+ } = Dedupe.filter(upload)
+ end
+end
diff --git a/test/web/upload/filter/exiftool_test.exs b/test/web/upload/filter/exiftool_test.exs
new file mode 100644
index 000000000..49f770c70
--- /dev/null
+++ b/test/web/upload/filter/exiftool_test.exs
@@ -0,0 +1,44 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.Filter.ExiftoolTest do
+ use Mobilizon.DataCase
+ alias Mobilizon.Web.Upload
+ alias Mobilizon.Web.Upload.Filter
+
+ test "apply exiftool filter" do
+ assert Mobilizon.Utils.command_available?("exiftool")
+
+ File.cp!(
+ "test/fixtures/DSCN0010.jpg",
+ "test/fixtures/DSCN0010_tmp.jpg"
+ )
+
+ upload = %Mobilizon.Web.Upload{
+ name: "image_with_GPS_data.jpg",
+ content_type: "image/jpeg",
+ path: Path.absname("test/fixtures/DSCN0010.jpg"),
+ tempfile: Path.absname("test/fixtures/DSCN0010_tmp.jpg")
+ }
+
+ assert Filter.Exiftool.filter(upload) == {:ok, :filtered}
+
+ {exif_original, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010.jpg"])
+ {exif_filtered, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010_tmp.jpg"])
+
+ refute exif_original == exif_filtered
+ assert String.match?(exif_original, ~r/GPS/)
+ refute String.match?(exif_filtered, ~r/GPS/)
+ end
+
+ test "verify webp files are skipped" do
+ upload = %Upload{
+ name: "sample.webp",
+ content_type: "image/webp"
+ }
+
+ assert Filter.Exiftool.filter(upload) == {:ok, :noop}
+ end
+end
diff --git a/test/web/upload/filter/filter_test.exs b/test/web/upload/filter/filter_test.exs
new file mode 100644
index 000000000..0bed838d1
--- /dev/null
+++ b/test/web/upload/filter/filter_test.exs
@@ -0,0 +1,35 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.FilterTest do
+ use Mobilizon.DataCase
+ use Mobilizon.Tests.Helpers
+
+ alias Mobilizon.Config
+ alias Mobilizon.Web.Upload.Filter
+
+ setup do: clear_config([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text])
+
+ test "applies filters" do
+ Config.put([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text], "custom-file.png")
+
+ File.cp!(
+ "test/fixtures/image.jpg",
+ "test/fixtures/image_tmp.jpg"
+ )
+
+ upload = %Mobilizon.Web.Upload{
+ name: "an… image.jpg",
+ content_type: "image/jpeg",
+ path: Path.absname("test/fixtures/image_tmp.jpg"),
+ tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+ }
+
+ assert Filter.filter([], upload) == {:ok, upload}
+
+ assert {:ok, upload} = Filter.filter([Mobilizon.Web.Upload.Filter.AnonymizeFilename], upload)
+ assert upload.name == "custom-file.png"
+ end
+end
diff --git a/test/web/upload/filter/mogrify_test.exs b/test/web/upload/filter/mogrify_test.exs
new file mode 100644
index 000000000..2c45e99c7
--- /dev/null
+++ b/test/web/upload/filter/mogrify_test.exs
@@ -0,0 +1,44 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.Filter.MogrifyTest do
+ use Mobilizon.DataCase
+ use Mobilizon.Tests.Helpers
+ import Mock
+
+ alias Mobilizon.Web.Upload
+ alias Mobilizon.Web.Upload.Filter
+
+ test "apply mogrify filter" do
+ clear_config(Filter.Mogrify, args: [{"tint", "40"}])
+
+ File.cp!(
+ "test/fixtures/image.jpg",
+ "test/fixtures/image_tmp.jpg"
+ )
+
+ upload = %Upload{
+ name: "an… image.jpg",
+ content_type: "image/jpeg",
+ path: Path.absname("test/fixtures/image_tmp.jpg"),
+ tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+ }
+
+ task =
+ Task.async(fn ->
+ assert_receive {:apply_filter, {_, "tint", "40"}}, 4_000
+ end)
+
+ with_mock Mogrify,
+ open: fn _f -> %Mogrify.Image{} end,
+ custom: fn _m, _a -> :ok end,
+ custom: fn m, a, o -> send(task.pid, {:apply_filter, {m, a, o}}) end,
+ save: fn _f, _o -> :ok end do
+ assert Filter.Mogrify.filter(upload) == {:ok, :filtered}
+ end
+
+ Task.await(task)
+ end
+end
diff --git a/test/web/upload/upload_test.exs b/test/web/upload/upload_test.exs
index 0500dd8f4..b9aeadbeb 100644
--- a/test/web/upload/upload_test.exs
+++ b/test/web/upload/upload_test.exs
@@ -79,7 +79,6 @@ defmodule Mobilizon.UploadTest do
assert data.name == "an [image.jpg"
end
- @tag :skip
test "fixes incorrect content type" do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
diff --git a/test/web/upload/uploader/local_test.exs b/test/web/upload/uploader/local_test.exs
new file mode 100644
index 000000000..5ac4243f6
--- /dev/null
+++ b/test/web/upload/uploader/local_test.exs
@@ -0,0 +1,58 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.Uploader.LocalTest do
+ use Mobilizon.DataCase
+ alias Mobilizon.Web.Upload
+ alias Mobilizon.Web.Upload.Uploader.Local
+
+ describe "get_file/1" do
+ test "it returns path to local folder for files" do
+ assert Local.get_file("") == {:ok, {:static_dir, "test/uploads"}}
+ end
+ end
+
+ describe "put_file/1" do
+ test "put file to local folder" do
+ File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+ file_path = "local_upload/files/image.jpg"
+
+ file = %Upload{
+ name: "image.jpg",
+ content_type: "image/jpeg",
+ path: file_path,
+ tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+ }
+
+ assert Local.put_file(file) == :ok
+
+ assert [Local.upload_path(), file_path]
+ |> Path.join()
+ |> File.exists?()
+ end
+ end
+
+ describe "remove_file/1" do
+ test "removes local file" do
+ File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+ file_path = "local_upload/files/image.jpg"
+
+ file = %Upload{
+ name: "image.jpg",
+ content_type: "image/jpeg",
+ path: file_path,
+ tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+ }
+
+ :ok = Local.put_file(file)
+ local_path = Path.join([Local.upload_path(), file_path])
+ assert File.exists?(local_path)
+
+ Local.remove_file(file_path)
+
+ refute File.exists?(local_path)
+ end
+ end
+end