Merge branch 'work-releases' into 'master'

Work with releases

Closes #569

See merge request framasoft/mobilizon!815

.dockerignore

@@ -17,3 +17,4 @@ SECURITY.md
 ssh_match_hostname
 support
 .js/package-lock.json
+js/node_modules

.gitignore

@@ -14,7 +14,6 @@ erl_crash.dump
 # secrets files as long as you replace their contents by environment
 # variables.
 /config/*.secret.exs
-/config/releases.exs
 
 /setup_db.psql
 

.gitlab-ci.yml

@@ -183,7 +183,7 @@ pages:
     - mkdir -p /kaniko/.docker
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$CI_REGISTRY_AUTH\",\"email\":\"$CI_REGISTRY_EMAIL\"}}}" > /kaniko/.docker/config.json
   script:
-    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/docker/production/Dockerfile --destination $DOCKER_IMAGE_NAME
+    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/docker/production/Dockerfile --destination $DOCKER_IMAGE_NAME --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --build-arg CI_COMMIT_TAG=$CI_COMMIT_TAG
 
 build-docker-master:
   <<: *docker
@@ -205,19 +205,32 @@ build-docker-tag:
 
 package-app:
   stage: package
-  variables:
+  variables: &release-variables
     MIX_ENV: "prod"
-  script:
+  script: &release-script
     - mix local.hex --force
     - mix local.rebar --force
-    - cp docker/production/releases.exs ./config/
     - mix deps.get
     - mix phx.digest
-    - mix release
+    - mix release --path release
+    - cd release && ln -s lib/mobilizon-*/priv priv
+  only:
+    - tags@framasoft/mobilizon
+  artifacts:
+    expire_in: never
+    paths:
+      - release
+
+package-app-dev:
+  stage: package
+  variables: *release-variables
+  script: *release-script
+  except:
+    - tags@framasoft/mobilizon
   artifacts:
     expire_in: 2 days
     paths:
-      - _build/prod/rel
+      - release
 
 release-upload:
   stage: upload

.sobelow-skips

@@ -1,2 +1,4 @@
 
 5048AE33D6269B15E21CF28C6F545AB6
+
+752C0E897CA81ACD81F4BB215FA5F8E4

config/config.exs

@@ -8,7 +8,7 @@ import Config
 # General application configuration
 config :mobilizon,
   ecto_repos: [Mobilizon.Storage.Repo],
-  env: Mix.env()
+  env: config_env()
 
 config :mobilizon, Mobilizon.Storage.Repo, types: Mobilizon.Storage.PostgresTypes
 
@@ -92,8 +92,7 @@ config :mobilizon, :media_proxy,
       follow_redirect: true,
       pool: :media
     ]
-  ],
-  whitelist: []
+  ]
 
 config :mobilizon, Mobilizon.Web.Email.Mailer,
   adapter: Bamboo.SMTPAdapter,
@@ -116,6 +115,10 @@ config :logger, :console,
   format: "$time $metadata[$level] $message\n",
   metadata: [:request_id]
 
+config :logger, Sentry.LoggerBackend,
+  level: :warn,
+  capture_log_messages: true
+
 config :mobilizon, Mobilizon.Web.Auth.Guardian, issuer: "mobilizon"
 
 config :guardian, Guardian.DB,
@@ -139,24 +142,6 @@ config :ueberauth,
 
 config :mobilizon, :auth, oauth_consumer_strategies: []
 
-config :mobilizon, :ldap,
-  enabled: System.get_env("LDAP_ENABLED") == "true",
-  host: System.get_env("LDAP_HOST") || "localhost",
-  port: String.to_integer(System.get_env("LDAP_PORT") || "389"),
-  ssl: System.get_env("LDAP_SSL") == "true",
-  sslopts: [],
-  tls: System.get_env("LDAP_TLS") == "true",
-  tlsopts: [],
-  base: System.get_env("LDAP_BASE") || "dc=example,dc=com",
-  uid: System.get_env("LDAP_UID") || "cn",
-  require_bind_for_search: !(System.get_env("LDAP_REQUIRE_BIND_FOR_SEARCH") == "false"),
-  # The full CN to filter by `memberOf`, or `false` if disabled
-  group: false,
-  # Either the admin UID matching the field in `uid`,
-  # Either a tuple with the fully qualified DN: {:full, uid=admin,dc=example.com,dc=local}
-  bind_uid: System.get_env("LDAP_BIND_UID"),
-  bind_password: System.get_env("LDAP_BIND_PASSWORD")
-
 config :geolix,
   databases: [
     %{
@@ -310,4 +295,4 @@ config :mobilizon, :external_resource_providers, %{
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
-import_config "#{Mix.env()}.exs"
+import_config "#{config_env()}.exs"

config/prod.exs

@@ -47,6 +47,5 @@ cond do
     import_config "prod.secret.exs"
 
   true ->
-    require Logger
-    Logger.error("No configuration file found")
+    :ok
 end

config/releases.exs

@@ -22,11 +22,9 @@ config :mobilizon, :instance,
   email_from: System.get_env("MOBILIZON_INSTANCE_EMAIL", "noreply@mobilizon.lan"),
   email_reply_to: System.get_env("MOBILIZON_REPLY_EMAIL", "noreply@mobilizon.lan")
 
-
 config :mobilizon, Mobilizon.Web.Upload.Uploader.Local,
   uploads: System.get_env("MOBILIZON_UPLOADS", "/app/uploads")
 
-
 config :mobilizon, Mobilizon.Storage.Repo,
   adapter: Ecto.Adapters.Postgres,
   username: System.get_env("MOBILIZON_DATABASE_USERNAME", "username"),
@@ -49,3 +47,14 @@ config :mobilizon, Mobilizon.Web.Email.Mailer,
   retries: 1,
   no_mx_lookups: false,
   auth: :if_available
+
+config :geolix,
+  databases: [
+    %{
+      id: :city,
+      adapter: Geolix.Adapter.MMDB2,
+      source: "/var/lib/mobilizon/geo_db/GeoLite2-City.mmdb"
+    }
+  ]
+
+config :mobilizon, Mobilizon.Web.Upload.Uploader.Local, uploads: "/var/lib/mobilizon/uploads"

config/test.exs

@@ -36,6 +36,24 @@ config :mobilizon, Mobilizon.Storage.Repo,
   port: System.get_env("MOBILIZON_DATABASE_PORT") || "5432",
   pool: Ecto.Adapters.SQL.Sandbox
 
+config :mobilizon, :ldap,
+  enabled: System.get_env("LDAP_ENABLED") == "true",
+  host: System.get_env("LDAP_HOST") || "localhost",
+  port: String.to_integer(System.get_env("LDAP_PORT") || "389"),
+  ssl: System.get_env("LDAP_SSL") == "true",
+  sslopts: [],
+  tls: System.get_env("LDAP_TLS") == "true",
+  tlsopts: [],
+  base: System.get_env("LDAP_BASE") || "dc=example,dc=com",
+  uid: System.get_env("LDAP_UID") || "cn",
+  require_bind_for_search: !(System.get_env("LDAP_REQUIRE_BIND_FOR_SEARCH") == "false"),
+  # The full CN to filter by `memberOf`, or `false` if disabled
+  group: false,
+  # Either the admin UID matching the field in `uid`,
+  # Either a tuple with the fully qualified DN: {:full, uid=admin,dc=example.com,dc=local}
+  bind_uid: System.get_env("LDAP_BIND_UID"),
+  bind_password: System.get_env("LDAP_BIND_PASSWORD")
+
 config :mobilizon, Mobilizon.Web.Email.Mailer, adapter: Bamboo.TestAdapter
 
 config :mobilizon, Mobilizon.Web.Upload, filters: [], link_name: false

docker/production/Dockerfile

@@ -1,5 +1,5 @@
 # First build the application assets
-FROM node:alpine as assets
+FROM node:14-alpine as assets
 
 RUN apk add --no-cache python build-base libwebp-tools bash imagemagick ncurses
 
@@ -8,7 +8,7 @@ RUN yarn install \
     && yarn run build
 
 # Then, build the application binary
-FROM elixir:alpine AS builder
+FROM elixir:1.11-alpine AS builder
 
 RUN apk add --no-cache build-base git cmake
 
@@ -20,9 +20,8 @@ RUN mix local.hex --force \
 
 COPY lib ./lib
 COPY priv ./priv
-COPY config ./config
+COPY config/config.exs config/prod.exs config/releases.exs ./config/
 COPY rel ./rel
-COPY docker/production/releases.exs ./config/
 COPY --from=assets ./priv/static ./priv/static
 
 RUN mix phx.digest \
@@ -31,13 +30,30 @@ RUN mix phx.digest \
 # Finally setup the app
 FROM alpine
 
+ARG BUILD_DATE
+ARG VCS_REF
+ARG CI_COMMIT_TAG
+ARG MOBILIZON_VERSION=${CI_COMMIT_TAG}
+
+LABEL org.opencontainers.image.title="mobilizon" \
+    org.opencontainers.image.description="Mobilizon for Docker" \
+    org.opencontainers.image.vendor="joinmobilizon.org" \
+    org.opencontainers.image.documentation="https://docs.joinmobilizon.org" \
+    org.opencontainers.image.licenses="AGPL-3.0" \
+    org.opencontainers.image.url="https://joinmobilizon.org" \
+    org.opencontainers.image.revision=$VCS_REF \
+    org.opencontainers.image.created=$BUILD_DATE
+
 RUN apk add --no-cache openssl ncurses-libs file postgresql-client
 
 RUN mkdir -p /app/uploads && chown nobody:nobody /app/uploads
+RUN mkdir -p /etc/mobilizon && chown nobody:nobody /etc/mobilizon
 
 USER nobody
 EXPOSE 4000
 
+ENV MOBILIZON_DOCKER=true
+
 COPY --from=builder --chown=nobody:nobody _build/prod/rel/mobilizon ./
 COPY docker/production/docker-entrypoint.sh ./
 

docker/tests/Dockerfile

@@ -1,7 +1,7 @@
 FROM elixir:latest
 LABEL maintainer="Thomas Citharel <tcit@tcit.fr>"
 
-ENV REFRESHED_AT=2020-10-22
+ENV REFRESHED_AT=2021-02-08
 RUN apt-get update -yq && apt-get install -yq build-essential inotify-tools postgresql-client git curl gnupg xvfb libgtk-3-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 cmake exiftool
 RUN curl -sL https://deb.nodesource.com/setup_12.x | bash && apt-get install nodejs -yq
 RUN npm install -g yarn wait-on

js/src/router/index.ts

@@ -158,9 +158,11 @@ const router = new Router({
 
 router.beforeEach(authGuardIfNeeded);
 router.afterEach(() => {
+  if (router.app.$children[0]) {
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-ignore
     router.app.$children[0].error = null;
+  }
 });
 
 export default router;

lib/config_provider.ex

@@ -0,0 +1,34 @@
+defmodule Mobilizon.ConfigProvider do
+  @moduledoc """
+  Module to provide configuration from a custom file
+  """
+  @behaviour Config.Provider
+
+  def init(path) when is_binary(path), do: path
+
+  def load(config, path) do
+    config_path = System.get_env("MOBILIZON_CONFIG_PATH") || path
+
+    cond do
+      File.exists?(config_path) ->
+        runtime_config = Config.Reader.read!(config_path)
+
+        Config.Reader.merge(config, runtime_config)
+
+      is_nil(System.get_env("MOBILIZON_DOCKER")) ->
+        warning = [
+          IO.ANSI.red(),
+          IO.ANSI.bright(),
+          "!!! #{config_path} not found! Please ensure it exists and that MOBILIZON_CONFIG_PATH is unset or points to an existing file",
+          IO.ANSI.reset()
+        ]
+
+        IO.puts(warning)
+        config
+
+      true ->
+        IO.puts("No runtime config file found, but using environment variables for Docker")
+        config
+    end
+  end
+end

lib/mix/tasks/mobilizon/instance.ex

@@ -61,7 +61,7 @@ defmodule Mix.Tasks.Mobilizon.Instance do
 
     paths =
       [config_path, psql_path] = [
-        Keyword.get(options, :output, "config/prod.secret.exs"),
+        Keyword.get(options, :output, "config/runtime.exs"),
         Keyword.get(options, :output_psql, "setup_db.psql")
       ]
 
@@ -146,7 +146,6 @@ defmodule Mix.Tasks.Mobilizon.Instance do
           database_port: Keyword.get(options, :dbport, 5432),
           database_username: dbuser,
           database_password: dbpass,
-          version: Mobilizon.Mixfile.project() |> Keyword.get(:version),
           instance_secret: instance_secret,
           auth_secret: auth_secret,
           listen_port: listen_port
@@ -160,12 +159,8 @@ defmodule Mix.Tasks.Mobilizon.Instance do
           database_password: dbpass
         )
 
-      shell_info("Writing config to #{config_path}.")
-
-      File.write(config_path, result_config)
-      shell_info("Writing #{psql_path}.")
-      File.write(psql_path, result_psql)
-
+      with :ok <- write_config(config_path, result_config),
+           :ok <- write_psql(psql_path, result_psql) do
         shell_info(
           "\n" <>
             """
@@ -176,6 +171,10 @@ defmodule Mix.Tasks.Mobilizon.Instance do
             }`.
             """
         )
+      else
+        {:error, err} -> exit(err)
+        _ -> exit(:unknown_error)
+      end
     else
       shell_error(
         "The task would have overwritten the following files:\n" <>
@@ -184,4 +183,36 @@ defmodule Mix.Tasks.Mobilizon.Instance do
       )
     end
   end
+
+  defp write_config(config_path, result_config) do
+    shell_info("Writing config to #{config_path}.")
+
+    case File.write(config_path, result_config) do
+      :ok ->
+        :ok
+
+      {:error, err} ->
+        shell_error(
+          "\nERROR: Unable to write config file to #{config_path}. Make sure you have permissions on the destination.\n"
+        )
+
+        {:error, err}
+    end
+  end
+
+  defp write_psql(psql_path, result_psql) do
+    shell_info("Writing #{psql_path}.")
+
+    case File.write(psql_path, result_psql) do
+      :ok ->
+        :ok
+
+      {:error, err} ->
+        shell_error(
+          "\nERROR: Unable to write psql file to #{psql_path}. Make sure you have permissions on the destination.\n"
+        )
+
+        {:error, err}
+    end
+  end
 end

lib/mobilizon/storage/repo.ex

@@ -11,6 +11,6 @@ defmodule Mobilizon.Storage.Repo do
   Dynamically loads the repository url from the DATABASE_URL environment variable.
   """
   def init(_, opts) do
-    {:ok, Keyword.put(opts, :url, System.get_env("DATABASE_URL"))}
+    {:ok, opts}
   end
 end

lib/web/endpoint.ex

@@ -2,9 +2,12 @@ defmodule Mobilizon.Web.Endpoint do
   @moduledoc """
   Endpoint for Mobilizon app
   """
+  if Application.fetch_env!(:mobilizon, :env) !== :test do
+    use Sentry.PlugCapture
+  end
+
   use Phoenix.Endpoint, otp_app: :mobilizon
   use Absinthe.Phoenix.Endpoint
-  use Sentry.Phoenix.Endpoint
 
   plug(Mobilizon.Web.Plugs.SetLocalePlug)
   plug(Mobilizon.Web.Plugs.HTTPSecurityPlug)
@@ -82,4 +85,8 @@ defmodule Mobilizon.Web.Endpoint do
   def websocket_url do
     String.replace_leading(url(), "http", "ws")
   end
+
+  if Application.fetch_env!(:mobilizon, :env) !== :test do
+    plug(Sentry.PlugContext)
+  end
 end

lib/web/media_proxy.ex

@@ -29,10 +29,6 @@ defmodule Mobilizon.Web.MediaProxy do
 
   def enabled?, do: Config.get([:media_proxy, :enabled], false)
 
-  # Note: media proxy must be enabled for media preview proxy in order to load all
-  #   non-local non-whitelisted URLs through it and be sure that body size constraint is preserved.
-  def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled])
-
   def local?(url), do: String.starts_with?(url, Web.Endpoint.url())
 
   defp base64_sig64(url) do

lib/web/router.ex

@@ -3,8 +3,6 @@ defmodule Mobilizon.Web.Router do
   Router for mobilizon app
   """
   use Mobilizon.Web, :router
-  use Plug.ErrorHandler
-  use Sentry.Plug
 
   pipeline :graphql do
     #    plug(:accepts, ["json"])

mix.exs

@@ -1,13 +1,13 @@
 defmodule Mobilizon.Mixfile do
   use Mix.Project
 
-  @version "1.0.6"
+  @version "1.1.0"
 
   def project do
     [
       app: :mobilizon,
       version: @version,
-      elixir: "~> 1.8",
+      elixir: "~> 1.11",
       elixirc_paths: elixirc_paths(Mix.env()),
       compilers: [:phoenix, :gettext] ++ Mix.compilers(),
       xref: [exclude: [:eldap]],
@@ -31,7 +31,8 @@ defmodule Mobilizon.Mixfile do
       docs: docs(),
       releases: [
         mobilizon: [
-          applications: [eldap: :transient]
+          applications: [eldap: :transient],
+          config_providers: [{Mobilizon.ConfigProvider, "/etc/mobilizon/config.exs"}]
         ]
       ]
     ]
@@ -141,7 +142,7 @@ defmodule Mobilizon.Mixfile do
       {:remote_ip, "~> 0.2.0"},
       {:ex_cldr_languages, "~> 0.2.1"},
       {:slugger, "~> 0.3"},
-      {:sentry, "~> 7.0"},
+      {:sentry, "~> 8.0"},
       # Dev and test dependencies
       {:phoenix_live_reload, "~> 1.2", only: [:dev, :e2e]},
       {:ex_machina, "~> 2.3", only: [:dev, :test]},

mix.lock

@@ -112,7 +112,7 @@
   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"},
   "remote_ip": {:hex, :remote_ip, "0.2.1", "cd27cd8ea54ecaaf3532776ff4c5e353b3804e710302e88c01eadeaaf42e7e24", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:inet_cidr, "~> 1.0", [hex: :inet_cidr, repo: "hexpm", optional: false]}, {:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2e7ab1a461cc3cd5719f37e116a08f45c8b8493923063631b164315d6b7ee8e0"},
   "rsa_ex": {:hex, :rsa_ex, "0.4.0", "e28dd7dc5236e156df434af0e4aa822384c8866c928e17b785d4edb7c253b558", [:mix], [], "hexpm", "40e1f08e8401da4be59a6dd0f4da30c42d5bb01703161f0208d839d97db27f4e"},
-  "sentry": {:hex, :sentry, "7.2.5", "570db92c3bbacd6ad02ac81cba8ac5af11235a55d65ac4375e3ec833975b83d3", [:mix], [{:hackney, "~> 1.8 or 1.6.5", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "ea84ed6848505ff2a246567df562f465d2b34c317d3ecba7c7df58daa56e5e5d"},
+  "sentry": {:hex, :sentry, "8.0.4", "6886ea584fa0dfc144dd3d8c4723570a4c0c6387a639077de34fce96b11995e2", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.3", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "781e4e0a8d4524c015f0f8d82e6bc30e6bf33df622da58d8c89d30c3cb943e67"},
   "shortuuid": {:hex, :shortuuid, "2.1.2", "14dbafdb2f6c7213fdfcc05c7572384b5051a7b1621170018ad4c05504bd96c1", [:mix], [], "hexpm", "d9b0c4f37500ea5199b6275ece872e213e9f45a015caf4aa777cec84f63ad353"},
   "sitemapper": {:hex, :sitemapper, "0.5.0", "23b0bb7b3888f03d4e4e5bedb7034e6d2979e169366372d960d6f433112b9bdf", [:mix], [{:ex_aws_s3, "~> 2.0", [hex: :ex_aws_s3, repo: "hexpm", optional: true]}, {:xml_builder, "~> 2.1.1", [hex: :xml_builder, repo: "hexpm", optional: false]}], "hexpm", "be7acff8d0245aa7ca125b9c4d0751009bbbca26ef866d888fef4fdf98670e41"},
   "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"},

priv/templates/config.template.eex

@@ -3,6 +3,7 @@
 import Config
 
 config :mobilizon, Mobilizon.Web.Endpoint,
+   server: true,
    url: [host: "<%= instance_domain %>"],
    http: [port: <%= listen_port %>],
    secret_key_base: "<%= instance_secret %>"