Merge branch 'ci-build' into 'main'

ci: Release on multiple distributions & fix Docker multiple-step build

Closes #1012

See merge request framasoft/mobilizon!1484
This commit is contained in:
Thomas Citharel 2023-11-23 13:35:45 +00:00
commit 42ddf3f653
9 changed files with 106 additions and 38 deletions

View file

@ -1,6 +1,10 @@
_build _build
CONTRIBUTING.md CONTRIBUTING.md
coverage
demo
deps deps
doc
docs
docker-compose.yml docker-compose.yml
Dockerfile Dockerfile
.elixir_ls .elixir_ls
@ -17,3 +21,6 @@ SECURITY.md
ssh_match_hostname ssh_match_hostname
package-lock.json package-lock.json
node_modules node_modules
playwright-report
test
tests

View file

@ -6,8 +6,7 @@ stages:
- build-js - build-js
- sentry - sentry
- test - test
- docker - build
- package
- upload - upload
- deploy - deploy
@ -125,7 +124,7 @@ deps:
exunit: exunit:
stage: test stage: test
services: services:
- name: postgis/postgis:14-3.2 - name: postgis/postgis:16-3.4
alias: postgres alias: postgres
variables: variables:
MIX_ENV: test MIX_ENV: test
@ -164,7 +163,7 @@ vitest:
e2e: e2e:
stage: test stage: test
services: services:
- name: postgis/postgis:14-3.2 - name: postgis/postgis:16-3.4
alias: postgres alias: postgres
variables: variables:
MIX_ENV: "e2e" MIX_ENV: "e2e"
@ -206,7 +205,7 @@ pages:
- public - public
.docker: &docker .docker: &docker
stage: docker stage: build
image: docker:24 image: docker:24
variables: variables:
DOCKER_TLS_CERTDIR: "/certs" DOCKER_TLS_CERTDIR: "/certs"
@ -214,6 +213,7 @@ pages:
DOCKER_TLS_VERIFY: 1 DOCKER_TLS_VERIFY: 1
DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
DOCKER_DRIVER: overlay2 DOCKER_DRIVER: overlay2
DOCKER_CLI_EXPERIMENTAL: enabled
services: services:
- docker:24-dind - docker:24-dind
cache: {} cache: {}
@ -229,6 +229,8 @@ pages:
# Install qemu/binfmt # Install qemu/binfmt
- docker pull tonistiigi/binfmt:latest - docker pull tonistiigi/binfmt:latest
- docker run --rm --privileged tonistiigi/binfmt:latest --install all - docker run --rm --privileged tonistiigi/binfmt:latest --install all
# Install jq
- apk --no-cache add jq
# Login to DockerHub # Login to DockerHub
- mkdir -p ~/.docker - mkdir -p ~/.docker
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$CI_REGISTRY_AUTH\",\"email\":\"$CI_REGISTRY_EMAIL\"}}}" > ~/.docker/config.json - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$CI_REGISTRY_AUTH\",\"email\":\"$CI_REGISTRY_EMAIL\"}}}" > ~/.docker/config.json
@ -242,9 +244,9 @@ build-docker-main:
when: never when: never
- if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_TRIGGERED == "true"' - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_TRIGGERED == "true"'
script: script:
- docker buildx build --push --platform linux/amd64 -t framasoft/mobilizon:main -f docker/production/Dockerfile . - docker buildx build --platform linux/amd64 -t framasoft/mobilizon:main -f docker/production/Dockerfile .
build-and-push-to-latest-docker-tag: build-docker-tag:
<<: *docker <<: *docker
rules: &release-tag-rules rules: &release-tag-rules
- if: '$CI_PROJECT_NAMESPACE != "framasoft"' - if: '$CI_PROJECT_NAMESPACE != "framasoft"'
@ -257,9 +259,9 @@ build-and-push-to-latest-docker-tag:
docker buildx build docker buildx build
--push --push
--platform linux/${ARCH} --platform linux/${ARCH}
--provenance=false
--build-arg="${ERL_FLAGS}" --build-arg="${ERL_FLAGS}"
-t framasoft/mobilizon:$CI_COMMIT_TAG -t framasoft/mobilizon:${CI_COMMIT_TAG}-${ARCH}
-t framasoft/mobilizon:latest
-f docker/production/Dockerfile . -f docker/production/Dockerfile .
parallel: parallel:
matrix: matrix:
@ -280,15 +282,49 @@ build-and-push-docker-tag:
script: script:
- > - >
docker buildx build docker buildx build
--push
--platform linux/amd64 --platform linux/amd64
-t framasoft/mobilizon:$CI_COMMIT_TAG --provenance=false
-t framasoft/mobilizon:$CI_COMMIT_TAG-amd64
-f docker/production/Dockerfile . -f docker/production/Dockerfile .
# Create manifest and push
docker-manifest-push:
<<: *docker
needs: ["build-docker-tag"]
rules: &release-tag-rules
- if: '$CI_PROJECT_NAMESPACE != "framasoft"'
when: never
- if: $CI_COMMIT_TAG != null
when: on_success
script:
- >
docker manifest create framasoft/mobilizon:${CI_COMMIT_TAG}
--amend framasoft/mobilizon:${CI_COMMIT_TAG}-amd64
--amend framasoft/mobilizon:${CI_COMMIT_TAG}-arm64
- docker manifest push --purge framasoft/mobilizon:${CI_COMMIT_TAG}
###
# Simply creating an alias to the tag doesn't work:
# « xxx is a manifest list »
# https://joonas.fi/2021/02/docker-multi-arch-image-tooling-buildx/
###
docker-latest:
<<: *docker
needs: ["docker-manifest-push"]
rules: &release-tag-rules
- if: '$CI_PROJECT_NAMESPACE != "framasoft"'
when: never
- if: $CI_COMMIT_TAG != null && $CI_COMMIT_TAG !~ /alpha|beta|rc/
when: on_success
script:
- echo docker manifest create framasoft/mobilizon:latest $(docker manifest inspect framasoft/mobilizon:$CI_COMMIT_TAG | jq '.manifests[] | .digest' | xargs -I {} echo framasoft/mobilizon@{})
- docker manifest create framasoft/mobilizon:latest $(docker manifest inspect framasoft/mobilizon:$CI_COMMIT_TAG | jq -r '.manifests[] | .digest' | xargs -I {} echo framasoft/mobilizon@{})
- docker manifest push --purge framasoft/mobilizon:latest
# Packaging app for amd64 # Packaging app for amd64
package-app: package-app:
image: mobilizon/buildpack:1.15.5-erlang-26.0.2-debian-buster image: mobilizon/buildpack:1.15.7-erlang-26.1.2-${SYSTEM}
stage: package stage: build
variables: &release-variables variables: &release-variables
MIX_ENV: "prod" MIX_ENV: "prod"
DEBIAN_FRONTEND: noninteractive DEBIAN_FRONTEND: noninteractive
@ -312,9 +348,23 @@ package-app:
expire_in: 2 days expire_in: 2 days
paths: paths:
- ${APP_ASSET} - ${APP_ASSET}
parallel:
matrix:
- SYSTEM:
[
"debian-bookworm",
"debian-bullseye",
"ubuntu-jammy",
"ubuntu-focal",
"ubuntu-bionic",
"alpine-3.17.5",
"alpine-3.18.4",
"fedora-38",
"fedora-39",
]
package-app-dev: package-app-dev:
stage: package stage: build
variables: *release-variables variables: *release-variables
script: *release-script script: *release-script
except: except:
@ -325,8 +375,8 @@ package-app-dev:
- ${APP_ASSET} - ${APP_ASSET}
# Packaging app for multi-arch # Packaging app for multi-arch
multi-arch-release: package-multi-arch-release:
stage: package stage: build
image: docker:24 image: docker:24
variables: variables:
DOCKER_TLS_CERTDIR: "/certs" DOCKER_TLS_CERTDIR: "/certs"
@ -366,6 +416,19 @@ multi-arch-release:
parallel: parallel:
matrix: matrix:
- ARCH: ["arm64"] - ARCH: ["arm64"]
## Currently not used as the hexpm base images do not have support for other architectures than amd64
# SYSTEM:
# [
# "debian-bookworm",
# "debian-bullseye",
# "ubuntu-jammy",
# "ubuntu-focal",
# "ubuntu-bionic",
# "alpine-3.17.5",
# "alpine-3.18.4",
# "fedora-38",
# "fedora-39",
# ]
rules: rules:
- if: '$CI_COMMIT_TAG != null || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_TRIGGERED == "true"' - if: '$CI_COMMIT_TAG != null || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_TRIGGERED == "true"'
timeout: 3h timeout: 3h

View file

@ -394,12 +394,6 @@ config :mobilizon, Mobilizon.Service.AntiSpam, service: Mobilizon.Service.AntiSp
config :mobilizon, Mobilizon.Service.SiteMap, config :mobilizon, Mobilizon.Service.SiteMap,
path: System.get_env("MOBILIZON_SITEMAP_PATH", "/var/lib/mobilizon/sitemap") path: System.get_env("MOBILIZON_SITEMAP_PATH", "/var/lib/mobilizon/sitemap")
config :sentry,
dsn: "",
environment_name: Mix.env(),
enable_source_code_context: true,
root_source_code_paths: [File.cwd!()]
# Import environment specific config. This must remain at the bottom # Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above. # of this file so it overrides the configuration defined above.
import_config "#{config_env()}.exs" import_config "#{config_env()}.exs"

View file

@ -1,4 +1,6 @@
FROM elixir:1.15 as build ARG IMAGE="elixir:1.15"
FROM ${IMAGE} as build
SHELL ["/bin/bash", "-c"] SHELL ["/bin/bash", "-c"]
ENV MIX_ENV prod ENV MIX_ENV prod
# ENV LANG en_US.UTF-8 # ENV LANG en_US.UTF-8
@ -12,7 +14,7 @@ ENV ERL_FLAGS=$ERL_FLAGS
# Set the right versions # Set the right versions
ENV ELIXIR_VERSION latest ENV ELIXIR_VERSION latest
ENV ERLANG_VERSION latest ENV ERLANG_VERSION latest
ENV NODE_VERSION 18 ENV NODE_VERSION 20
# Install system dependencies # Install system dependencies
RUN apt-get update -yq && apt-get install -yq build-essential cmake postgresql-client git curl gnupg unzip exiftool webp imagemagick gifsicle RUN apt-get update -yq && apt-get install -yq build-essential cmake postgresql-client git curl gnupg unzip exiftool webp imagemagick gifsicle

View file

@ -1,5 +1,5 @@
# First build the application assets # First build the application assets
FROM node:18-alpine as assets FROM node:20-alpine as assets
RUN apk add --no-cache python3 build-base libwebp-tools bash imagemagick ncurses RUN apk add --no-cache python3 build-base libwebp-tools bash imagemagick ncurses
WORKDIR /build WORKDIR /build
@ -51,7 +51,7 @@ LABEL org.opencontainers.image.title="mobilizon" \
org.opencontainers.image.created=$BUILD_DATE org.opencontainers.image.created=$BUILD_DATE
RUN apk add --no-cache curl openssl ca-certificates ncurses-libs file postgresql-client libgcc libstdc++ imagemagick python3 py3-pip py3-pillow py3-cffi py3-brotli gcc g++ musl-dev python3-dev pango libxslt-dev ttf-cantarell openssl1.1-compat RUN apk add --no-cache curl openssl ca-certificates ncurses-libs file postgresql-client libgcc libstdc++ imagemagick python3 py3-pip py3-pillow py3-cffi py3-brotli gcc g++ musl-dev python3-dev pango libxslt-dev ttf-cantarell openssl1.1-compat
RUN pip install weasyprint pyexcel-ods3 RUN pip --no-cache-dir install weasyprint pyexcel-ods3
# Create every data directory # Create every data directory
RUN mkdir -p /var/lib/mobilizon/uploads && chown nobody:nobody /var/lib/mobilizon/uploads RUN mkdir -p /var/lib/mobilizon/uploads && chown nobody:nobody /var/lib/mobilizon/uploads
@ -60,14 +60,14 @@ RUN mkdir -p /var/lib/mobilizon/tzdata && chown nobody:nobody /var/lib/mobilizon
RUN mkdir -p /var/lib/mobilizon/sitemap && chown nobody:nobody /var/lib/mobilizon/sitemap RUN mkdir -p /var/lib/mobilizon/sitemap && chown nobody:nobody /var/lib/mobilizon/sitemap
RUN mkdir -p /var/lib/mobilizon/uploads/exports/{csv,pdf,ods} && chown -R nobody:nobody /var/lib/mobilizon/uploads/exports RUN mkdir -p /var/lib/mobilizon/uploads/exports/{csv,pdf,ods} && chown -R nobody:nobody /var/lib/mobilizon/uploads/exports
# Get timezone geodata
RUN curl -L 'https://packages.joinmobilizon.org/tz_world/timezones-geodata.dets' -o /var/lib/mobilizon/timezones/timezones-geodata.dets
RUN chown -R nobody:nobody /var/lib/mobilizon/timezones
# Create configuration directory # Create configuration directory
RUN mkdir -p /etc/mobilizon && chown nobody:nobody /etc/mobilizon RUN mkdir -p /etc/mobilizon && chown nobody:nobody /etc/mobilizon
USER nobody USER nobody
# Get timezone geodata
RUN curl -L 'https://packages.joinmobilizon.org/tz_world/timezones-geodata.dets' -o /var/lib/mobilizon/timezones/timezones-geodata.dets
EXPOSE 4000 EXPOSE 4000
ENV MOBILIZON_DOCKER=true ENV MOBILIZON_DOCKER=true

View file

@ -1,11 +1,11 @@
FROM elixir:latest FROM elixir:latest
LABEL maintainer="Thomas Citharel <tcit@tcit.fr>" LABEL maintainer="Thomas Citharel <thomas.citharel@framasoft.org>"
ENV REFRESHED_AT=2023-08-17 ENV REFRESHED_AT=2023-11-20
RUN apt-get update -yq && apt-get install -yq build-essential inotify-tools postgresql-client git curl gnupg xvfb libgtk-3-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 cmake exiftool python3-pip python3-setuptools RUN apt-get update -yq && apt-get install -yq ca-certificates build-essential inotify-tools postgresql-client git curl gnupg xvfb libgtk-3-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 cmake exiftool python3-pip python3-setuptools
RUN curl -sL https://deb.nodesource.com/setup_20.x | bash && apt-get install nodejs -yq RUN mkdir -p /etc/apt/keyrings && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && apt-get update && apt-get install nodejs -yq
RUN npm install -g wait-on RUN npm install -g wait-on
RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
RUN mix local.hex --force && mix local.rebar --force RUN mix local.hex --force && mix local.rebar --force
RUN pip3 install -Iv weasyprint pyexcel_ods3 RUN pip3 --no-cache-dir install -Iv weasyprint pyexcel_ods3
RUN curl https://dbip.mirror.framasoft.org/files/dbip-city-lite-latest.mmdb --output GeoLite2-City.mmdb -s && mkdir -p /usr/share/GeoIP && mv GeoLite2-City.mmdb /usr/share/GeoIP/ RUN curl https://dbip.mirror.framasoft.org/files/dbip-city-lite-latest.mmdb --output GeoLite2-City.mmdb -s && mkdir -p /usr/share/GeoIP && mv GeoLite2-City.mmdb /usr/share/GeoIP/

View file

@ -53,6 +53,8 @@ defmodule Mobilizon.Web.ErrorView do
locale = locale =
Mobilizon.Config.instance_config() Mobilizon.Config.instance_config()
|> Keyword.get(:default_language, "en") |> Keyword.get(:default_language, "en")
# Just in case
|> Kernel.||("en")
Gettext.put_locale(locale) Gettext.put_locale(locale)

View file

@ -201,7 +201,7 @@ defmodule Mobilizon.Mixfile do
{:remote_ip, "~> 1.1"}, {:remote_ip, "~> 1.1"},
{:ex_cldr_languages, "~> 0.3.0"}, {:ex_cldr_languages, "~> 0.3.0"},
{:slugger, "~> 0.3"}, {:slugger, "~> 0.3"},
{:sentry, "~> 9.0"}, {:sentry, "~> 8.1"},
{:html_entities, "~> 0.5"}, {:html_entities, "~> 0.5"},
{:sweet_xml, "~> 0.7"}, {:sweet_xml, "~> 0.7"},
{:web_push_encryption, github: "danhper/elixir-web-push-encryption"}, {:web_push_encryption, github: "danhper/elixir-web-push-encryption"},

View file

@ -125,7 +125,7 @@
"ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
"remote_ip": {:hex, :remote_ip, "1.1.0", "cb308841595d15df3f9073b7c39243a1dd6ca56e5020295cb012c76fbec50f2d", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "616ffdf66aaad6a72fc546dabf42eed87e2a99e97b09cbd92b10cc180d02ed74"}, "remote_ip": {:hex, :remote_ip, "1.1.0", "cb308841595d15df3f9073b7c39243a1dd6ca56e5020295cb012c76fbec50f2d", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "616ffdf66aaad6a72fc546dabf42eed87e2a99e97b09cbd92b10cc180d02ed74"},
"replug": {:hex, :replug, "0.1.0", "61d35f8c873c0078a23c49579a48f36e45789414b1ec0daee3fd5f4e34221f23", [:mix], [{:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f71f7a57e944e854fe4946060c6964098e53958074c69fb844b96e0bd58cfa60"}, "replug": {:hex, :replug, "0.1.0", "61d35f8c873c0078a23c49579a48f36e45789414b1ec0daee3fd5f4e34221f23", [:mix], [{:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f71f7a57e944e854fe4946060c6964098e53958074c69fb844b96e0bd58cfa60"},
"sentry": {:hex, :sentry, "9.1.0", "8689b85774003ddcebfd9d48a93bc3f3bf72223983514521aa30645c6f204f86", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.3", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "d70c88ab0c6a511594856ae2244d1bd70b8b7a4a42201a3569880f1dd2a3adec"}, "sentry": {:hex, :sentry, "8.1.0", "8d235b62fce5f8e067ea1644e30939405b71a5e1599d9529ff82899d11d03f2b", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.3", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "f9fc7641ef61e885510f5e5963c2948b9de1de597c63f781e9d3d6c9c8681ab4"},
"shortuuid": {:hex, :shortuuid, "3.0.0", "028684d9eeed0ad4b800e8481afd854e1a61c526f35952455b2ee4248601e7b8", [:mix], [], "hexpm", "dfd8f80f514cbb91622cb83f4ac0d6e2f06d98cc6d4aeba94444a212289d0d39"}, "shortuuid": {:hex, :shortuuid, "3.0.0", "028684d9eeed0ad4b800e8481afd854e1a61c526f35952455b2ee4248601e7b8", [:mix], [], "hexpm", "dfd8f80f514cbb91622cb83f4ac0d6e2f06d98cc6d4aeba94444a212289d0d39"},
"sitemapper": {:hex, :sitemapper, "0.7.0", "4aee7930327a9a01b1c9b81d1d42f60c1a295e9f420108eb2d130c317415abd7", [:mix], [{:ex_aws_s3, "~> 2.0", [hex: :ex_aws_s3, repo: "hexpm", optional: true]}, {:xml_builder, "~> 2.1", [hex: :xml_builder, repo: "hexpm", optional: false]}], "hexpm", "60f7a684e5e9fe7f10ac5b69f48b0be2bcbba995afafcb3c143fc0c8ef1f223f"}, "sitemapper": {:hex, :sitemapper, "0.7.0", "4aee7930327a9a01b1c9b81d1d42f60c1a295e9f420108eb2d130c317415abd7", [:mix], [{:ex_aws_s3, "~> 2.0", [hex: :ex_aws_s3, repo: "hexpm", optional: true]}, {:xml_builder, "~> 2.1", [hex: :xml_builder, repo: "hexpm", optional: false]}], "hexpm", "60f7a684e5e9fe7f10ac5b69f48b0be2bcbba995afafcb3c143fc0c8ef1f223f"},
"sleeplocks": {:hex, :sleeplocks, "1.1.2", "d45aa1c5513da48c888715e3381211c859af34bee9b8290490e10c90bb6ff0ca", [:rebar3], [], "hexpm", "9fe5d048c5b781d6305c1a3a0f40bb3dfc06f49bf40571f3d2d0c57eaa7f59a5"}, "sleeplocks": {:hex, :sleeplocks, "1.1.2", "d45aa1c5513da48c888715e3381211c859af34bee9b8290490e10c90bb6ff0ca", [:rebar3], [], "hexpm", "9fe5d048c5b781d6305c1a3a0f40bb3dfc06f49bf40571f3d2d0c57eaa7f59a5"},