From ef2bc8ea261838cf31fe4fe11b2954a19c864295 Mon Sep 17 00:00:00 2001
From: David Roetzel <david@roetzel.de>
Date: Wed, 4 Sep 2024 16:10:45 +0200
Subject: [PATCH] Add redis sentinel support to ruby part of code (#31744)
---
lib/mastodon/redis_configuration.rb | 102 ++++++++++--------
spec/lib/mastodon/redis_configuration_spec.rb | 56 ++++++++++
2 files changed, 112 insertions(+), 46 deletions(-)
diff --git a/lib/mastodon/redis_configuration.rb b/lib/mastodon/redis_configuration.rb
index 3cd121e4a..9139d8758 100644
--- a/lib/mastodon/redis_configuration.rb
+++ b/lib/mastodon/redis_configuration.rb
@@ -1,34 +1,33 @@
# frozen_string_literal: true
class Mastodon::RedisConfiguration
+ DEFAULTS = {
+ host: 'localhost',
+ port: 6379,
+ db: 0,
+ }.freeze
+
def base
- @base ||= {
- url: setup_base_redis_url,
- driver: driver,
- namespace: base_namespace,
- }
+ @base ||= setup_config(prefix: nil, defaults: DEFAULTS)
+ .merge(namespace: base_namespace)
end
def sidekiq
- @sidekiq ||= {
- url: setup_prefixed_redis_url(:sidekiq),
- driver: driver,
- namespace: sidekiq_namespace,
- }
+ @sidekiq ||= setup_config(prefix: 'SIDEKIQ_')
+ .merge(namespace: sidekiq_namespace)
end
def cache
- @cache ||= {
- url: setup_prefixed_redis_url(:cache),
- driver: driver,
- namespace: cache_namespace,
- expires_in: 10.minutes,
- connect_timeout: 5,
- pool: {
- size: Sidekiq.server? ? Sidekiq[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
- timeout: 5,
- },
- }
+ @cache ||= setup_config(prefix: 'CACHE_')
+ .merge({
+ namespace: cache_namespace,
+ expires_in: 10.minutes,
+ connect_timeout: 5,
+ pool: {
+ size: Sidekiq.server? ? Sidekiq[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
+ timeout: 5,
+ },
+ })
end
private
@@ -55,42 +54,53 @@ class Mastodon::RedisConfiguration
namespace ? "#{namespace}_cache" : 'cache'
end
- def setup_base_redis_url
- url = ENV.fetch('REDIS_URL', nil)
- return url if url.present?
+ def setup_config(prefix: nil, defaults: {})
+ prefix = "#{prefix}REDIS_"
- user = ENV.fetch('REDIS_USER', '')
- password = ENV.fetch('REDIS_PASSWORD', '')
- host = ENV.fetch('REDIS_HOST', 'localhost')
- port = ENV.fetch('REDIS_PORT', 6379)
- db = ENV.fetch('REDIS_DB', 0)
+ url = ENV.fetch("#{prefix}URL", nil)
+ user = ENV.fetch("#{prefix}USER", nil)
+ password = ENV.fetch("#{prefix}PASSWORD", nil)
+ host = ENV.fetch("#{prefix}HOST", defaults[:host])
+ port = ENV.fetch("#{prefix}PORT", defaults[:port])
+ db = ENV.fetch("#{prefix}DB", defaults[:db])
+ name = ENV.fetch("#{prefix}SENTINEL_MASTER", nil)
+ sentinels = parse_sentinels(ENV.fetch("#{prefix}SENTINELS", nil))
- construct_uri(host, port, db, user, password)
- end
+ return { url:, driver: } if url
- def setup_prefixed_redis_url(prefix)
- prefix = "#{prefix.to_s.upcase}_"
- url = ENV.fetch("#{prefix}REDIS_URL", nil)
-
- return url if url.present?
-
- user = ENV.fetch("#{prefix}REDIS_USER", nil)
- password = ENV.fetch("#{prefix}REDIS_PASSWORD", nil)
- host = ENV.fetch("#{prefix}REDIS_HOST", nil)
- port = ENV.fetch("#{prefix}REDIS_PORT", nil)
- db = ENV.fetch("#{prefix}REDIS_DB", nil)
-
- if host.nil?
- base[:url]
+ if name.present? && sentinels.present?
+ host = name
+ port = nil
+ db ||= 0
else
- construct_uri(host, port, db, user, password)
+ sentinels = nil
+ end
+
+ url = construct_uri(host, port, db, user, password)
+
+ if url.present?
+ { url:, driver:, name:, sentinels: }
+ else
+ # Fall back to base config. This has defaults for the URL
+ # so this cannot lead to an endless loop.
+ base
end
end
def construct_uri(host, port, db, user, password)
+ return nil if host.blank?
+
Addressable::URI.parse("redis://#{host}:#{port}/#{db}").tap do |uri|
uri.user = user if user.present?
uri.password = password if password.present?
end.normalize.to_str
end
+
+ def parse_sentinels(sentinels_string)
+ (sentinels_string || '').split(',').map do |sentinel|
+ host, port = sentinel.split(':')
+ port = port.present? ? port.to_i : 26_379
+ { host: host, port: port }
+ end.presence
+ end
end
diff --git a/spec/lib/mastodon/redis_configuration_spec.rb b/spec/lib/mastodon/redis_configuration_spec.rb
index c7326fd41..a48ffc80e 100644
--- a/spec/lib/mastodon/redis_configuration_spec.rb
+++ b/spec/lib/mastodon/redis_configuration_spec.rb
@@ -45,6 +45,20 @@ RSpec.describe Mastodon::RedisConfiguration do
it 'uses the url from the base config' do
expect(subject[:url]).to eq 'redis://localhost:6379/0'
end
+
+ context 'when the base config uses sentinel' do
+ around do |example|
+ ClimateControl.modify REDIS_SENTINELS: '192.168.0.1:3000,192.168.0.2:4000', REDIS_SENTINEL_MASTER: 'mainsentinel' do
+ example.run
+ end
+ end
+
+ it 'uses the sentinel configuration from base config' do
+ expect(subject[:url]).to eq 'redis://mainsentinel/0'
+ expect(subject[:name]).to eq 'mainsentinel'
+ expect(subject[:sentinels]).to contain_exactly({ host: '192.168.0.1', port: 3000 }, { host: '192.168.0.2', port: 4000 })
+ end
+ end
end
context "when the `#{prefix}_REDIS_URL` environment variable is present" do
@@ -72,6 +86,39 @@ RSpec.describe Mastodon::RedisConfiguration do
end
end
+ shared_examples 'sentinel support' do |prefix = nil|
+ prefix = prefix ? "#{prefix}_" : ''
+
+ context 'when configuring sentinel support' do
+ around do |example|
+ ClimateControl.modify "#{prefix}REDIS_PASSWORD": 'testpass1', "#{prefix}REDIS_HOST": 'redis2.example.com', "#{prefix}REDIS_SENTINELS": '192.168.0.1:3000,192.168.0.2:4000', "#{prefix}REDIS_SENTINEL_MASTER": 'mainsentinel' do
+ example.run
+ end
+ end
+
+ it 'constructs the url using the sentinel master name' do
+ expect(subject[:url]).to eq 'redis://:testpass1@mainsentinel/0'
+ end
+
+ it 'includes the sentinel master name and list of sentinels' do
+ expect(subject[:name]).to eq 'mainsentinel'
+ expect(subject[:sentinels]).to contain_exactly({ host: '192.168.0.1', port: 3000 }, { host: '192.168.0.2', port: 4000 })
+ end
+ end
+
+ context 'when giving sentinels without port numbers' do
+ around do |example|
+ ClimateControl.modify "#{prefix}REDIS_SENTINELS": '192.168.0.1,192.168.0.2', "#{prefix}REDIS_SENTINEL_MASTER": 'mainsentinel' do
+ example.run
+ end
+ end
+
+ it 'uses the default sentinel port' do
+ expect(subject[:sentinels]).to contain_exactly({ host: '192.168.0.1', port: 26_379 }, { host: '192.168.0.2', port: 26_379 })
+ end
+ end
+ end
+
describe '#base' do
subject { redis_environment.base }
@@ -81,6 +128,8 @@ RSpec.describe Mastodon::RedisConfiguration do
url: 'redis://localhost:6379/0',
driver: :hiredis,
namespace: nil,
+ name: nil,
+ sentinels: nil,
})
end
end
@@ -113,12 +162,15 @@ RSpec.describe Mastodon::RedisConfiguration do
url: 'redis://:testpass@redis.example.com:3333/3',
driver: :hiredis,
namespace: nil,
+ name: nil,
+ sentinels: nil,
})
end
end
include_examples 'setting a different driver'
include_examples 'setting a namespace'
+ include_examples 'sentinel support'
end
describe '#sidekiq' do
@@ -127,6 +179,7 @@ RSpec.describe Mastodon::RedisConfiguration do
include_examples 'secondary configuration', 'SIDEKIQ'
include_examples 'setting a different driver'
include_examples 'setting a namespace'
+ include_examples 'sentinel support', 'SIDEKIQ'
end
describe '#cache' do
@@ -139,6 +192,8 @@ RSpec.describe Mastodon::RedisConfiguration do
namespace: 'cache',
expires_in: 10.minutes,
connect_timeout: 5,
+ name: nil,
+ sentinels: nil,
pool: {
size: 5,
timeout: 5,
@@ -166,5 +221,6 @@ RSpec.describe Mastodon::RedisConfiguration do
include_examples 'secondary configuration', 'CACHE'
include_examples 'setting a different driver'
+ include_examples 'sentinel support', 'CACHE'
end
end