From ac0eb0533eb97686b2dca6f31d0b87d8c6fd5c31 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 21 Aug 2023 16:50:22 +0200
Subject: [PATCH] Add Elasticsearch cluster health check and indexes mismatch
check to dashboard (#26448)
---
app/chewy/instances_index.rb | 2 +-
.../admin/system_check/elasticsearch_check.rb | 61 ++++++++++++++++++-
config/locales/en.yml | 14 +++++
.../system_check/elasticsearch_check_spec.rb | 35 ++++++++++-
4 files changed, 105 insertions(+), 7 deletions(-)
diff --git a/app/chewy/instances_index.rb b/app/chewy/instances_index.rb
index 0d58167dc..8f10d13b6 100644
--- a/app/chewy/instances_index.rb
+++ b/app/chewy/instances_index.rb
@@ -6,7 +6,7 @@ class InstancesIndex < Chewy::Index
index_scope ::Instance.searchable
root date_detection: false do
- field :domain, type: 'text', index_prefixes: { min_chars: 1 }
+ field :domain, type: 'text', index_prefixes: { min_chars: 1, max_chars: 5 }
field :accounts_count, type: 'long'
end
end
diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb
index 0b55be350..a6f1f164a 100644
--- a/app/lib/admin/system_check/elasticsearch_check.rb
+++ b/app/lib/admin/system_check/elasticsearch_check.rb
@@ -1,6 +1,13 @@
# frozen_string_literal: true
class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
+ INDEXES = [
+ InstancesIndex,
+ AccountsIndex,
+ TagsIndex,
+ StatusesIndex,
+ ].freeze
+
def skip?
!current_user.can?(:view_devops)
end
@@ -8,11 +15,15 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
def pass?
return true unless Chewy.enabled?
- running_version.present? && compatible_version?
+ running_version.present? && compatible_version? && cluster_health['status'] == 'green' && indexes_match? && preset_matches?
+ rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error
+ false
end
def message
- if running_version.present?
+ if running_version.blank?
+ Admin::SystemCheck::Message.new(:elasticsearch_running_check)
+ elsif !compatible_version?
Admin::SystemCheck::Message.new(
:elasticsearch_version_check,
I18n.t(
@@ -21,13 +32,32 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
required_version: required_version
)
)
+ elsif !indexes_match?
+ Admin::SystemCheck::Message.new(
+ :elasticsearch_index_mismatch,
+ mismatched_indexes.join(' ')
+ )
+ elsif cluster_health['status'] == 'red'
+ Admin::SystemCheck::Message.new(:elasticsearch_health_red)
+ elsif cluster_health['number_of_nodes'] < 2 && es_preset != 'single_node_cluster'
+ Admin::SystemCheck::Message.new(:elasticsearch_preset_single_node, nil, 'https://docs.joinmastodon.org/admin/optional/elasticsearch/#scaling')
+ elsif Chewy.client.indices.get_settings['chewy_specifications'].dig('settings', 'index', 'number_of_replicas')&.to_i&.positive? && es_preset == 'single_node_cluster'
+ Admin::SystemCheck::Message.new(:elasticsearch_reset_chewy)
+ elsif cluster_health['status'] == 'yellow'
+ Admin::SystemCheck::Message.new(:elasticsearch_health_yellow)
else
- Admin::SystemCheck::Message.new(:elasticsearch_running_check)
+ Admin::SystemCheck::Message.new(:elasticsearch_preset, nil, 'https://docs.joinmastodon.org/admin/optional/elasticsearch/#scaling')
end
+ rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error
+ Admin::SystemCheck::Message.new(:elasticsearch_running_check)
end
private
+ def cluster_health
+ @cluster_health ||= Chewy.client.cluster.health
+ end
+
def running_version
@running_version ||= begin
Chewy.client.info['version']['number']
@@ -49,5 +79,30 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
Gem::Version.new(running_version) >= Gem::Version.new(required_version) ||
Gem::Version.new(compatible_wire_version) >= Gem::Version.new(required_version)
+ rescue ArgumentError
+ false
+ end
+
+ def mismatched_indexes
+ @mismatched_indexes ||= INDEXES.filter_map do |klass|
+ klass.index_name if Chewy.client.indices.get_mapping[klass.index_name]&.deep_symbolize_keys != klass.mappings_hash
+ end
+ end
+
+ def indexes_match?
+ mismatched_indexes.empty?
+ end
+
+ def es_preset
+ ENV.fetch('ES_PRESET', 'single_node_cluster')
+ end
+
+ def preset_matches?
+ case es_preset
+ when 'single_node_cluster'
+ cluster_health['number_of_nodes'] == 1
+ else
+ cluster_health['number_of_nodes'] > 1
+ end
end
end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 4a052e000..389b7aa66 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -814,6 +814,20 @@ en:
system_checks:
database_schema_check:
message_html: There are pending database migrations. Please run them to ensure the application behaves as expected
+ elasticsearch_health_red:
+ message_html: Elasticsearch cluster is unhealthy (red status), search features are unavailable
+ elasticsearch_health_yellow:
+ message_html: Elasticsearch cluster is unhealthy (yellow status), you may want to investigate the reason
+ elasticsearch_index_mismatch:
+ message_html: Elasticsearch index mappings are outdated. Please run <code>tootctl search deploy --only=%{value}</code>
+ elasticsearch_preset:
+ action: See documentation
+ message_html: Your Elasticsearch cluster has more than one node, but Mastodon is not configured to use them.
+ elasticsearch_preset_single_node:
+ action: See documentation
+ message_html: Your Elasticsearch cluster has only one node, <code>ES_PRESET</code> should be set to <code>single_node_cluster</code>.
+ elasticsearch_reset_chewy:
+ message_html: Your Elasticsearch system index is outdated due to a setting change. Please run <code>tootctl search deploy --reset-chewy</code> to update it.
elasticsearch_running_check:
message_html: Could not connect to Elasticsearch. Please check that it is running, or disable full-text search
elasticsearch_version_check:
diff --git a/spec/lib/admin/system_check/elasticsearch_check_spec.rb b/spec/lib/admin/system_check/elasticsearch_check_spec.rb
index 498215926..bf518b56e 100644
--- a/spec/lib/admin/system_check/elasticsearch_check_spec.rb
+++ b/spec/lib/admin/system_check/elasticsearch_check_spec.rb
@@ -11,7 +11,25 @@ describe Admin::SystemCheck::ElasticsearchCheck do
describe 'pass?' do
context 'when chewy is enabled' do
- before { allow(Chewy).to receive(:enabled?).and_return(true) }
+ before do
+ allow(Chewy).to receive(:enabled?).and_return(true)
+ allow(Chewy.client.cluster).to receive(:health).and_return({ 'status' => 'green', 'number_of_nodes' => 1 })
+ allow(Chewy.client.indices).to receive(:get_mapping).and_return({
+ AccountsIndex.index_name => AccountsIndex.mappings_hash.deep_stringify_keys,
+ StatusesIndex.index_name => StatusesIndex.mappings_hash.deep_stringify_keys,
+ InstancesIndex.index_name => InstancesIndex.mappings_hash.deep_stringify_keys,
+ TagsIndex.index_name => TagsIndex.mappings_hash.deep_stringify_keys,
+ })
+ allow(Chewy.client.indices).to receive(:get_settings).and_return({
+ 'chewy_specifications' => {
+ 'settings' => {
+ 'index' => {
+ 'number_of_replicas' => 0,
+ },
+ },
+ },
+ })
+ end
context 'when running version is present and high enough' do
before do
@@ -67,8 +85,19 @@ describe Admin::SystemCheck::ElasticsearchCheck do
end
describe 'message' do
+ before do
+ allow(Chewy).to receive(:enabled?).and_return(true)
+ allow(Chewy.client.cluster).to receive(:health).and_return({ 'status' => 'green', 'number_of_nodes' => 1 })
+ allow(Chewy.client.indices).to receive(:get_mapping).and_return({
+ AccountsIndex.index_name => AccountsIndex.mappings_hash.deep_stringify_keys,
+ StatusesIndex.index_name => StatusesIndex.mappings_hash.deep_stringify_keys,
+ InstancesIndex.index_name => InstancesIndex.mappings_hash.deep_stringify_keys,
+ TagsIndex.index_name => TagsIndex.mappings_hash.deep_stringify_keys,
+ })
+ end
+
context 'when running version is present' do
- before { allow(Chewy.client).to receive(:info).and_return({ 'version' => { 'number' => '999.99.9' } }) }
+ before { allow(Chewy.client).to receive(:info).and_return({ 'version' => { 'number' => '1.2.3' } }) }
it 'sends class name symbol to message instance' do
allow(Admin::SystemCheck::Message).to receive(:new)
@@ -77,7 +106,7 @@ describe Admin::SystemCheck::ElasticsearchCheck do
check.message
expect(Admin::SystemCheck::Message).to have_received(:new)
- .with(:elasticsearch_version_check, 'Elasticsearch 999.99.9 is running while 7.x is required')
+ .with(:elasticsearch_version_check, 'Elasticsearch 1.2.3 is running while 7.x is required')
end
end