Fix serverless compatibility: nil params in pool requests and unsupported template settings

Backport of fix-serverless-incompatible-issues to 11.x branch.

- Guard against nil params in perform_request_to_url with params ||= {}
- Replace nil with {} in http_client pool.put calls (template_put,
  rollover_alias_put, ilm_policy_put) to prevent NoMethodError on
  serverless when merging EAV headers
- Strip serverless-incompatible template settings (number_of_shards,
  number_of_replicas) before installing templates on serverless ES

Author: alexcams

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 11.22.16
+ - Fix serverless compatibility: guard against nil params in pool requests and strip unsupported template settings (number_of_shards, number_of_replicas) on serverless
+
 ## 11.22.15
  - Improves the logging experience when DLQ used [#1258](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1258)
 

lib/logstash/outputs/elasticsearch/http_client.rb

@@ -433,7 +433,7 @@ def template_exists?(template_endpoint, name)
     def template_put(template_endpoint, name, template)
       path = "#{template_endpoint}/#{name}"
       logger.info("Installing Elasticsearch template", name: name)
-      @pool.put(path, nil, LogStash::Json.dump(template))
+      @pool.put(path, {}, LogStash::Json.dump(template))
     rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
       raise e unless e.response_code == 404
     end
@@ -447,7 +447,7 @@ def rollover_alias_exists?(name)
 
     # Create a new rollover alias
     def rollover_alias_put(alias_name, alias_definition)
-      @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
+      @pool.put(CGI::escape(alias_name), {}, LogStash::Json.dump(alias_definition))
       logger.info("Created rollover alias", name: alias_name)
       # If the rollover alias already exists, ignore the error that comes back from Elasticsearch
     rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
@@ -473,7 +473,7 @@ def ilm_policy_exists?(name)
     def ilm_policy_put(name, policy)
       path = "_ilm/policy/#{name}"
       logger.info("Installing ILM policy #{policy}", name: name)
-      @pool.put(path, nil, LogStash::Json.dump(policy))
+      @pool.put(path, {}, LogStash::Json.dump(policy))
     end
 
 

lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -28,6 +28,7 @@ def self.install_template(plugin)
       end
 
       add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
+      strip_serverless_incompatible_settings(template) if plugin.serverless?
       plugin.logger.debug("Attempting to install template", template: template)
       install(plugin.client, template_endpoint(plugin), template_name(plugin), template, plugin.template_overwrite)
     end
@@ -129,5 +130,19 @@ def self.index_template_api?(plugin)
       end
     end
 
+    SERVERLESS_INCOMPATIBLE_SETTINGS = %w[
+      number_of_shards
+      index.number_of_shards
+      number_of_replicas
+      index.number_of_replicas
+    ].freeze
+
+    def self.strip_serverless_incompatible_settings(template)
+      settings = template.dig('template', 'settings') || template['settings']
+      return unless settings
+
+      SERVERLESS_INCOMPATIBLE_SETTINGS.each { |s| settings.delete(s) }
+    end
+
   end
 end end end

spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -186,4 +186,75 @@
 
     end
   end
+
+  describe ".strip_serverless_incompatible_settings" do
+    context "with composable template (template.settings)" do
+      let(:template) do
+        {
+          "template" => {
+            "settings" => {
+              "index.refresh_interval" => "5s",
+              "number_of_shards" => 1,
+              "number_of_replicas" => 0
+            }
+          }
+        }
+      end
+
+      it "removes number_of_shards and number_of_replicas" do
+        described_class.strip_serverless_incompatible_settings(template)
+        expect(template["template"]["settings"]).not_to have_key("number_of_shards")
+        expect(template["template"]["settings"]).not_to have_key("number_of_replicas")
+      end
+
+      it "preserves compatible settings" do
+        described_class.strip_serverless_incompatible_settings(template)
+        expect(template["template"]["settings"]["index.refresh_interval"]).to eq("5s")
+      end
+    end
+
+    context "with index-prefixed setting keys" do
+      let(:template) do
+        {
+          "template" => {
+            "settings" => {
+              "index.number_of_shards" => 1,
+              "index.number_of_replicas" => 0
+            }
+          }
+        }
+      end
+
+      it "removes index-prefixed variants" do
+        described_class.strip_serverless_incompatible_settings(template)
+        expect(template["template"]["settings"]).not_to have_key("index.number_of_shards")
+        expect(template["template"]["settings"]).not_to have_key("index.number_of_replicas")
+      end
+    end
+
+    context "with legacy template (top-level settings)" do
+      let(:template) do
+        {
+          "settings" => {
+            "index.refresh_interval" => "5s",
+            "number_of_shards" => 1
+          }
+        }
+      end
+
+      it "removes incompatible settings from top-level settings" do
+        described_class.strip_serverless_incompatible_settings(template)
+        expect(template["settings"]).not_to have_key("number_of_shards")
+        expect(template["settings"]["index.refresh_interval"]).to eq("5s")
+      end
+    end
+
+    context "with no settings" do
+      let(:template) { { "mappings" => {} } }
+
+      it "does not raise" do
+        expect { described_class.strip_serverless_incompatible_settings(template) }.not_to raise_error
+      end
+    end
+  end
 end

version

@@ -1 +1 @@
-11.22.15
+11.22.16