Merge branch 'main' into fix/fetch-phase-breaker-lifecycle

Author: drempapis

docs/changelog/138047.yaml

@@ -0,0 +1,6 @@
+pr: 138047
+summary: Add configurable `max_batch_size` for `GoogleVertexAI` embedding service
+  settings
+area: Machine Learning
+type: bug
+issues: []

muted-tests.yml

@@ -424,18 +424,9 @@ tests:
 - class: org.elasticsearch.xpack.security.authc.TokenServiceTests
   method: testSupersedingTokenEncryption
   issue: https://github.com/elastic/elasticsearch/issues/139031
-- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
-  method: test {p0=search/370_profile/dfs knn vector profiling collector name}
-  issue: https://github.com/elastic/elasticsearch/issues/139065
-- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
-  method: test {p0=search.highlight/50_synthetic_source/text multi unified from vectors}
-  issue: https://github.com/elastic/elasticsearch/issues/139066
 - class: org.elasticsearch.xpack.search.AsyncSearchConcurrentStatusIT
   method: testConcurrentStatusFetchWhileTaskCloses
   issue: https://github.com/elastic/elasticsearch/issues/138543
-- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
-  method: test {p0=search.vectors/130_knn_query_nested_search/nested kNN search pre-filtered on alias with filter on top level fields}
-  issue: https://github.com/elastic/elasticsearch/issues/139131
 - class: org.elasticsearch.xpack.logsdb.RandomizedRollingUpgradeIT
   method: testIndexingSyntheticSource
   issue: https://github.com/elastic/elasticsearch/issues/139152
@@ -445,9 +436,6 @@ tests:
 - class: org.elasticsearch.xpack.core.security.authc.AuthenticationTests
   method: testMaybeRewriteForOlderVersionWithCrossClusterAccessRewritesAuthenticationInMetadata
   issue: https://github.com/elastic/elasticsearch/issues/139167
-- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
-  method: test {p0=eql/10_basic/Sequence checking correct join key ordering.}
-  issue: https://github.com/elastic/elasticsearch/issues/139173
 - class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
   method: test {p0=search.retrievers/result-diversification/10_mmr_result_diversification_retriever/Test MMR result diversification single index float type}
   issue: https://github.com/elastic/elasticsearch/issues/139195
@@ -457,12 +445,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {csv-spec:spatial.ConvertFromStringParseError}
   issue: https://github.com/elastic/elasticsearch/issues/139213
-- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
-  method: test {p0=search.vectors/180_update_dense_vector_type/Test update flat --> bbq_flat --> bbq_hnsw}
-  issue: https://github.com/elastic/elasticsearch/issues/139253
-- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
-  method: test {p0=search.vectors/41_knn_search_half_byte_quantized_bfloat16/Knn search with mip}
-  issue: https://github.com/elastic/elasticsearch/issues/139254
 - class: org.elasticsearch.xpack.inference.integration.AuthorizationTaskExecutorIT
   method: testCreatesEisChatCompletionEndpoint
   issue: https://github.com/elastic/elasticsearch/issues/138764
@@ -478,9 +460,12 @@ tests:
 - class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
   method: test {yaml=search.retrievers/result-diversification/10_mmr_result_diversification_retriever/Test MMR result diversification single index float type}
   issue: https://github.com/elastic/elasticsearch/issues/139335
-- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
-  method: test {p0=search.vectors/200_dense_vector_docvalue_fields/dense_vector docvalues with bfloat16}
-  issue: https://github.com/elastic/elasticsearch/issues/139368
+- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
+  method: test {p0=search/140_pre_filter_search_shards/prefilter on non-indexed date fields}
+  issue: https://github.com/elastic/elasticsearch/issues/139381
+- class: org.elasticsearch.xpack.downsample.DownsampleIT
+  method: testAggregateMethod
+  issue: https://github.com/elastic/elasticsearch/issues/139382
 
 # Examples:
 #

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/200_dense_vector_docvalue_fields.yml

@@ -298,38 +298,38 @@ setup:
   - length: {hits.hits.0.fields.vector7.0: 5}
   - length: {hits.hits.0.fields.vector8.0: 5}
 
-  - close_to: { hits.hits.0.fields.vector7.0.0: { value: 230.0, error: 0.4 } }
-  - close_to: { hits.hits.0.fields.vector7.0.1: { value: 300.33, error: 0.4 } }
-  - close_to: { hits.hits.0.fields.vector7.0.2: { value: -34.8988, error: 0.4 } }
-  - close_to: { hits.hits.0.fields.vector7.0.3: { value: 15.555, error: 0.4 } }
-  - close_to: { hits.hits.0.fields.vector7.0.4: { value: -200.0, error: 0.4 } }
-
-  - close_to: { hits.hits.0.fields.vector8.0.0: { value: 130.0, error: 0.4 } }
-  - close_to: { hits.hits.0.fields.vector8.0.1: { value: 115.0, error: 0.4 } }
-  - close_to: { hits.hits.0.fields.vector8.0.2: { value: -1.02, error: 0.4 } }
-  - close_to: { hits.hits.0.fields.vector8.0.3: { value: 15.555, error: 0.4 } }
-  - close_to: { hits.hits.0.fields.vector8.0.4: { value: -100.0, error: 0.4 } }
+  - close_to: { hits.hits.0.fields.vector7.0.0: { value: 230.0, error: 1 } }
+  - close_to: { hits.hits.0.fields.vector7.0.1: { value: 300.33, error: 1 } }
+  - close_to: { hits.hits.0.fields.vector7.0.2: { value: -34.8988, error: 0.2 } }
+  - close_to: { hits.hits.0.fields.vector7.0.3: { value: 15.555, error: 0.2 } }
+  - close_to: { hits.hits.0.fields.vector7.0.4: { value: -200.0, error: 1 } }
+
+  - close_to: { hits.hits.0.fields.vector8.0.0: { value: 130.0, error: 1 } }
+  - close_to: { hits.hits.0.fields.vector8.0.1: { value: 115.0, error: 1 } }
+  - close_to: { hits.hits.0.fields.vector8.0.2: { value: -1.02, error: 0.2 } }
+  - close_to: { hits.hits.0.fields.vector8.0.3: { value: 15.555, error: 0.2 } }
+  - close_to: { hits.hits.0.fields.vector8.0.4: { value: -100.0, error: 1 } }
 
   - match: {hits.hits.1._id: "2"}
   - match: {hits.hits.1.fields.name.0: "moose.jpg"}
 
   - length: {hits.hits.1.fields.vector7.0: 5}
   - match: {hits.hits.1.fields.vector8: null}
 
-  - close_to: { hits.hits.1.fields.vector7.0.0: { value: -0.5, error: 0.4 } }
-  - close_to: { hits.hits.1.fields.vector7.0.1: { value: 100.0, error: 0.4 } }
-  - close_to: { hits.hits.1.fields.vector7.0.2: { value: -13, error: 0.4 } }
-  - close_to: { hits.hits.1.fields.vector7.0.3: { value: 14.8, error: 0.4 } }
-  - close_to: { hits.hits.1.fields.vector7.0.4: { value: -156.0, error: 0.4 } }
+  - close_to: { hits.hits.1.fields.vector7.0.0: { value: -0.5, error: 0.2 } }
+  - close_to: { hits.hits.1.fields.vector7.0.1: { value: 100.0, error: 1 } }
+  - close_to: { hits.hits.1.fields.vector7.0.2: { value: -13, error: 0.2 } }
+  - close_to: { hits.hits.1.fields.vector7.0.3: { value: 14.8, error: 0.2 } }
+  - close_to: { hits.hits.1.fields.vector7.0.4: { value: -156.0, error: 1 } }
 
   - match: {hits.hits.2._id: "3"}
   - match: {hits.hits.2.fields.name.0: "rabbit.jpg"}
 
   - length: {hits.hits.2.fields.vector8.0: 5}
   - match: {hits.hits.2.fields.vector7: null}
 
-  - close_to: { hits.hits.2.fields.vector8.0.0: { value: 130.0, error: 0.4 } }
-  - close_to: { hits.hits.2.fields.vector8.0.1: { value: 115.0, error: 0.4 } }
-  - close_to: { hits.hits.2.fields.vector8.0.2: { value: -1.02, error: 0.4 } }
-  - close_to: { hits.hits.2.fields.vector8.0.3: { value: 15.555, error: 0.4 } }
-  - close_to: { hits.hits.2.fields.vector8.0.4: { value: -100.0, error: 0.4 } }
+  - close_to: { hits.hits.2.fields.vector8.0.0: { value: 130.0, error: 1 } }
+  - close_to: { hits.hits.2.fields.vector8.0.1: { value: 115.0, error: 1 } }
+  - close_to: { hits.hits.2.fields.vector8.0.2: { value: -1.02, error: 0.2 } }
+  - close_to: { hits.hits.2.fields.vector8.0.3: { value: 15.555, error: 0.2 } }
+  - close_to: { hits.hits.2.fields.vector8.0.4: { value: -100.0, error: 1 } }

server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java

@@ -55,7 +55,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         final OpenPointInTimeRequest openRequest = new OpenPointInTimeRequest(indices);
         final boolean crossProjectEnabled = crossProjectModeDecider.crossProjectEnabled();
         if (crossProjectEnabled) {
-            openRequest.projectRouting(request.param("project_routing", null));
             openRequest.indicesOptions(IndicesOptions.fromRequest(request, OpenPointInTimeRequest.DEFAULT_CPS_INDICES_OPTIONS));
         } else {
             openRequest.indicesOptions(IndicesOptions.fromRequest(request, OpenPointInTimeRequest.DEFAULT_INDICES_OPTIONS));

server/src/main/java/org/elasticsearch/index/engine/ThreadPoolMergeExecutorService.java

@@ -175,6 +175,9 @@ public Iterator<Setting<?>> settings() {
      * Initial value for IO write rate limit of individual merge tasks when doAutoIOThrottle is true
      */
     static final ByteSizeValue START_IO_RATE = ByteSizeValue.ofMb(20L);
+
+    private static final Logger logger = LogManager.getLogger(ThreadPoolMergeExecutorService.class);
+
     /**
      * Total number of submitted merge tasks that support IO auto throttling and that have not yet been run (or aborted).
      * This includes merge tasks that are currently running and that are backlogged (by their respective merge schedulers).
@@ -863,13 +866,15 @@ private static long newTargetIORateBytesPerSec(
                 MIN_IO_RATE.getBytes(),
                 currentTargetIORateBytesPerSec - currentTargetIORateBytesPerSec / 10L
             );
+            logger.debug("Decreasing target IO rate for merges to {}", newTargetIORateBytesPerSec);
         } else if (currentlySubmittedIOThrottledMergeTasks > concurrentMergesCeilLimitForThrottling
             && currentTargetIORateBytesPerSec < MAX_IO_RATE.getBytes()) {
                 // increase target IO rate by 20% (capped)
                 newTargetIORateBytesPerSec = Math.min(
                     MAX_IO_RATE.getBytes(),
                     currentTargetIORateBytesPerSec + currentTargetIORateBytesPerSec / 5L
                 );
+                logger.debug("Increasing target IO rate for merges to {}", newTargetIORateBytesPerSec);
             } else {
                 newTargetIORateBytesPerSec = currentTargetIORateBytesPerSec;
             }

server/src/main/java/org/elasticsearch/inference/ServiceSettings.java

@@ -14,6 +14,8 @@
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.xcontent.ToXContentObject;
 
+import java.util.Map;
+
 public interface ServiceSettings extends ToXContentObject, VersionedNamedWriteable, FilteredXContent {
 
     /**
@@ -61,4 +63,8 @@ default DenseVectorFieldMapper.ElementType elementType() {
      */
     @Nullable
     String modelId();
+
+    default ServiceSettings updateServiceSettings(Map<String, Object> serviceSettings) {
+        return this;
+    }
 }

server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java

@@ -62,7 +62,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
 
         final boolean crossProjectEnabled = crossProjectModeDecider.crossProjectEnabled();
         if (crossProjectEnabled) {
-            fieldRequest.projectRouting(request.param("project_routing", null));
             // Setting includeResolvedTo to always include index resolution data structure in the linked project responses,
             // in order to allow the coordinating node to call CrossProjectIndexResolutionValidator#validate
             fieldRequest.includeResolvedTo(true);

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResolveIndexAction.java

@@ -9,9 +9,11 @@
 
 package org.elasticsearch.rest.action.admin.indices;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.indices.resolve.ResolveIndexAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexMode;
@@ -20,22 +22,26 @@
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.search.crossproject.CrossProjectModeDecider;
+import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestResolveIndexAction extends BaseRestHandler {
     private static final Set<String> CAPABILITIES = Set.of("mode_filter");
-    private final Settings settings;
+    private final CrossProjectModeDecider crossProjectModeDecider;
 
     public RestResolveIndexAction(Settings settings) {
-        this.settings = settings;
+        this.crossProjectModeDecider = new CrossProjectModeDecider(settings);
     }
 
     @Override
@@ -45,7 +51,7 @@ public String getName() {
 
     @Override
     public List<Route> routes() {
-        return List.of(new Route(GET, "/_resolve/index/{name}"));
+        return List.of(new Route(GET, "/_resolve/index/{name}"), new Route(POST, "/_resolve/index/{name}"));
     }
 
     @Override
@@ -57,17 +63,28 @@ public Set<String> supportedCapabilities() {
     protected BaseRestHandler.RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         String[] indices = Strings.splitStringByCommaToArray(request.param("name"));
         String modeParam = request.param("mode");
-        final boolean crossProjectEnabled = settings != null && settings.getAsBoolean("serverless.cross_project.enabled", false);
-        String projectRouting = null;
-        if (crossProjectEnabled) {
-            projectRouting = request.param("project_routing");
-        }
+
+        final boolean crossProjectEnabled = crossProjectModeDecider.crossProjectEnabled();
+        AtomicReference<String> projectRouting = new AtomicReference<>();
         IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, ResolveIndexAction.Request.DEFAULT_INDICES_OPTIONS);
+
         if (crossProjectEnabled) {
+            request.withContentOrSourceParamParserOrNull(parser -> {
+                try {
+                    // If parser is null, there's no request body. projectRouting will then yield `null`.
+                    if (parser != null) {
+                        projectRouting.set(parseProjectRouting(parser));
+                    }
+                } catch (Exception e) {
+                    throw new ElasticsearchException("Couldn't parse request body", e);
+                }
+            });
+
             indicesOptions = IndicesOptions.builder(indicesOptions)
                 .crossProjectModeOptions(new IndicesOptions.CrossProjectModeOptions(true))
                 .build();
         }
+
         ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request(
             indices,
             indicesOptions,
@@ -76,8 +93,44 @@ protected BaseRestHandler.RestChannelConsumer prepareRequest(RestRequest request
                 : Arrays.stream(modeParam.split(","))
                     .map(IndexMode::fromString)
                     .collect(() -> EnumSet.noneOf(IndexMode.class), EnumSet::add, EnumSet::addAll),
-            projectRouting
+            projectRouting.get()
         );
         return channel -> client.admin().indices().resolveIndex(resolveRequest, new RestToXContentListener<>(channel));
     }
+
+    private static String parseProjectRouting(XContentParser parser) throws ParsingException {
+        try {
+            XContentParser.Token first = parser.nextToken();
+            if (first == null) {
+                return null;
+            }
+
+            if (first != XContentParser.Token.START_OBJECT) {
+                throw new ParsingException(
+                    parser.getTokenLocation(),
+                    "Expected [" + XContentParser.Token.START_OBJECT + "] but found [" + first + "]",
+                    parser.getTokenLocation()
+                );
+            }
+
+            String projectRouting = null;
+            for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    String currentName = parser.currentName();
+                    if ("project_routing".equals(currentName)) {
+                        parser.nextToken();
+                        projectRouting = parser.text();
+                    } else {
+                        throw new ParsingException(parser.getTokenLocation(), "request does not support [" + parser.currentName() + "]");
+                    }
+                }
+            }
+
+            return projectRouting;
+        } catch (ParsingException e) {
+            throw e;
+        } catch (Exception e) {
+            throw new ParsingException(parser == null ? null : parser.getTokenLocation(), "Failed to parse", e);
+        }
+    }
 }

server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java

@@ -108,9 +108,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         request.param("min_compatible_shard_node");
 
         final boolean crossProjectEnabled = crossProjectModeDecider.crossProjectEnabled();
-        if (crossProjectEnabled) {
-            searchRequest.setProjectRouting(request.param("project_routing"));
-        }
 
         /*
          * We have to pull out the call to `source().size(size)` because

server/src/main/resources/transport/definitions/referable/google_vertex_ai_configurable_max_batch_size.csv

@@ -0,0 +1 @@
+9241000