Merge branch 'main' into access-controller

Author: cwperks

CHANGELOG.md

@@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Use QueryCoordinatorContext for the rewrite in validate API. ([#18272](https://github.com/opensearch-project/OpenSearch/pull/18272))
 - Upgrade crypto kms plugin dependencies for AWS SDK v2.x. ([#18268](https://github.com/opensearch-project/OpenSearch/pull/18268))
 - Add support for `matched_fields` with the unified highlighter ([#18164](https://github.com/opensearch-project/OpenSearch/issues/18164))
+- [repository-s3] Add support for SSE-KMS and S3 bucket owner verification ([#18312](https://github.com/opensearch-project/OpenSearch/pull/18312))
 - Create equivalents of JSM's AccessController in the java agent ([#18346](https://github.com/opensearch-project/OpenSearch/issues/18346))
 
 ### Changed
@@ -54,6 +55,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Deprecated
 
 ### Removed
+- [repository-s3] Removed existing ineffective `server_side_encryption` setting ([#18312](https://github.com/opensearch-project/OpenSearch/pull/18312))
 
 ### Fixed
 - Fix simultaneously creating a snapshot and updating the repository can potentially trigger an infinite loop ([#17532](https://github.com/opensearch-project/OpenSearch/pull/17532))
@@ -64,6 +66,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Null check field names in QueryStringQueryBuilder ([#18194](https://github.com/opensearch-project/OpenSearch/pull/18194))
 - Avoid NPE if on SnapshotInfo if 'shallow' boolean not present ([#18187](https://github.com/opensearch-project/OpenSearch/issues/18187))
 - Fix 'system call filter not installed' caused when network.host: 0.0.0.0 ([#18309](https://github.com/opensearch-project/OpenSearch/pull/18309))
+- Fix MatrixStatsAggregator reuse when mode parameter changes ([#18242](https://github.com/opensearch-project/OpenSearch/issues/18242))
 
 ### Security
 

MAINTAINERS.md

@@ -26,6 +26,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Owais Kazi        | [owaiskazi19](https://github.com/owaiskazi19)           | Amazon      |
 | Pan Guixin        | [bugmakerrrrrr](https://github.com/bugmakerrrrrr)       | ByteDance   |
 | Peter Nied        | [peternied](https://github.com/peternied)               | Amazon      |
+| Rishabh Maurya    | [rishabhmaurya](https://github.com/rishabhmaurya)       | Amazon      |
 | Rishikesh Pasham  | [Rishikesh1159](https://github.com/Rishikesh1159)       | Amazon      |
 | Sachin Kale       | [sachinpkale](https://github.com/sachinpkale)           | Amazon      |
 | Sarat Vemulapalli | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon      |

distribution/tools/plugin-cli/build.gradle

@@ -37,6 +37,7 @@ base {
 dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:opensearch-cli")
+  api project(":libs:agent-sm:agent-policy")
   api "org.bouncycastle:bc-fips:${versions.bouncycastle_jce}"
   api "org.bouncycastle:bcpg-fips:${versions.bouncycastle_pg}"
   testImplementation project(":test:framework")

distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/PluginSecurity.java

@@ -37,16 +37,15 @@
 import org.opensearch.cli.Terminal.Verbosity;
 import org.opensearch.cli.UserException;
 import org.opensearch.common.util.io.IOUtils;
+import org.opensearch.secure_sm.policy.PolicyFile;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.security.NoSuchAlgorithmException;
 import java.security.Permission;
 import java.security.PermissionCollection;
 import java.security.Permissions;
 import java.security.Policy;
-import java.security.URIParameter;
 import java.security.UnresolvedPermission;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -143,22 +142,12 @@ static Set<String> parsePermissions(Path file, Path tmpDir) throws IOException {
         // 2. read permission to the code itself (e.g. jar file of the code)
 
         Path emptyPolicyFile = Files.createTempFile(tmpDir, "empty", "tmp");
-        final Policy emptyPolicy;
-        try {
-            emptyPolicy = Policy.getInstance("JavaPolicy", new URIParameter(emptyPolicyFile.toUri()));
-        } catch (NoSuchAlgorithmException e) {
-            throw new RuntimeException(e);
-        }
+        final Policy emptyPolicy = new PolicyFile(emptyPolicyFile.toUri().toURL());
         IOUtils.rm(emptyPolicyFile);
 
         // parse the plugin's policy file into a set of permissions
-        final Policy policy;
-        try {
-            policy = Policy.getInstance("JavaPolicy", new URIParameter(file.toUri()));
-        } catch (NoSuchAlgorithmException e) {
-            throw new RuntimeException(e);
-        }
-        PermissionCollection permissions = policy.getPermissions(PluginSecurity.class.getProtectionDomain());
+        final Policy policy = new PolicyFile(file.toUri().toURL());
+        final PermissionCollection permissions = policy.getPermissions(PluginSecurity.class.getProtectionDomain());
         // this method is supported with the specific implementation we use, but just check for safety.
         if (permissions == Policy.UNSUPPORTED_EMPTY_COLLECTION) {
             throw new UnsupportedOperationException("JavaPolicy implementation does not support retrieving permissions");

gradle/code-coverage.gradle

@@ -19,7 +19,7 @@ repositories {
 
 allprojects {
   plugins.withId('jacoco') {
-    jacoco.toolVersion = '0.8.12'
+    jacoco.toolVersion = '0.8.13'
   }
 }
 

libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PolicyFile.java

@@ -125,8 +125,32 @@ private void addGrantEntry(GrantEntry grantEntry, List<PolicyEntry> entries) thr
         entries.add(new PolicyEntry(codesource, permissions));
     }
 
+    /**
+     * Expands known system properties like ${java.home} and ${user.home} to their absolute
+     * path equivalents.
+     */
+    private static String expandKnownSystemProperty(final String property, final String value) {
+        final int index = value.indexOf("${" + property + "}/");
+        final String path = System.getProperty(property);
+        if (path.endsWith(File.pathSeparator)) {
+            return path + value.substring(index + property.length() + 4 /* replace the path separator */);
+        } else {
+            return path + value.substring(index + property.length() + 3 /* keep the path separator */);
+        }
+    }
+
     private static PermissionEntry expandPermissionName(PermissionEntry pe) {
-        if (pe.name() == null || !pe.name().contains("${{")) {
+        if (pe.name() == null) {
+            return pe;
+        }
+
+        if (pe.name().startsWith("${java.home}")) {
+            return new PermissionEntry(pe.permission(), expandKnownSystemProperty("java.home", pe.name()), pe.action());
+        } else if (pe.name().startsWith("${user.home}")) {
+            return new PermissionEntry(pe.permission(), expandKnownSystemProperty("user.home", pe.name()), pe.action());
+        }
+
+        if (!pe.name().contains("${{")) {
             return pe;
         }
 

libs/agent-sm/agent/src/test/java/org/opensearch/javaagent/FileInterceptorIntegTests.java

@@ -22,6 +22,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.security.Permission;
 import java.security.PermissionCollection;
 import java.security.Permissions;
 import java.security.Policy;
@@ -56,6 +57,17 @@ public PermissionCollection getPermissions(ProtectionDomain domain) {
                 permissions.add(new FilePermission(System.getProperty("user.dir") + "/-", "read,write,delete"));
                 return permissions;
             }
+
+            @Override
+            public boolean implies(ProtectionDomain domain, Permission permission) {
+                final PermissionCollection pc = getPermissions(domain);
+
+                if (pc == null) {
+                    return false;
+                }
+
+                return pc.implies(permission);
+            }
         };
         AgentPolicy.setPolicy(policy);
         Files.createDirectories(getTestDir());

modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java

@@ -31,6 +31,7 @@
 
 package org.opensearch.search.aggregations.matrix.stats;
 
+import org.opensearch.Version;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ToXContent;
@@ -45,6 +46,7 @@
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Objects;
 
 public class MatrixStatsAggregationBuilder extends ArrayValuesSourceAggregationBuilder.LeafOnly<MatrixStatsAggregationBuilder> {
     public static final String NAME = "matrix_stats";
@@ -74,11 +76,18 @@ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBu
      */
     public MatrixStatsAggregationBuilder(StreamInput in) throws IOException {
         super(in);
+        if (in.getVersion().onOrAfter(Version.V_3_1_0)) {
+            this.multiValueMode = in.readEnum(MultiValueMode.class);
+        } else {
+            this.multiValueMode = MultiValueMode.AVG;
+        }
     }
 
     @Override
-    protected void innerWriteTo(StreamOutput out) {
-        // Do nothing, no extra state to write to stream
+    protected void innerWriteTo(StreamOutput out) throws IOException {
+        if (out.getVersion().onOrAfter(Version.V_3_1_0)) {
+            out.writeEnum(multiValueMode);
+        }
     }
 
     public MatrixStatsAggregationBuilder multiValueMode(MultiValueMode multiValueMode) {
@@ -110,4 +119,18 @@ public XContentBuilder doXContentBody(XContentBuilder builder, ToXContent.Params
     public String getType() {
         return NAME;
     }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+        MatrixStatsAggregationBuilder other = (MatrixStatsAggregationBuilder) obj;
+        return multiValueMode == other.multiValueMode;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), multiValueMode);
+    }
 }

modules/aggs-matrix-stats/src/test/java/org/opensearch/search/aggregations/matrix/MatrixStatsIT.java

@@ -0,0 +1,93 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.aggregations.matrix;
+
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.cache.request.RequestCacheStats;
+import org.opensearch.indices.IndicesRequestCache;
+import org.opensearch.search.MultiValueMode;
+import org.opensearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
+import org.opensearch.transport.client.Client;
+
+import java.util.List;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, supportsDedicatedMasters = false)
+public class MatrixStatsIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase {
+    public MatrixStatsIT(Settings nodeSettings) {
+        super(nodeSettings);
+    }
+
+    public void testMatrixStatsMultiValueModeEffect() throws Exception {
+        String index = "test_matrix_stats_multimode";
+        Client client = client();
+
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate(index)
+                .setSettings(
+                    Settings.builder()
+                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                )
+                .get()
+        );
+
+        client.prepareIndex(index).setId("1").setSource("num", List.of(10, 30), "num2", List.of(40, 60)).setWaitForActiveShards(1).get();
+        client.admin().indices().prepareRefresh(index).get();
+
+        MatrixStatsAggregationBuilder avgAgg = new MatrixStatsAggregationBuilder("agg_avg").fields(List.of("num", "num2"))
+            .multiValueMode(MultiValueMode.AVG);
+
+        client.prepareSearch(index).setSize(0).setRequestCache(true).addAggregation(avgAgg).get();
+
+        RequestCacheStats stats1 = getRequestCacheStats(client, index);
+        long hit1 = stats1.getHitCount();
+        long miss1 = stats1.getMissCount();
+
+        client.prepareSearch(index).setSize(0).setRequestCache(true).addAggregation(avgAgg).get();
+
+        RequestCacheStats stats2 = getRequestCacheStats(client, index);
+        long hit2 = stats2.getHitCount();
+        long miss2 = stats2.getMissCount();
+
+        MatrixStatsAggregationBuilder minAgg = new MatrixStatsAggregationBuilder("agg_min").fields(List.of("num", "num2"))
+            .multiValueMode(MultiValueMode.MIN);
+
+        client.prepareSearch(index).setSize(0).setRequestCache(true).addAggregation(minAgg).get();
+
+        RequestCacheStats stats3 = getRequestCacheStats(client, index);
+        long hit3 = stats3.getHitCount();
+        long miss3 = stats3.getMissCount();
+
+        client.prepareSearch(index).setSize(0).setRequestCache(true).addAggregation(minAgg).get();
+
+        RequestCacheStats stats4 = getRequestCacheStats(client, index);
+        long hit4 = stats4.getHitCount();
+        long miss4 = stats4.getMissCount();
+
+        assertEquals("Expected 1 cache miss for first AVG request", 1, miss1);
+        assertEquals("Expected 1 cache hit for second AVG request", hit1 + 1, hit2);
+        assertEquals("Expected 1 cache miss for first MIN request", miss1 + 1, miss3);
+        assertEquals("Expected 1 cache hit for second MIN request", hit2 + 1, hit4);
+        assertEquals("Expected no additional cache misses for second MIN request", miss3, miss4);
+    }
+
+    private static RequestCacheStats getRequestCacheStats(Client client, String index) {
+        return client.admin().indices().prepareStats(index).setRequestCache(true).get().getTotal().getRequestCache();
+    }
+}