Performance fixes (#50)

* Performance fixes

- HTTP_CLIENT is now static and includes a connection timeout
- StackableAuthorizer only constructs a single instance of StackableAccessControlEnforcer
- StackableAccessControlEnforcer does not serialize to JSON twice anymore
- Spotless formatting applied to code

* Add changelog entry

* Fix changelog entry

* chore: Update to HDFS 3.4.0 and fix Tilt workflow

* Add shell for benchmarks

---------

Co-authored-by: Sebastian Bernauer <[email protected]>

Author: lfrancke

CHANGELOG.md

@@ -7,8 +7,10 @@ All notable changes to this project will be documented in this file.
 ### Changed
 
 - Bump okio to 1.17.6 to get rid of CVE-2023-3635 ([#46])
+- Performance fixes ([#50])
 
 [#46]: https://github.com/stackabletech/hdfs-utils/pull/46
+[#50]: https://github.com/stackabletech/hdfs-utils/pull/50
 
 ## [0.3.0] - 2024-07-04
 

Dockerfile

@@ -1,4 +1,8 @@
-FROM docker.stackable.tech/stackable/hadoop:3.3.6-stackable0.0.0-dev
+FROM docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev
 
-COPY --chown=stackable:stackable ./hdfs-utils-*.jar /stackable/hadoop/share/hadoop/tools/lib/
-COPY --chown=stackable:stackable ./bom.json /stackable/hadoop/share/hadoop/tools/lib/hdfs-utils.cdx.json
+# Remove existing hdfs-utils jars, so we can ship our custom one
+RUN rm -f /stackable/hadoop/share/hadoop/common/lib/hdfs-utils-*.jar
+RUN rm -f /stackable/hadoop/share/hadoop/tools/lib/hdfs-utils-*.jar
+
+COPY --chown=stackable:stackable ./hdfs-utils-*.jar /stackable/hadoop/share/hadoop/common/lib/
+COPY --chown=stackable:stackable ./bom.json /stackable/hadoop/share/hadoop/common/lib/hdfs-utils.cdx.json

pom.xml

@@ -35,6 +35,7 @@
     <maven.compiler.release>${java.version}</maven.compiler.release>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 
+    <!-- Tip: Use "mvn versions:display-dependency-updates" to check for updates -->
     <cleanthat.version>2.17</cleanthat.version>
     <error-prone.version>2.28.0</error-prone.version>
     <google-java-format.version>1.19.2</google-java-format.version>

src/main/java/tech/stackable/hadoop/StackableAccessControlEnforcer.java

@@ -7,26 +7,30 @@
 
 public class StackableAuthorizer extends INodeAttributeProvider {
 
-    private static final Logger LOG = LoggerFactory.getLogger(StackableAuthorizer.class);
-
-    @Override
-    public void start() {
-        LOG.debug("Starting HdfsOpaAuthorizer");
-    }
-
-    @Override
-    public void stop() {
-        LOG.debug("Stopping HdfsOpaAuthorizer");
-    }
-
-    @Override
-    public INodeAttributes getAttributes(String[] strings, INodeAttributes iNodeAttributes) {
-        // No special attributes needed
-        return iNodeAttributes;
-    }
-
-    @Override
-    public AccessControlEnforcer getExternalAccessControlEnforcer(AccessControlEnforcer defaultEnforcer) {
-        return new StackableAccessControlEnforcer();
-    }
+  private static final Logger LOG = LoggerFactory.getLogger(StackableAuthorizer.class);
+
+  private static final StackableAccessControlEnforcer ENFORCER =
+      new StackableAccessControlEnforcer();
+
+  @Override
+  public void start() {
+    LOG.debug("Starting HdfsOpaAuthorizer");
+  }
+
+  @Override
+  public void stop() {
+    LOG.debug("Stopping HdfsOpaAuthorizer");
+  }
+
+  @Override
+  public INodeAttributes getAttributes(String[] strings, INodeAttributes iNodeAttributes) {
+    // No special attributes needed
+    return iNodeAttributes;
+  }
+
+  @Override
+  public AccessControlEnforcer getExternalAccessControlEnforcer(
+      AccessControlEnforcer defaultEnforcer) {
+    return ENFORCER;
+  }
 }

src/main/java/tech/stackable/hadoop/StackableAuthorizer.java

@@ -8,12 +8,9 @@
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
-import java.util.HashMap;
+import java.time.Duration;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
-
-import com.fasterxml.jackson.databind.type.TypeFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.GroupMappingServiceProvider;
 import org.slf4j.Logger;
@@ -23,7 +20,8 @@ public class StackableGroupMapper implements GroupMappingServiceProvider {
 
   public static final String OPA_MAPPING_URL_PROP = "hadoop.security.group.mapping.opa.policy.url";
   private static final Logger LOG = LoggerFactory.getLogger(StackableGroupMapper.class);
-  private final HttpClient httpClient = HttpClient.newHttpClient();
+  private static final HttpClient HTTP_CLIENT =
+      HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(30)).build();
   private final ObjectMapper json;
   private URI opaUri;
 
@@ -80,13 +78,13 @@ public List<String> getGroups(String user) {
     HttpResponse<String> response = null;
     try {
       response =
-          httpClient.send(
+          HTTP_CLIENT.send(
               HttpRequest.newBuilder(opaUri)
                   .header("Content-Type", "application/json")
                   .POST(HttpRequest.BodyPublishers.ofString(body))
                   .build(),
               HttpResponse.BodyHandlers.ofString());
-      LOG.debug("Opa response: {}", response.body());
+      LOG.debug("OPA response: {}", response.body());
     } catch (Exception e) {
       LOG.error(e.getMessage());
       throw new OpaException.QueryFailed(e);

src/main/java/tech/stackable/hadoop/StackableGroupMapper.java

@@ -25,7 +25,7 @@ metadata:
   name: simple-hdfs
 spec:
   image:
-    productVersion: 3.3.6
+    productVersion: 3.4.0
     custom: hdfs # Will be overwritten by Tilt
     pullPolicy: IfNotPresent
   clusterConfig:

test/stack/20-hdfs.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: test-hdfs-permissions
-          image: docker.stackable.tech/stackable/hadoop:3.3.6-stackable0.0.0-dev
+          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev
           env:
             - name: HADOOP_CONF_DIR
               value: /stackable/conf/hdfs

test/stack/30-test-hdfs-permissions.yaml

@@ -0,0 +1,71 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: benchmark-shell
+spec:
+  template:
+    spec:
+      containers:
+        - name: benchmark-shell
+          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev
+          env:
+            - name: HADOOP_CONF_DIR
+              value: /stackable/conf/hdfs
+            - name: KRB5_CONFIG
+              value: /stackable/kerberos/krb5.conf
+            - name: HADOOP_OPTS
+              value: -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf
+          command:
+            - /bin/bash
+            - -c
+            - |
+              set -ex
+              klist -k /stackable/kerberos/keytab
+
+              log_in () { kdestroy; kinit -kt /stackable/kerberos/keytab $1/benchmark-shell.default.svc.cluster.local; }
+
+              log_in admin
+
+              bin/hdfs dfs -mkdir -p /bench
+              bin/hdfs dfs -ls /bench
+
+              # for i in $(seq 0 10); do echo "Creating $i" && bin/hdfs dfs -put -f /etc/hosts /bench/$i; done
+
+              # Watch out for the exact command you are using! (e.g. don't use "du -h /""). Checl the NameNode logs to
+              # make sure you actually produce enough OPA calls.
+              # time bin/hdfs dfs -du -h /bench
+
+              # So that you can run the benchmark manually
+              sleep infinity
+
+              exit 0
+          volumeMounts:
+            - name: hdfs-config
+              mountPath: /stackable/conf/hdfs
+            - name: kerberos
+              mountPath: /stackable/kerberos
+      volumes:
+        - name: hdfs-config
+          configMap:
+            name: simple-hdfs
+        - name: kerberos
+          ephemeral:
+            volumeClaimTemplate:
+              metadata:
+                annotations:
+                  secrets.stackable.tech/class: kerberos-default
+                  secrets.stackable.tech/scope: service=benchmark-shell
+                  secrets.stackable.tech/kerberos.service.names: admin,alice,bob
+              spec:
+                storageClassName: secrets.stackable.tech
+                accessModes:
+                  - ReadWriteOnce
+                resources:
+                  requests:
+                    storage: "1"
+      securityContext:
+        fsGroup: 1000
+        runAsGroup: 1000
+        runAsUser: 1000
+      restartPolicy: OnFailure

test/stack/31-benchmark-shell.yaml

@@ -25,7 +25,7 @@ metadata:
   name: simple-hdfs
 spec:
   image:
-    productVersion: 3.3.6
+    productVersion: 3.4.0
     custom: hdfs # updated by tilt
     pullPolicy: IfNotPresent
   clusterConfig: