MANTA-5523 Authorization layer must be aware of bucket scoped accesskeys

boot/setup.sh

@@ -193,6 +193,22 @@ function setup_rotation_cron {
     echo "JWT rotation cron job installed successfully"
 }
 
+function manta_setup_poll_interval {
+    echo "Configuring UFDS_POLL_INTERVAL for authcache replicator"
+    local current_interval=""
+    current_interval=$(get_sapi_metadata UFDS_POLL_INTERVAL)
+
+    if [[ -z "$current_interval" ]]; then
+        echo "Setting UFDS_POLL_INTERVAL to 2000ms (authcache hot path)"
+        if ! $SVC_ROOT/boot/set-sapi-metadata.sh UFDS_POLL_INTERVAL "2000"; then
+            echo "Warning: Failed to set UFDS_POLL_INTERVAL in SAPI" >&2
+            echo "Replicator will use template default (2000ms)" >&2
+        fi
+    else
+        echo "UFDS_POLL_INTERVAL already set to ${current_interval}ms"
+    fi
+}
+
 function manta_setup_auth {
     svccfg import $SVC_ROOT/smf/manifests/mahi.xml
     svcadm enable mahi
@@ -240,6 +256,9 @@ if [[ ${FLAVOR} == "manta" ]]; then
     echo "Setting up JWT rotation cron job"
     setup_rotation_cron
 
+    echo "Setting up replicator poll interval"
+    manta_setup_poll_interval
+
     # set up log rotation for mahiv2 first so logadm rotates logs properly
     manta_add_logadm_entry "mahi-replicator"
     manta_add_logadm_entry "mahi-server"

docs/operator-endpoints.md

@@ -0,0 +1,97 @@
+# Operator Endpoints
+
+Mahi exposes two internal endpoints for direct Redis manipulation.
+These are on the mahi server port (80) and are not exposed to
+external S3 clients.
+
+## POST /cache-push/:accesskeyid
+
+Writes an access key directly to Redis, bypassing the replicator
+poll interval (~2s).  Called by CloudAPI after creating or updating
+keys for immediate availability.
+
+**Request body:**
+
+| Field | Type | Required | Description |
+|-------|------|----------|-------------|
+| accesskeysecret | string | yes | Secret access key |
+| ownerUuid | string | yes | Owner account UUID |
+| status | string | no | Key status (default: "Active") |
+| scope | string\|null | no | Scope JSON string or null |
+
+**Behavior:**
+
+- Active keys are written in the unified format via the shared
+  builder (`redis-accesskey-format.js`), identical to the
+  replicator output.
+- Inactive keys are removed from Redis (reverse lookup deleted,
+  key entry removed from user record).
+- Uses `Date.now()` as the write version, which is always greater
+  than the replicator's changenumber.  This ensures cachePush
+  always wins over a concurrent replicator write.
+- Idempotent.  Repeated calls overwrite the previous entry.
+- Best-effort: failure is logged at warn level by the CloudAPI
+  caller but does not block the CloudAPI response.
+
+**Example:**
+
+```
+curl -X POST http://authcache.coal.joyent.us/cache-push/abc123 \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "accesskeysecret": "tdc_...",
+    "ownerUuid": "fe3617d8-...",
+    "status": "Active",
+    "scope": "{\"version\":1,\"permissions\":[{\"bucket\":\"my-bucket\",\"level\":\"read\"}]}"
+  }'
+```
+
+## POST /key-revoke/:accesskeyid
+
+Removes an access key from Redis immediately and writes a
+revocation tombstone with a 24-hour TTL.  The replicator checks
+for the tombstone before writing a key — if present, the write
+is skipped, making the revocation durable across replication
+cycles.
+
+Use for emergency revocation of compromised keys without waiting
+for the UFDS delete to propagate through the replicator.
+
+**Request body:** None required.
+
+**Behavior:**
+
+- Deletes the key entry from the user record in Redis.
+- Deletes the reverse lookup at `/accesskey/:accesskeyid`.
+- Writes a tombstone at `/revoked/:accesskeyid` with a 24-hour
+  TTL (`SETEX`).
+- Repeated calls renew the tombstone TTL.
+- Returns 404 if the key is not found in Redis.
+
+**Important:** Revocation is temporary.  The replicator will
+re-add the key on its next cycle once the tombstone expires
+(24 hours) if the key still exists in UFDS.  To permanently
+revoke a key:
+
+1. Call `DELETE /:account/accesskeys/:id` via CloudAPI
+   (which deletes from UFDS and calls key-revoke automatically).
+2. Or: call key-revoke for immediate effect, then delete
+   from UFDS within 24 hours.
+
+**Example:**
+
+```
+curl -X POST http://authcache.coal.joyent.us/key-revoke/abc123
+```
+
+**Response:**
+
+```json
+{
+  "revoked": true,
+  "accessKeyId": "abc123",
+  "userUuid": "fe3617d8-...",
+  "tombstoneTtlSeconds": 86400,
+  "replicationWarning": "Key removed from Redis cache and revocation tombstone set (86400s TTL). Delete from UFDS via CloudAPI to permanently revoke."
+}
+```

lib/redis-accesskey-format.js

@@ -0,0 +1,158 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+/*
+ * Copyright 2026 Edgecast Cloud LLC.
+ */
+
+/*
+ * Canonical Redis entry builders for permanent access keys.
+ *
+ * Two code paths write permanent key data to Redis:
+ *   1. Replicator transforms (UFDS → Redis sync)
+ *   2. cachePush endpoint (CloudAPI → Redis shortcut)
+ *
+ * Both must produce identical structures.  This module
+ * is the single source of truth for the Redis format
+ * so the invariant is enforced by construction, not by
+ * convention.
+ *
+ * Permanent key format (in /uuid/{uuid}.accesskeys):
+ *   { secret: string, scope: string|null }
+ *
+ * Reverse lookup format (in /accesskey/{keyId}):
+ *   { type: "accesskey", accessKeyId, userUuid,
+ *     credentialType: "permanent", scope: string|null }
+ *
+ * Scope values for permanent keys:
+ *   null      — key is unscoped (unrestricted access)
+ *   JSON str  — key is scoped (e.g. '{"version":1,...}')
+ *   ""        — preserved as-is; downstream parseScope
+ *               returns null, causing fail-closed deny
+ *
+ * Scope values for STS temporary credentials (in sts.js,
+ * NOT built by this module):
+ *   "none"    — parent key was explicitly unscoped
+ *   JSON str  — inherited from scoped parent key
+ *   null      — legacy pre-sentinel temp credential
+ */
+
+
+/**
+ * @brief Build the accesskeys map entry for a permanent key
+ *
+ * Stored at /uuid/{ownerUuid}.accesskeys[accessKeyId].
+ *
+ * @param {string} secret - Secret access key
+ * @param {string|null} scope - Scope JSON string or null
+ * @param {number} [version] - Write version (changenumber or
+ *   0 for unversioned).  Used by the replicator and cachePush
+ *   to prevent stale writes from overwriting newer data.
+ * @return {Object} { secret, scope, version }
+ */
+function buildPermanentKeyEntry(secret, scope, version) {
+    /*
+     * Warn on malformed scope JSON but store the value
+     * as-is.  Downstream parseScope() will fail to parse
+     * it and enforceBucketScope will deny the request
+     * (fail-closed).  We do NOT replace with null because
+     * null means "unrestricted" (fail-open).  We do NOT
+     * throw because callers are in async callback chains
+     * without try/catch guards.
+     */
+    if (scope != null && scope !== '') {
+        try {
+            JSON.parse(scope);
+        } catch (e) {
+            /* global console */
+            console.error(
+                'buildPermanentKeyEntry: invalid ' +
+                'scope JSON (storing as-is, will ' +
+                'fail-closed on enforcement): ' +
+                e.message);
+        }
+    }
+    return ({
+        secret: secret,
+        scope: (scope != null) ? scope : null,
+        version: (version != null) ? version : 0
+    });
+}
+
+
+/**
+ * @brief Build the reverse-lookup entry for a permanent key
+ *
+ * Stored at /accesskey/{accessKeyId}.
+ *
+ * @param {string} accessKeyId - Access key ID
+ * @param {string} userUuid - Owner UUID
+ * @param {string|null} scope - Scope JSON string or null
+ * @param {number} [version] - Write version (see
+ *   buildPermanentKeyEntry)
+ * @return {Object} Reverse-lookup entry
+ */
+function buildPermanentKeyLookup(accessKeyId, userUuid, scope, version) {
+    return ({
+        type: 'accesskey',
+        accessKeyId: accessKeyId,
+        userUuid: userUuid,
+        credentialType: 'permanent',
+        scope: (scope != null) ? scope : null,
+        version: (version != null) ? version : 0
+    });
+}
+
+
+/*
+ * Revocation tombstone constants and helpers.
+ *
+ * When an operator calls POST /scope-revoke/:accesskeyid,
+ * a tombstone key is written to Redis with a TTL.  The
+ * replicator checks for the tombstone before writing a key
+ * to Redis — if present, the write is skipped, making the
+ * revocation durable across replication cycles.
+ *
+ * The tombstone auto-expires after REVOKE_TTL_SECONDS.
+ * The operator should delete the key from UFDS (via CloudAPI)
+ * before the tombstone expires to make revocation permanent.
+ * Repeated scope-revoke calls renew the tombstone TTL.
+ */
+var REVOKE_TTL_SECONDS = 86400; // 24 hours
+
+
+/**
+ * @brief Build the Redis key path for a revocation tombstone
+ *
+ * @param {string} accesskeyid - Access key ID
+ * @return {string} Redis key path
+ */
+function revokedKeyPath(accesskeyid) {
+    return ('/revoked/' + accesskeyid);
+}
+
+
+/**
+ * @brief Build the revocation tombstone value
+ *
+ * @param {string} userUuid - Owner UUID of the revoked key
+ * @return {Object} Tombstone data
+ */
+function buildRevocationTombstone(userUuid) {
+    return ({
+        revokedAt: Date.now(),
+        userUuid: userUuid
+    });
+}
+
+
+module.exports = {
+    buildPermanentKeyEntry: buildPermanentKeyEntry,
+    buildPermanentKeyLookup: buildPermanentKeyLookup,
+    REVOKE_TTL_SECONDS: REVOKE_TTL_SECONDS,
+    revokedKeyPath: revokedKeyPath,
+    buildRevocationTombstone: buildRevocationTombstone
+};

lib/replicator/main.js

@@ -96,6 +96,16 @@ function main() {
         level: process.env.LOG_LEVEL || 'info'
     });
 
+    // Defensive: default to 2s if interval is missing or invalid.
+    // Only the authcache (manta) template uses {{UFDS_POLL_INTERVAL}};
+    // SDC hardcodes 10000 in its template, so this path only fires
+    // on authcache where 2s is the correct value for the S3 hot path.
+    if (!ufdsConfig.interval || (typeof (ufdsConfig.interval)) !== 'number') {
+        log.warn({configInterval: ufdsConfig.interval},
+            'ufds.interval missing or invalid in config, defaulting to 2000ms');
+        ufdsConfig.interval = 2000;
+    }
+
     ufdsConfig.url = opts['ufds-url'] || ufdsConfig.url;
     redisConfig.host = opts['redis-host'] || redisConfig.host;
     redisConfig.port = opts['redis-port'] || redisConfig.port;