Merge pull request #116 from cipherstash/enable-index-on-hmac_256

Enable simpler indexing on encrypted records with hmac_256

Author: tobyhede

README.md

@@ -103,8 +103,8 @@ SELECT eql_v2.add_column('users', 'encrypted_email');
 After modifying configurations, activate them by running:
 
 ```sql
-SELECT eql_v2.encrypt();
-SELECT eql_v2.activate();
+SELECT eql_v2.migrate_config();
+SELECT eql_v2.activate_config();
 ```
 
 **Important:** These functions must be run after any modifications to the configuration.
@@ -219,8 +219,8 @@ SELECT eql_v2.add_search_config(
 After adding an index, you have to activate the configuration:
 
 ```sql
-SELECT eql_v2.encrypt();
-SELECT eql_v2.activate();
+SELECT eql_v2.migrate_config();
+SELECT eql_v2.activate_config();
 ```
 
 ## Searching data with EQL

src/blake3/functions.sql

@@ -10,8 +10,11 @@ CREATE FUNCTION eql_v2.blake3(val jsonb)
   IMMUTABLE STRICT PARALLEL SAFE
 AS $$
 	BEGIN
+    IF val IS NULL THEN
+      RETURN NULL;
+    END IF;
 
-    IF NOT (val ? 'b3') NULL THEN
+    IF NOT (val ? 'b3') THEN
         RAISE 'Expected a blake3 index (b3) value in json: %', val;
     END IF;
 
@@ -34,3 +37,24 @@ AS $$
     RETURN (SELECT eql_v2.blake3(val.data));
   END;
 $$ LANGUAGE plpgsql;
+
+
+CREATE FUNCTION eql_v2.has_blake3(val jsonb)
+  RETURNS boolean
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    RETURN val ? 'b3';
+  END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE FUNCTION eql_v2.has_blake3(val eql_v2_encrypted)
+  RETURNS boolean
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    RETURN eql_v2.has_blake3(val.data);
+  END;
+$$ LANGUAGE plpgsql;
+

src/bloom_filter/functions.sql

@@ -8,9 +8,14 @@ CREATE FUNCTION eql_v2.bloom_filter(val jsonb)
   IMMUTABLE STRICT PARALLEL SAFE
 AS $$
 	BEGIN
+    IF val IS NULL THEN
+      RETURN NULL;
+    END IF;
+
     IF val ? 'bf' THEN
       RETURN ARRAY(SELECT jsonb_array_elements(val->'bf'))::eql_v2.bloom_filter;
     END IF;
+
     RAISE 'Expected a match index (bf) value in json: %', val;
   END;
 $$ LANGUAGE plpgsql;
@@ -26,3 +31,23 @@ AS $$
     RETURN (SELECT eql_v2.bloom_filter(val.data));
   END;
 $$ LANGUAGE plpgsql;
+
+
+CREATE FUNCTION eql_v2.has_bloom_filter(val jsonb)
+  RETURNS boolean
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    RETURN val ? 'bf';
+  END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE FUNCTION eql_v2.has_bloom_filter(val eql_v2_encrypted)
+  RETURNS boolean
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    RETURN eql_v2.has_bloom_filter(val.data);
+  END;
+$$ LANGUAGE plpgsql;

src/config/functions.sql

@@ -144,7 +144,7 @@ $$ LANGUAGE plpgsql;
 -- Raises an exception if the configuration is already `encrypting` or if there is no `pending` configuration to encrypt.
 --
 
-CREATE FUNCTION eql_v2.encrypt()
+CREATE FUNCTION eql_v2.migrate_config()
   RETURNS boolean
 AS $$
 	BEGIN
@@ -168,7 +168,7 @@ $$ LANGUAGE plpgsql;
 
 
 
-CREATE FUNCTION eql_v2.activate()
+CREATE FUNCTION eql_v2.activate_config()
   RETURNS boolean
 AS $$
 	BEGIN

src/encryptindex/functions_test.sql

@@ -155,7 +155,7 @@ CREATE TABLE users
 DO $$
   BEGIN
     PERFORM eql_v2.add_search_config('users', 'name', 'match');
-    PERFORM eql_v2.encrypt();
+    PERFORM eql_v2.migrate_config();
 
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting'));
@@ -205,7 +205,7 @@ CREATE TABLE users
 DO $$
   BEGIN
     PERFORM eql_v2.add_search_config('users', 'name', 'match');
-    PERFORM eql_v2.encrypt();
+    PERFORM eql_v2.migrate_config();
 
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting'));
@@ -256,8 +256,8 @@ DO $$
   BEGIN
     PERFORM eql_v2.add_search_config('users', 'name', 'match');
 
-    PERFORM eql_v2.encrypt(); -- need to encrypt first
-    PERFORM eql_v2.activate();
+    PERFORM eql_v2.migrate_config(); -- need to encrypt first
+    PERFORM eql_v2.activate_config();
 
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'inactive'));

src/hmac_256/functions.sql

@@ -8,6 +8,10 @@ CREATE FUNCTION eql_v2.hmac_256(val jsonb)
   IMMUTABLE STRICT PARALLEL SAFE
 AS $$
 	BEGIN
+    IF val IS NULL THEN
+      RETURN NULL;
+    END IF;
+
     IF val ? 'hm' THEN
       RETURN val->>'hm';
     END IF;
@@ -16,6 +20,27 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
+CREATE FUNCTION eql_v2.has_hmac_256(val jsonb)
+  RETURNS boolean
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    RETURN val ? 'hm';
+  END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE FUNCTION eql_v2.has_hmac_256(val eql_v2_encrypted)
+  RETURNS boolean
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    RETURN eql_v2.has_hmac_256(val.data);
+  END;
+$$ LANGUAGE plpgsql;
+
+
+
 -- extracts hmac_256 index from an encrypted column
 
 CREATE FUNCTION eql_v2.hmac_256(val eql_v2_encrypted)

src/hmac_256/functions_test.sql

@@ -12,3 +12,15 @@ DO $$
 
   END;
 $$ LANGUAGE plpgsql;
+
+
+DO $$
+  DECLARE
+   e eql_v2_encrypted;
+  BEGIN
+    e := create_encrypted_json(1, 'hm');
+
+    ASSERT eql_v2.has_hmac_256(e);
+  END;
+$$ LANGUAGE plpgsql;
+

src/operators/operator_class.sql

@@ -10,9 +10,66 @@
 -- REQUIRE: src/operators/>.sql
 
 
+--
+-- Compare two eql_v2_encrypted values
+-- Uses `ore_block_u64_8_256` or `has_hmac_256` index terms for comparison if defined on ONE of the compared value
+--
+-- Important note: -- Index order of operations is reversed from equality operator.
+-- In equality operations, `has_hmac_256` is preferred as it reduces to a text comparison and is more efficient
+-- As compare is used for ordering, `ore_block_u64_8_256` provides more complete ordering and is checked first.
+-- THe assumption is that if you add ore you are adding it because you want to use it specifically for comparison.
+
+-- Thusly, the logic for determining which index term to use:
+--    Use ORE if BOTH parameters have ore index
+--    Fallback to hmac if BOTH parameters have hmac index
+--    Fallback to ORE if ONE of the parameters has ore index (will compare against a NULL term for the other parameter)
+--    Fallback to hmac if ONE of the parameters has hmac index (will compare against a NULL term term for the other parameter)
+--
+-- As a general rule, columns should have the same index terms as they are encrypted with the same configuration.
+-- Index terms should only be different during an encryption config migration.
+-- eg, when adding an ore index to a column any existing values will NOT have the ore index until encryptindexed/migrated
+--
 CREATE FUNCTION eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted)
   RETURNS integer
   IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  BEGIN
+
+    -- PERFORM eql_v2.log('eql_v2.has_hmac_256(a)', eql_v2.has_hmac_256(a)::text);
+    -- PERFORM eql_v2.log('eql_v2.has_hmac_256(b)', eql_v2.has_hmac_256(b)::text);
+    -- PERFORM eql_v2.log('eql_v2.has_ore_block_u64_8_256(b)', eql_v2.has_ore_block_u64_8_256(b)::text);
+    -- PERFORM eql_v2.log('eql_v2.has_ore_block_u64_8_256(b)', eql_v2.has_ore_block_u64_8_256(b)::text);
+
+
+    -- Use ORE if BOTH parameters have ore index
+    IF eql_v2.has_ore_block_u64_8_256(a) AND eql_v2.has_ore_block_u64_8_256(b) THEN
+      RETURN eql_v2.compare_ore_block_u64_8_256(a, b);
+    END IF;
+
+    -- Fallback to hmac if BOTH parameters have hmac index
+    IF eql_v2.has_hmac_256(a) AND eql_v2.has_hmac_256(b) THEN
+      RETURN eql_v2.compare_hmac(a, b);
+    END IF;
+
+    -- Fallback to ORE if one of the parameters has ore index
+    IF eql_v2.has_ore_block_u64_8_256(a) OR eql_v2.has_ore_block_u64_8_256(b) THEN
+      RETURN eql_v2.compare_ore_block_u64_8_256(a, b);
+    END IF;
+
+    -- Fallback to hmac if ONE of the parameters has hmac index
+    IF eql_v2.has_hmac_256(a) OR eql_v2.has_hmac_256(b) THEN
+      RETURN eql_v2.compare_hmac(a, b);
+    END IF;
+
+    RAISE 'Expected an hmac_256 (hm) or ore_block_u64_8_256 (ob) value in json: %', val;
+  END;
+$$ LANGUAGE plpgsql;
+
+--------------------
+
+CREATE FUNCTION eql_v2.compare_ore_block_u64_8_256(a eql_v2_encrypted, b eql_v2_encrypted)
+  RETURNS integer
+  IMMUTABLE STRICT PARALLEL SAFE
 AS $$
   DECLARE
     a_ore eql_v2.ore_block_u64_8_256;
@@ -38,6 +95,51 @@ AS $$
   END;
 $$ LANGUAGE plpgsql;
 
+
+--------------------
+
+CREATE FUNCTION eql_v2.compare_hmac(a eql_v2_encrypted, b eql_v2_encrypted)
+  RETURNS integer
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  DECLARE
+    a_hmac eql_v2.hmac_256;
+    b_hmac eql_v2.hmac_256;
+  BEGIN
+
+    a_hmac = eql_v2.hmac_256(a);
+    b_hmac = eql_v2.hmac_256(b);
+
+    IF a_hmac IS NULL AND b_hmac IS NULL THEN
+      RETURN 0;
+    END IF;
+
+    IF a_hmac IS NULL THEN
+      RETURN -1;
+    END IF;
+
+    IF b_hmac IS NULL THEN
+      RETURN 1;
+    END IF;
+
+    IF a_hmac = b_hmac THEN
+      RETURN 0;
+    END IF;
+
+    IF a_hmac < b_hmac THEN
+      RETURN -1;
+    END IF;
+
+    IF a_hmac > b_hmac THEN
+      RETURN 1;
+    END IF;
+
+  END;
+$$ LANGUAGE plpgsql;
+
+
+--------------------
+
 CREATE OPERATOR FAMILY eql_v2.encrypted_operator USING btree;
 
 CREATE OPERATOR CLASS eql_v2.encrypted_operator DEFAULT FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_operator AS
@@ -48,3 +150,28 @@ CREATE OPERATOR CLASS eql_v2.encrypted_operator DEFAULT FOR TYPE eql_v2_encrypte
   OPERATOR 5 >,
   FUNCTION 1 eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted);
 
+
+--------------------
+
+-- CREATE OPERATOR FAMILY eql_v2.encrypted_operator_ore_block_u64_8_256 USING btree;
+
+-- CREATE OPERATOR CLASS eql_v2.encrypted_operator_ore_block_u64_8_256 FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_operator_ore_block_u64_8_256 AS
+--   OPERATOR 1 <,
+--   OPERATOR 2 <=,
+--   OPERATOR 3 =,
+--   OPERATOR 4 >=,
+--   OPERATOR 5 >,
+--   FUNCTION 1 eql_v2.compare_ore_block_u64_8_256(a eql_v2_encrypted, b eql_v2_encrypted);
+
+-- --------------------
+
+-- CREATE OPERATOR FAMILY eql_v2.encrypted_hmac_256_operator USING btree;
+
+-- CREATE OPERATOR CLASS eql_v2.encrypted_hmac_256_operator FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_hmac_256_operator AS
+--   OPERATOR 1 <,
+--   OPERATOR 2 <=,
+--   OPERATOR 3 =,
+--   OPERATOR 4 >=,
+--   OPERATOR 5 >,
+--   FUNCTION 1 eql_v2.compare_hmac(a eql_v2_encrypted, b eql_v2_encrypted);
+