Can someone offer some clarity about Expression Bindings?

[CanyonCasa]

ref: Node 12.15.0; jsonnata 1.8.3

Since jsonata represents a native query language for JSON, I have assumed that expression bindings are analogous to named replacement parameters in SQL statements, but as the following code cases show this isn't always the case. Apparently, they can't be used in place of or inside regular expressions. I would think the bindings would be applied prior to parsing, which works when I have done it manually, or recognized by the parser for later substitution, but that doesn't appear to be the case. Or am I just fundamentally doing something wrong?

// jsonata Bindings test case...

// define mock data...
var data = 
{
  "recipes": [
  {
    "name": "safeUsers",
    "expression": "users[][username ~> /.+/].$sift(function($v, $k) {$k != 'credentials'})"
  },
  {
    "name": "safeUserByUsername",
    "expression": "users[][username ~> /jd/].$sift(function($v, $k) {$k != 'credentials'})"
  },
  {
    "name": "safeUserList",
    "expression": "users[][username ~> /$pattern/].$sift(function($v, $k) {$k != 'credentials'})"
  },
  {
    "name": "safeUserList2",
    "expression": "users[][username ~> $pattern].$sift(function($v, $k) {$k != 'credentials'})"
  },
  {
    "name": "safeUserList3",
    "expression": "users[][username ~> /?/].$sift(function($v, $k) {$k != 'credentials'})"
  },
  {
    "name": "safeUser",
    "expression": "users[username ~> /$name/] ~> | $ | {}, 'credentials' |"
  }
  ],
  "users": [
    {
      "username": "jd",
      "credentials": {
        "hash": "$2a$11$526GoWnCeCvVadf.T2IkXufWbQvN2cm5jitHmor1TXnOCtYU6RSe2",
        "code": {
          "code": "946779",
          "iat": 1589829069
        }
      },
      "member": [
        "admin",
        "users",
        "data",
        "red"
      ],
      "fullname": "John Doe",
      "email": "john@example.net",
      "phone": "5555555555",
      "ref": null,
      "status": "ACTIVE"
    },
    {
      "username": "janey",
      "credentials": {
        "hash": "$2a$11$vZ1gATw79gqMpXPhkDF1c.QPtfG5i2bwKTa3ytcwj9wbOr6MMNVXe",
        "code": {}
      },
      "email": "janey@example.net",
      "member": [
        "users"
      ],
      "fullname": "Jane Doe",
      "phone": "0000000000",
      "ref": null,
      "status": "PENDING"
    }
  ]
};

const jsonata = require('jsonata');

for (let r of data.recipes) {
  console.log("Recipe:", r.name);
  console.log("  Expr:", r.expression);
  switch (r.name) {
    case 'safeUsers':
    case 'safeUserByUsername':
      // no bindings, works ...
      let result = jsonata(r.expression).evaluate(data);
      console.log("Result:", JSON.stringify(result,null,2));
      break;
    case 'safeUserList': 
      // this approach, passing bindings doesn't work...
      let patterns = { wildcard:".+", fixed: "janey" };
      for (let p in patterns) {
        console.log("Pattern:", p);
        let bindings = { pattern: patterns[p] };
        console.log("Bindings:", bindings);
        let bindingResult = jsonata(r.expression).evaluate(data,bindings);
        console.log("Binding Result:", JSON.stringify(bindingResult,null,2));
        let expr = jsonata(r.expression);
        console.log("Resolved Expression:", expr);
        expr.assign('pattern',patterns[p]);
        console.log("Bound Expression:", expr);
        let altResult = expr.evaluate(data,bindings);
        console.log("Alt Result:", JSON.stringify(altResult,null,2));
      };
      break;
    case 'safeUserList2':
      // this approach only throws errors
/*
      let regex = { wildcardRE: /.+/, wildcard: '/.+/', fixed: "/janey/" };
      for (let re in regex) {
        console.log("RegExp:", re);
        let bindings = { pattern: regex[re] };
        console.log("Bindings:", bindings);
        // following all throws errors...
        let bindingResult = jsonata(r.expression).evaluate(data,bindings);
        console.log("Binding Result:", JSON.stringify(bindingResult,null,2));
        let expr2 = jsonata(r.expression);
        expr2.assign('RegExp',regex[re]);
        let altResult = expr2.evaluate(data,bindings);
        console.log("Alt Result:", JSON.stringify(altResult,null,2));
      };
*/
      break;
    case 'safeUserList3':
      // ordered parameters pre-substitution works
      let params = { wildcard: ['.+'], fixed: ["janey"] };
      for (let p in params) {
        console.log("Param:", params[p]);
        let exp = r.expression;
        while (exp.includes('?')) exp = exp.replace('?',params[p].shift()||'');
        console.log("Resolved Expression:",exp);
        let bindingResult = jsonata(exp).evaluate(data);
        console.log("Binding Result:", JSON.stringify(bindingResult,null,2));
      };
      break;
    case 'safeUser':
      // pre substitution binding works...
      let bindings = { one: "jd", two: ".+" };
      for (let b in bindings) {
        let bx = { name: bindings[b] };
        console.log("Binding:", bx);
        let preBindExpr = r.expression.replace('$name',bindings[b]);
        console.log("Pre bind Expression:", preBindExpr);
        let preresult = jsonata(preBindExpr).evaluate(data);
        console.log("Binding Result:", JSON.stringify(preresult,null,2));
      };
      break;
    default:
  };  
};

Outputs...

Recipe: safeUsers
Expr: users[][username ~> /.+/].$sift(function($v, $k) {$k != 'credentials'})
Result: [
{
"username": "jd",
"member": [
"admin",
"users",
"data",
"red"
],
"fullname": "John Doe",
"email": "john@example.net",
"phone": "5555555555",
"ref": null,
"status": "ACTIVE"
},
{
"username": "janey",
"email": "janey@example.net",
"member": [
"users"
],
"fullname": "Jane Doe",
"phone": "0000000000",
"ref": null,
"status": "PENDING"
}
]
Recipe: safeUserByUsername
Expr: users[][username ~> /jd/].$sift(function($v, $k) {$k != 'credentials'})
Result: [
{
"username": "jd",
"member": [
"admin",
"users",
"data",
"red"
],
"fullname": "John Doe",
"email": "john@example.net",
"phone": "5555555555",
"ref": null,
"status": "ACTIVE"
}
]
Recipe: safeUserList
Expr: users[][username ~> /$pattern/].$sift(function($v, $k) {$k != 'credentials'})
Pattern: wildcard
Bindings: { pattern: '.+' }
Binding Result: undefined
Resolved Expression: {
evaluate: [Function: evaluate],
assign: [Function: assign],
registerFunction: [Function: registerFunction],
ast: [Function: ast],
errors: [Function: errors]
}
Bound Expression: {
evaluate: [Function: evaluate],
assign: [Function: assign],
registerFunction: [Function: registerFunction],
ast: [Function: ast],
errors: [Function: errors]
}
Alt Result: undefined
Pattern: fixed
Bindings: { pattern: 'janey' }
Binding Result: undefined
Resolved Expression: {
evaluate: [Function: evaluate],
assign: [Function: assign],
registerFunction: [Function: registerFunction],
ast: [Function: ast],
errors: [Function: errors]
}
Bound Expression: {
evaluate: [Function: evaluate],
assign: [Function: assign],
registerFunction: [Function: registerFunction],
ast: [Function: ast],
errors: [Function: errors]
}
Alt Result: undefined
Recipe: safeUserList2
Expr: users[][username ~> $pattern].$sift(function($v, $k) {$k != 'credentials'})
Recipe: safeUserList3
Expr: users[][username ~> /?/].$sift(function($v, $k) {$k != 'credentials'})
Param: [ '.+' ]
Resolved Expression: users[][username ~> /.+/].$sift(function($v, $k) {$k != 'credentials'})
Binding Result: [
{
"username": "jd",
"member": [
"admin",
"users",
"data",
"red"
],
"fullname": "John Doe",
"email": "john@example.net",
"phone": "5555555555",
"ref": null,
"status": "ACTIVE"
},
{
"username": "janey",
"email": "janey@example.net",
"member": [
"users"
],
"fullname": "Jane Doe",
"phone": "0000000000",
"ref": null,
"status": "PENDING"
}
]
Param: [ 'janey' ]
Resolved Expression: users[][username ~> /janey/].$sift(function($v, $k) {$k != 'credentials'})
Binding Result: [
{
"username": "janey",
"email": "janey@example.net",
"member": [
"users"
],
"fullname": "Jane Doe",
"phone": "0000000000",
"ref": null,
"status": "PENDING"
}
]
Recipe: safeUser
Expr: users[username ~> /$name/] ~> | $ | {}, 'credentials' |
Binding: { name: 'jd' }
Pre bind Expression: users[username ~> /jd/] ~> | $ | {}, 'credentials' |
Binding Result: {
"username": "jd",
"member": [
"admin",
"users",
"data",
"red"
],
"fullname": "John Doe",
"email": "john@example.net",
"phone": "5555555555",
"ref": null,
"status": "ACTIVE"
}
Binding: { name: '.+' }
Pre bind Expression: users[username ~> /.+/] ~> | $ | {}, 'credentials' |
Binding Result: [
{
"username": "jd",
"member": [
"admin",
"users",
"data",
"red"
],
"fullname": "John Doe",
"email": "john@example.net",
"phone": "5555555555",
"ref": null,
"status": "ACTIVE"
},
{
"username": "janey",
"email": "janey@example.net",
"member": [
"users"
],
"fullname": "Jane Doe",
"phone": "0000000000",
"ref": null,
"status": "PENDING"
}
]

[andrew-coleman]

Bindings are dynamic, so they are evaluated at runtime rather than compile time.
The regex between the /.../ delimiters must be valid regex syntax. I guess you are looking for a type of macro expansion mechanism when building the expression.

[CanyonCasa]

Ok, if I had thought about it more, it makes sense that the expression has already been compiled when bindings get resolved. I don't know that I would say I'm looking for a "macro expansion mechanism", but more that regular expressions simply be "bindings-aware".

For example, the predicate [username = $name], where $name is a string, works as expected, as does the (hard coded) regular expression [username ~> /.+/], but neither the case [username ~> /$name/], where $name='.+', nor [username ~> $name], where $name=/.+/ work at all. So, it would suggest the regular expression (function) is unaware of resolving the binding in its context as an input pattern before the predicate test. Maybe internally that comes down to a macro expansion mechanism but it seems like a simpler pattern resolution action.

Is there a workaround to predicate pattern/wildcard matching without using the obvious regular expression mechanism?

[andrew-coleman]

This workaround is slightly messy, but you could dynamically build the regular expression using string concatenation, then use the $eval() function to create the regex object (which is itself a function). For example:

(
    $regex := $eval('/' & $$.Account.regex & '/');
    Account.Order.Product[`Product Name` ~> $regex]
)

See https://try.jsonata.org/BG0WXiJGy

[CanyonCasa]

That works. It's a bit extra, but since my queries are all "predefined recipes" it all becomes transparent to the results. Thanks.

I'd still vote to have regular expressions become "bindings-aware" in the future. In the short term, I'd suggest some clarification of regular expression documentation relative to bindings.