From 8bcd33812541f8706c34b2a83aafc30ff24442d6 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Mon, 9 Sep 2024 12:09:53 -0800 Subject: [PATCH 01/99] Update Jenkinsfile --- Jenkinsfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index afa02be1..2ed4bf3b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -245,10 +245,11 @@ pipeline { choice( name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_SERVER_VERSION', choices: [ - '3.11', // Current Apache Cassandra - '4.0', // Development Apache Cassandra + '3.11', // Previous Apache Cassandra + '4.0', // Previous Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise - 'dse-6.8.30', // Development DataStax Enterprise + 'dse-6.8.30', // Previoius DataStax Enterprise + 'dse-6.9.0', // Current DataStax Enterprise 'ALL'], description: '''Apache Cassandra and DataStax Enterprise server version to use for adhoc BUILD-AND-EXECUTE-TESTS ONLY! @@ -330,6 +331,7 @@ pipeline { '4.0', // Development Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.8.30' // Development DataStax Enterprise + 'dse-6.9.0', // Current DataStax Enterprise } axis { name 'NODEJS_VERSION' @@ -381,7 +383,7 @@ pipeline { } stage('Execute-Examples') { when { - expression { env.CASSANDRA_VERSION == 'dse-6.8.30' } + expression { env.CASSANDRA_VERSION == 'dse-6.8.30' or } } steps { executeExamples() From 86da0c00eda44303533bb08f1ad3f8fa12ab9424 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Mon, 9 Sep 2024 13:33:12 -0800 Subject: [PATCH 02/99] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2ed4bf3b..321066a9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -330,7 +330,7 @@ pipeline { values '3.11', // Current Apache Cassandra '4.0', // Development Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise - 'dse-6.8.30' // Development DataStax Enterprise + 'dse-6.8.30', // Development DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise } axis { From 34d50240d9a2e956642f8b46a33680b99d2ec892 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Mon, 9 Sep 2024 13:34:03 -0800 Subject: [PATCH 03/99] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 321066a9..43a4ec47 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -331,7 +331,7 @@ pipeline { '4.0', // Development Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.8.30', // Development DataStax Enterprise - 'dse-6.9.0', // Current DataStax Enterprise + 'dse-6.9.0' // Current DataStax Enterprise } axis { name 'NODEJS_VERSION' From 0699654eb1eb7b93fd93f536773088d02699e802 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Mon, 9 Sep 2024 13:34:28 -0800 Subject: [PATCH 04/99] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 43a4ec47..0770cf1e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -383,7 +383,7 @@ pipeline { } stage('Execute-Examples') { when { - expression { env.CASSANDRA_VERSION == 'dse-6.8.30' or } + expression { env.CASSANDRA_VERSION == 'dse-6.8.30' } } steps { executeExamples() From 5a5a9d838e6e6217fc6afb0976079d127b7899b8 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Mon, 9 Sep 2024 16:17:15 -0800 Subject: [PATCH 05/99] Update Jenkinsfile --- Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 0770cf1e..777c6f33 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,6 +19,9 @@ def initializeEnvironment() { env.GITHUB_COMMIT_URL = "${GITHUB_PROJECT_URL}/commit/${env.GIT_COMMIT}" env.NODEJS_VERSION_FULL = nodeVersions[env.NODEJS_VERSION] + env.JAVA8_HOME="${JABBA_HOME}/jdk/1.8" + env.JAVA11_HOME="${JABBA_HOME}/jdk/openjdk@1.11" + sh label: 'Assign Node.js global environment', script: '''#!/bin/bash -lex nodenv versions echo "Using Node.js runtime ${NODEJS_VERSION} (${NODEJS_VERSION_FULL})" From e715972bc53ec6f58cc9f34f3f540d2c73e90fe6 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Mon, 9 Sep 2024 17:04:10 -0800 Subject: [PATCH 06/99] Update Jenkinsfile --- Jenkinsfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 777c6f33..d028328f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -330,8 +330,9 @@ pipeline { axes { axis { name 'CASSANDRA_VERSION' - values '3.11', // Current Apache Cassandra - '4.0', // Development Apache Cassandra + values '3.11', // Previous Apache Cassandra + '4.0', // Previous Apache Cassandra + '5.0.0', // Latest Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.8.30', // Development DataStax Enterprise 'dse-6.9.0' // Current DataStax Enterprise @@ -515,10 +516,12 @@ pipeline { axes { axis { name 'CASSANDRA_VERSION' - values '3.11', // Current Apache Cassandra - '4.0', // Development Apache Cassandra - 'dse-5.1.35', // Legacy DataStax Enterprise - 'dse-6.8.30' // Development DataStax Enterprise + values '3.11', // Previous Apache Cassandra + '4.0', // Previous Apache Cassandra + '5.0.0', // Current Apache Cassandra + 'dse-5.1.35', // Previous DataStax Enterprise + 'dse-6.8.30', // Previous DataStax Enterprise + 'dse-6.9.0' // Current DataStax Enterprise } axis { name 'NODEJS_VERSION' From 406a8b640c736e33e8814da6dde767666b930567 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 10 Sep 2024 11:21:32 -0800 Subject: [PATCH 07/99] Update Jenkinsfile --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index d028328f..25dc864a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -332,6 +332,7 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra + '4.1', // Previous Apache Cassandra '5.0.0', // Latest Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.8.30', // Development DataStax Enterprise From 7653d957453ae6b5fc42207af500725cd85d049c Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 10 Sep 2024 11:31:51 -0800 Subject: [PATCH 08/99] Update test-helper.js --- test/test-helper.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test-helper.js b/test/test-helper.js index ee5eab5d..956d6e71 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -1269,6 +1269,7 @@ helper.ccm.resumeNode = function (nodeIndex, callback) { }; helper.ccm.exec = function (params, callback) { + console.log("Executing ccm command: ", params); helper.ccm.spawn('ccm', params, callback); }; From e33ba763da66031e291cfc5883912d3604cbfd76 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 10 Sep 2024 12:04:27 -0800 Subject: [PATCH 09/99] Update test-helper.js --- test/test-helper.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test-helper.js b/test/test-helper.js index 956d6e71..551d6ca9 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -1269,6 +1269,7 @@ helper.ccm.resumeNode = function (nodeIndex, callback) { }; helper.ccm.exec = function (params, callback) { + // eslint-disable-next-line no-console, no-undef console.log("Executing ccm command: ", params); helper.ccm.spawn('ccm', params, callback); }; From ddcdec494796a945f996a14bacaa056c726e963d Mon Sep 17 00:00:00 2001 From: janehe Date: Tue, 10 Sep 2024 23:51:33 +0000 Subject: [PATCH 10/99] fixyaml --- test/test-helper.js | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/test/test-helper.js b/test/test-helper.js index ee5eab5d..5b084c77 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -32,6 +32,7 @@ const defaultOptions = require('../lib/client-options').defaultOptions; const { Host, HostMap } = require('../lib/host'); const OperationState = require('../lib/operation-state'); const promiseUtils = require('../lib/promise-utils'); +const { Console } = require('console'); util.inherits(RetryMultipleTimes, policies.retry.RetryPolicy); @@ -80,7 +81,7 @@ const helper = { * @param {String} [options.keyspace] Name of the keyspace to create. * @param {Number} [options.replicationFactor] Keyspace replication factor. * @param {Array} [options.queries] Queries to run after client creation. - * @param {Boolean} [options.removeClusterAfter=true] Determines whether ccm remove should be called on after(). + * @param {Boolean} [options.removeClusterAfter=false] Determines whether ccm remove should be called on after(). */ setup: function (nodeLength, options) { options = options || utils.emptyObject; @@ -491,7 +492,7 @@ const helper = { } return (function (l) { if (levels.indexOf(l) >= 0) { - // eslint-disable-next-line no-console, no-undef + console.log.apply(console, arguments); } }); @@ -987,6 +988,33 @@ const helper = { ], callback); }, + /** + * + * @param {Array.} yamlToFix + */ + fixYaml : function (yamlToFix) { + if (helper.isCassandraGreaterThan("4.1.0")) { + // fix the yaml options that turned obsolete since 4.1.0 + yamlToFix = yamlToFix.map(keyValue => { + const [key, value] = keyValue.split(':'); + var m = /^(\w+)_in_ms$/.exec(key); + if (m) { + return `${m[1]}:${value}ms`; + } + m = /^(\w+)_in_kb$/.exec(key); + if (m) { + return `${m[1]}:${value}KiB`; + } + m = /enable_(\w+)$/.exec(key); + if (m) { + return `${m[1]}_enabled:${value}`; + } + return keyValue + }); + } + return yamlToFix; + }, + /** * Makes a http request and returns the body. * @param {{host, port, path}} requestOptions @@ -1135,6 +1163,7 @@ helper.ccm.startAll = function (nodeLength, options, callback) { if (!options.yaml || !options.yaml.length) { return next(); } + options.yaml = helper.fixYaml(options.yaml); helper.trace('With cassandra yaml options', options.yaml); self.exec(['updateconf'].concat(options.yaml), next); }, @@ -1269,6 +1298,8 @@ helper.ccm.resumeNode = function (nodeIndex, callback) { }; helper.ccm.exec = function (params, callback) { + // eslint-disable-next-line no-console, no-undef + console.log("Executing ccm command: ", params); helper.ccm.spawn('ccm', params, callback); }; @@ -1564,7 +1595,6 @@ helper.ads.getKrb5ConfigPath = function() { return path.join(this.dir, 'krb5.conf'); }; - /** * A retry policy for testing purposes only, retries for a number of times * @param {Number} times From 3796202f9f451d32092eaed93f900a62dbcc23dc Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 10 Sep 2024 15:52:21 -0800 Subject: [PATCH 11/99] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 25dc864a..6673447d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -332,7 +332,7 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra - '4.1', // Previous Apache Cassandra + '4.1.0', // Previous Apache Cassandra '5.0.0', // Latest Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.8.30', // Development DataStax Enterprise From c59e57cf4bd28e4ed260d336d34924795969f5db Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 11 Sep 2024 00:14:58 +0000 Subject: [PATCH 12/99] fix eslint --- test/test-helper.js | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/test/test-helper.js b/test/test-helper.js index 5b084c77..a2a8c362 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -32,7 +32,6 @@ const defaultOptions = require('../lib/client-options').defaultOptions; const { Host, HostMap } = require('../lib/host'); const OperationState = require('../lib/operation-state'); const promiseUtils = require('../lib/promise-utils'); -const { Console } = require('console'); util.inherits(RetryMultipleTimes, policies.retry.RetryPolicy); @@ -492,7 +491,7 @@ const helper = { } return (function (l) { if (levels.indexOf(l) >= 0) { - + // eslint-disable-next-line no-console, no-undef console.log.apply(console, arguments); } }); @@ -997,19 +996,19 @@ const helper = { // fix the yaml options that turned obsolete since 4.1.0 yamlToFix = yamlToFix.map(keyValue => { const [key, value] = keyValue.split(':'); - var m = /^(\w+)_in_ms$/.exec(key); - if (m) { + let a = /^(\w+)_in_ms$/.exec(key); + if (a) { return `${m[1]}:${value}ms`; } - m = /^(\w+)_in_kb$/.exec(key); - if (m) { + let b = /^(\w+)_in_kb$/.exec(key); + if (b) { return `${m[1]}:${value}KiB`; } - m = /enable_(\w+)$/.exec(key); - if (m) { + let c = /enable_(\w+)$/.exec(key); + if (c) { return `${m[1]}_enabled:${value}`; } - return keyValue + return keyValue; }); } return yamlToFix; From dc66e2e4a8a6923d777067687675197a8c27f447 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 11 Sep 2024 00:26:52 +0000 Subject: [PATCH 13/99] fix eslint --- test/test-helper.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test-helper.js b/test/test-helper.js index a2a8c362..1a30159d 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -996,17 +996,17 @@ const helper = { // fix the yaml options that turned obsolete since 4.1.0 yamlToFix = yamlToFix.map(keyValue => { const [key, value] = keyValue.split(':'); - let a = /^(\w+)_in_ms$/.exec(key); + const a = /^(\w+)_in_ms$/.exec(key); if (a) { - return `${m[1]}:${value}ms`; + return `${a[1]}:${value}ms`; } - let b = /^(\w+)_in_kb$/.exec(key); + const b = /^(\w+)_in_kb$/.exec(key); if (b) { - return `${m[1]}:${value}KiB`; + return `${b[1]}:${value}KiB`; } - let c = /enable_(\w+)$/.exec(key); + const c = /enable_(\w+)$/.exec(key); if (c) { - return `${m[1]}_enabled:${value}`; + return `${c[1]}_enabled:${value}`; } return keyValue; }); From 28fe313c7dc28ae5945321fc504d8f70c9b626e1 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 11 Sep 2024 06:28:18 +0000 Subject: [PATCH 14/99] fix virtual table --- test/integration/short/metadata-tests.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/integration/short/metadata-tests.js b/test/integration/short/metadata-tests.js index 85bb05d7..440ac7da 100644 --- a/test/integration/short/metadata-tests.js +++ b/test/integration/short/metadata-tests.js @@ -1029,6 +1029,9 @@ describe('metadata @SERVER_API', function () { }, done); }); vit('4.0', 'should retrieve the metadata of a virtual table', () => { + if(helper.isCassandraGreaterThan('4.0')){ + this.skip(); + } const client = setupInfo.client; return client.metadata.getTable('system_views', 'clients') .then((table) => { @@ -1043,6 +1046,21 @@ describe('metadata @SERVER_API', function () { assert.deepEqual(table.clusteringKeys.map(c => c.name), ['port']); }); }); + vit('4.1', 'should retrieve the metadata of a virtual table above 4.1 version', () => { + const client = setupInfo.client; + return client.metadata.getTable('system_views', 'clients') + .then((table) => { + assert.ok(table); + assert.ok(table.virtual); + assert.strictEqual(table.name, 'clients'); + assert.deepEqual(table.columns.map(c => c.name), ['address', 'client_options', 'connection_stage', 'driver_name', + 'driver_version', 'hostname', 'port', 'protocol_version', 'request_count', 'ssl_cipher_sui`te', + 'ssl_enabled', 'ssl_protocol', 'username']); + assert.deepEqual(table.clusteringOrder, ['ASC']); + assert.deepEqual(table.partitionKeys.map(c => c.name), ['address']); + assert.deepEqual(table.clusteringKeys.map(c => c.name), ['port']); + }); + }); it('should retrieve the updated metadata after a schema change', function (done) { const client = newInstance(); const nonSyncClient = newInstance({isMetadataSyncEnabled: false}); From 594be6016a2e23643a43a372db7b830762f96cc3 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 11 Sep 2024 06:32:33 +0000 Subject: [PATCH 15/99] typo ` --- test/integration/short/metadata-tests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/short/metadata-tests.js b/test/integration/short/metadata-tests.js index 440ac7da..1d8a9ad3 100644 --- a/test/integration/short/metadata-tests.js +++ b/test/integration/short/metadata-tests.js @@ -1054,7 +1054,7 @@ describe('metadata @SERVER_API', function () { assert.ok(table.virtual); assert.strictEqual(table.name, 'clients'); assert.deepEqual(table.columns.map(c => c.name), ['address', 'client_options', 'connection_stage', 'driver_name', - 'driver_version', 'hostname', 'port', 'protocol_version', 'request_count', 'ssl_cipher_sui`te', + 'driver_version', 'hostname', 'port', 'protocol_version', 'request_count', 'ssl_cipher_suite', 'ssl_enabled', 'ssl_protocol', 'username']); assert.deepEqual(table.clusteringOrder, ['ASC']); assert.deepEqual(table.partitionKeys.map(c => c.name), ['address']); From 2a4575d9a0855f74e92e2621a9354c31e8357ceb Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 11 Sep 2024 07:07:05 +0000 Subject: [PATCH 16/99] 5.0-beta1, virtual table --- Jenkinsfile | 5 +-- test/integration/short/metadata-tests.js | 40 ++++++++++++------------ 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6673447d..639fd8de 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -333,7 +333,7 @@ pipeline { values '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra '4.1.0', // Previous Apache Cassandra - '5.0.0', // Latest Apache Cassandra + '5.0-beta1', // Latest Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.8.30', // Development DataStax Enterprise 'dse-6.9.0' // Current DataStax Enterprise @@ -519,7 +519,8 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra - '5.0.0', // Current Apache Cassandra + '4.1.0', // Previous Apache Cassandra + '5.0-beta1', // Current Apache Cassandra 'dse-5.1.35', // Previous DataStax Enterprise 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0' // Current DataStax Enterprise diff --git a/test/integration/short/metadata-tests.js b/test/integration/short/metadata-tests.js index 1d8a9ad3..f2cc32c5 100644 --- a/test/integration/short/metadata-tests.js +++ b/test/integration/short/metadata-tests.js @@ -1028,8 +1028,8 @@ describe('metadata @SERVER_API', function () { }); }, done); }); - vit('4.0', 'should retrieve the metadata of a virtual table', () => { - if(helper.isCassandraGreaterThan('4.0')){ + it('should retrieve the metadata of a virtual table', function () { + if (!helper.isCassandraGreaterThan('3.9')){ this.skip(); } const client = setupInfo.client; @@ -1038,24 +1038,24 @@ describe('metadata @SERVER_API', function () { assert.ok(table); assert.ok(table.virtual); assert.strictEqual(table.name, 'clients'); - assert.deepEqual(table.columns.map(c => c.name), ['address', 'connection_stage', 'driver_name', - 'driver_version', 'hostname', 'port', 'protocol_version', 'request_count', 'ssl_cipher_suite', - 'ssl_enabled', 'ssl_protocol', 'username']); - assert.deepEqual(table.clusteringOrder, ['ASC']); - assert.deepEqual(table.partitionKeys.map(c => c.name), ['address']); - assert.deepEqual(table.clusteringKeys.map(c => c.name), ['port']); - }); - }); - vit('4.1', 'should retrieve the metadata of a virtual table above 4.1 version', () => { - const client = setupInfo.client; - return client.metadata.getTable('system_views', 'clients') - .then((table) => { - assert.ok(table); - assert.ok(table.virtual); - assert.strictEqual(table.name, 'clients'); - assert.deepEqual(table.columns.map(c => c.name), ['address', 'client_options', 'connection_stage', 'driver_name', - 'driver_version', 'hostname', 'port', 'protocol_version', 'request_count', 'ssl_cipher_suite', - 'ssl_enabled', 'ssl_protocol', 'username']); + if (helper.isCassandraGreaterThan('4.0')) { + if (helper.isCassandraGreaterThan('4.1')) { + // 5.0 and above + assert.deepEqual(table.columns.map(c => c.name), ['address', 'client_options', 'connection_stage', 'driver_name', + 'driver_version', 'hostname', 'keyspace_name', 'port', 'protocol_version', 'request_count', 'ssl_cipher_suite', + 'ssl_enabled', 'ssl_protocol', 'username']); + }else{ + // 4.1 + assert.deepEqual(table.columns.map(c => c.name), ['address', 'client_options', 'connection_stage', 'driver_name', + 'driver_version', 'hostname', 'port', 'protocol_version', 'request_count', 'ssl_cipher_suite', + 'ssl_enabled', 'ssl_protocol', 'username']); + } + }else{ + // 4.0 + assert.deepEqual(table.columns.map(c => c.name), ['address', 'connection_stage', 'driver_name', + 'driver_version', 'hostname', 'port', 'protocol_version', 'request_count', 'ssl_cipher_suite', + 'ssl_enabled', 'ssl_protocol', 'username']); + } assert.deepEqual(table.clusteringOrder, ['ASC']); assert.deepEqual(table.partitionKeys.map(c => c.name), ['address']); assert.deepEqual(table.clusteringKeys.map(c => c.name), ['port']); From 3f5143d7721a8742d9591572e6aa848c26219bc5 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 10 Sep 2024 23:41:52 -0800 Subject: [PATCH 17/99] method too large??? --- Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 639fd8de..86de4cd1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -333,7 +333,6 @@ pipeline { values '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra '4.1.0', // Previous Apache Cassandra - '5.0-beta1', // Latest Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.8.30', // Development DataStax Enterprise 'dse-6.9.0' // Current DataStax Enterprise @@ -520,7 +519,6 @@ pipeline { values '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra '4.1.0', // Previous Apache Cassandra - '5.0-beta1', // Current Apache Cassandra 'dse-5.1.35', // Previous DataStax Enterprise 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0' // Current DataStax Enterprise From ed616b5ab46a4a776cc883664abde5805311b336 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 10 Sep 2024 23:43:12 -0800 Subject: [PATCH 18/99] Update Jenkinsfile --- Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 86de4cd1..ade1ce3c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -333,9 +333,10 @@ pipeline { values '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra '4.1.0', // Previous Apache Cassandra - 'dse-5.1.35', // Legacy DataStax Enterprise - 'dse-6.8.30', // Development DataStax Enterprise - 'dse-6.9.0' // Current DataStax Enterprise + '5.0-beat1 + // 'dse-5.1.35', // Legacy DataStax Enterprise + // 'dse-6.8.30', // Development DataStax Enterprise + // 'dse-6.9.0' // Current DataStax Enterprise } axis { name 'NODEJS_VERSION' From d4bcb1438898fb6c1ee22e3e89c575a18398fc65 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Wed, 11 Sep 2024 09:26:40 -0800 Subject: [PATCH 19/99] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ade1ce3c..691ce47b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -333,7 +333,7 @@ pipeline { values '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra '4.1.0', // Previous Apache Cassandra - '5.0-beat1 + '5.0-beta1' // 'dse-5.1.35', // Legacy DataStax Enterprise // 'dse-6.8.30', // Development DataStax Enterprise // 'dse-6.9.0' // Current DataStax Enterprise From 552589e440a2a0d6d6d0a5ba8099050715feccd6 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 11 Sep 2024 18:22:26 +0000 Subject: [PATCH 20/99] fix isCassandraGreaterThan --- test/integration/short/metadata-tests.js | 46 ++++++++++++------------ 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/test/integration/short/metadata-tests.js b/test/integration/short/metadata-tests.js index f2cc32c5..4b20d7d9 100644 --- a/test/integration/short/metadata-tests.js +++ b/test/integration/short/metadata-tests.js @@ -1029,37 +1029,39 @@ describe('metadata @SERVER_API', function () { }, done); }); it('should retrieve the metadata of a virtual table', function () { - if (!helper.isCassandraGreaterThan('3.9')){ - this.skip(); - } - const client = setupInfo.client; - return client.metadata.getTable('system_views', 'clients') - .then((table) => { - assert.ok(table); - assert.ok(table.virtual); - assert.strictEqual(table.name, 'clients'); - if (helper.isCassandraGreaterThan('4.0')) { - if (helper.isCassandraGreaterThan('4.1')) { + if (helper.isCassandraGreaterThan('4.0')) { + const client = setupInfo.client; + return client.metadata.getTable('system_views', 'clients') + .then((table) => { + assert.ok(table); + assert.ok(table.virtual); + assert.strictEqual(table.name, 'clients'); + assert.deepEqual(table.clusteringOrder, ['ASC']); + assert.deepEqual(table.partitionKeys.map(c => c.name), ['address']); + assert.deepEqual(table.clusteringKeys.map(c => c.name), ['port']); + + if (helper.isCassandraGreaterThan('5.0')) { // 5.0 and above assert.deepEqual(table.columns.map(c => c.name), ['address', 'client_options', 'connection_stage', 'driver_name', 'driver_version', 'hostname', 'keyspace_name', 'port', 'protocol_version', 'request_count', 'ssl_cipher_suite', 'ssl_enabled', 'ssl_protocol', 'username']); - }else{ + } else if (helper.isCassandraGreaterThan('4.1')) { // 4.1 assert.deepEqual(table.columns.map(c => c.name), ['address', 'client_options', 'connection_stage', 'driver_name', 'driver_version', 'hostname', 'port', 'protocol_version', 'request_count', 'ssl_cipher_suite', 'ssl_enabled', 'ssl_protocol', 'username']); + } else { + // 4.0 + assert.deepEqual(table.columns.map(c => c.name), ['address', 'connection_stage', 'driver_name', + 'driver_version', 'hostname', 'port', 'protocol_version', 'request_count', 'ssl_cipher_suite', + 'ssl_enabled', 'ssl_protocol', 'username']); } - }else{ - // 4.0 - assert.deepEqual(table.columns.map(c => c.name), ['address', 'connection_stage', 'driver_name', - 'driver_version', 'hostname', 'port', 'protocol_version', 'request_count', 'ssl_cipher_suite', - 'ssl_enabled', 'ssl_protocol', 'username']); - } - assert.deepEqual(table.clusteringOrder, ['ASC']); - assert.deepEqual(table.partitionKeys.map(c => c.name), ['address']); - assert.deepEqual(table.clusteringKeys.map(c => c.name), ['port']); - }); + }); + } else { + // lower than 4.0, skip + this.skip(); + } + }); it('should retrieve the updated metadata after a schema change', function (done) { const client = newInstance(); From 5e91c298fbc8da14827fb6028ca4c14ca8392723 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 11 Sep 2024 18:33:34 +0000 Subject: [PATCH 21/99] eslint --- test/integration/short/metadata-tests.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/integration/short/metadata-tests.js b/test/integration/short/metadata-tests.js index 4b20d7d9..9941e1a3 100644 --- a/test/integration/short/metadata-tests.js +++ b/test/integration/short/metadata-tests.js @@ -1057,11 +1057,8 @@ describe('metadata @SERVER_API', function () { 'ssl_enabled', 'ssl_protocol', 'username']); } }); - } else { - // lower than 4.0, skip - this.skip(); - } - + } + // else, lower than 4.0, skip }); it('should retrieve the updated metadata after a schema change', function (done) { const client = newInstance(); From 51a00a95ce5680660091ac40c7da11b2258f1759 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Wed, 11 Sep 2024 21:27:04 -0800 Subject: [PATCH 22/99] Update Jenkinsfile --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 691ce47b..6f297945 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -331,12 +331,12 @@ pipeline { axis { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra - '4.0', // Previous Apache Cassandra + // '4.0', // Previous Apache Cassandra '4.1.0', // Previous Apache Cassandra - '5.0-beta1' + '5.0-beta1', // 'dse-5.1.35', // Legacy DataStax Enterprise - // 'dse-6.8.30', // Development DataStax Enterprise - // 'dse-6.9.0' // Current DataStax Enterprise + 'dse-6.8.30', // Development DataStax Enterprise + 'dse-6.9.0' // Current DataStax Enterprise } axis { name 'NODEJS_VERSION' From 72e6fa8e49742951411aab9675a1f6f46bdefaa0 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 13 Sep 2024 18:40:45 +0000 Subject: [PATCH 23/99] c45-working --- test/integration/short/geometry/polygon-tests.js | 7 ++++++- test/test-helper.js | 14 ++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/test/integration/short/geometry/polygon-tests.js b/test/integration/short/geometry/polygon-tests.js index 6fd81830..41715487 100644 --- a/test/integration/short/geometry/polygon-tests.js +++ b/test/integration/short/geometry/polygon-tests.js @@ -130,7 +130,12 @@ vdescribe('dse-5.0', 'Polygon @SERVER_API', function () { normalizedCoordinates.push([c[0], c[2], c[1], c[3]]); } } - assert.deepEqual(normalizedCoordinates, polygon.toJSON().coordinates); + if (helper.isDseGreaterThan('6.9')) { + //TODO: find documentation + assert.deepEqual(value, polygon.toJSON()); + }else{ + assert.deepEqual(normalizedCoordinates, polygon.toJSON().coordinates); + } eachNext(); }); }); diff --git a/test/test-helper.js b/test/test-helper.js index 1a30159d..413acb05 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -41,7 +41,8 @@ const cassandraVersionByDse = { '5.1': '3.11', '6.0': '3.11', '6.7': '3.11', - '6.8': '3.11' + '6.8': '3.11', + '6.9': '3.11' }; const afterNextHandlers = []; @@ -104,7 +105,11 @@ const helper = { after(client.shutdown.bind(client)); } if (options.removeClusterAfter !== false) { - after(helper.ccmHelper.remove); + after(function (callback) { + if (this.currentTest.state !== 'failed') { + helper.ccmHelper.remove(callback); + } + }); } return { @@ -415,7 +420,8 @@ const helper = { getServerInfo: function () { return { version: process.env['CCM_VERSION'] || '3.11.4', - isDse: process.env['CCM_IS_DSE'] === 'true' + isDse: process.env['CCM_IS_DSE'] === 'true', + isHcd: process.env['CCM_IS_HCD'] === 'true' }; }, @@ -1346,7 +1352,7 @@ helper.ccm.spawn = function (processName, params, callback) { }; helper.ccm.remove = function (callback) { - helper.ccm.exec(['remove'], callback); + // helper.ccm.exec(['remove'], callback); }; helper.ccm.removeIfAny = function (callback) { From ffd0ded13babc0050b03ce18fbb1bf7bfdf84912 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 13 Sep 2024 22:24:26 +0000 Subject: [PATCH 24/99] add hcd --- Jenkinsfile | 19 +++++++++++--- test/test-helper.js | 61 ++++++++++++++++++++++++++++++++------------- 2 files changed, 59 insertions(+), 21 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 691ce47b..d4a29a05 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,13 +41,25 @@ CCM_CASSANDRA_VERSION=${DSE_FIXED_VERSION} # maintain for backwards compatibilit CCM_VERSION=${DSE_FIXED_VERSION} CCM_SERVER_TYPE=dse DSE_VERSION=${DSE_FIXED_VERSION} -CCM_IS_DSE=true +CCM_DISTRIBUTION=dse CCM_BRANCH=${DSE_FIXED_VERSION} DSE_BRANCH=${DSE_FIXED_VERSION} ENVIRONMENT_EOF ''' } + if (env.CASSANDRA_VERSION.split('-')[0] == 'hcd'){ + env.HCD_FIXED_VERSION = env.CASSANDRA_VERSION.split('-')[1] + sh label: 'Update environment for HCD', script: '''#!/bin/bash -le + cat >> ${HOME}/environment.txt << ENVIRONMENT_EOF +CCM_PATH=${HOME}/ccm +CCM_CASSANDRA_VERSION=${HCD_FIXED_VERSION} # maintain for backwards compatibility +CCM_VERSION=${HCD_FIXED_VERSION} +CCM_DISTRIBUTION=hcd +ENVIRONMENT_EOF + ''' + } + sh label: 'Display Node.js and environment information', script: '''#!/bin/bash -le # Load CCM environment variables set -o allexport @@ -332,11 +344,12 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra - '4.1.0', // Previous Apache Cassandra - '5.0-beta1' + // '4.1.0', // Previous Apache Cassandra + '5.0-beta1', // 'dse-5.1.35', // Legacy DataStax Enterprise // 'dse-6.8.30', // Development DataStax Enterprise // 'dse-6.9.0' // Current DataStax Enterprise + 'hcd-1.0.0' // HCD } axis { name 'NODEJS_VERSION' diff --git a/test/test-helper.js b/test/test-helper.js index 413acb05..88c77e67 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -45,6 +45,10 @@ const cassandraVersionByDse = { '6.9': '3.11' }; +const cassandraVersionByHcd = { + '1.0': '4.0', +} + const afterNextHandlers = []; let testUnhandledError = null; @@ -81,7 +85,7 @@ const helper = { * @param {String} [options.keyspace] Name of the keyspace to create. * @param {Number} [options.replicationFactor] Keyspace replication factor. * @param {Array} [options.queries] Queries to run after client creation. - * @param {Boolean} [options.removeClusterAfter=false] Determines whether ccm remove should be called on after(). + * @param {Boolean} [options.removeClusterAfter=true] Determines whether ccm remove should be called on after(). */ setup: function (nodeLength, options) { options = options || utils.emptyObject; @@ -104,14 +108,14 @@ const helper = { } after(client.shutdown.bind(client)); } - if (options.removeClusterAfter !== false) { - after(function (callback) { - if (this.currentTest.state !== 'failed') { - helper.ccmHelper.remove(callback); - } - }); - } - + after(function (callback) { + if (this.currentTest?.state !== 'failed') { + console.log('Test status: ', this.currentTest.state); + helper.ccmHelper.remove(callback); + }else{ + callback(); + } + }); return { client: client, keyspace: keyspace @@ -400,28 +404,34 @@ const helper = { /** * Gets the Apache Cassandra version. - * When the server is DSE, gets the Apache Cassandra equivalent. + * When the server is DSE/HCD, gets the Apache Cassandra equivalent. */ getCassandraVersion: function () { const serverInfo = this.getServerInfo(); - if (!serverInfo.isDse) { + if (serverInfo.distribution === 'cassandra') { return serverInfo.version; } - - const dseVersion = serverInfo.version.split('.').slice(0, 2).join('.'); - return cassandraVersionByDse[dseVersion] || cassandraVersionByDse['6.7']; + const literalVersion = serverInfo.version.split('.').slice(0, 2).join('.'); + if (serverInfo.distribution === 'hcd') { + return cassandraVersionByHcd[literalVersion] || cassandraVersionByHcd['1.0']; + } + if (serverInfo.distribution === 'dse') { + return cassandraVersionByDse[literalVersion] || cassandraVersionByDse['6.7']; + } + throw new Error('Unknown distribution ' + serverInfo.distribution); }, /** * Gets the server version and type. - * @return {{version, isDse}} + * @return {{version: String, isDse: Boolean, isHcd: Boolean, distribution: String}} */ getServerInfo: function () { return { version: process.env['CCM_VERSION'] || '3.11.4', - isDse: process.env['CCM_IS_DSE'] === 'true', - isHcd: process.env['CCM_IS_HCD'] === 'true' + isDse: process.env['CCM_DISTRIBUTION'] === 'dse', + isHcd: process.env['CCM_DISTRIBUTION'] === 'hcd', + distribution: process.env['CCM_DISTRIBUTION'] || 'cassandra' }; }, @@ -456,6 +466,11 @@ const helper = { return this.getServerInfo().isDse; }, + /** Determines if the current server is a HCD instance. */ + isHcd: function () { + return this.getServerInfo().isHcd; + }, + /** * Determines if the current C* or DSE instance version is greater than or equals to the C* version provided * @param {String} version The version in string format, dot separated. @@ -1136,6 +1151,10 @@ helper.ccm.startAll = function (nodeLength, options, callback) { create.push('--dse'); } + if(serverInfo.isHcd) { + create.push('--hcd'); + } + create.push('-v', serverInfo.version); if (process.env['CCM_INSTALL_DIR']) { @@ -1240,6 +1259,10 @@ helper.ccm.bootstrapNode = function (options, callback) { ccmArgs.push('--dse'); } + if (helper.getServerInfo().isHcd) { + ccmArgs.push('--hcd'); + } + if (options.dc) { ccmArgs.push('-d', options.dc); } @@ -1352,10 +1375,12 @@ helper.ccm.spawn = function (processName, params, callback) { }; helper.ccm.remove = function (callback) { - // helper.ccm.exec(['remove'], callback); + console.log("helper.ccm.remove"); + helper.ccm.exec(['remove'], callback); }; helper.ccm.removeIfAny = function (callback) { + console.log("helper.ccm.removeIfAny"); helper.ccm.exec(['remove'], function () { // Ignore errors callback(); From 824eca65ea16bc507bcdbcae8010f73c8379b445 Mon Sep 17 00:00:00 2001 From: janehe Date: Sat, 14 Sep 2024 00:26:39 +0000 Subject: [PATCH 25/99] eslint --- test/test-helper.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/test-helper.js b/test/test-helper.js index 88c77e67..9d7add09 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -47,7 +47,7 @@ const cassandraVersionByDse = { const cassandraVersionByHcd = { '1.0': '4.0', -} +}; const afterNextHandlers = []; let testUnhandledError = null; @@ -109,8 +109,7 @@ const helper = { after(client.shutdown.bind(client)); } after(function (callback) { - if (this.currentTest?.state !== 'failed') { - console.log('Test status: ', this.currentTest.state); + if (this.currentTest && this.currentTest.state !== 'failed') { helper.ccmHelper.remove(callback); }else{ callback(); @@ -1375,12 +1374,10 @@ helper.ccm.spawn = function (processName, params, callback) { }; helper.ccm.remove = function (callback) { - console.log("helper.ccm.remove"); helper.ccm.exec(['remove'], callback); }; helper.ccm.removeIfAny = function (callback) { - console.log("helper.ccm.removeIfAny"); helper.ccm.exec(['remove'], function () { // Ignore errors callback(); From 3873621f6c8d18a60ede3f4161e32b591ea34ce8 Mon Sep 17 00:00:00 2001 From: janehe Date: Sat, 14 Sep 2024 07:06:43 +0000 Subject: [PATCH 26/99] extend wait time --- test/integration/short/control-connection-tests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/short/control-connection-tests.js b/test/integration/short/control-connection-tests.js index e2b3d260..da3b4cb6 100644 --- a/test/integration/short/control-connection-tests.js +++ b/test/integration/short/control-connection-tests.js @@ -121,7 +121,7 @@ describe('ControlConnection', function () { // While the host is started, it's not a given that it will have been connected and marked up, // wait for that to be the case. - await helper.wait.forNodeToBeAdded(cc.hosts, 3); + await helper.wait.forNodeToBeAdded(cc.hosts, 3, 50000, 200); await helper.wait.forNodeUp(cc.hosts, 3); const countUp = cc.hosts.values().reduce((value, host) => value + (host.isUp() ? 1 : 0), 0); From db633d499969816da5e00c28681198a000398ea1 Mon Sep 17 00:00:00 2001 From: janehe Date: Mon, 16 Sep 2024 20:45:25 +0000 Subject: [PATCH 27/99] deleting -j port --- test/test-helper.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/test-helper.js b/test/test-helper.js index 9d7add09..1f912372 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -1249,8 +1249,6 @@ helper.ccm.bootstrapNode = function (options, callback) { 'node' + options.nodeIndex, '-i', ipPrefix + options.nodeIndex, - '-j', - (7000 + 100 * options.nodeIndex).toString(), '-b' ]; @@ -1299,7 +1297,7 @@ helper.ccm.setWorkload = function (nodeIndex, workloads, callback) { * @param {Function} callback */ helper.ccm.startNode = function (nodeIndex, callback) { - const args = ['node' + nodeIndex, 'start', '--wait-other-notice', '--wait-for-binary-proto']; + const args = ['node' + nodeIndex, 'start', '--wait-for-binary-proto']; if (helper.isWin() && helper.isCassandraGreaterThan('2.2.4')) { args.push('--quiet-windows'); From 70a95e386cf07057c718ebeb8a78836ca9efc800 Mon Sep 17 00:00:00 2001 From: janehe Date: Mon, 16 Sep 2024 21:17:32 +0000 Subject: [PATCH 28/99] change test helper ccm commands to look exactly the same as lukasz's --- test/integration/short/control-connection-tests.js | 2 +- test/test-helper.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/integration/short/control-connection-tests.js b/test/integration/short/control-connection-tests.js index da3b4cb6..e2b3d260 100644 --- a/test/integration/short/control-connection-tests.js +++ b/test/integration/short/control-connection-tests.js @@ -121,7 +121,7 @@ describe('ControlConnection', function () { // While the host is started, it's not a given that it will have been connected and marked up, // wait for that to be the case. - await helper.wait.forNodeToBeAdded(cc.hosts, 3, 50000, 200); + await helper.wait.forNodeToBeAdded(cc.hosts, 3); await helper.wait.forNodeUp(cc.hosts, 3); const countUp = cc.hosts.values().reduce((value, host) => value + (host.isUp() ? 1 : 0), 0); diff --git a/test/test-helper.js b/test/test-helper.js index 1f912372..e00005cd 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -1146,6 +1146,8 @@ helper.ccm.startAll = function (nodeLength, options, callback) { const clusterName = helper.getRandomName('test'); let create = ['create', clusterName]; + create.push('-i', '127.0.0.'); + if (serverInfo.isDse) { create.push('--dse'); } @@ -1249,7 +1251,6 @@ helper.ccm.bootstrapNode = function (options, callback) { 'node' + options.nodeIndex, '-i', ipPrefix + options.nodeIndex, - '-b' ]; if (helper.getServerInfo().isDse) { From ffcc2409c8b7c5c8ba9fa123e222e9332efa5fdd Mon Sep 17 00:00:00 2001 From: janehe Date: Tue, 17 Sep 2024 02:01:11 +0000 Subject: [PATCH 29/99] wait for longer --- test/integration/short/control-connection-tests.js | 6 +++--- test/test-helper.js | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/integration/short/control-connection-tests.js b/test/integration/short/control-connection-tests.js index e2b3d260..9e7a05a5 100644 --- a/test/integration/short/control-connection-tests.js +++ b/test/integration/short/control-connection-tests.js @@ -122,7 +122,7 @@ describe('ControlConnection', function () { // While the host is started, it's not a given that it will have been connected and marked up, // wait for that to be the case. await helper.wait.forNodeToBeAdded(cc.hosts, 3); - await helper.wait.forNodeUp(cc.hosts, 3); + await helper.wait.forNodeUp(cc.hosts, 3, 5000, 200); const countUp = cc.hosts.values().reduce((value, host) => value + (host.isUp() ? 1 : 0), 0); assert.strictEqual(countUp, 3); @@ -220,7 +220,7 @@ describe('ControlConnection', function () { // restart node 2 and make sure it comes up. await util.promisify(helper.ccmHelper.startNode)(2); - await helper.wait.forNodeUp(cc.hosts, 2); + await helper.wait.forNodeUp(cc.hosts, 2, 5000, 200); // check that host 1 is down, host 2 is up and the control connection is to host 2. cc.hosts.forEach(h => { @@ -232,7 +232,7 @@ describe('ControlConnection', function () { } }); - await helper.wait.until(() => cc.host); + await helper.wait.until(() => cc.host, 5000, 200); assert.strictEqual(helper.lastOctetOf(cc.host), '2'); }); diff --git a/test/test-helper.js b/test/test-helper.js index e00005cd..9d7add09 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -1146,8 +1146,6 @@ helper.ccm.startAll = function (nodeLength, options, callback) { const clusterName = helper.getRandomName('test'); let create = ['create', clusterName]; - create.push('-i', '127.0.0.'); - if (serverInfo.isDse) { create.push('--dse'); } @@ -1251,6 +1249,9 @@ helper.ccm.bootstrapNode = function (options, callback) { 'node' + options.nodeIndex, '-i', ipPrefix + options.nodeIndex, + '-j', + (7000 + 100 * options.nodeIndex).toString(), + '-b' ]; if (helper.getServerInfo().isDse) { @@ -1298,7 +1299,7 @@ helper.ccm.setWorkload = function (nodeIndex, workloads, callback) { * @param {Function} callback */ helper.ccm.startNode = function (nodeIndex, callback) { - const args = ['node' + nodeIndex, 'start', '--wait-for-binary-proto']; + const args = ['node' + nodeIndex, 'start', '--wait-other-notice', '--wait-for-binary-proto']; if (helper.isWin() && helper.isCassandraGreaterThan('2.2.4')) { args.push('--quiet-windows'); From d67111aa1c7cc764c07af297ef4f26fdc960289e Mon Sep 17 00:00:00 2001 From: janehe Date: Tue, 17 Sep 2024 02:38:30 +0000 Subject: [PATCH 30/99] forNodeToBeAdded instead of forNodeUp --- test/integration/short/control-connection-tests.js | 4 ++-- test/test-helper.js | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/test/integration/short/control-connection-tests.js b/test/integration/short/control-connection-tests.js index 9e7a05a5..b4810216 100644 --- a/test/integration/short/control-connection-tests.js +++ b/test/integration/short/control-connection-tests.js @@ -121,8 +121,8 @@ describe('ControlConnection', function () { // While the host is started, it's not a given that it will have been connected and marked up, // wait for that to be the case. - await helper.wait.forNodeToBeAdded(cc.hosts, 3); - await helper.wait.forNodeUp(cc.hosts, 3, 5000, 200); + await helper.wait.forNodeToBeAdded(cc.hosts, 3, 5000, 200); + await helper.wait.forNodeUp(cc.hosts, 3); const countUp = cc.hosts.values().reduce((value, host) => value + (host.isUp() ? 1 : 0), 0); assert.strictEqual(countUp, 3); diff --git a/test/test-helper.js b/test/test-helper.js index 9d7add09..e00005cd 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -1146,6 +1146,8 @@ helper.ccm.startAll = function (nodeLength, options, callback) { const clusterName = helper.getRandomName('test'); let create = ['create', clusterName]; + create.push('-i', '127.0.0.'); + if (serverInfo.isDse) { create.push('--dse'); } @@ -1249,9 +1251,6 @@ helper.ccm.bootstrapNode = function (options, callback) { 'node' + options.nodeIndex, '-i', ipPrefix + options.nodeIndex, - '-j', - (7000 + 100 * options.nodeIndex).toString(), - '-b' ]; if (helper.getServerInfo().isDse) { @@ -1299,7 +1298,7 @@ helper.ccm.setWorkload = function (nodeIndex, workloads, callback) { * @param {Function} callback */ helper.ccm.startNode = function (nodeIndex, callback) { - const args = ['node' + nodeIndex, 'start', '--wait-other-notice', '--wait-for-binary-proto']; + const args = ['node' + nodeIndex, 'start', '--wait-for-binary-proto']; if (helper.isWin() && helper.isCassandraGreaterThan('2.2.4')) { args.push('--quiet-windows'); From 441a3a884d755ad67cf53f353658f979b433ca81 Mon Sep 17 00:00:00 2001 From: janehe Date: Tue, 17 Sep 2024 05:49:11 +0000 Subject: [PATCH 31/99] add all distributions to jenkinsfil --- Jenkinsfile | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d4a29a05..d6487f80 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -344,11 +344,11 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra - // '4.1.0', // Previous Apache Cassandra - '5.0-beta1', - // 'dse-5.1.35', // Legacy DataStax Enterprise - // 'dse-6.8.30', // Development DataStax Enterprise - // 'dse-6.9.0' // Current DataStax Enterprise + '4.1.0', // Previous Apache Cassandra + '5.0-beta1', + 'dse-5.1.35', // Legacy DataStax Enterprise + 'dse-6.8.30', // Development DataStax Enterprise + 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD } axis { @@ -439,10 +439,11 @@ pipeline { axes { axis { name 'CASSANDRA_VERSION' - values '3.11', // Current Apache Cassandra - '4.0', // Development Apache Cassandra - 'dse-5.1.35', // Legacy DataStax Enterprise - 'dse-6.8.30' // Development DataStax Enterprise + values '3.11', // Previous Apache Cassandra + '4.1.0', // Previous Apache Cassandra + '5.0-beta1', // Current Apache Cassandra + 'dse-6.9.0', // Current DataStax Enterprise + 'hcd-1.0.0' // HCD } axis { name 'NODEJS_VERSION' @@ -530,12 +531,15 @@ pipeline { axes { axis { name 'CASSANDRA_VERSION' - values '3.11', // Previous Apache Cassandra - '4.0', // Previous Apache Cassandra - '4.1.0', // Previous Apache Cassandra - 'dse-5.1.35', // Previous DataStax Enterprise - 'dse-6.8.30', // Previous DataStax Enterprise - 'dse-6.9.0' // Current DataStax Enterprise + values '3.11', // Previous Apache Cassandra + '4.0', // Previous Apache Cassandra + '4.1.0', // Previous Apache Cassandra + '5.0-beta1', + 'dse-5.1.35', // Legacy DataStax Enterprise + 'dse-6.8.30', // Development DataStax Enterprise + 'dse-6.9.0', // Current DataStax Enterprise + 'hcd-1.0.0' // HCD + } axis { name 'NODEJS_VERSION' From 2618f4203245ccb00ccb8803defd45adaea8ee3a Mon Sep 17 00:00:00 2001 From: janehe Date: Tue, 17 Sep 2024 05:50:00 +0000 Subject: [PATCH 32/99] method too large? --- Jenkinsfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d6487f80..002576ec 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -343,11 +343,8 @@ pipeline { axis { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra - '4.0', // Previous Apache Cassandra '4.1.0', // Previous Apache Cassandra '5.0-beta1', - 'dse-5.1.35', // Legacy DataStax Enterprise - 'dse-6.8.30', // Development DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD } From d062967aac9be52b57b1bbbbd555a9931f184a8a Mon Sep 17 00:00:00 2001 From: janehe Date: Tue, 17 Sep 2024 05:52:12 +0000 Subject: [PATCH 33/99] method too large2 --- Jenkinsfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 002576ec..a879c42a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -342,9 +342,8 @@ pipeline { axes { axis { name 'CASSANDRA_VERSION' - values '3.11', // Previous Apache Cassandra - '4.1.0', // Previous Apache Cassandra - '5.0-beta1', + values '4.1.0', // Previous Apache Cassandra + '5.0-beta1', // Current Apache Cassandra 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD } From 44ca6afe5ccf09f9de10943db4700ceb6425421c Mon Sep 17 00:00:00 2001 From: janehe Date: Tue, 17 Sep 2024 05:53:12 +0000 Subject: [PATCH 34/99] method too large3 --- Jenkinsfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a879c42a..d3c4c94c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -527,12 +527,8 @@ pipeline { axes { axis { name 'CASSANDRA_VERSION' - values '3.11', // Previous Apache Cassandra - '4.0', // Previous Apache Cassandra - '4.1.0', // Previous Apache Cassandra + values '4.1.0', // Previous Apache Cassandra '5.0-beta1', - 'dse-5.1.35', // Legacy DataStax Enterprise - 'dse-6.8.30', // Development DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD From 653a30fa980a7768d766cefb2f125b14c3566b70 Mon Sep 17 00:00:00 2001 From: janehe Date: Tue, 17 Sep 2024 05:54:18 +0000 Subject: [PATCH 35/99] at least add 3.11 --- Jenkinsfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d3c4c94c..a91a2fb0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -342,7 +342,8 @@ pipeline { axes { axis { name 'CASSANDRA_VERSION' - values '4.1.0', // Previous Apache Cassandra + values '3.11', // Previous Apache Cassandra + '4.1.0', // Previous Apache Cassandra '5.0-beta1', // Current Apache Cassandra 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD @@ -527,8 +528,9 @@ pipeline { axes { axis { name 'CASSANDRA_VERSION' - values '4.1.0', // Previous Apache Cassandra - '5.0-beta1', + values '3.11', // Previous Apache Cassandra + '4.1.0', // Previous Apache Cassandra + '5.0-beta1', // Current Apache Cassandra 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD From 845b0fadafbf89bf31d5e0f56bd31db0f9829615 Mon Sep 17 00:00:00 2001 From: janehe Date: Tue, 17 Sep 2024 06:01:28 +0000 Subject: [PATCH 36/99] delete some scaffold code --- test/test-helper.js | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/test/test-helper.js b/test/test-helper.js index e00005cd..c342d1e3 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -108,13 +108,9 @@ const helper = { } after(client.shutdown.bind(client)); } - after(function (callback) { - if (this.currentTest && this.currentTest.state !== 'failed') { - helper.ccmHelper.remove(callback); - }else{ - callback(); - } - }); + if (options.removeClusterAfter !== false) { + after(helper.ccmHelper.remove); + } return { client: client, keyspace: keyspace @@ -1146,8 +1142,6 @@ helper.ccm.startAll = function (nodeLength, options, callback) { const clusterName = helper.getRandomName('test'); let create = ['create', clusterName]; - create.push('-i', '127.0.0.'); - if (serverInfo.isDse) { create.push('--dse'); } @@ -1251,6 +1245,9 @@ helper.ccm.bootstrapNode = function (options, callback) { 'node' + options.nodeIndex, '-i', ipPrefix + options.nodeIndex, + '-j', + (7000 + 100 * options.nodeIndex).toString(), + '-b' ]; if (helper.getServerInfo().isDse) { @@ -1324,8 +1321,6 @@ helper.ccm.resumeNode = function (nodeIndex, callback) { }; helper.ccm.exec = function (params, callback) { - // eslint-disable-next-line no-console, no-undef - console.log("Executing ccm command: ", params); helper.ccm.spawn('ccm', params, callback); }; From fcd0144b9d2ab829c5980b101c5813fee2884e8d Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 19 Sep 2024 06:56:01 +0000 Subject: [PATCH 37/99] add nodenv install -list --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index afa02be1..cb30ed4e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -72,6 +72,7 @@ def installDriverAndDependencies() { def executeLinter() { sh label: 'Perform static analysis of source code', script: '''#!/bin/bash -lex npm run eslint + nodenv install -list ''' } From 6b42d4c436f9ce73539b0bf71e198b469de7dc89 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 19 Sep 2024 07:03:26 +0000 Subject: [PATCH 38/99] delete node v16 --- Jenkinsfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fb2f2af4..3171830a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ def initializeEnvironment() { - def nodeVersions = ['16': '16.20.2', '18': '18.17.1', '20': '20.5.1'] + def nodeVersions = ['18': '18.17.1', '20': '20.5.1'] env.DRIVER_DISPLAY_NAME = 'Cassandra Node.js Driver' env.DRIVER_METRIC_TYPE = 'oss' if (env.GIT_URL.contains('riptano/nodejs-driver')) { @@ -87,7 +87,6 @@ def installDriverAndDependencies() { def executeLinter() { sh label: 'Perform static analysis of source code', script: '''#!/bin/bash -lex npm run eslint - nodenv install -list ''' } @@ -256,7 +255,7 @@ pipeline {
''') choice( name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_NODEJS_VERSION', - choices: ['16', '18', '20', 'ALL'], + choices: ['18', '20', '22', 'ALL'], description: 'Node.js version to use for adhoc BUILD-AND-EXECUTE-TESTS ONLY!') choice( name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_SERVER_VERSION', @@ -351,7 +350,7 @@ pipeline { } axis { name 'NODEJS_VERSION' - values '16', '18', '20' + values '18', '20' } } @@ -445,7 +444,7 @@ pipeline { } axis { name 'NODEJS_VERSION' - values '16', '18', '20' + values '18', '20' } } @@ -538,7 +537,7 @@ pipeline { } axis { name 'NODEJS_VERSION' - values '16', '18', '20' + values '18', '20' } } when { From 90a3722f7ea4714e6955870e0f3587c1dee0fb81 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 19 Sep 2024 20:53:44 +0000 Subject: [PATCH 39/99] update jenkinsfile for node 22 --- Jenkinsfile | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3171830a..6daa961c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ def initializeEnvironment() { - def nodeVersions = ['18': '18.17.1', '20': '20.5.1'] + def nodeVersions = ['18': '18.20.4', '20': '20.17.0', '22': '22.8.0'] env.DRIVER_DISPLAY_NAME = 'Cassandra Node.js Driver' env.DRIVER_METRIC_TYPE = 'oss' if (env.GIT_URL.contains('riptano/nodejs-driver')) { @@ -262,9 +262,12 @@ pipeline { choices: [ '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra + '4.1.0', // Previous Apache Cassandra + '5.0-beta1', // Current Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.8.30', // Previoius DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise + 'hcd-1.0.0', // HCD 'ALL'], description: '''Apache Cassandra and DataStax Enterprise server version to use for adhoc BUILD-AND-EXECUTE-TESTS ONLY! @@ -280,7 +283,15 @@ pipeline { - + + + + + + + + + @@ -288,7 +299,15 @@ pipeline { - + + + + + + + + +
4.0Apache Cassandra v4.x (CURRENTLY UNDER DEVELOPMENT)Apache Cassandra v4.0
4.1Apache Cassandra v4.1
5.0-beta1Apache Cassandra v5.0
dse-5.1
dse-6.8DataStax Enterprise v6.8.x (CURRENTLY UNDER DEVELOPMENT)DataStax Enterprise v6.8
dse-6.9DataStax Enterprise v6.9
hcd-1.0.0Hyper-Converged Database v1.0.0
''') booleanParam( @@ -345,12 +364,13 @@ pipeline { values '3.11', // Previous Apache Cassandra '4.1.0', // Previous Apache Cassandra '5.0-beta1', // Current Apache Cassandra + 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD } axis { name 'NODEJS_VERSION' - values '18', '20' + values '18', '20', '22' } } @@ -439,12 +459,13 @@ pipeline { values '3.11', // Previous Apache Cassandra '4.1.0', // Previous Apache Cassandra '5.0-beta1', // Current Apache Cassandra + 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD } axis { name 'NODEJS_VERSION' - values '18', '20' + values '18', '20', '22' } } @@ -531,13 +552,14 @@ pipeline { values '3.11', // Previous Apache Cassandra '4.1.0', // Previous Apache Cassandra '5.0-beta1', // Current Apache Cassandra + 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD } axis { name 'NODEJS_VERSION' - values '18', '20' + values '18', '20', '22' } } when { From 7bfdb9d23c00f308999daf73d71dddfe9f7ec62a Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 19 Sep 2024 21:52:58 +0000 Subject: [PATCH 40/99] del description --- Jenkinsfile | 42 +----------------------------------------- 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6daa961c..c0b34a50 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -269,47 +269,7 @@ pipeline { 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0', // HCD 'ALL'], - description: '''Apache Cassandra and DataStax Enterprise server version to use for adhoc BUILD-AND-EXECUTE-TESTS ONLY! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ChoiceDescription
3.11Apache Cassandra v3.11.x
4.0Apache Cassandra v4.0
4.1Apache Cassandra v4.1
5.0-beta1Apache Cassandra v5.0
dse-5.1DataStax Enterprise v5.1.x
dse-6.8DataStax Enterprise v6.8
dse-6.9DataStax Enterprise v6.9
hcd-1.0.0Hyper-Converged Database v1.0.0
''') + description: '''Apache Cassandra and DataStax Enterprise server version to use for adhoc BUILD-AND-EXECUTE-TESTS ONLY!''') booleanParam( name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_EXECUTE_EXAMPLES', defaultValue: false, From 54ae08b08d8cdeaca89055af0664bda27fd864e3 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 19 Sep 2024 21:55:20 +0000 Subject: [PATCH 41/99] comment out dse 6.8 --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c0b34a50..696f5670 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -324,7 +324,7 @@ pipeline { values '3.11', // Previous Apache Cassandra '4.1.0', // Previous Apache Cassandra '5.0-beta1', // Current Apache Cassandra - 'dse-6.8.30', // Previous DataStax Enterprise + // 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD } From 4155270f0a0b796e49059f9b0b580e1df7617716 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 19 Sep 2024 22:08:03 +0000 Subject: [PATCH 42/99] delete notify slack --- Jenkinsfile | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 696f5670..7cb2c9bc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -386,20 +386,6 @@ pipeline { } } } - post { - aborted { - notifySlack('aborted') - } - success { - notifySlack('completed') - } - unstable { - notifySlack('unstable') - } - failure { - notifySlack('FAILED') - } - } } stage('Scheduled-Testing') { From 54633fe2dc43806a6e9a1591532dfa191deb8668 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 19 Sep 2024 22:15:43 +0000 Subject: [PATCH 43/99] uncomment 6.8.30 --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7cb2c9bc..c4b30472 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -324,7 +324,7 @@ pipeline { values '3.11', // Previous Apache Cassandra '4.1.0', // Previous Apache Cassandra '5.0-beta1', // Current Apache Cassandra - // 'dse-6.8.30', // Previous DataStax Enterprise + 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD } From cd583f48f1d784c44682e60824fbf44789422432 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 20 Sep 2024 01:07:26 +0000 Subject: [PATCH 44/99] add back notify slack --- Jenkinsfile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c4b30472..5aad26ec 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -378,7 +378,7 @@ pipeline { } stage('Execute-Examples') { when { - expression { env.CASSANDRA_VERSION == 'dse-6.8.30' } + expression { env.CASSANDRA_VERSION == 'dse-6.8.30' } } steps { executeExamples() @@ -386,6 +386,20 @@ pipeline { } } } + post { + aborted { + notifySlack('aborted') + } + success { + notifySlack('completed') + } + unstable { + notifySlack('unstable') + } + failure { + notifySlack('FAILED') + } + } } stage('Scheduled-Testing') { From d9ea8f587d2af98c65ad655f87856c2ad36ee09d Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 20 Sep 2024 01:12:39 +0000 Subject: [PATCH 45/99] delete notifyslack when success or unstable --- Jenkinsfile | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5aad26ec..8dfbb2ca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -390,12 +390,6 @@ pipeline { aborted { notifySlack('aborted') } - success { - notifySlack('completed') - } - unstable { - notifySlack('unstable') - } failure { notifySlack('FAILED') } @@ -485,12 +479,6 @@ pipeline { aborted { notifySlack('aborted') } - success { - notifySlack('completed') - } - unstable { - notifySlack('unstable') - } failure { notifySlack('FAILED') } From 23b267bff8b8132081930eae2b0bb88059cff8ec Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 24 Sep 2024 16:53:41 -0800 Subject: [PATCH 46/99] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6fbfd049..de2c1f43 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "types": "./index.d.ts", "dependencies": { "@types/long": "~5.0.0", - "@types/node": ">=8", + "@types/node": "^18.20.4", "adm-zip": "~0.5.10", "long": "~5.2.3" }, From 0b32897a5dbbf17f01a1c92d13844a5722da57ad Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 25 Sep 2024 01:31:59 +0000 Subject: [PATCH 47/99] typing not working, annoying --- test/integration/short/vector-tests.js | 51 ++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 test/integration/short/vector-tests.js diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js new file mode 100644 index 00000000..bbc869ff --- /dev/null +++ b/test/integration/short/vector-tests.js @@ -0,0 +1,51 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; +const assert = require('assert'); +const util = require('util'); +const helper = require('../../test-helper.js'); +const Client = require('../../../lib/client.js'); +const ExecutionProfile = require('../../../lib/execution-profile.js').ExecutionProfile; + +const types = require('../../../lib/types/index.js'); +const utils = require('../../../lib/utils.js'); +const errors = require('../../../lib/errors.js'); +const vit = helper.vit; +const vdescribe = helper.vdescribe; +const numericTests = require('./numeric-tests.js'); +const pagingTests = require('./paging-tests.js'); + +vdescribe('5.0.0', 'Vector tests', function () { + this.timeout(120000); + describe('#execute with vectors', function () { + const keyspace = helper.getRandomName('ks'); + const table = keyspace + '.' + helper.getRandomName('table'); + const createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY, v1 vector)`; + + const setupInfo = helper.setup(1, { + keyspace: keyspace, + queries: [ createTableCql ] + }); + it('should insert and select vectors', function(done){ + const client = setupInfo.client; + // if client undefined, raise error + if(!client) return done(new Error('client is not defined')); + const id = types.types.Uuid.random(); + const v1 = new types.Vector([1.1, 2.2, 3.3]); + done(); + }); + }); +}); \ No newline at end of file From 37255d9e055db02c57c8aabccfa2016ebda9ac89 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 24 Sep 2024 19:44:51 -0800 Subject: [PATCH 48/99] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index de2c1f43..7b116e2f 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "types": "./index.d.ts", "dependencies": { "@types/long": "~5.0.0", - "@types/node": "^18.20.4", + "@types/node": "^18.11.18", "adm-zip": "~0.5.10", "long": "~5.2.3" }, From ec4d008aa69950379c8b8ad0e2d6c5d48b888cb3 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 25 Sep 2024 05:15:08 +0000 Subject: [PATCH 49/99] most simple vector integration test working --- lib/encoder.js | 4 ++-- test/integration/short/vector-tests.js | 22 +++++++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index 74219f77..d7593631 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -989,7 +989,7 @@ function defineInstanceMembers() { const argsLength = typeName.length - (stringToExclude.length + 2); const params = parseParams(typeName, argsStartIndex, argsLength); if (params.length === 2) { - return {subtype: subtypeResolveFn(params[0]), dimensions: parseInt(params[1], 10)}; + return {subtype: subtypeResolveFn(params[0].trim()), dimensions: parseInt(params[1].trim(), 10)}; } throw new TypeError('Not a valid type ' + typeName); }; @@ -1938,7 +1938,7 @@ function encodeDateRange(value) { * @param {Number} length * @param {String} [open] * @param {String} [close] - * @returns {Array} + * @returns {Array} * @private */ function parseParams(value, startIndex, length, open, close) { diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index bbc869ff..b49ac40f 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -20,7 +20,7 @@ const helper = require('../../test-helper.js'); const Client = require('../../../lib/client.js'); const ExecutionProfile = require('../../../lib/execution-profile.js').ExecutionProfile; -const types = require('../../../lib/types/index.js'); +const { types } = require('../../../index.js'); const utils = require('../../../lib/utils.js'); const errors = require('../../../lib/errors.js'); const vit = helper.vit; @@ -33,7 +33,7 @@ vdescribe('5.0.0', 'Vector tests', function () { describe('#execute with vectors', function () { const keyspace = helper.getRandomName('ks'); const table = keyspace + '.' + helper.getRandomName('table'); - const createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY, v1 vector)`; + const createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY, v1 vector);`; const setupInfo = helper.setup(1, { keyspace: keyspace, @@ -43,9 +43,21 @@ vdescribe('5.0.0', 'Vector tests', function () { const client = setupInfo.client; // if client undefined, raise error if(!client) return done(new Error('client is not defined')); - const id = types.types.Uuid.random(); - const v1 = new types.Vector([1.1, 2.2, 3.3]); - done(); + const id = types.Uuid.random(); + const v1 = new Float32Array([1.1, 2.2, 3.3]); + const query = `INSERT INTO ${table} (id, v1) VALUES (${id}, [${v1.toString()}])`; + client.execute(query, [], {}, function(err){ + if (err) return done(err); + client.execute(`SELECT v1 FROM ${table} WHERE id = ?`, [id], { prepare: true }, function(err, result){ + if (err) return done(err); + assert.strictEqual(result.rows.length, 1); + assert.strictEqual(result.rows[0].v1.length, 3); + assert.strictEqual(result.rows[0].v1[0], v1[0]); + assert.strictEqual(result.rows[0].v1[1], v1[1]); + assert.strictEqual(result.rows[0].v1[2], v1[2]); + done(); + }); + }); }); }); }); \ No newline at end of file From e1b3573bb14691a6ff28e1d49834e31f29a6b4d9 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 26 Sep 2024 05:26:22 +0000 Subject: [PATCH 50/99] value?.get not a function --- lib/encoder.js | 37 +++++--- lib/types/index.js | 1 + lib/types/vector.js | 108 +++++++++++++++++++++++ lib/types/vector.ts | 114 +++++++++++++++++++++++++ package.json | 3 +- test/integration/short/vector-tests.js | 6 +- 6 files changed, 255 insertions(+), 14 deletions(-) create mode 100644 lib/types/vector.js create mode 100644 lib/types/vector.ts diff --git a/lib/encoder.js b/lib/encoder.js index d7593631..9effff62 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -26,6 +26,7 @@ const utils = require('./utils'); const token = require('./token'); const { DateRange } = require('./datastax/search'); const geo = require('./geometry'); +const Vector = require('./types/vector'); const Geometry = geo.Geometry; const LineString = geo.LineString; const Point = geo.Point; @@ -128,7 +129,7 @@ const zeroLengthTypesSupported = new Set([ /** * Serializes and deserializes to and from a CQL type and a Javascript Type. * @param {Number} protocolVersion - * @param {ClientOptions} options + * @param {import('./client.js').ClientOptions} options * @constructor */ function Encoder(protocolVersion, options) { @@ -908,6 +909,12 @@ function defineInstanceMembers() { return Buffer.concat(parts, totalLength); }; + /** + * + * @param {Buffer} buffer + * @param {{subtype: String, dimensions: Number}} params + * @returns {import('./types').Vector} + */ this.decodeVector = function(buffer, params) { const subtype = params["subtype"]; const dimensions = params["dimensions"]; @@ -922,20 +929,21 @@ function defineInstanceMembers() { offset = i * elemLength; rv[i] = this.decode(buffer.slice(offset, offset + elemLength), subtype); } - return new Float32Array(rv); + return new Vector(rv); }; /** - * @param {CqlVector} value - * @param {Object} params + * @param {Vector} value + * @param {{subtype: any, dimensions: Number}} params + * @returns {Buffer} */ this.encodeVector = function(value, params) { // Evaluate params to encodeVector(), returning the computed subtype function evalParams() { - if (!(value instanceof Float32Array)) { - throw new TypeError("Driver only supports vectors of 4 byte floating point values"); + if (!(value instanceof Vector)) { + throw new TypeError("Driver only supports Vector type"); } // Perform client-side validation iff we were actually supplied with meaningful type info. In practice @@ -997,7 +1005,7 @@ function defineInstanceMembers() { /** * If not provided, it uses the array of buffers or the parameters and hints to build the routingKey * @param {Array} params - * @param {ExecutionOptions} execOptions + * @param {import('..').ExecutionOptions} execOptions * @param [keys] parameter keys and positions in the params array * @throws TypeError * @internal @@ -1698,7 +1706,7 @@ Encoder.prototype.encode = function (value, typeInfo) { /** * Try to guess the Cassandra type to be stored, based on the javascript value type * @param value - * @returns {{code: number, info: object}|null} + * @returns {{code: number, info: string? }|null} * @ignore * @internal */ @@ -1753,10 +1761,19 @@ Encoder.guessDataType = function (value) { info = customTypeNames.duration; } // Map JS TypedArrays onto vectors - else if (Encoder.isTypedArray(value)) { + else if (value instanceof types.Vector) { code = dataTypes.custom; // TODO: another area that we have to generalize if we ever need to support vector subtypes other than float - info = buildParameterizedCustomType(customTypeNames.vector, [singleTypeNamesByDataType[dataTypes.float], value.length]); + if (value?.get(0)) { + const subtype = this.guessDataType(value.get(0)); + if (subtype?.code) { + info = buildParameterizedCustomType(customTypeNames.vector, [singleTypeNamesByDataType[subtype.code], value.length]); + } else { + throw new TypeError("Cannot guess subtype from element " + value.get(0)); + } + } else { + throw new TypeError("Cannot guess subtype of empty vector"); + } } else if (Array.isArray(value)) { code = dataTypes.list; diff --git a/lib/types/index.js b/lib/types/index.js index 953295a0..3063740a 100644 --- a/lib/types/index.js +++ b/lib/types/index.js @@ -624,6 +624,7 @@ exports.DriverError = errors.DriverError; exports.TimeoutError = TimeoutError; exports.TimeUuid = TimeUuid; exports.Tuple = require('./tuple'); +exports.Vector = require('./vector'); exports.Uuid = Uuid; exports.unset = unset; exports.generateTimestamp = generateTimestamp; diff --git a/lib/types/vector.js b/lib/types/vector.js new file mode 100644 index 00000000..e163ed7e --- /dev/null +++ b/lib/types/vector.js @@ -0,0 +1,108 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +"use strict"; +/** @module types */ +/** + * Creates a new instance of Cql Vector, also compatible with Float32Array. + * @class + */ +var Vector = /** @class */ (function () { + /** + * + * @param {Float32Array | Array} elements + */ + function Vector(elements) { + if (elements instanceof Float32Array) { + this.elements = Array.from(elements); + } + else if (Array.isArray(elements)) { + this.elements = elements; + } + else { + throw new TypeError('Vector must be constructed with a Float32Array or an Array'); + } + if (this.elements.length === 0) { + throw new TypeError('Vector must contain at least one value'); + } + /** + * Returns the number of the elements. + * @type Number + */ + this.length = this.elements.length; + return new Proxy(this, { + get: function (obj, key) { + if (typeof (key) === 'string' && (Number.isInteger(Number(key)))) // key is an index + return obj.elements[key]; + else + return obj[key]; + }, + set: function (obj, key, value) { + if (typeof (key) === 'string' && (Number.isInteger(Number(key)))) // key is an index + return obj.elements[key] = value; + else + return obj[key] = value; + } + }); + } + /** + * Returns the string representation of the vector. + * @returns {string} + */ + Vector.prototype.toString = function () { + return "[".concat(this.elements.toString(), "]"); + }; + /** + * + * @param {number} index + */ + Vector.prototype.at = function (index) { + return this.elements[index]; + }; + /** + * + * @param {...any} elements + * @returns + */ + Vector.of = function () { + var elements = []; + for (var _i = 0; _i < arguments.length; _i++) { + elements[_i] = arguments[_i]; + } + return new Vector(elements); + }; + /** + * ` + * @returns {ArrayIterator} + */ + Vector.prototype[Symbol.iterator] = function () { + return this.elements[Symbol.iterator](); + }; + Object.defineProperty(Vector, Symbol.species, { + /** + * instanceof + */ + get: function () { + return Vector; + }, + enumerable: false, + configurable: true + }); + return Vector; +}()); +Object.defineProperty(Vector, Symbol.hasInstance, { + value: function (i) { return typeof (i) === typeof (Vector) || i instanceof Float32Array; } +}); +module.exports = Vector; diff --git a/lib/types/vector.ts b/lib/types/vector.ts new file mode 100644 index 00000000..2d531550 --- /dev/null +++ b/lib/types/vector.ts @@ -0,0 +1,114 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +"use strict"; + +/** @module types */ + +/** + * Creates a new instance of Cql Vector, also compatible with Float32Array. + * @class + */ + +class Vector implements ArrayLike { + length: number; + elements: any[]; + /** + * + * @param {Float32Array | Array} elements + */ + constructor(elements) { + if (elements instanceof Float32Array) { + this.elements = Array.from(elements); + }else if(Array.isArray(elements)){ + this.elements = elements; + }else{ + throw new TypeError('Vector must be constructed with a Float32Array or an Array'); + } + + if (this.elements.length === 0) { + throw new TypeError('Vector must contain at least one value'); + } + + /** + * Returns the number of the elements. + * @type Number + */ + this.length = this.elements.length; + + return new Proxy(this, { + get: (obj, key) => { + if (typeof(key) === 'string' && (Number.isInteger(Number(key)))) // key is an index + return obj.elements[key] + else + return obj[key] + }, + set: (obj, key, value) => { + if (typeof(key) === 'string' && (Number.isInteger(Number(key)))) // key is an index + return obj.elements[key] = value + else + return obj[key] = value + } + }) + } + + /** + * Returns the string representation of the vector. + * @returns {string} + */ + toString() { + return `[${this.elements.toString()}]`; + } + + /** + * + * @param {number} index + */ + at(index) { + return this.elements[index]; + } + + /** + * + * @param {...any} elements + * @returns + */ + static of(...elements) { + return new Vector(elements); + } + + /** + * ` + * @returns {ArrayIterator} + */ + [Symbol.iterator]() { + return this.elements[Symbol.iterator](); + } + + [n: number]: T; + + /** + * instanceof + */ + static get [Symbol.species]() { + return Vector; + } +} + +Object.defineProperty(Vector, Symbol.hasInstance, { + value: function (i: any) { return typeof(i) === typeof(Vector) || i instanceof Float32Array } +}) +module.exports = Vector; \ No newline at end of file diff --git a/package.json b/package.json index 6fbfd049..e2466273 100644 --- a/package.json +++ b/package.json @@ -21,11 +21,12 @@ "types": "./index.d.ts", "dependencies": { "@types/long": "~5.0.0", - "@types/node": ">=8", + "@types/node": "^18.11.18", "adm-zip": "~0.5.10", "long": "~5.2.3" }, "devDependencies": { + "@types/mocha": "^10.0.8", "chai": "~4.3.8", "kerberos": "~2.0.3", "mocha": "~10.2.0", diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index b49ac40f..9eefc06f 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -45,14 +45,14 @@ vdescribe('5.0.0', 'Vector tests', function () { if(!client) return done(new Error('client is not defined')); const id = types.Uuid.random(); const v1 = new Float32Array([1.1, 2.2, 3.3]); - const query = `INSERT INTO ${table} (id, v1) VALUES (${id}, [${v1.toString()}])`; - client.execute(query, [], {}, function(err){ + const query = `INSERT INTO ${table} (id, v1) VALUES (?, ?)`; + client.execute(query, [id, v1], {prepare : true}, function(err){ if (err) return done(err); client.execute(`SELECT v1 FROM ${table} WHERE id = ?`, [id], { prepare: true }, function(err, result){ if (err) return done(err); assert.strictEqual(result.rows.length, 1); assert.strictEqual(result.rows[0].v1.length, 3); - assert.strictEqual(result.rows[0].v1[0], v1[0]); + assert.strictEqual(result.rows[0].v1[0], v1[0]); assert.strictEqual(result.rows[0].v1[1], v1[1]); assert.strictEqual(result.rows[0].v1[2], v1[2]); done(); From 7f02b84e2145b6db714fd835f90f3fce87520898 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 26 Sep 2024 06:08:51 +0000 Subject: [PATCH 51/99] Skip g.V().count)() related tests on dse6.9.0 --- .../short/graph/graph-olap-tests.js | 18 +++++----- test/integration/short/graph/graph-tests.js | 34 ++++++++++--------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/test/integration/short/graph/graph-olap-tests.js b/test/integration/short/graph/graph-olap-tests.js index 87292fff..857122dd 100644 --- a/test/integration/short/graph/graph-olap-tests.js +++ b/test/integration/short/graph/graph-olap-tests.js @@ -134,15 +134,17 @@ vdescribe('dse-5.0', 'Client with spark workload', function () { {loadBalancing: new DefaultLoadBalancingPolicy(), graphOptions: {source: 'a'}})]}, true) ); context('with no callback specified', function () { - it('should return a promise for OLAP query', function () { - const client = newInstance(); - const p = client.executeGraph('g.V().count()', { graphSource: 'a' }); - helper.assertInstanceOf(p, Promise); - return p.then(function (result) { - helper.assertInstanceOf(result, graphModule.GraphResultSet); - assert.strictEqual(typeof result.first(), 'number'); + // if dse-6.9.0, skip this test because NODEJS-676 and DSP-24336 + (helper.getServerInfo().version === '6.9.0' && helper.getServerInfo().isDse) ? + it.skip : it('should return a promise for OLAP query', function () { + const client = newInstance(); + const p = client.executeGraph('g.V().count()', { graphSource: 'a' }); + helper.assertInstanceOf(p, Promise); + return p.then(function (result) { + helper.assertInstanceOf(result, graphModule.GraphResultSet); + assert.strictEqual(typeof result.first(), 'number'); + }); }); - }); }); }); }); diff --git a/test/integration/short/graph/graph-tests.js b/test/integration/short/graph/graph-tests.js index 55ecada4..0705d5eb 100644 --- a/test/integration/short/graph/graph-tests.js +++ b/test/integration/short/graph/graph-tests.js @@ -449,21 +449,23 @@ vdescribe('dse-5.0', 'Client @SERVER_API', function () { done(); }); })); - it('should retrieve a Int64 scalar', wrapClient(function (client, done) { - const query = JSON.stringify({ - '@type': 'g:Bytecode', - '@value': { - 'step': [["V"], ["count"]] - } - }); - client.executeGraph(query, null, { graphLanguage: 'bytecode-json' }, function (err, result) { - assert.ifError(err); - helper.assertInstanceOf(result, graphModule.GraphResultSet); - const count = result.first(); - helper.assertInstanceOf(count, types.Long); - done(); - }); - })); + // if dse-6.9.0, skip this test because NODEJS-676 and DSP-24336 + (helper.getServerInfo().version === '6.9.0' && helper.getServerInfo().isDse) ? + it.skip : it('should retrieve a Int64 scalar', wrapClient(function (client, done) { + const query = JSON.stringify({ + '@type': 'g:Bytecode', + '@value': { + 'step': [["V"], ["count"]] + } + }); + client.executeGraph(query, null, { graphLanguage: 'bytecode-json' }, function (err, result) { + assert.ifError(err); + helper.assertInstanceOf(result, graphModule.GraphResultSet); + const count = result.first(); + helper.assertInstanceOf(count, types.Long); + done(); + }); + })); it('should allow graph language to be set from the execution profile', wrapClient(function (client, done) { const query = JSON.stringify({ '@type': 'g:Bytecode', @@ -480,7 +482,7 @@ vdescribe('dse-5.0', 'Client @SERVER_API', function () { }); done(); }); - }, { profiles: [ new ExecutionProfile('graph-profile1', { graphOptions: { language: 'bytecode-json' } }) ]})); + }, { profiles: [new ExecutionProfile('graph-profile1', { graphOptions: { language: 'bytecode-json' } })] })); }); }); it('should use list as a parameter', wrapClient(function(client, done) { From 101726162da3b0a02575cfb62183f7bbb5e2bbc1 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 26 Sep 2024 06:29:28 +0000 Subject: [PATCH 52/99] use 5.0 instead of 5.0-beta1, use openjdk@17 --- Jenkinsfile | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8dfbb2ca..ef2e7f57 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,6 +21,7 @@ def initializeEnvironment() { env.JAVA8_HOME="${JABBA_HOME}/jdk/1.8" env.JAVA11_HOME="${JABBA_HOME}/jdk/openjdk@1.11" + env.JAVA17_HOME="${JABBA_HOME}/jdk/openjdk@1.17" sh label: 'Assign Node.js global environment', script: '''#!/bin/bash -lex nodenv versions @@ -91,6 +92,10 @@ def executeLinter() { } def executeTests() { + def javaVersion = '1.8' + if (env.CASSANDRA_VERSION == '5.0') { + javaVersion = 'openjdk@1.17' + } sh label: 'Execute tests', script: '''#!/bin/bash -lex # Load CCM environment variables set -o allexport @@ -101,7 +106,7 @@ def executeTests() { # TODO: This should last us through testing against Cassandra 4.1.x at least but # will eventually need to be made more generic. . ${JABBA_SHELL} - jabba use 1.8 + jabba use ''' + javaVersion + ''' npm run ci_jenkins ''' @@ -262,8 +267,8 @@ pipeline { choices: [ '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra - '4.1.0', // Previous Apache Cassandra - '5.0-beta1', // Current Apache Cassandra + '4.1', // Previous Apache Cassandra + '5.0', // Current Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.8.30', // Previoius DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise @@ -322,8 +327,8 @@ pipeline { axis { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra - '4.1.0', // Previous Apache Cassandra - '5.0-beta1', // Current Apache Cassandra + '4.1', // Previous Apache Cassandra + '5.0', // Current Apache Cassandra 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD @@ -411,8 +416,8 @@ pipeline { axis { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra - '4.1.0', // Previous Apache Cassandra - '5.0-beta1', // Current Apache Cassandra + '4.1', // Previous Apache Cassandra + '5.0', // Current Apache Cassandra 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD @@ -498,8 +503,8 @@ pipeline { axis { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra - '4.1.0', // Previous Apache Cassandra - '5.0-beta1', // Current Apache Cassandra + '4.1', // Previous Apache Cassandra + '5.0', // Current Apache Cassandra 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD From 6aaf093237df11d672f47ac94648d4672d87abde Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 26 Sep 2024 06:40:18 +0000 Subject: [PATCH 53/99] lint --- .../short/graph/graph-olap-tests.js | 22 +++++++++++-------- test/integration/short/graph/graph-tests.js | 7 ++++-- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/test/integration/short/graph/graph-olap-tests.js b/test/integration/short/graph/graph-olap-tests.js index 857122dd..41789bf1 100644 --- a/test/integration/short/graph/graph-olap-tests.js +++ b/test/integration/short/graph/graph-olap-tests.js @@ -135,16 +135,20 @@ vdescribe('dse-5.0', 'Client with spark workload', function () { ); context('with no callback specified', function () { // if dse-6.9.0, skip this test because NODEJS-676 and DSP-24336 - (helper.getServerInfo().version === '6.9.0' && helper.getServerInfo().isDse) ? - it.skip : it('should return a promise for OLAP query', function () { - const client = newInstance(); - const p = client.executeGraph('g.V().count()', { graphSource: 'a' }); - helper.assertInstanceOf(p, Promise); - return p.then(function (result) { - helper.assertInstanceOf(result, graphModule.GraphResultSet); - assert.strictEqual(typeof result.first(), 'number'); + if (helper.getServerInfo().version === '6.9.0' && helper.getServerInfo().isDse) { + xit('should return a promise for OLAP query'); + } else { + it( + 'should return a promise for OLAP query', function () { + const client = newInstance(); + const p = client.executeGraph('g.V().count()', { graphSource: 'a' }); + helper.assertInstanceOf(p, Promise); + return p.then(function (result) { + helper.assertInstanceOf(result, graphModule.GraphResultSet); + assert.strictEqual(typeof result.first(), 'number'); + }); }); - }); + } }); }); }); diff --git a/test/integration/short/graph/graph-tests.js b/test/integration/short/graph/graph-tests.js index 0705d5eb..81e641e7 100644 --- a/test/integration/short/graph/graph-tests.js +++ b/test/integration/short/graph/graph-tests.js @@ -450,8 +450,10 @@ vdescribe('dse-5.0', 'Client @SERVER_API', function () { }); })); // if dse-6.9.0, skip this test because NODEJS-676 and DSP-24336 - (helper.getServerInfo().version === '6.9.0' && helper.getServerInfo().isDse) ? - it.skip : it('should retrieve a Int64 scalar', wrapClient(function (client, done) { + if (helper.getServerInfo().version === '6.9.0' && helper.getServerInfo().isDse) { + it.skip('should retrieve a Int64 scalar'); + } else { + it('should retrieve a Int64 scalar', wrapClient(function (client, done) { const query = JSON.stringify({ '@type': 'g:Bytecode', '@value': { @@ -466,6 +468,7 @@ vdescribe('dse-5.0', 'Client @SERVER_API', function () { done(); }); })); + } it('should allow graph language to be set from the execution profile', wrapClient(function (client, done) { const query = JSON.stringify({ '@type': 'g:Bytecode', From bd49d080230bb2492fb3e338ca5e367a3b434bc9 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 26 Sep 2024 07:01:56 +0000 Subject: [PATCH 54/99] revert to 5.0-beta1 --- Jenkinsfile | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ef2e7f57..0bbb9482 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,7 +21,6 @@ def initializeEnvironment() { env.JAVA8_HOME="${JABBA_HOME}/jdk/1.8" env.JAVA11_HOME="${JABBA_HOME}/jdk/openjdk@1.11" - env.JAVA17_HOME="${JABBA_HOME}/jdk/openjdk@1.17" sh label: 'Assign Node.js global environment', script: '''#!/bin/bash -lex nodenv versions @@ -93,9 +92,6 @@ def executeLinter() { def executeTests() { def javaVersion = '1.8' - if (env.CASSANDRA_VERSION == '5.0') { - javaVersion = 'openjdk@1.17' - } sh label: 'Execute tests', script: '''#!/bin/bash -lex # Load CCM environment variables set -o allexport @@ -106,7 +102,7 @@ def executeTests() { # TODO: This should last us through testing against Cassandra 4.1.x at least but # will eventually need to be made more generic. . ${JABBA_SHELL} - jabba use ''' + javaVersion + ''' + jabba use 1.8 npm run ci_jenkins ''' @@ -268,7 +264,7 @@ pipeline { '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra '4.1', // Previous Apache Cassandra - '5.0', // Current Apache Cassandra + '5.0-beta1', // Current Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.8.30', // Previoius DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise @@ -328,7 +324,7 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.1', // Previous Apache Cassandra - '5.0', // Current Apache Cassandra + '5.0-beta1', // Current Apache Cassandra 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD @@ -417,7 +413,7 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.1', // Previous Apache Cassandra - '5.0', // Current Apache Cassandra + '5.0-beta1', // Current Apache Cassandra 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD @@ -504,7 +500,7 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.1', // Previous Apache Cassandra - '5.0', // Current Apache Cassandra + '5.0-beta1', // Current Apache Cassandra 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD From 4a8f8498565ec2a76ae1f04a9b059892460c0c22 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 26 Sep 2024 08:02:14 +0000 Subject: [PATCH 55/99] merge three functions into one --- Jenkinsfile | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0bbb9482..1505f3c4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -217,6 +217,12 @@ def describeAdhocTestingStage() { } } +def describeInstallAndLint(){ + describePerCommitStage() + installDriverAndDependencies() + executeLinter() +} + // branch pattern for cron def branchPatternCron() { ~"(master)" @@ -350,19 +356,9 @@ pipeline { } } } - stage('Describe-Build') { + stage('Describe-Install-And-Lint') { steps { - describePerCommitStage() - } - } - stage('Install-Driver-And-Dependencies') { - steps { - installDriverAndDependencies() - } - } - stage('Execute-Linter') { - steps { - executeLinter() + describeInstallAndLint() } } stage('Execute-Tests') { @@ -391,6 +387,12 @@ pipeline { aborted { notifySlack('aborted') } + success { + notifySlack('completed') + } + unstable { + notifySlack('unstable') + } failure { notifySlack('FAILED') } @@ -480,6 +482,12 @@ pipeline { aborted { notifySlack('aborted') } + success { + notifySlack('completed') + } + unstable { + notifySlack('unstable') + } failure { notifySlack('FAILED') } From 80112a3ba0b278d59a0e6440c1bfab8df4175ecf Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 26 Sep 2024 08:22:48 +0000 Subject: [PATCH 56/99] delete notifySlack for ScehduledTesting --- Jenkinsfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1505f3c4..7eff3e67 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -482,12 +482,6 @@ pipeline { aborted { notifySlack('aborted') } - success { - notifySlack('completed') - } - unstable { - notifySlack('unstable') - } failure { notifySlack('FAILED') } From 2c5b5e0880458718568d8029a088a3e34bd30d5f Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 26 Sep 2024 08:24:58 +0000 Subject: [PATCH 57/99] merge three functions into one --- Jenkinsfile | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7eff3e67..1f8491e1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -441,19 +441,9 @@ pipeline { } } } - stage('Describe-Build') { - steps { - describeScheduledTestingStage() - } - } - stage('Install-Driver-And-Dependencies') { - steps { - installDriverAndDependencies() - } - } - stage('Execute-Linter') { + stage('Describe-Install-And-Lint') { steps { - executeLinter() + describeInstallAndLint() } } stage('Execute-Tests') { From 7b6f29daa84ed24b2ec99aa8d94444d93353112f Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 26 Sep 2024 08:27:59 +0000 Subject: [PATCH 58/99] add back long description --- Jenkinsfile | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1f8491e1..00a192ea 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -276,7 +276,31 @@ pipeline { 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0', // HCD 'ALL'], - description: '''Apache Cassandra and DataStax Enterprise server version to use for adhoc BUILD-AND-EXECUTE-TESTS ONLY!''') + description: '''Apache Cassandra and DataStax Enterprise server version to use for adhoc BUILD-AND-EXECUTE-TESTS ONLY! + + + + + + + + + + + + + + + + + + + + + + + +
ChoiceDescription
3.11Apache Cassandra v3.11.x
4.0Apache Cassandra v4.x (CURRENTLY UNDER DEVELOPMENT)
dse-5.1DataStax Enterprise v5.1.x
dse-6.8DataStax Enterprise v6.8.x (CURRENTLY UNDER DEVELOPMENT)
''') booleanParam( name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_EXECUTE_EXAMPLES', defaultValue: false, From 2f2a0ea8d7f6fa77fe76a4372878925db0a8468b Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 26 Sep 2024 22:11:46 +0000 Subject: [PATCH 59/99] change package-lock.json --- package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index c30bacf4..70248559 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@types/long": "^5.0.0", - "@types/node": ">=8", + "@types/node": "^18.11.18", "adm-zip": "^0.5.10", "long": "^5.2.3" }, From b61d998756de0a4e3935713ee720e490c70053d0 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 27 Sep 2024 02:00:27 +0000 Subject: [PATCH 60/99] float becomes double? --- lib/encoder.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index 9effff62..f47a349c 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -16,7 +16,7 @@ 'use strict'; const util = require('util'); -const types = require('./types'); +const { types } = require('../index.js'); const dataTypes = types.dataTypes; const Long = types.Long; const Integer = types.Integer; @@ -955,9 +955,6 @@ function defineInstanceMembers() { if (value.length !== dimensions) { throw new TypeError(`Expected vector with ${dimensions} dimensions, observed size of ${value.length}`); } - if (subtype.code !== dataTypes.float) { - throw new TypeError("Driver only supports vectors of 4 byte floating point values"); - } return subtype; } @@ -1764,9 +1761,9 @@ Encoder.guessDataType = function (value) { else if (value instanceof types.Vector) { code = dataTypes.custom; // TODO: another area that we have to generalize if we ever need to support vector subtypes other than float - if (value?.get(0)) { - const subtype = this.guessDataType(value.get(0)); - if (subtype?.code) { + if (value && value[0]) { + const subtype = this.guessDataType(value[0]); + if (subtype?.code !== null && subtype?.code !== undefined) { info = buildParameterizedCustomType(customTypeNames.vector, [singleTypeNamesByDataType[subtype.code], value.length]); } else { throw new TypeError("Cannot guess subtype from element " + value.get(0)); From 51a3c6c1f4208af159a7c5ff3ff89228e585f887 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 27 Sep 2024 06:43:21 +0000 Subject: [PATCH 61/99] methods are considered enumerable --- lib/types/vector.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/types/vector.js b/lib/types/vector.js index e163ed7e..4707c322 100644 --- a/lib/types/vector.js +++ b/lib/types/vector.js @@ -19,6 +19,7 @@ * Creates a new instance of Cql Vector, also compatible with Float32Array. * @class */ +const util = require('node:util'); var Vector = /** @class */ (function () { /** * @@ -44,7 +45,9 @@ var Vector = /** @class */ (function () { this.length = this.elements.length; return new Proxy(this, { get: function (obj, key) { - if (typeof (key) === 'string' && (Number.isInteger(Number(key)))) // key is an index + if (key === 'IDENTITY'){ + return 'Vector'; + } else if (typeof (key) === 'string' && (Number.isInteger(Number(key)))) // key is an index return obj.elements[key]; else return obj[key]; @@ -90,7 +93,8 @@ var Vector = /** @class */ (function () { Vector.prototype[Symbol.iterator] = function () { return this.elements[Symbol.iterator](); }; - Object.defineProperty(Vector, Symbol.species, { + + Vector.prototype[Symbol.species] = { /** * instanceof */ @@ -99,10 +103,14 @@ var Vector = /** @class */ (function () { }, enumerable: false, configurable: true - }); + }; + return Vector; }()); + Object.defineProperty(Vector, Symbol.hasInstance, { - value: function (i) { return typeof (i) === typeof (Vector) || i instanceof Float32Array; } + value: function (i) { + return (util.types.isProxy(i) && i.IDENTITY === 'Vector') || i instanceof Float32Array; } }); + module.exports = Vector; From f10cda6fcfa7cd739f4d3f72daf0b60eeb7d5a76 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 27 Sep 2024 06:47:56 +0000 Subject: [PATCH 62/99] refactor to class --- lib/types/vector.js | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/lib/types/vector.js b/lib/types/vector.js index 4707c322..ed10a74a 100644 --- a/lib/types/vector.js +++ b/lib/types/vector.js @@ -20,12 +20,12 @@ * @class */ const util = require('node:util'); -var Vector = /** @class */ (function () { +class Vector { /** * * @param {Float32Array | Array} elements */ - function Vector(elements) { + constructor (elements) { if (elements instanceof Float32Array) { this.elements = Array.from(elements); } @@ -64,37 +64,25 @@ var Vector = /** @class */ (function () { * Returns the string representation of the vector. * @returns {string} */ - Vector.prototype.toString = function () { + toString() { return "[".concat(this.elements.toString(), "]"); - }; + } /** * * @param {number} index */ - Vector.prototype.at = function (index) { + at(index) { return this.elements[index]; }; /** - * - * @param {...any} elements - * @returns - */ - Vector.of = function () { - var elements = []; - for (var _i = 0; _i < arguments.length; _i++) { - elements[_i] = arguments[_i]; - } - return new Vector(elements); - }; - /** - * ` + * * @returns {ArrayIterator} */ - Vector.prototype[Symbol.iterator] = function () { + [Symbol.iterator]() { return this.elements[Symbol.iterator](); - }; + } - Vector.prototype[Symbol.species] = { + [Symbol.species] = { /** * instanceof */ @@ -104,9 +92,7 @@ var Vector = /** @class */ (function () { enumerable: false, configurable: true }; - - return Vector; -}()); +}; Object.defineProperty(Vector, Symbol.hasInstance, { value: function (i) { From ab35ee7aa192382737df97939e775bc76bed597f Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 27 Sep 2024 07:13:29 +0000 Subject: [PATCH 63/99] pass all existing tests --- lib/encoder.js | 14 +++-- lib/types/vector.js | 11 ++++ lib/types/vector.ts | 114 ------------------------------------- test/unit/encoder-tests.js | 4 +- 4 files changed, 22 insertions(+), 121 deletions(-) delete mode 100644 lib/types/vector.ts diff --git a/lib/encoder.js b/lib/encoder.js index f47a349c..e20dd7e7 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -1762,11 +1762,15 @@ Encoder.guessDataType = function (value) { code = dataTypes.custom; // TODO: another area that we have to generalize if we ever need to support vector subtypes other than float if (value && value[0]) { - const subtype = this.guessDataType(value[0]); - if (subtype?.code !== null && subtype?.code !== undefined) { - info = buildParameterizedCustomType(customTypeNames.vector, [singleTypeNamesByDataType[subtype.code], value.length]); - } else { - throw new TypeError("Cannot guess subtype from element " + value.get(0)); + if (value instanceof Float32Array) { + info = buildParameterizedCustomType(customTypeNames.vector, [singleTypeNamesByDataType[dataTypes.float], value.length]); + }else{ + const subtype = this.guessDataType(value[0]); + if (subtype?.code !== null && subtype?.code !== undefined) { + info = buildParameterizedCustomType(customTypeNames.vector, [singleTypeNamesByDataType[subtype.code], value.length]); + } else { + throw new TypeError("Cannot guess subtype from element " + value.get(0)); + } } } else { throw new TypeError("Cannot guess subtype of empty vector"); diff --git a/lib/types/vector.js b/lib/types/vector.js index ed10a74a..909b3eb3 100644 --- a/lib/types/vector.js +++ b/lib/types/vector.js @@ -57,6 +57,17 @@ class Vector { return obj.elements[key] = value; else return obj[key] = value; + }, + ownKeys: function (obj) { + return Reflect.ownKeys(elements); + }, + getOwnPropertyDescriptor(target, key) { + if (typeof (key) === 'string' && (Number.isInteger(Number(key)))){ + // array index + return { enumerable: true, configurable: true}; + }else{ + return Reflect.getOwnPropertyDescriptor(target, key); + } } }); } diff --git a/lib/types/vector.ts b/lib/types/vector.ts deleted file mode 100644 index 2d531550..00000000 --- a/lib/types/vector.ts +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -"use strict"; - -/** @module types */ - -/** - * Creates a new instance of Cql Vector, also compatible with Float32Array. - * @class - */ - -class Vector implements ArrayLike { - length: number; - elements: any[]; - /** - * - * @param {Float32Array | Array} elements - */ - constructor(elements) { - if (elements instanceof Float32Array) { - this.elements = Array.from(elements); - }else if(Array.isArray(elements)){ - this.elements = elements; - }else{ - throw new TypeError('Vector must be constructed with a Float32Array or an Array'); - } - - if (this.elements.length === 0) { - throw new TypeError('Vector must contain at least one value'); - } - - /** - * Returns the number of the elements. - * @type Number - */ - this.length = this.elements.length; - - return new Proxy(this, { - get: (obj, key) => { - if (typeof(key) === 'string' && (Number.isInteger(Number(key)))) // key is an index - return obj.elements[key] - else - return obj[key] - }, - set: (obj, key, value) => { - if (typeof(key) === 'string' && (Number.isInteger(Number(key)))) // key is an index - return obj.elements[key] = value - else - return obj[key] = value - } - }) - } - - /** - * Returns the string representation of the vector. - * @returns {string} - */ - toString() { - return `[${this.elements.toString()}]`; - } - - /** - * - * @param {number} index - */ - at(index) { - return this.elements[index]; - } - - /** - * - * @param {...any} elements - * @returns - */ - static of(...elements) { - return new Vector(elements); - } - - /** - * ` - * @returns {ArrayIterator} - */ - [Symbol.iterator]() { - return this.elements[Symbol.iterator](); - } - - [n: number]: T; - - /** - * instanceof - */ - static get [Symbol.species]() { - return Vector; - } -} - -Object.defineProperty(Vector, Symbol.hasInstance, { - value: function (i: any) { return typeof(i) === typeof(Vector) || i instanceof Float32Array } -}) -module.exports = Vector; \ No newline at end of file diff --git a/test/unit/encoder-tests.js b/test/unit/encoder-tests.js index 12b12395..921534a5 100644 --- a/test/unit/encoder-tests.js +++ b/test/unit/encoder-tests.js @@ -20,7 +20,7 @@ const util = require('util'); const utils = require('../../lib/utils'); const tokenizer = require('../../lib/tokenizer'); const token = require('../../lib/token'); - +const Vector = require('../../lib/types/vector'); const Encoder = require('../../lib/encoder'); const types = require('../../lib/types'); const ExecutionOptions = require('../../lib/execution-options').ExecutionOptions; @@ -685,7 +685,7 @@ describe('encoder', function () { const guessedTypeObj = Encoder.guessDataType(refVal); const encoded = encoder.encode(refVal, guessedTypeObj); const decoded = encoder.decode(encoded, guessedTypeObj); - helper.assertInstanceOf(decoded, Float32Array); + helper.assertInstanceOf(decoded, Vector); for (const k in decoded) { if (decoded.hasOwnProperty(k)) { assert.equal(decoded[k],refVal[k]); From cecd5309ada240c31e5c1b603b3dba3541201d59 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 27 Sep 2024 14:18:34 +0000 Subject: [PATCH 64/99] pass existing tests --- test/unit/encoder-tests.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/unit/encoder-tests.js b/test/unit/encoder-tests.js index 921534a5..03b2f66c 100644 --- a/test/unit/encoder-tests.js +++ b/test/unit/encoder-tests.js @@ -703,7 +703,7 @@ describe('encoder', function () { const typeObj = {code: dataTypes.custom, info: typeName}; const encoded = encoder.encode(refVal, typeObj); const decoded = encoder.decode(encoded, typeObj); - helper.assertInstanceOf(decoded, Float32Array); + helper.assertInstanceOf(decoded, Vector); for (const k in decoded) { if (decoded.hasOwnProperty(k)) { assert.equal(decoded[k],refVal[k]); @@ -735,12 +735,6 @@ describe('encoder', function () { assert.throws(function() { encoder.encode(refVal, {code: dataTypes.custom, info: typeName}); }, TypeError); }); - it('should fail to encode if full type provided and subtype is not FloatType', function () { - const encoder = new Encoder(4, {}); - const refVal = new Float32Array([1, 2, 3]); - const typeName = 'org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.Int32Type,3)'; - assert.throws(function() { encoder.encode(refVal, {code: dataTypes.custom, info: typeName}); }, TypeError); - }); }); describe('#encode()', function () { From 2a503ddf1ad23ac483002030536d74fdc4732734 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 27 Sep 2024 20:58:16 +0000 Subject: [PATCH 65/99] update package-lock --- package-lock.json | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 70248559..d35a5d25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,22 +9,22 @@ "version": "4.7.2", "license": "Apache-2.0", "dependencies": { - "@types/long": "^5.0.0", + "@types/long": "~5.0.0", "@types/node": "^18.11.18", - "adm-zip": "^0.5.10", - "long": "^5.2.3" + "adm-zip": "~0.5.10", + "long": "~5.2.3" }, "devDependencies": { - "chai": "^4.3.8", - "kerberos": "^2.0.3", - "mocha": "^10.2.0", - "mocha-jenkins-reporter": "^0.4.8", + "chai": "~4.3.8", + "kerberos": "~2.0.3", + "mocha": "~10.2.0", + "mocha-jenkins-reporter": "~0.4.8", "proxyquire": "~2.1.3", - "sinon": "^15.2.0", + "sinon": "~15.2.0", "temp": ">= 0.8.3" }, "engines": { - "node": ">=8" + "node": ">=16" } }, "node_modules/@sinonjs/commons": { @@ -81,9 +81,12 @@ } }, "node_modules/@types/node": { - "version": "20.5.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.9.tgz", - "integrity": "sha512-PcGNd//40kHAS3sTlzKB9C9XL4K0sTup8nbG5lC14kzEteTNuAFh9u5nA0o5TWnSG2r/JNPRXFVcHJIIeRlmqQ==" + "version": "18.19.53", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.53.tgz", + "integrity": "sha512-GLxgUgHhDKO1Edw9Q0lvMbiO/IQXJwJlMaqxSGBXMpPy8uhkCs2iiPFaB2Q/gmobnFkckD3rqTBMVjXdwq+nKg==", + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/adm-zip": { "version": "0.5.10", @@ -1712,6 +1715,11 @@ "node": ">=4" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", From 70cfb4039674c1ea29583cfb1ad6341c81a6e802 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 1 Oct 2024 00:18:15 -0800 Subject: [PATCH 66/99] Update Jenkinsfile --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 00a192ea..057e49ed 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -355,9 +355,9 @@ pipeline { values '3.11', // Previous Apache Cassandra '4.1', // Previous Apache Cassandra '5.0-beta1', // Current Apache Cassandra - 'dse-6.8.30', // Previous DataStax Enterprise - 'dse-6.9.0', // Current DataStax Enterprise - 'hcd-1.0.0' // HCD + 'dse-6.8', // Previous DataStax Enterprise + 'dse-6.9', // Current DataStax Enterprise + 'hcd-1.0' // HCD } axis { name 'NODEJS_VERSION' From ab9eb827060220884b28b0faf842fb9be6a8f578 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 1 Oct 2024 00:18:50 -0800 Subject: [PATCH 67/99] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 057e49ed..46eb1f36 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -354,7 +354,7 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.1', // Previous Apache Cassandra - '5.0-beta1', // Current Apache Cassandra + '5.0', // Current Apache Cassandra 'dse-6.8', // Previous DataStax Enterprise 'dse-6.9', // Current DataStax Enterprise 'hcd-1.0' // HCD From 60ea99b150980a780fb3b5b7f150165c630cb7c6 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 1 Oct 2024 00:39:42 -0800 Subject: [PATCH 68/99] Update Jenkinsfile --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 46eb1f36..18f5e5b4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -354,9 +354,9 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.1', // Previous Apache Cassandra - '5.0', // Current Apache Cassandra - 'dse-6.8', // Previous DataStax Enterprise - 'dse-6.9', // Current DataStax Enterprise + '5.0.1', // Current Apache Cassandra + 'dse-6.8.51', // Previous DataStax Enterprise + 'dse-6.9.2', // Current DataStax Enterprise 'hcd-1.0' // HCD } axis { From 4c908ba9646627a39e83fc3a277ff219e1914e65 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 1 Oct 2024 11:46:32 -0800 Subject: [PATCH 69/99] Update Jenkinsfile --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 18f5e5b4..1ad51fba 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -355,9 +355,9 @@ pipeline { values '3.11', // Previous Apache Cassandra '4.1', // Previous Apache Cassandra '5.0.1', // Current Apache Cassandra - 'dse-6.8.51', // Previous DataStax Enterprise - 'dse-6.9.2', // Current DataStax Enterprise - 'hcd-1.0' // HCD + 'dse-6.8.34', // Previous DataStax Enterprise + 'dse-6.8.33', // Current DataStax Enterprise + 'hcd-1.0.0' // HCD } axis { name 'NODEJS_VERSION' From a736b3ab82558e7ca81dc8434309ca82df2f46cd Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 1 Oct 2024 17:43:59 -0800 Subject: [PATCH 70/99] Update Jenkinsfile to 5.0.1 --- Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1ad51fba..2b161acb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -270,7 +270,7 @@ pipeline { '3.11', // Previous Apache Cassandra '4.0', // Previous Apache Cassandra '4.1', // Previous Apache Cassandra - '5.0-beta1', // Current Apache Cassandra + '5.0', // Current Apache Cassandra 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.8.30', // Previoius DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise @@ -354,9 +354,9 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.1', // Previous Apache Cassandra - '5.0.1', // Current Apache Cassandra - 'dse-6.8.34', // Previous DataStax Enterprise - 'dse-6.8.33', // Current DataStax Enterprise + '5.0', // Current Apache Cassandra + 'dse-6.8.30', // Previous DataStax Enterprise + 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD } axis { @@ -439,7 +439,7 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.1', // Previous Apache Cassandra - '5.0-beta1', // Current Apache Cassandra + '5.0', // Current Apache Cassandra 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD @@ -516,7 +516,7 @@ pipeline { name 'CASSANDRA_VERSION' values '3.11', // Previous Apache Cassandra '4.1', // Previous Apache Cassandra - '5.0-beta1', // Current Apache Cassandra + '5.0', // Current Apache Cassandra 'dse-6.8.30', // Previous DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // HCD From 9c356715d92907a333725713c1fe836f41e0345e Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Tue, 1 Oct 2024 21:35:28 -0800 Subject: [PATCH 71/99] delete java version --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2b161acb..7d7b74a9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -91,7 +91,6 @@ def executeLinter() { } def executeTests() { - def javaVersion = '1.8' sh label: 'Execute tests', script: '''#!/bin/bash -lex # Load CCM environment variables set -o allexport From e61f695514747c0abcd4b272095f9d02165c3808 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 4 Oct 2024 01:47:14 +0000 Subject: [PATCH 72/99] massive refactor of typedef ColumnInfo --- examples/package-lock.json | 50 ++++ lib/encoder.js | 234 +++++++++++++------ lib/errors.js | 17 +- lib/types/duration.js | 198 +--------------- lib/types/vector.js | 4 +- lib/utils.js | 268 +++++++++++++++++++++- package-lock.json | 41 ++-- test/integration/short/vector-tests-ts.js | 64 ++++++ test/integration/short/vector-tests-ts.ts | 64 ++++++ test/integration/short/vector-tests.js | 1 + test/unit/encoder-tests.js | 17 ++ 11 files changed, 674 insertions(+), 284 deletions(-) create mode 100644 examples/package-lock.json create mode 100644 test/integration/short/vector-tests-ts.js create mode 100644 test/integration/short/vector-tests-ts.ts diff --git a/examples/package-lock.json b/examples/package-lock.json new file mode 100644 index 00000000..6659ea47 --- /dev/null +++ b/examples/package-lock.json @@ -0,0 +1,50 @@ +{ + "name": "cassandra-driver-examples", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cassandra-driver-examples", + "version": "0.0.1", + "license": "Apache-2.0", + "dependencies": { + "async": "^1.5.2", + "cassandra-driver": "file:../" + } + }, + "..": { + "name": "cassandra-driver", + "version": "4.7.2", + "license": "Apache-2.0", + "dependencies": { + "@types/long": "~5.0.0", + "@types/node": "^18.11.18", + "adm-zip": "~0.5.10", + "long": "~5.2.3" + }, + "devDependencies": { + "@types/mocha": "^10.0.8", + "chai": "~4.3.8", + "kerberos": "~2.0.3", + "mocha": "~10.2.0", + "mocha-jenkins-reporter": "~0.4.8", + "proxyquire": "~2.1.3", + "sinon": "~15.2.0", + "temp": ">= 0.8.3" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha512-nSVgobk4rv61R9PUSDtYt7mPVB2olxNR5RWJcAsH676/ef11bUZwvu7+RGYrYauVdDPcO519v68wRhXQtxsV9w==" + }, + "node_modules/cassandra-driver": { + "resolved": "..", + "link": true + } + } +} diff --git a/lib/encoder.js b/lib/encoder.js index e20dd7e7..71d2a661 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -126,6 +126,19 @@ const zeroLengthTypesSupported = new Set([ dataTypes.blob ]); +/** + * @typedef {(singleTypeNames[keyof singleTypeNames])} SingleTypeCodes + * @typedef {{code : SingleTypeCodes }} SingleColumnInfo + * @typedef {{code : (types.dataTypes.custom), customTypeName : ('point' | 'polygon' | 'duration' | 'lineString' | 'dateRange')}} CustomSimpleColumnInfo + * @typedef {{code : (types.dataTypes.map), info : [ColumnInfo, ColumnInfo], options?: {frozen?: Boolean, reversed?: Boolean}}} MapColumnInfo + * @typedef {{code : (types.dataTypes.tuple), info : Array, options?: {frozen?: Boolean, reversed?: Boolean}}} TupleColumnInfo + * @typedef {{code : (types.dataTypes.list | types.dataTypes.set), info : ColumnInfo, options?: {frozen?: Boolean, reversed?: Boolean}}} ListSetColumnInfo + * TODO: udt type + * @typedef {{code : (types.dataTypes.custom), customTypeName : ('vector'), info : ColumnInfo, dimension : number}} VectorColumnInfo + * @typedef {{code : (types.dataTypes.custom), info : string}} OtherCustomColumnInfo + * @typedef {SingleColumnInfo | CustomSimpleColumnInfo | MapColumnInfo | TupleColumnInfo | ListSetColumnInfo | VectorColumnInfo | OtherCustomColumnInfo} ColumnInfo If this is a simple type, info is null; if this is a collection type with a simple subtype, info is a string, if this is a nested collection type, info is a ColumnInfo object + */ + /** * Serializes and deserializes to and from a CQL type and a Javascript Type. * @param {Number} protocolVersion @@ -912,8 +925,8 @@ function defineInstanceMembers() { /** * * @param {Buffer} buffer - * @param {{subtype: String, dimensions: Number}} params - * @returns {import('./types').Vector} + * @param {{subtype: ColumnInfo, dimensions: Number}} params + * @returns {import('../index.js').types.Vector} */ this.decodeVector = function(buffer, params) { const subtype = params["subtype"]; @@ -932,49 +945,99 @@ function defineInstanceMembers() { return new Vector(rv); }; + /** + * @param {{code: Number, info: String?}} cqlType + * @returns {Number} + */ + this.serializationSizeIfFixed = function (cqlType) { + switch (cqlType.code) { + case dataTypes.bigint: + return 8; + case dataTypes.boolean: + return 1; + case dataTypes.timestamp: + return 8; + case dataTypes.double: + return 8; + case dataTypes.float: + return 4; + case dataTypes.int: + return 4; + case dataTypes.timeuuid: + return 16; + case dataTypes.uuid: + return 16; + + case dataTypes.ascii: + case dataTypes.tinyint: + case dataTypes.blob: + case dataTypes.counter: + case dataTypes.decimal: + case dataTypes.duration: + case dataTypes.inet: + case dataTypes.varint: + case dataTypes.smallint: + case dataTypes.date: + case dataTypes.time: + case dataTypes.text: + case dataTypes.varchar: + case dataTypes.list: + case dataTypes.map: + case dataTypes.set: + case dataTypes.udt: + case dataTypes.tuple: + return -1; + case dataTypes.custom: + if (cqlType.info?.startsWith(cqlNames.vector) || cqlType.info?.startsWith(customTypeNames.vector)) { + let vectorParams; + if (cqlType.info?.startsWith(cqlNames.vector)){ + // it's a vector with "vector" + /** @type {{subtype: string, dimensions : number}} */ + vectorParams = this.parseVectorTypeArgs(cqlType.info, cqlNames.vector, (arg) => arg); + }else{ + // it's a vector with "org.apache.cassandra.db.marshal.VectorType" + vectorParams = this.parseVectorTypeArgs(cqlType.info, customTypeNames.vector, this.parseFqTypeName); + } + + /** @type {{code: number, info: Object|Array|null}} */ + const subtypeParams = this.parseFqTypeName(vectorParams.subtype.info); + const subtypeSerialSize = this.serializationSizeIfFixed(subtypeParams); + if (subtypeSerialSize === -1){ + return -1; + }else{ + return subtypeSerialSize * vectorParams.dimensions; + } + } + return -1; + default: + throw new TypeError('Cannot calculate size from ' + cqlType.code + ' ' + cqlType.info); + } + } /** * @param {Vector} value - * @param {{subtype: any, dimensions: Number}} params + * @param {ColumnInfo} params * @returns {Buffer} */ this.encodeVector = function(value, params) { - // Evaluate params to encodeVector(), returning the computed subtype - function evalParams() { - - if (!(value instanceof Vector)) { - throw new TypeError("Driver only supports Vector type"); - } - - // Perform client-side validation iff we were actually supplied with meaningful type info. In practice - // this will only occur when using prepared statements. - if (params.hasOwnProperty("subtype") && params.hasOwnProperty("dimensions")) { - - const subtype = params["subtype"]; - const dimensions = params["dimensions"]; - if (value.length !== dimensions) { - throw new TypeError(`Expected vector with ${dimensions} dimensions, observed size of ${value.length}`); - } - return subtype; - } - - return { code: dataTypes.float }; + if (!(value instanceof Vector)) { + throw new TypeError("Driver only supports Vector type when encoding a vector"); } - if (!Encoder.isTypedArray(value)) { - throw new TypeError('Expected TypedArray subclass, obtained ' + util.inspect(value)); + const dimension = params["dimension"]; + if (value.length !== dimension) { + throw new TypeError(`Expected vector with ${dimension} dimensions, observed size of ${value.length}`); } + if (value.length === 0) { throw new TypeError("Cannot encode empty array as vector"); } - const subtype = evalParams(); - // TypedArrays are _not_ JS arrays so explicitly convert them here before trying to write them // into a buffer const elems = []; for (const elem of value) { - elems.push(this.encode(elem, subtype)); + elems.push(this.encode(elem, params.info)); } return Buffer.concat(elems); }; @@ -985,7 +1048,7 @@ function defineInstanceMembers() { * @param {String} typeName * @param {String} stringToExclude Leading string indicating this is a vector type (to be excluded when eval'ing args) * @param {Function} subtypeResolveFn Function used to resolve subtype type; varies depending on type naming convention - * @returns {Object} + * @returns {ColumnInfo} * @internal */ this.parseVectorTypeArgs = function(typeName, stringToExclude, subtypeResolveFn) { @@ -994,7 +1057,9 @@ function defineInstanceMembers() { const argsLength = typeName.length - (stringToExclude.length + 2); const params = parseParams(typeName, argsStartIndex, argsLength); if (params.length === 2) { - return {subtype: subtypeResolveFn(params[0].trim()), dimensions: parseInt(params[1].trim(), 10)}; + /** @type {ColumnInfo} */ + const columnInfo = { code: dataTypes.custom, info: subtypeResolveFn(params[0].trim()), customTypeName : 'vector', dimension: parseInt(params[1].trim(), 10 )}; + return columnInfo; } throw new TypeError('Not a valid type ' + typeName); }; @@ -1150,8 +1215,10 @@ function defineInstanceMembers() { * @param {Number} startIndex * @param {Number|null} length * @param {Function} udtResolver - * @returns {Promise<{err, info, options}>} callback Callback invoked with err and {{code: number, info: Object|Array|null, options: {frozen: Boolean}}} + * @async + * @returns {Promise.} callback Callback invoked with err and {{code: number, info: Object|Array|null, options: {frozen: Boolean}}} * @internal + * @throws {Error} * @ignore */ this.parseTypeName = async function (keyspace, typeName, startIndex, length, udtResolver) { @@ -1160,6 +1227,7 @@ function defineInstanceMembers() { length = typeName.length; } + /** @type {ColumnInfo & {options : {frozen : boolean}}} */ const dataType = { code: 0, info: null, @@ -1321,20 +1389,14 @@ function defineInstanceMembers() { * @param {String} typeName * @param {Number} [startIndex] * @param {Number} [length] - * @throws TypeError - * @returns {{code: number, info: Object|Array|null, options: {frozen: Boolean, reversed: Boolean}}} + * @throws {TypeError} + * @returns {ColumnInfo} * @internal * @ignore */ this.parseFqTypeName = function (typeName, startIndex, length) { - const dataType = { - code: 0, - info: null, - options: { - reversed: false, - frozen: false - } - }; + let frozen = false; + let reversed = false; startIndex = startIndex || 0; let params; if (!length) { @@ -1344,19 +1406,21 @@ function defineInstanceMembers() { //Remove the reversed token startIndex += complexTypeNames.reversed.length + 1; length -= complexTypeNames.reversed.length + 2; - dataType.options.reversed = true; + reversed = true; } if (length > complexTypeNames.frozen.length && typeName.indexOf(complexTypeNames.frozen, startIndex) === startIndex) { //Remove the frozen token startIndex += complexTypeNames.frozen.length + 1; length -= complexTypeNames.frozen.length + 2; - dataType.options.frozen = true; + frozen = true; } if (typeName === complexTypeNames.empty) { //set as custom - dataType.info = 'empty'; - return dataType; + return { + code: dataTypes.custom, + info: 'empty' + }; } //Quick check if its a single type if (length <= singleFqTypeNamesLength) { @@ -1365,8 +1429,7 @@ function defineInstanceMembers() { } const typeCode = singleTypeNames[typeName]; if (typeof typeCode === 'number') { - dataType.code = typeCode; - return dataType; + return {code : typeCode}; } throw new TypeError('Not a valid type "' + typeName + '"'); } @@ -1380,9 +1443,15 @@ function defineInstanceMembers() { if (params.length !== 1) { throw new TypeError('Not a valid type ' + typeName); } - dataType.code = dataTypes.list; - dataType.info = this.parseFqTypeName(params[0]); - return dataType; + const info = this.parseFqTypeName(params[0]); + return { + code: dataTypes.list, + info: info, + options: { + frozen: frozen, + reversed: reversed + } + }; } if (typeName.indexOf(complexTypeNames.set, startIndex) === startIndex) { //Its a set @@ -1395,9 +1464,14 @@ function defineInstanceMembers() { { throw new TypeError('Not a valid type ' + typeName); } - dataType.code = dataTypes.set; - dataType.info = this.parseFqTypeName(params[0]); - return dataType; + const info = this.parseFqTypeName(params[0]); + return { + code : dataTypes.set, + info : info, + options : { + frozen : frozen, + reversed : reversed} + }; } if (typeName.indexOf(complexTypeNames.map, startIndex) === startIndex) { //org.apache.cassandra.db.marshal.MapType(keyType,valueType) @@ -1409,9 +1483,15 @@ function defineInstanceMembers() { if (params.length !== 2) { throw new TypeError('Not a valid type ' + typeName); } - dataType.code = dataTypes.map; - dataType.info = [this.parseFqTypeName(params[0]), this.parseFqTypeName(params[1])]; - return dataType; + const info1 = this.parseFqTypeName(params[0]) + const info2 = this.parseFqTypeName(params[1]); + return { + code : dataTypes.map, + info : [info1, info2], + options : { + frozen : frozen, + reversed : reversed} + }; } if (typeName.indexOf(complexTypeNames.udt, startIndex) === startIndex) { //move cursor across the name and bypass the parenthesis @@ -1427,21 +1507,27 @@ function defineInstanceMembers() { if (params.length < 1) { throw new TypeError('Not a valid type ' + typeName); } - dataType.code = dataTypes.tuple; - dataType.info = params.map(x => this.parseFqTypeName(x)); - return dataType; + const info = params.map(x => this.parseFqTypeName(x)); + return { + code : dataTypes.tuple, + info : info, + options : { + frozen : frozen, + reversed : reversed} + }; } if (typeName.indexOf(customTypeNames.vector, startIndex) === startIndex) { // It's a vector, so record the subtype and dimension. - dataType.code = dataTypes.custom; - dataType.info = this.parseVectorTypeArgs(typeName, customTypeNames.vector, this.parseFqTypeName); - return dataType; + return this.parseVectorTypeArgs(typeName, customTypeNames.vector, this.parseFqTypeName); } // Assume custom type if cannot be parsed up to this point. - dataType.info = typeName.substr(startIndex, length); - return dataType; + const info = typeName.substr(startIndex, length); + return { + code: dataTypes.custom, + info: info + }; }; /** * Parses type names with composites @@ -1632,7 +1718,7 @@ Encoder.prototype.decode = function (buffer, type) { * This is part of an experimental API, this can be changed future releases. *

* @param {*} value The value to be converted. - * @param {{code: number, info: *|Object}|String|Number} [typeInfo] The type information. + * @param {ColumnInfo} typeInfo The type information. *

It can be either a:

*
    *
  • A String representing the data type.
  • @@ -1662,7 +1748,7 @@ Encoder.prototype.encode = function (value, typeInfo) { return value; } - /** @type {{code: Number, info: object}} */ + /** @type {ColumnInfo | {code : null, info : null}} */ let type = { code: null, info: null @@ -1685,9 +1771,11 @@ Encoder.prototype.encode = function (value, typeInfo) { } else { //Lets guess - type = Encoder.guessDataType(value); - if (!type) { + const guessedType = Encoder.guessDataType(value); + if (!guessedType) { throw new TypeError('Target data type could not be guessed, you should use prepared statements for accurate type mapping. Value: ' + util.inspect(value)); + }else{ + type = guessedType; } } @@ -1761,13 +1849,13 @@ Encoder.guessDataType = function (value) { else if (value instanceof types.Vector) { code = dataTypes.custom; // TODO: another area that we have to generalize if we ever need to support vector subtypes other than float - if (value && value[0]) { + if (value && value.length > 0) { if (value instanceof Float32Array) { info = buildParameterizedCustomType(customTypeNames.vector, [singleTypeNamesByDataType[dataTypes.float], value.length]); }else{ const subtype = this.guessDataType(value[0]); - if (subtype?.code !== null && subtype?.code !== undefined) { - info = buildParameterizedCustomType(customTypeNames.vector, [singleTypeNamesByDataType[subtype.code], value.length]); + if (subtype?.info) { + info = buildParameterizedCustomType(customTypeNames.vector, [subtype.info, value.length]); } else { throw new TypeError("Cannot guess subtype from element " + value.get(0)); } @@ -2010,6 +2098,10 @@ function concatRoutingKey(parts, totalLength) { return routingKey; } +/** + * @param {string} customTypeName + * @param {Array} args + */ function buildParameterizedCustomType(customTypeName, args) { return `${customTypeName}(${args.join(',')})`; } diff --git a/lib/errors.js b/lib/errors.js index 8ec192f5..bbb30d31 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -14,6 +14,8 @@ * limitations under the License. */ 'use strict'; +const exp = require('constants'); +const Long = require('long'); const util = require('util'); /** * Contains the error classes exposed by the driver. @@ -164,6 +166,18 @@ function BusyConnectionError(address, maxRequestsPerConnection, connectionLength util.inherits(BusyConnectionError, DriverError); +/** + * + * @param {Long} long + */ +function VIntOutOfRangeException(long){ + const message = `Value ${long.toString} is out of range for a JavaScript Number`; + DriverError.call(this, message, this.constructor); + this.info = 'Represents a run-time exception when attempting to decode a vint and the JavaScript Number doesn\'t have enough space to fit the value that was decoded'; +} + +util.inherits(VIntOutOfRangeException, DriverError); + exports.ArgumentError = ArgumentError; exports.AuthenticationError = AuthenticationError; exports.BusyConnectionError = BusyConnectionError; @@ -172,4 +186,5 @@ exports.OperationTimedOutError = OperationTimedOutError; exports.DriverInternalError = DriverInternalError; exports.NoHostAvailableError = NoHostAvailableError; exports.NotSupportedError = NotSupportedError; -exports.ResponseError = ResponseError; \ No newline at end of file +exports.ResponseError = ResponseError; +exports.VIntOutOfRangeException = VIntOutOfRangeException; \ No newline at end of file diff --git a/lib/types/duration.js b/lib/types/duration.js index aa1b4197..4dc084f0 100644 --- a/lib/types/duration.js +++ b/lib/types/duration.js @@ -17,6 +17,7 @@ const Long = require('long'); const util = require('util'); const utils = require('../utils'); +const VIntCoding = utils.VIntCoding; /** @module types */ @@ -514,201 +515,4 @@ Builder.prototype.build = function () { new Duration(this._months, this._days, this._nanoseconds)); }; -/** - * Contains the methods for reading and writing vints into binary format. - * Exposes only 2 internal methods, the rest are hidden. - * @private - */ -const VIntCoding = (function () { - /** @param {Long} n */ - function encodeZigZag64(n) { - // (n << 1) ^ (n >> 63); - return n.toUnsigned().shiftLeft(1).xor(n.shiftRight(63)); - } - - /** @param {Long} n */ - function decodeZigZag64(n) { - // (n >>> 1) ^ -(n & 1); - return n.shiftRightUnsigned(1).xor(n.and(Long.ONE).negate()); - } - - /** - * @param {Long} value - * @param {Buffer} buffer - * @returns {Number} - */ - function writeVInt(value, buffer) { - return writeUnsignedVInt(encodeZigZag64(value), buffer); - } - - /** - * @param {Long} value - * @param {Buffer} buffer - * @returns {number} - */ - function writeUnsignedVInt(value, buffer) { - const size = computeUnsignedVIntSize(value); - if (size === 1) { - buffer[0] = value.getLowBits(); - return 1; - } - encodeVInt(value, size, buffer); - return size; - } - - /** - * @param {Long} value - * @returns {number} - */ - function computeUnsignedVIntSize(value) { - const magnitude = numberOfLeadingZeros(value.or(Long.ONE)); - return (639 - magnitude * 9) >> 6; - } - - /** - * @param {Long} value - * @param {Number} size - * @param {Buffer} buffer - */ - function encodeVInt(value, size, buffer) { - const extraBytes = size - 1; - let intValue = value.getLowBits(); - let i; - let intBytes = 4; - for (i = extraBytes; i >= 0 && (intBytes--) > 0; i--) { - buffer[i] = 0xFF & intValue; - intValue >>= 8; - } - intValue = value.getHighBits(); - for (; i >= 0; i--) { - buffer[i] = 0xFF & intValue; - intValue >>= 8; - } - buffer[0] |= encodeExtraBytesToRead(extraBytes); - } - /** - * Returns the number of zero bits preceding the highest-order one-bit in the binary representation of the value. - * @param {Long} value - * @returns {Number} - */ - function numberOfLeadingZeros(value) { - if (value.equals(Long.ZERO)) { - return 64; - } - let n = 1; - let x = value.getHighBits(); - if (x === 0) { - n += 32; - x = value.getLowBits(); - } - if (x >>> 16 === 0) { - n += 16; - x <<= 16; - } - if (x >>> 24 === 0) { - n += 8; - x <<= 8; - } - if (x >>> 28 === 0) { - n += 4; - x <<= 4; - } - if (x >>> 30 === 0) { - n += 2; - x <<= 2; - } - n -= x >>> 31; - return n; - } - - - function encodeExtraBytesToRead(extraBytesToRead) { - return ~(0xff >> extraBytesToRead); - } - - /** - * @param {Buffer} buffer - * @param {{value: number}} offset - * @returns {Long} - */ - function readVInt(buffer, offset) { - return decodeZigZag64(readUnsignedVInt(buffer, offset)); - } - - /** - * @param {Buffer} input - * @param {{ value: number}} offset - * @returns {Long} - */ - function readUnsignedVInt(input, offset) { - const firstByte = input[offset.value++]; - if ((firstByte & 0x80) === 0) { - return Long.fromInt(firstByte); - } - const sByteInt = fromSignedByteToInt(firstByte); - const size = numberOfExtraBytesToRead(sByteInt); - let result = Long.fromInt(sByteInt & firstByteValueMask(size)); - for (let ii = 0; ii < size; ii++) { - const b = Long.fromInt(input[offset.value++]); - // (result << 8) | b - result = result.shiftLeft(8).or(b); - } - return result; - } - - function fromSignedByteToInt(value) { - if (value > 0x7f) { - return value - 0x0100; - } - return value; - } - - function numberOfLeadingZerosInt32(i) { - if (i === 0) { - return 32; - } - let n = 1; - if (i >>> 16 === 0) { - n += 16; - i <<= 16; - } - if (i >>> 24 === 0) { - n += 8; - i <<= 8; - } - if (i >>> 28 === 0) { - n += 4; - i <<= 4; - } - if (i >>> 30 === 0) { - n += 2; - i <<= 2; - } - n -= i >>> 31; - return n; - } - - /** - * @param {Number} firstByte - * @returns {Number} - */ - function numberOfExtraBytesToRead(firstByte) { - // Instead of counting 1s of the byte, we negate and count 0 of the byte - return numberOfLeadingZerosInt32(~firstByte) - 24; - } - - /** - * @param {Number} extraBytesToRead - * @returns {Number} - */ - function firstByteValueMask(extraBytesToRead) { - return 0xff >> extraBytesToRead; - } - - return { - readVInt: readVInt, - writeVInt: writeVInt - }; -})(); - module.exports = Duration; diff --git a/lib/types/vector.js b/lib/types/vector.js index 909b3eb3..c8fdbc28 100644 --- a/lib/types/vector.js +++ b/lib/types/vector.js @@ -24,8 +24,9 @@ class Vector { /** * * @param {Float32Array | Array} elements + * @param {String?} subtype */ - constructor (elements) { + constructor (elements, subtype = null) { if (elements instanceof Float32Array) { this.elements = Array.from(elements); } @@ -43,6 +44,7 @@ class Vector { * @type Number */ this.length = this.elements.length; + this.subtype = subtype; return new Proxy(this, { get: function (obj, key) { if (key === 'IDENTITY'){ diff --git a/lib/utils.js b/lib/utils.js index 1ce7c4b5..b17f36b9 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -15,6 +15,7 @@ */ 'use strict'; +const Long = require('long'); const util = require('util'); const net = require('net'); const { EventEmitter } = require('events'); @@ -1040,6 +1041,270 @@ function whilst(condition, fn, callback) { } } + +/** + * Contains the methods for reading and writing vints into binary format. + * Exposes only 2 internal methods, the rest are hidden. + */ +const VIntCoding = (function () { + /** @param {Long} n */ + function encodeZigZag64(n) { + // (n << 1) ^ (n >> 63); + return n.toUnsigned().shiftLeft(1).xor(n.shiftRight(63)); + } + + /** @param {Long} n */ + function decodeZigZag64(n) { + // (n >>> 1) ^ -(n & 1); + return n.shiftRightUnsigned(1).xor(n.and(Long.ONE).negate()); + } + + /** + * @param {Long} value + * @param {Buffer} buffer + * @returns {Number} + */ + function writeVInt(value, buffer) { + return writeUnsignedVInt(encodeZigZag64(value), buffer); + } + + /** + * @param {Long} value + * @param {Buffer} buffer + * @returns {number} + */ + function writeUnsignedVInt(value, buffer) { + const size = computeUnsignedVIntSize(value); + if (size === 1) { + buffer[0] = value.getLowBits(); + return 1; + } + encodeVInt(value, size, buffer); + return size; + } + + /** + * @param {Long} value + * @returns {number} + */ + function computeUnsignedVIntSize(value) { + const magnitude = numberOfLeadingZeros(value.or(Long.ONE)); + return (639 - magnitude * 9) >> 6; + } + + /** + * @param {Long} value + * @param {Number} size + * @param {Buffer} buffer + */ + function encodeVInt(value, size, buffer) { + const extraBytes = size - 1; + let intValue = value.getLowBits(); + let i; + let intBytes = 4; + for (i = extraBytes; i >= 0 && (intBytes--) > 0; i--) { + buffer[i] = 0xFF & intValue; + intValue >>= 8; + } + intValue = value.getHighBits(); + for (; i >= 0; i--) { + buffer[i] = 0xFF & intValue; + intValue >>= 8; + } + buffer[0] |= encodeExtraBytesToRead(extraBytes); + } + /** + * Returns the number of zero bits preceding the highest-order one-bit in the binary representation of the value. + * @param {Long} value + * @returns {Number} + */ + function numberOfLeadingZeros(value) { + if (value.equals(Long.ZERO)) { + return 64; + } + let n = 1; + let x = value.getHighBits(); + if (x === 0) { + n += 32; + x = value.getLowBits(); + } + if (x >>> 16 === 0) { + n += 16; + x <<= 16; + } + if (x >>> 24 === 0) { + n += 8; + x <<= 8; + } + if (x >>> 28 === 0) { + n += 4; + x <<= 4; + } + if (x >>> 30 === 0) { + n += 2; + x <<= 2; + } + n -= x >>> 31; + return n; + } + + + function encodeExtraBytesToRead(extraBytesToRead) { + return ~(0xff >> extraBytesToRead); + } + + /** + * @param {Buffer} buffer + * @param {{value: number}} offset + * @returns {Long} + */ + function readVInt(buffer, offset) { + return decodeZigZag64(readUnsignedVInt(buffer, offset)); + } + + /** + * @param {Buffer} input + * @param {{ value: number}} offset + * @returns {Long} + */ + function readUnsignedVInt(input, offset) { + const firstByte = input[offset.value++]; + if ((firstByte & 0x80) === 0) { + return Long.fromInt(firstByte); + } + const sByteInt = fromSignedByteToInt(firstByte); + const size = numberOfExtraBytesToRead(sByteInt); + let result = Long.fromInt(sByteInt & firstByteValueMask(size)); + for (let ii = 0; ii < size; ii++) { + const b = Long.fromInt(input[offset.value++]); + // (result << 8) | b + result = result.shiftLeft(8).or(b); + } + return result; + } + + function fromSignedByteToInt(value) { + if (value > 0x7f) { + return value - 0x0100; + } + return value; + } + + function numberOfLeadingZerosInt32(i) { + if (i === 0) { + return 32; + } + let n = 1; + if (i >>> 16 === 0) { + n += 16; + i <<= 16; + } + if (i >>> 24 === 0) { + n += 8; + i <<= 8; + } + if (i >>> 28 === 0) { + n += 4; + i <<= 4; + } + if (i >>> 30 === 0) { + n += 2; + i <<= 2; + } + n -= i >>> 31; + return n; + } + + /** + * @param {Number} firstByte + * @returns {Number} + */ + function numberOfExtraBytesToRead(firstByte) { + // Instead of counting 1s of the byte, we negate and count 0 of the byte + return numberOfLeadingZerosInt32(~firstByte) - 24; + } + + /** + * @param {Number} extraBytesToRead + * @returns {Number} + */ + function firstByteValueMask(extraBytesToRead) { + return 0xff >> extraBytesToRead; + } + + /** + * @param {Number} value + * @param {Buffer} output + * @returns {void} + */ + function writeUnsignedVInt32(value, output) { + writeUnsignedVInt(Long.fromNumber(value), output); + } + + /** + * Read up to a 32-bit integer back, using the unsigned (no zigzag) encoding. + * + *

    Note this method is the same as {@link #readUnsignedVInt(DataInput)}, except that we do + * *not* block if there are not enough bytes in the buffer to reconstruct the value. + * + * @param {Buffer} input + * @param {Number} readerIndex + * @returns {Number} + * @throws VIntOutOfRangeException If the vint doesn't fit into a 32-bit integer + */ + function getUnsignedVInt32(input, readerIndex) { + return checkedCast(getUnsignedVInt(input, readerIndex, input.length)); + } + + /** + * + * @param {Buffer} input + * @param {Number} readerIndex + * @param {Number} readerLimit + * @returns {Long} + */ + function getUnsignedVInt(input, readerIndex, readerLimit) { + if (readerIndex < 0) + throw new errors.ArgumentError( + "Reader index should be non-negative, but was " + readerIndex); + + if (readerIndex >= readerLimit) return Long.fromNumber(-1); + + const firstByte = /** @type {Number} */ (input.at(readerIndex++)); + + // Bail out early if this is one byte, necessary or it fails later + if (firstByte >= 0) return Long.fromNumber(firstByte); + + const size = numberOfExtraBytesToRead(firstByte); + if (readerIndex + size > readerLimit) return Long.fromNumber(-1); + + let retval = Long.fromNumber(firstByte & firstByteValueMask(size)); + for (let ii = 0; ii < size; ii++) { + const b = /** @type {Number} */ (input.at(readerIndex++)); + retval.shiftLeft(8); + retval.or(b & 0xff); + } + + return retval; + } + + /** + * + * @param {Long} value + * @returns {Number} + */ + function checkedCast(value) { + const result = value.toInt(); + if (value.notEquals(result)) throw new errors.VIntOutOfRangeException(value); + return result; + } + + return { + readVInt: readVInt, + writeVInt: writeVInt + }; +})(); + exports.adaptNamedParamsPrepared = adaptNamedParamsPrepared; exports.adaptNamedParamsWithHints = adaptNamedParamsWithHints; exports.AddressResolver = AddressResolver; @@ -1084,4 +1349,5 @@ exports.timesSeries = timesSeries; exports.totalLength = totalLength; exports.validateFn = validateFn; exports.whilst = whilst; -exports.HashSet = HashSet; \ No newline at end of file +exports.HashSet = HashSet; +exports.VIntCoding = VIntCoding; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index c30bacf4..f1e89630 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,22 +9,23 @@ "version": "4.7.2", "license": "Apache-2.0", "dependencies": { - "@types/long": "^5.0.0", - "@types/node": ">=8", - "adm-zip": "^0.5.10", - "long": "^5.2.3" + "@types/long": "~5.0.0", + "@types/node": "^18.11.18", + "adm-zip": "~0.5.10", + "long": "~5.2.3" }, "devDependencies": { - "chai": "^4.3.8", - "kerberos": "^2.0.3", - "mocha": "^10.2.0", - "mocha-jenkins-reporter": "^0.4.8", + "@types/mocha": "^10.0.8", + "chai": "~4.3.8", + "kerberos": "~2.0.3", + "mocha": "~10.2.0", + "mocha-jenkins-reporter": "~0.4.8", "proxyquire": "~2.1.3", - "sinon": "^15.2.0", + "sinon": "~15.2.0", "temp": ">= 0.8.3" }, "engines": { - "node": ">=8" + "node": ">=16" } }, "node_modules/@sinonjs/commons": { @@ -80,10 +81,19 @@ "long": "*" } }, + "node_modules/@types/mocha": { + "version": "10.0.8", + "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.8.tgz", + "integrity": "sha512-HfMcUmy9hTMJh66VNcmeC9iVErIZJli2bszuXc6julh5YGuRb/W5OnkHjwLNYdFlMis0sY3If5SEAp+PktdJjw==", + "dev": true + }, "node_modules/@types/node": { - "version": "20.5.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.9.tgz", - "integrity": "sha512-PcGNd//40kHAS3sTlzKB9C9XL4K0sTup8nbG5lC14kzEteTNuAFh9u5nA0o5TWnSG2r/JNPRXFVcHJIIeRlmqQ==" + "version": "18.19.54", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.54.tgz", + "integrity": "sha512-+BRgt0G5gYjTvdLac9sIeE0iZcJxi4Jc4PV5EUzqi+88jmQLr+fRZdv2tCTV7IHKSGxM6SaLoOXQWWUiLUItMw==", + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/adm-zip": { "version": "0.5.10", @@ -1712,6 +1722,11 @@ "node": ">=4" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/test/integration/short/vector-tests-ts.js b/test/integration/short/vector-tests-ts.js new file mode 100644 index 00000000..e49e3237 --- /dev/null +++ b/test/integration/short/vector-tests-ts.js @@ -0,0 +1,64 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; +var assert = require('assert'); +var util = require('util'); +var helper = require('../../test-helper.js'); +var Client = require('../../../lib/client.js'); +var ExecutionProfile = require('../../../lib/execution-profile.js').ExecutionProfile; +var types = require('../../../index.js').types; +var utils = require('../../../lib/utils.js'); +var errors = require('../../../lib/errors.js'); +var vit = helper.vit; +var vdescribe = helper.vdescribe; +var numericTests = require('./numeric-tests.js'); +var pagingTests = require('./paging-tests.js'); +vdescribe('5.0.0', 'Vector tests TypeScript', function () { + this.timeout(120000); + describe('#execute with vectors', function () { + var keyspace = helper.getRandomName('ks'); + var table = keyspace + '.' + helper.getRandomName('table'); + var createTableCql = "CREATE TABLE ".concat(table, " (id uuid PRIMARY KEY, v1 vector);"); + var setupInfo = helper.setup(1, { + keyspace: keyspace, + queries: [createTableCql] + }); + it('should insert and select vectors', function (done) { + var client = setupInfo.client; + // if client undefined, raise error + if (!client) + return done(new Error('client is not defined')); + var id = types.Uuid.random(); + var v1 = new Float32Array([1.1, 2.2, 3.3]); + var query = "INSERT INTO ".concat(table, " (id, v1) VALUES (?, ?)"); + client.execute(query, [id, v1], { prepare: true }, function (err) { + if (err) + return done(err); + client.execute("SELECT v1 FROM ".concat(table, " WHERE id = ?"), [id], { prepare: true }, function (err, result) { + if (err) + return done(err); + var v1 = result.rows[0].v1; + assert.strictEqual(result.rows.length, 1); + assert.strictEqual(v1.length, 3); + assert.strictEqual(v1[0], v1[0]); + assert.strictEqual(v1[1], v1[1]); + assert.strictEqual(v1[2], v1[2]); + done(); + }); + }); + }); + }); +}); diff --git a/test/integration/short/vector-tests-ts.ts b/test/integration/short/vector-tests-ts.ts new file mode 100644 index 00000000..a857f5c0 --- /dev/null +++ b/test/integration/short/vector-tests-ts.ts @@ -0,0 +1,64 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; +const assert = require('assert'); +const util = require('util'); +const helper = require('../../test-helper.js'); +const Client = require('../../../lib/client.js'); +const ExecutionProfile = require('../../../lib/execution-profile.js').ExecutionProfile; + +const { types } = require('../../../index.js'); +const utils = require('../../../lib/utils.js'); +const errors = require('../../../lib/errors.js'); +const vit = helper.vit; +const vdescribe = helper.vdescribe; +const numericTests = require('./numeric-tests.js'); +const pagingTests = require('./paging-tests.js'); + +vdescribe('5.0.0', 'Vector tests TypeScript', function () { + this.timeout(120000); + describe('#execute with vectors', function () { + const keyspace = helper.getRandomName('ks'); + const table = keyspace + '.' + helper.getRandomName('table'); + const createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY, v1 vector);`; + + const setupInfo = helper.setup(1, { + keyspace: keyspace, + queries: [ createTableCql ] + }); + it('should insert and select vectors', function(done){ + const client = setupInfo.client; + // if client undefined, raise error + if(!client) return done(new Error('client is not defined')); + const id = types.Uuid.random(); + const v1 = new Float32Array([1.1, 2.2, 3.3]); + const query = `INSERT INTO ${table} (id, v1) VALUES (?, ?)`; + client.execute(query, [id, v1], {prepare : true}, function(err){ + if (err) return done(err); + client.execute(`SELECT v1 FROM ${table} WHERE id = ?`, [id], { prepare: true }, function(err, result){ + if (err) return done(err); + const v1 : Float32Array = result.rows[0].v1; + assert.strictEqual(result.rows.length, 1); + assert.strictEqual(v1.length, 3); + assert.strictEqual(v1[0], v1[0]); + assert.strictEqual(v1[1], v1[1]); + assert.strictEqual(v1[2], v1[2]); + done(); + }); + }); + }); + }); +}); \ No newline at end of file diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index 9eefc06f..a418cf7d 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -27,6 +27,7 @@ const vit = helper.vit; const vdescribe = helper.vdescribe; const numericTests = require('./numeric-tests.js'); const pagingTests = require('./paging-tests.js'); +const Vector = require('../../../lib/types/vector.js'); vdescribe('5.0.0', 'Vector tests', function () { this.timeout(120000); diff --git a/test/unit/encoder-tests.js b/test/unit/encoder-tests.js index 03b2f66c..688343b0 100644 --- a/test/unit/encoder-tests.js +++ b/test/unit/encoder-tests.js @@ -735,6 +735,23 @@ describe('encoder', function () { assert.throws(function() { encoder.encode(refVal, {code: dataTypes.custom, info: typeName}); }, TypeError); }); + it('should encode/decode nested type as vector, encoder guesses type', function () { + const encoder = new Encoder(4, {}); + const refVal = new Vector([new Float32Array([1.2, 3.4, 5.6]), new Float32Array([7.8, 9.0, 11.2])]); + const guessedTypeObj = Encoder.guessDataType(refVal); + const encoded = encoder.encode(refVal, guessedTypeObj); + const decoded = encoder.decode(encoded, guessedTypeObj); + helper.assertInstanceOf(decoded, Vector); + // for (const k in decoded) { + // if (decoded.hasOwnProperty(k)) { + // assert.equal(decoded[k],refVal[k]); + // } + // else { + // assert.fail(); + // } + // } + }); + }); describe('#encode()', function () { From 8f2d5f22ecfca932edbeb99cf276f6c42d0b60e4 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 4 Oct 2024 02:15:36 +0000 Subject: [PATCH 73/99] refactor parseTypeName --- lib/encoder.js | 118 +++++++++++++++++++++++++++++++------------------ 1 file changed, 75 insertions(+), 43 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index 71d2a661..bbd0319f 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -27,6 +27,7 @@ const token = require('./token'); const { DateRange } = require('./datastax/search'); const geo = require('./geometry'); const Vector = require('./types/vector'); +const { conditionalDescribe } = require('../test/test-helper.js'); const Geometry = geo.Geometry; const LineString = geo.LineString; const Point = geo.Point; @@ -128,15 +129,17 @@ const zeroLengthTypesSupported = new Set([ /** * @typedef {(singleTypeNames[keyof singleTypeNames])} SingleTypeCodes + * @typedef {('point' | 'polygon' | 'duration' | 'lineString' | 'dateRange')} CustomSimpleTypeCodes + * @typedef {(customTypeNames[CustomSimpleTypeCodes]) | CustomSimpleTypeCodes | 'empty'} CustomSimpleTypeNames * @typedef {{code : SingleTypeCodes }} SingleColumnInfo - * @typedef {{code : (types.dataTypes.custom), customTypeName : ('point' | 'polygon' | 'duration' | 'lineString' | 'dateRange')}} CustomSimpleColumnInfo + * @typedef {{code : (types.dataTypes.custom), info : CustomSimpleTypeNames}} CustomSimpleColumnInfo * @typedef {{code : (types.dataTypes.map), info : [ColumnInfo, ColumnInfo], options?: {frozen?: Boolean, reversed?: Boolean}}} MapColumnInfo * @typedef {{code : (types.dataTypes.tuple), info : Array, options?: {frozen?: Boolean, reversed?: Boolean}}} TupleColumnInfo * @typedef {{code : (types.dataTypes.list | types.dataTypes.set), info : ColumnInfo, options?: {frozen?: Boolean, reversed?: Boolean}}} ListSetColumnInfo - * TODO: udt type + * @typedef {{code : (types.dataTypes.udt), info : {name : string, fields : Array<{name : string, type : ColumnInfo}>}}} UdtColumnInfo * @typedef {{code : (types.dataTypes.custom), customTypeName : ('vector'), info : ColumnInfo, dimension : number}} VectorColumnInfo * @typedef {{code : (types.dataTypes.custom), info : string}} OtherCustomColumnInfo - * @typedef {SingleColumnInfo | CustomSimpleColumnInfo | MapColumnInfo | TupleColumnInfo | ListSetColumnInfo | VectorColumnInfo | OtherCustomColumnInfo} ColumnInfo If this is a simple type, info is null; if this is a collection type with a simple subtype, info is a string, if this is a nested collection type, info is a ColumnInfo object + * @typedef {SingleColumnInfo | CustomSimpleColumnInfo | MapColumnInfo | TupleColumnInfo | ListSetColumnInfo | VectorColumnInfo | OtherCustomColumnInfo | UdtColumnInfo} ColumnInfo If this is a simple type, info is null; if this is a collection type with a simple subtype, info is a string, if this is a nested collection type, info is a ColumnInfo object */ /** @@ -925,7 +928,7 @@ function defineInstanceMembers() { /** * * @param {Buffer} buffer - * @param {{subtype: ColumnInfo, dimensions: Number}} params + * @param {VectorColumnInfo} params * @returns {import('../index.js').types.Vector} */ this.decodeVector = function(buffer, params) { @@ -946,7 +949,7 @@ function defineInstanceMembers() { }; /** - * @param {{code: Number, info: String?}} cqlType + * @param {ColumnInfo} cqlType * @returns {Number} */ this.serializationSizeIfFixed = function (cqlType) { @@ -1015,7 +1018,7 @@ function defineInstanceMembers() { } /** * @param {Vector} value - * @param {ColumnInfo} params + * @param {VectorColumnInfo} params * @returns {Buffer} */ this.encodeVector = function(value, params) { @@ -1227,21 +1230,25 @@ function defineInstanceMembers() { length = typeName.length; } - /** @type {ColumnInfo & {options : {frozen : boolean}}} */ - const dataType = { - code: 0, - info: null, - options: { - frozen: false - } - }; + // /** @type {ColumnInfo & {options : {frozen : boolean}}} */ + // const dataType = { + // code: 0, + // info: null, + // options: { + // frozen: false + // } + // }; let innerTypes; + let frozen = false; if (typeName.indexOf("'", startIndex) === startIndex) { //If quoted, this is a custom type. - dataType.info = typeName.substr(startIndex+1, length-2); - return dataType; + const info = typeName.substr(startIndex+1, length-2); + return { + code: dataTypes.custom, + info: info + }; } if (!length) { @@ -1252,7 +1259,7 @@ function defineInstanceMembers() { //Remove the frozen token startIndex += cqlNames.frozen.length + 1; length -= cqlNames.frozen.length + 2; - dataType.options.frozen = true; + frozen = true; } if (typeName.indexOf(cqlNames.list, startIndex) === startIndex) { @@ -1265,9 +1272,14 @@ function defineInstanceMembers() { throw new TypeError('Not a valid type ' + typeName); } - dataType.code = dataTypes.list; - dataType.info = await this.parseTypeName(keyspace, innerTypes[0], 0, null, udtResolver); - return dataType; + const info = await this.parseTypeName(keyspace, innerTypes[0], 0, null, udtResolver); + return { + code: dataTypes.list, + info: info, + options: { + frozen: frozen + } + }; } if (typeName.indexOf(cqlNames.set, startIndex) === startIndex) { @@ -1280,9 +1292,14 @@ function defineInstanceMembers() { throw new TypeError('Not a valid type ' + typeName); } - dataType.code = dataTypes.set; - dataType.info = await this.parseTypeName(keyspace, innerTypes[0], 0, null, udtResolver); - return dataType; + const info = await this.parseTypeName(keyspace, innerTypes[0], 0, null, udtResolver); + return { + code: dataTypes.set, + info: info, + options: { + frozen: frozen + } + }; } if (typeName.indexOf(cqlNames.map, startIndex) === startIndex) { @@ -1296,9 +1313,14 @@ function defineInstanceMembers() { throw new TypeError('Not a valid type ' + typeName); } - dataType.code = dataTypes.map; - dataType.info = await this._parseChildTypes(keyspace, innerTypes, udtResolver); - return dataType; + const info = await this._parseChildTypes(keyspace, innerTypes, udtResolver); + return { + code: dataTypes.map, + info: info, + options: { + frozen: frozen + } + }; } if (typeName.indexOf(cqlNames.tuple, startIndex) === startIndex) { @@ -1311,22 +1333,22 @@ function defineInstanceMembers() { throw new TypeError('Not a valid type ' + typeName); } - dataType.code = dataTypes.tuple; - dataType.info = await this._parseChildTypes(keyspace, innerTypes, udtResolver); - return dataType; + const info = await this._parseChildTypes(keyspace, innerTypes, udtResolver); + return { + code: dataTypes.tuple, + info: info + }; } if (typeName.indexOf(cqlNames.vector, startIndex) === startIndex) { // It's a vector, so record the subtype and dimension. - dataType.code = dataTypes.custom; // parseVectorTypeArgs is not an async function but we are. To keep things simple let's ask the // function to just return whatever it finds for an arg and we'll eval it after the fact const params = this.parseVectorTypeArgs(typeName, cqlNames.vector, (arg) => arg ); params["subtype"] = await this.parseTypeName(keyspace, params["subtype"]); - dataType.info = params; - return dataType; + return params; } const quoted = typeName.indexOf('"', startIndex) === startIndex; @@ -1348,26 +1370,24 @@ function defineInstanceMembers() { const typeCode = dataTypes[typeName]; if (typeof typeCode === 'number') { - dataType.code = typeCode; - return dataType; + return {code : typeCode}; } if (typeName === cqlNames.duration) { - dataType.info = customTypeNames.duration; - return dataType; + return {code : dataTypes.custom, info : customTypeNames.duration}; } if (typeName === cqlNames.empty) { // Set as custom - dataType.info = 'empty'; - return dataType; + return {code : dataTypes.custom, info : 'empty'}; } const udtInfo = await udtResolver(keyspace, typeName); if (udtInfo) { - dataType.code = dataTypes.udt; - dataType.info = udtInfo; - return dataType; + return { + code: dataTypes.udt, + info: udtInfo, + }; } throw new TypeError('Not a valid type "' + typeName + '"'); @@ -1593,6 +1613,13 @@ function defineInstanceMembers() { isComposite: isComposite }; }; + /** + * + * @param {string} typeName + * @param {number} startIndex + * @param {number} length + * @returns {UdtColumnInfo} + */ this._parseUdtName = function (typeName, startIndex, length) { const udtParams = parseParams(typeName, startIndex, length); if (udtParams.length < 2) { @@ -1603,6 +1630,9 @@ function defineInstanceMembers() { code: dataTypes.udt, info: null }; + /** + * @type {{keyspace: String, name: String, fields: Array}} + */ const udtInfo = { keyspace: udtParams[0], name: utils.allocBufferFromString(udtParams[1], 'hex').toString(), @@ -1617,8 +1647,10 @@ function defineInstanceMembers() { type: fieldType }); } - dataType.info = udtInfo; - return dataType; + return { + code : dataTypes.udt, + info : udtInfo + }; }; } From 35abc6c9f31b4b50371bc8394409ab8823871845 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 4 Oct 2024 02:42:15 +0000 Subject: [PATCH 74/99] refactor guessDataType --- lib/encoder.js | 105 +++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 55 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index bbd0319f..29929d06 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -128,18 +128,19 @@ const zeroLengthTypesSupported = new Set([ ]); /** - * @typedef {(singleTypeNames[keyof singleTypeNames])} SingleTypeCodes + * @typedef {(singleTypeNames[keyof singleTypeNames] | types.dataTypes.duration | types.dataTypes.text)} SingleTypeCodes * @typedef {('point' | 'polygon' | 'duration' | 'lineString' | 'dateRange')} CustomSimpleTypeCodes * @typedef {(customTypeNames[CustomSimpleTypeCodes]) | CustomSimpleTypeCodes | 'empty'} CustomSimpleTypeNames * @typedef {{code : SingleTypeCodes }} SingleColumnInfo * @typedef {{code : (types.dataTypes.custom), info : CustomSimpleTypeNames}} CustomSimpleColumnInfo * @typedef {{code : (types.dataTypes.map), info : [ColumnInfo, ColumnInfo], options?: {frozen?: Boolean, reversed?: Boolean}}} MapColumnInfo - * @typedef {{code : (types.dataTypes.tuple), info : Array, options?: {frozen?: Boolean, reversed?: Boolean}}} TupleColumnInfo + * @typedef {{code : (types.dataTypes.tuple), info : Array, options?: {frozen?: Boolean, reversed?: Boolean}}} TupleColumnInfo + * @typedef {{code : (types.dataTypes.tuple | types.dataTypes.list)}} TupleListColumnInfoWithoutSubtype TODO: guessDataType can return null on tuple/list info, why? * @typedef {{code : (types.dataTypes.list | types.dataTypes.set), info : ColumnInfo, options?: {frozen?: Boolean, reversed?: Boolean}}} ListSetColumnInfo * @typedef {{code : (types.dataTypes.udt), info : {name : string, fields : Array<{name : string, type : ColumnInfo}>}}} UdtColumnInfo * @typedef {{code : (types.dataTypes.custom), customTypeName : ('vector'), info : ColumnInfo, dimension : number}} VectorColumnInfo * @typedef {{code : (types.dataTypes.custom), info : string}} OtherCustomColumnInfo - * @typedef {SingleColumnInfo | CustomSimpleColumnInfo | MapColumnInfo | TupleColumnInfo | ListSetColumnInfo | VectorColumnInfo | OtherCustomColumnInfo | UdtColumnInfo} ColumnInfo If this is a simple type, info is null; if this is a collection type with a simple subtype, info is a string, if this is a nested collection type, info is a ColumnInfo object + * @typedef {SingleColumnInfo | CustomSimpleColumnInfo | MapColumnInfo | TupleColumnInfo | ListSetColumnInfo | VectorColumnInfo | OtherCustomColumnInfo | UdtColumnInfo | TupleListColumnInfoWithoutSubtype} ColumnInfo If this is a simple type, info is null; if this is a collection type with a simple subtype, info is a string, if this is a nested collection type, info is a ColumnInfo object */ /** @@ -991,29 +992,17 @@ function defineInstanceMembers() { case dataTypes.tuple: return -1; case dataTypes.custom: - if (cqlType.info?.startsWith(cqlNames.vector) || cqlType.info?.startsWith(customTypeNames.vector)) { - let vectorParams; - if (cqlType.info?.startsWith(cqlNames.vector)){ - // it's a vector with "vector" - /** @type {{subtype: string, dimensions : number}} */ - vectorParams = this.parseVectorTypeArgs(cqlType.info, cqlNames.vector, (arg) => arg); - }else{ - // it's a vector with "org.apache.cassandra.db.marshal.VectorType" - vectorParams = this.parseVectorTypeArgs(cqlType.info, customTypeNames.vector, this.parseFqTypeName); - } - - /** @type {{code: number, info: Object|Array|null}} */ - const subtypeParams = this.parseFqTypeName(vectorParams.subtype.info); - const subtypeSerialSize = this.serializationSizeIfFixed(subtypeParams); + if ('customTypeName' in cqlType && cqlType.customTypeName === 'vector'){ + const subtypeSerialSize = this.serializationSizeIfFixed(cqlType.info); if (subtypeSerialSize === -1){ return -1; }else{ - return subtypeSerialSize * vectorParams.dimensions; + return subtypeSerialSize * cqlType.dimension; } } return -1; default: - throw new TypeError('Cannot calculate size from ' + cqlType.code + ' ' + cqlType.info); + throw new TypeError('Cannot calculate size'); } } /** @@ -1051,7 +1040,7 @@ function defineInstanceMembers() { * @param {String} typeName * @param {String} stringToExclude Leading string indicating this is a vector type (to be excluded when eval'ing args) * @param {Function} subtypeResolveFn Function used to resolve subtype type; varies depending on type naming convention - * @returns {ColumnInfo} + * @returns {VectorColumnInfo} * @internal */ this.parseVectorTypeArgs = function(typeName, stringToExclude, subtypeResolveFn) { @@ -1823,71 +1812,78 @@ Encoder.prototype.encode = function (value, typeInfo) { /** * Try to guess the Cassandra type to be stored, based on the javascript value type * @param value - * @returns {{code: number, info: string? }|null} + * @returns {ColumnInfo | null} * @ignore * @internal */ Encoder.guessDataType = function (value) { - let code = null; - let info = null; const esTypeName = (typeof value); if (esTypeName === 'number') { - code = dataTypes.double; + return {code : dataTypes.double}; } else if (esTypeName === 'string') { - code = dataTypes.text; if (value.length === 36 && uuidRegex.test(value)){ - code = dataTypes.uuid; + return {code : dataTypes.uuid}; + }else{ + return {code : dataTypes.text}; } } else if (esTypeName === 'boolean') { - code = dataTypes.boolean; + return {code : dataTypes.boolean}; } else if (value instanceof Buffer) { - code = dataTypes.blob; + return {code : dataTypes.blob}; } else if (value instanceof Date) { - code = dataTypes.timestamp; + return {code : dataTypes.timestamp}; } else if (value instanceof Long) { - code = dataTypes.bigint; + return {code : dataTypes.bigint}; } else if (value instanceof Integer) { - code = dataTypes.varint; + return {code : dataTypes.varint}; } else if (value instanceof BigDecimal) { - code = dataTypes.decimal; + return {code : dataTypes.decimal}; } else if (value instanceof types.Uuid) { - code = dataTypes.uuid; + return {code : dataTypes.uuid}; } else if (value instanceof types.InetAddress) { - code = dataTypes.inet; + return {code : dataTypes.inet}; } else if (value instanceof types.Tuple) { - code = dataTypes.tuple; + return {code : dataTypes.tuple}; } else if (value instanceof types.LocalDate) { - code = dataTypes.date; + return {code : dataTypes.date}; } else if (value instanceof types.LocalTime) { - code = dataTypes.time; + return {code : dataTypes.time}; } else if (value instanceof types.Duration) { - code = dataTypes.custom; - info = customTypeNames.duration; + return {code : dataTypes.custom, + info : customTypeNames.duration}; } // Map JS TypedArrays onto vectors else if (value instanceof types.Vector) { - code = dataTypes.custom; - // TODO: another area that we have to generalize if we ever need to support vector subtypes other than float if (value && value.length > 0) { if (value instanceof Float32Array) { - info = buildParameterizedCustomType(customTypeNames.vector, [singleTypeNamesByDataType[dataTypes.float], value.length]); + return { + code: dataTypes.custom, + customTypeName: 'vector', + info: { code: dataTypes.float }, + dimension: value.length + } }else{ const subtype = this.guessDataType(value[0]); - if (subtype?.info) { - info = buildParameterizedCustomType(customTypeNames.vector, [subtype.info, value.length]); + if (subtype) { + return { + code: dataTypes.custom, + customTypeName: 'vector', + info: subtype, + dimension: value.length + } } else { throw new TypeError("Cannot guess subtype from element " + value.get(0)); } @@ -1897,27 +1893,26 @@ Encoder.guessDataType = function (value) { } } else if (Array.isArray(value)) { - code = dataTypes.list; + return {code : dataTypes.list}; } else if (value instanceof Geometry) { - code = dataTypes.custom; if (value instanceof LineString) { - info = customTypeNames.lineString; + return {code : dataTypes.custom, + info : customTypeNames.lineString}; } else if (value instanceof Point) { - info = customTypeNames.point; + return {code : dataTypes.custom, + info : customTypeNames.point}; } else if (value instanceof Polygon) { - info = customTypeNames.polygon; + return {code : dataTypes.custom, + info : customTypeNames.polygon}; } } else if (value instanceof DateRange) { - code = dataTypes.custom; - info = customTypeNames.dateRange; + return {code : dataTypes.custom, + info : customTypeNames.dateRange}; } - if (code === null) { - return null; - } - return { code: code, info: info }; + return null; }; /** From cbe002c9984c5f49bcfaf262027b18f5e8072a89 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 4 Oct 2024 20:04:55 +0000 Subject: [PATCH 75/99] fixed several tests --- lib/encoder.js | 89 ++++++++++++++++++++++++-------------- lib/types/index.js | 15 +++++-- test/unit/encoder-tests.js | 26 +++++++---- 3 files changed, 86 insertions(+), 44 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index 29929d06..e51836aa 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -137,7 +137,7 @@ const zeroLengthTypesSupported = new Set([ * @typedef {{code : (types.dataTypes.tuple), info : Array, options?: {frozen?: Boolean, reversed?: Boolean}}} TupleColumnInfo * @typedef {{code : (types.dataTypes.tuple | types.dataTypes.list)}} TupleListColumnInfoWithoutSubtype TODO: guessDataType can return null on tuple/list info, why? * @typedef {{code : (types.dataTypes.list | types.dataTypes.set), info : ColumnInfo, options?: {frozen?: Boolean, reversed?: Boolean}}} ListSetColumnInfo - * @typedef {{code : (types.dataTypes.udt), info : {name : string, fields : Array<{name : string, type : ColumnInfo}>}}} UdtColumnInfo + * @typedef {{code : (types.dataTypes.udt), info : {name : string, fields : Array<{name : string, type : ColumnInfo}>}, options? : {frozen?: Boolean, reversed?: Boolean}}} UdtColumnInfo * @typedef {{code : (types.dataTypes.custom), customTypeName : ('vector'), info : ColumnInfo, dimension : number}} VectorColumnInfo * @typedef {{code : (types.dataTypes.custom), info : string}} OtherCustomColumnInfo * @typedef {SingleColumnInfo | CustomSimpleColumnInfo | MapColumnInfo | TupleColumnInfo | ListSetColumnInfo | VectorColumnInfo | OtherCustomColumnInfo | UdtColumnInfo | TupleListColumnInfoWithoutSubtype} ColumnInfo If this is a simple type, info is null; if this is a collection type with a simple subtype, info is a string, if this is a nested collection type, info is a ColumnInfo object @@ -206,7 +206,15 @@ function defineInstanceMembers() { this.decodeBlob = function (bytes) { return this.handleBuffer(bytes); }; - this.decodeCustom = function (bytes, typeName) { + + /** + * + * @param {Buffer} bytes + * @param {string} typeName + * @param {ColumnInfo | null} columnInfo + * @returns + */ + this.decodeCustom = function (bytes, typeName, columnInfo = null) { // Make sure we actually have something to process in typeName before we go any further if (!typeName || typeName.length === 0) { @@ -214,8 +222,8 @@ function defineInstanceMembers() { } // Special handling for vector custom types (since they have args) - if (typeName.startsWith(customTypeNames.vector)) { - return this.decodeVector(bytes, this.parseVectorTypeArgs(typeName, customTypeNames.vector, this.parseFqTypeName)); + if (columnInfo != null && 'customTypeName' in columnInfo && columnInfo.customTypeName === 'vector') { + return this.decodeVector(bytes, columnInfo); } const handler = customDecoders[typeName]; if (handler) { @@ -223,6 +231,7 @@ function defineInstanceMembers() { } return this.handleBuffer(bytes); }; + this.decodeUtf8String = function (bytes) { return bytes.toString('utf8'); }; @@ -726,11 +735,19 @@ function defineInstanceMembers() { } return value; }; - this.encodeCustom = function (value, customTypeName) { + + /** + * + * @param {any} value + * @param {string} customTypeName + * @param {ColumnInfo | null} columnInfo + * @returns + */ + this.encodeCustom = function (value, customTypeName, columnInfo = null) { // Special handling for vector custom types (since they have args) - if (customTypeName.startsWith(customTypeNames.vector)) { - return this.encodeVector(value, this.parseVectorTypeArgs(customTypeName, customTypeNames.vector, this.parseFqTypeName)); + if (columnInfo != null && 'customTypeName' in columnInfo && columnInfo.customTypeName === 'vector') { + return this.encodeVector(value, columnInfo); } const handler = customEncoders[customTypeName]; if (handler) { @@ -771,7 +788,7 @@ function defineInstanceMembers() { return buf; }; /** - * @param {Number|String} value + * @param {Number} value * @private */ this.encodeTinyint = function (value) { @@ -933,16 +950,20 @@ function defineInstanceMembers() { * @returns {import('../index.js').types.Vector} */ this.decodeVector = function(buffer, params) { - const subtype = params["subtype"]; - const dimensions = params["dimensions"]; - const elemLength = 4; // TODO: figure this out based on the subtype + const subtype = params.info; + const dimension = params.dimension; + // const elemLength = this.serializationSizeIfFixed(subtype); + // if (elemLength === -1) { + // throw new Error('Var size not implmented yet'); + // } + const elemLength = 4; const expectedLength = buffer.length / elemLength; - if ((elemLength * dimensions) !== buffer.length) { - throw new TypeError(`Expected buffer of subtype ${subtype} with dimensions ${dimensions} to be of size ${expectedLength}, observed size ${buffer.length}`); + if ((elemLength * dimension) !== buffer.length) { + throw new TypeError(`Expected buffer of subtype ${subtype} with dimensions ${dimension} to be of size ${expectedLength}, observed size ${buffer.length}`); } const rv = []; let offset = 0; - for (let i = 0; i < dimensions; i++) { + for (let i = 0; i < dimension; i++) { offset = i * elemLength; rv[i] = this.decode(buffer.slice(offset, offset + elemLength), subtype); } @@ -1335,7 +1356,7 @@ function defineInstanceMembers() { // parseVectorTypeArgs is not an async function but we are. To keep things simple let's ask the // function to just return whatever it finds for an arg and we'll eval it after the fact const params = this.parseVectorTypeArgs(typeName, cqlNames.vector, (arg) => arg ); - params["subtype"] = await this.parseTypeName(keyspace, params["subtype"]); + params["info"] = await this.parseTypeName(keyspace, params["info"]); return params; } @@ -1376,6 +1397,9 @@ function defineInstanceMembers() { return { code: dataTypes.udt, info: udtInfo, + options: { + frozen: frozen + } }; } @@ -1506,7 +1530,12 @@ function defineInstanceMembers() { //move cursor across the name and bypass the parenthesis startIndex += complexTypeNames.udt.length + 1; length -= complexTypeNames.udt.length + 2; - return this._parseUdtName(typeName, startIndex, length); + const udtType = this._parseUdtName(typeName, startIndex, length); + udtType.options = { + frozen : frozen, + reversed : reversed + }; + return udtType; } if (typeName.indexOf(complexTypeNames.tuple, startIndex) === startIndex) { //move cursor across the name and bypass the parenthesis @@ -1715,9 +1744,7 @@ function setEncoders() { * This is part of an experimental API, this can be changed future releases. *

    * @param {Buffer} buffer Raw buffer to be decoded. - * @param {Object} type An object containing the data type code and info. - * @param {Number} type.code Type code. - * @param {Object} [type.info] Additional information on the type for complex / nested types. + * @param {ColumnInfo} type */ Encoder.prototype.decode = function (buffer, type) { if (buffer === null || (buffer.length === 0 && !zeroLengthTypesSupported.has(type.code))) { @@ -1730,7 +1757,7 @@ Encoder.prototype.decode = function (buffer, type) { throw new Error('Unknown data type: ' + type.code); } - return decoder.call(this, buffer, type.info); + return decoder.call(this, buffer, 'info' in type? type.info : null, 'dimension' in type ? type : null); }; /** @@ -1739,7 +1766,7 @@ Encoder.prototype.decode = function (buffer, type) { * This is part of an experimental API, this can be changed future releases. *

    * @param {*} value The value to be converted. - * @param {ColumnInfo} typeInfo The type information. + * @param {ColumnInfo | Number | String} typeInfo The type information. *

    It can be either a:

    *
      *
    • A String representing the data type.
    • @@ -1769,24 +1796,22 @@ Encoder.prototype.encode = function (value, typeInfo) { return value; } - /** @type {ColumnInfo | {code : null, info : null}} */ - let type = { - code: null, - info: null - }; + /** @type {ColumnInfo | null} */ + let type = null; if (typeInfo) { if (typeof typeInfo === 'number') { - type.code = typeInfo; + type = { + code: typeInfo + }; } else if (typeof typeInfo === 'string') { type = dataTypes.getByName(typeInfo); } - if (typeof typeInfo.code === 'number') { - type.code = typeInfo.code; - type.info = typeInfo.info; + else if (typeof typeInfo.code === 'number') { + type = typeInfo; } - if (typeof type.code !== 'number') { + if (type == null || typeof type.code !== 'number') { throw new TypeError('Type information not valid, only String and Number values are valid hints'); } } @@ -1806,7 +1831,7 @@ Encoder.prototype.encode = function (value, typeInfo) { throw new Error('Type not supported ' + type.code); } - return encoder.call(this, value, type.info); + return encoder.call(this, value, 'info' in type ? type.info : null, 'dimension' in type ? type : null); }; /** diff --git a/lib/types/index.js b/lib/types/index.js index 3063740a..dab2ee35 100644 --- a/lib/types/index.js +++ b/lib/types/index.js @@ -135,8 +135,8 @@ const dataTypes = { tuple: 0x0031, /** * Returns the typeInfo of a given type name - * @param name - * @returns {{code: number, info: *|Object}} + * @param {string} name + * @returns {import('../encoder').ColumnInfo} */ getByName: function(name) { name = name.toLowerCase(); @@ -161,8 +161,17 @@ const dataTypes = { return this.getByName(x.trim()); }, this)}; } + const vectorMatches = /^(vector)<*(.+) *, *(\d+)>$/.exec(name); + if(vectorMatches){ + return { + code: this.custom, + customTypeName: 'vector', + info: this.getByName(vectorMatches[2]), + dimension: parseInt(vectorMatches[3]) + } + } } - const typeInfo = { code: this[name], info: null}; + const typeInfo = { code: this[name]}; if (typeof typeInfo.code !== 'number') { throw new TypeError('Data type with name ' + name + ' not valid'); } diff --git a/test/unit/encoder-tests.js b/test/unit/encoder-tests.js index 688343b0..53c71225 100644 --- a/test/unit/encoder-tests.js +++ b/test/unit/encoder-tests.js @@ -22,7 +22,7 @@ const tokenizer = require('../../lib/tokenizer'); const token = require('../../lib/token'); const Vector = require('../../lib/types/vector'); const Encoder = require('../../lib/encoder'); -const types = require('../../lib/types'); +const { types } = require('../../index.js'); const ExecutionOptions = require('../../lib/execution-options').ExecutionOptions; const dataTypes = types.dataTypes; const helper = require('../test-helper'); @@ -683,6 +683,10 @@ describe('encoder', function () { const encoder = new Encoder(4, {}); const refVal = new Float32Array([1.2, 3.4, 5.6]); const guessedTypeObj = Encoder.guessDataType(refVal); + if (guessedTypeObj == null){ + assert.fail(); + return; + } const encoded = encoder.encode(refVal, guessedTypeObj); const decoded = encoder.decode(encoded, guessedTypeObj); helper.assertInstanceOf(decoded, Vector); @@ -699,8 +703,8 @@ describe('encoder', function () { it('should encode/decode FloatArray as vector, encoder provided with full type', function () { const encoder = new Encoder(4, {}); const refVal = new Float32Array([1.2, 3.4, 5.6]); - const typeName = 'org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.FloatType,3)'; - const typeObj = {code: dataTypes.custom, info: typeName}; + /** @type {import('../../lib/encoder').VectorColumnInfo} */ + const typeObj = {code: dataTypes.custom, info: {code : dataTypes.float}, dimension : 3, customTypeName : 'vector'}; const encoded = encoder.encode(refVal, typeObj); const decoded = encoder.decode(encoded, typeObj); helper.assertInstanceOf(decoded, Vector); @@ -739,6 +743,10 @@ describe('encoder', function () { const encoder = new Encoder(4, {}); const refVal = new Vector([new Float32Array([1.2, 3.4, 5.6]), new Float32Array([7.8, 9.0, 11.2])]); const guessedTypeObj = Encoder.guessDataType(refVal); + if (guessedTypeObj == null){ + assert.fail(); + return; + } const encoded = encoder.encode(refVal, guessedTypeObj); const decoded = encoder.decode(encoded, guessedTypeObj); helper.assertInstanceOf(decoded, Vector); @@ -1101,9 +1109,9 @@ describe('encoder', function () { type = encoder.parseFqTypeName('org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.FloatType,10)'); assert.strictEqual(dataTypes.custom, type.code); assert.ok(typeof type.info === 'object'); - assert.strictEqual(Object.keys(type.info).length, 2); - assert.strictEqual(dataTypes.float, type.info["subtype"].code); - assert.strictEqual(10, type.info["dimensions"]); + assert.strictEqual(Object.keys(type.info).length, 1); // {code: dataTypes.float} + assert.strictEqual(dataTypes.float, type.info.code); + assert.strictEqual(10, type["dimension"]); }); it('should parse frozen types', function () { @@ -1219,7 +1227,7 @@ describe('encoder', function () { ['tuple', dataTypes.tuple, [dataTypes.varchar, dataTypes.int]], ['frozen>', dataTypes.list, dataTypes.timeuuid], ['map>>', dataTypes.map, [dataTypes.text, dataTypes.list]], - ['vector', dataTypes.custom, {subtype: {code: dataTypes.float}, dimensions: 20}] + ['vector', dataTypes.custom, {code: dataTypes.float, dimension: 20}] ]; for (const item of items) { @@ -1235,8 +1243,8 @@ describe('encoder', function () { }); } else if (typeof item[2] === 'object') { - assert.strictEqual(dataType.info.subtype.code, item[2].subtype.code); - assert.strictEqual(dataType.info.dimensions, item[2].dimensions); + assert.strictEqual(dataType.info.code, item[2].code); + assert.strictEqual(dataType.dimension, item[2].dimension); } else { assert.strictEqual(dataType.info.code, item[2]); From 706e7c97510ed3b0991c7c63c2ba7713be4ff07a Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 4 Oct 2024 21:03:36 +0000 Subject: [PATCH 76/99] fix eslint --- lib/encoder.js | 113 ++++++++----------- lib/errors.js | 2 - lib/types/index.js | 4 +- lib/types/vector.js | 131 ++++++++++------------ lib/utils.js | 16 +-- test/integration/short/vector-tests-ts.js | 64 ----------- test/integration/short/vector-tests-ts.ts | 8 -- test/integration/short/vector-tests.js | 65 +++++------ test/unit/encoder-tests.js | 1 - 9 files changed, 149 insertions(+), 255 deletions(-) delete mode 100644 test/integration/short/vector-tests-ts.js diff --git a/lib/encoder.js b/lib/encoder.js index e51836aa..d86a7325 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -16,7 +16,8 @@ 'use strict'; const util = require('util'); -const { types } = require('../index.js'); +// const { types } = require('../index.js'); +const types = require('./types'); const dataTypes = types.dataTypes; const Long = types.Long; const Integer = types.Integer; @@ -27,7 +28,6 @@ const token = require('./token'); const { DateRange } = require('./datastax/search'); const geo = require('./geometry'); const Vector = require('./types/vector'); -const { conditionalDescribe } = require('../test/test-helper.js'); const Geometry = geo.Geometry; const LineString = geo.LineString; const Point = geo.Point; @@ -96,6 +96,8 @@ const singleTypeNames = Object.freeze({ 'org.apache.cassandra.db.marshal.IntegerType': dataTypes.varint, 'org.apache.cassandra.db.marshal.CounterColumnType': dataTypes.counter }); + +// eslint-disable-next-line no-unused-vars const singleTypeNamesByDataType = invertObject(singleTypeNames); const singleFqTypeNamesLength = Object.keys(singleTypeNames).reduce(function (previous, current) { return current.length > previous ? current.length : previous; @@ -131,15 +133,15 @@ const zeroLengthTypesSupported = new Set([ * @typedef {(singleTypeNames[keyof singleTypeNames] | types.dataTypes.duration | types.dataTypes.text)} SingleTypeCodes * @typedef {('point' | 'polygon' | 'duration' | 'lineString' | 'dateRange')} CustomSimpleTypeCodes * @typedef {(customTypeNames[CustomSimpleTypeCodes]) | CustomSimpleTypeCodes | 'empty'} CustomSimpleTypeNames - * @typedef {{code : SingleTypeCodes }} SingleColumnInfo - * @typedef {{code : (types.dataTypes.custom), info : CustomSimpleTypeNames}} CustomSimpleColumnInfo + * @typedef {{code : SingleTypeCodes, options? : {frozen?:boolean, reversed?:boolean} }} SingleColumnInfo + * @typedef {{code : (types.dataTypes.custom), info : CustomSimpleTypeNames, options? : {frozen?:boolean, reversed?:boolean}}} CustomSimpleColumnInfo * @typedef {{code : (types.dataTypes.map), info : [ColumnInfo, ColumnInfo], options?: {frozen?: Boolean, reversed?: Boolean}}} MapColumnInfo * @typedef {{code : (types.dataTypes.tuple), info : Array, options?: {frozen?: Boolean, reversed?: Boolean}}} TupleColumnInfo * @typedef {{code : (types.dataTypes.tuple | types.dataTypes.list)}} TupleListColumnInfoWithoutSubtype TODO: guessDataType can return null on tuple/list info, why? * @typedef {{code : (types.dataTypes.list | types.dataTypes.set), info : ColumnInfo, options?: {frozen?: Boolean, reversed?: Boolean}}} ListSetColumnInfo * @typedef {{code : (types.dataTypes.udt), info : {name : string, fields : Array<{name : string, type : ColumnInfo}>}, options? : {frozen?: Boolean, reversed?: Boolean}}} UdtColumnInfo - * @typedef {{code : (types.dataTypes.custom), customTypeName : ('vector'), info : ColumnInfo, dimension : number}} VectorColumnInfo - * @typedef {{code : (types.dataTypes.custom), info : string}} OtherCustomColumnInfo + * @typedef {{code : (types.dataTypes.custom), customTypeName : ('vector'), info : ColumnInfo, dimension : number, options? : {frozen?:boolean, reversed?:boolean}}} VectorColumnInfo + * @typedef {{code : (types.dataTypes.custom), info : string, options? : {frozen?:boolean, reversed?:boolean}}} OtherCustomColumnInfo * @typedef {SingleColumnInfo | CustomSimpleColumnInfo | MapColumnInfo | TupleColumnInfo | ListSetColumnInfo | VectorColumnInfo | OtherCustomColumnInfo | UdtColumnInfo | TupleListColumnInfoWithoutSubtype} ColumnInfo If this is a simple type, info is null; if this is a collection type with a simple subtype, info is a string, if this is a nested collection type, info is a ColumnInfo object */ @@ -952,11 +954,10 @@ function defineInstanceMembers() { this.decodeVector = function(buffer, params) { const subtype = params.info; const dimension = params.dimension; - // const elemLength = this.serializationSizeIfFixed(subtype); - // if (elemLength === -1) { - // throw new Error('Var size not implmented yet'); - // } - const elemLength = 4; + const elemLength = this.serializationSizeIfFixed(subtype); + if (elemLength === -1) { + throw new Error('Var size not implmented yet'); + } const expectedLength = buffer.length / elemLength; if ((elemLength * dimension) !== buffer.length) { throw new TypeError(`Expected buffer of subtype ${subtype} with dimensions ${dimension} to be of size ${expectedLength}, observed size ${buffer.length}`); @@ -1017,15 +1018,15 @@ function defineInstanceMembers() { const subtypeSerialSize = this.serializationSizeIfFixed(cqlType.info); if (subtypeSerialSize === -1){ return -1; - }else{ - return subtypeSerialSize * cqlType.dimension; } + return subtypeSerialSize * cqlType.dimension; + } return -1; default: throw new TypeError('Cannot calculate size'); } - } + }; /** * @param {Vector} value * @param {VectorColumnInfo} params @@ -1037,7 +1038,7 @@ function defineInstanceMembers() { throw new TypeError("Driver only supports Vector type when encoding a vector"); } - const dimension = params["dimension"]; + const dimension = params.dimension; if (value.length !== dimension) { throw new TypeError(`Expected vector with ${dimension} dimensions, observed size of ${value.length}`); } @@ -1448,11 +1449,16 @@ function defineInstanceMembers() { length -= complexTypeNames.frozen.length + 2; frozen = true; } + const options = { + frozen: frozen, + reversed: reversed + }; if (typeName === complexTypeNames.empty) { //set as custom return { code: dataTypes.custom, - info: 'empty' + info: 'empty', + options: options }; } //Quick check if its a single type @@ -1462,7 +1468,7 @@ function defineInstanceMembers() { } const typeCode = singleTypeNames[typeName]; if (typeof typeCode === 'number') { - return {code : typeCode}; + return {code : typeCode, options : options}; } throw new TypeError('Not a valid type "' + typeName + '"'); } @@ -1480,10 +1486,7 @@ function defineInstanceMembers() { return { code: dataTypes.list, info: info, - options: { - frozen: frozen, - reversed: reversed - } + options: options }; } if (typeName.indexOf(complexTypeNames.set, startIndex) === startIndex) { @@ -1501,9 +1504,7 @@ function defineInstanceMembers() { return { code : dataTypes.set, info : info, - options : { - frozen : frozen, - reversed : reversed} + options: options }; } if (typeName.indexOf(complexTypeNames.map, startIndex) === startIndex) { @@ -1516,14 +1517,12 @@ function defineInstanceMembers() { if (params.length !== 2) { throw new TypeError('Not a valid type ' + typeName); } - const info1 = this.parseFqTypeName(params[0]) + const info1 = this.parseFqTypeName(params[0]); const info2 = this.parseFqTypeName(params[1]); return { code : dataTypes.map, info : [info1, info2], - options : { - frozen : frozen, - reversed : reversed} + options: options }; } if (typeName.indexOf(complexTypeNames.udt, startIndex) === startIndex) { @@ -1531,10 +1530,7 @@ function defineInstanceMembers() { startIndex += complexTypeNames.udt.length + 1; length -= complexTypeNames.udt.length + 2; const udtType = this._parseUdtName(typeName, startIndex, length); - udtType.options = { - frozen : frozen, - reversed : reversed - }; + udtType.options = options; return udtType; } if (typeName.indexOf(complexTypeNames.tuple, startIndex) === startIndex) { @@ -1549,22 +1545,23 @@ function defineInstanceMembers() { return { code : dataTypes.tuple, info : info, - options : { - frozen : frozen, - reversed : reversed} + options: options }; } if (typeName.indexOf(customTypeNames.vector, startIndex) === startIndex) { // It's a vector, so record the subtype and dimension. - return this.parseVectorTypeArgs(typeName, customTypeNames.vector, this.parseFqTypeName); + const params = this.parseVectorTypeArgs(typeName, customTypeNames.vector, this.parseFqTypeName); + params.options = options; + return params; } // Assume custom type if cannot be parsed up to this point. const info = typeName.substr(startIndex, length); return { code: dataTypes.custom, - info: info + info: info, + options: options }; }; /** @@ -1644,10 +1641,6 @@ function defineInstanceMembers() { //It should contain at least the keyspace, name of the udt and a type throw new TypeError('Not a valid type ' + typeName); } - const dataType = { - code: dataTypes.udt, - info: null - }; /** * @type {{keyspace: String, name: String, fields: Array}} */ @@ -1849,9 +1842,9 @@ Encoder.guessDataType = function (value) { else if (esTypeName === 'string') { if (value.length === 36 && uuidRegex.test(value)){ return {code : dataTypes.uuid}; - }else{ - return {code : dataTypes.text}; } + return {code : dataTypes.text}; + } else if (esTypeName === 'boolean') { return {code : dataTypes.boolean}; @@ -1899,20 +1892,20 @@ Encoder.guessDataType = function (value) { customTypeName: 'vector', info: { code: dataTypes.float }, dimension: value.length - } - }else{ - const subtype = this.guessDataType(value[0]); - if (subtype) { - return { - code: dataTypes.custom, - customTypeName: 'vector', - info: subtype, - dimension: value.length - } - } else { - throw new TypeError("Cannot guess subtype from element " + value.get(0)); - } + }; } + const subtype = this.guessDataType(value[0]); + if (subtype) { + return { + code: dataTypes.custom, + customTypeName: 'vector', + info: subtype, + dimension: value.length + }; + } + throw new TypeError("Cannot guess subtype from element " + value.get(0)); + + } else { throw new TypeError("Cannot guess subtype of empty vector"); } @@ -2150,14 +2143,6 @@ function concatRoutingKey(parts, totalLength) { return routingKey; } -/** - * @param {string} customTypeName - * @param {Array} args - */ -function buildParameterizedCustomType(customTypeName, args) { - return `${customTypeName}(${args.join(',')})`; -} - function invertObject(obj) { const rv = {}; for(const k in obj){ diff --git a/lib/errors.js b/lib/errors.js index bbb30d31..970c528b 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -14,8 +14,6 @@ * limitations under the License. */ 'use strict'; -const exp = require('constants'); -const Long = require('long'); const util = require('util'); /** * Contains the error classes exposed by the driver. diff --git a/lib/types/index.js b/lib/types/index.js index dab2ee35..1542c6fa 100644 --- a/lib/types/index.js +++ b/lib/types/index.js @@ -167,8 +167,8 @@ const dataTypes = { code: this.custom, customTypeName: 'vector', info: this.getByName(vectorMatches[2]), - dimension: parseInt(vectorMatches[3]) - } + dimension: parseInt(vectorMatches[3], 10) + }; } } const typeInfo = { code: this[name]}; diff --git a/lib/types/vector.js b/lib/types/vector.js index c8fdbc28..6ae1824b 100644 --- a/lib/types/vector.js +++ b/lib/types/vector.js @@ -21,95 +21,86 @@ */ const util = require('node:util'); class Vector { - /** + /** * * @param {Float32Array | Array} elements * @param {String?} subtype */ - constructor (elements, subtype = null) { - if (elements instanceof Float32Array) { - this.elements = Array.from(elements); - } - else if (Array.isArray(elements)) { - this.elements = elements; - } - else { - throw new TypeError('Vector must be constructed with a Float32Array or an Array'); - } - if (this.elements.length === 0) { - throw new TypeError('Vector must contain at least one value'); - } - /** + constructor (elements, subtype = null) { + if (elements instanceof Float32Array) { + this.elements = Array.from(elements); + } + else if (Array.isArray(elements)) { + this.elements = elements; + } + else { + throw new TypeError('Vector must be constructed with a Float32Array or an Array'); + } + if (this.elements.length === 0) { + throw new TypeError('Vector must contain at least one value'); + } + /** * Returns the number of the elements. * @type Number */ - this.length = this.elements.length; - this.subtype = subtype; - return new Proxy(this, { - get: function (obj, key) { - if (key === 'IDENTITY'){ - return 'Vector'; - } else if (typeof (key) === 'string' && (Number.isInteger(Number(key)))) // key is an index - return obj.elements[key]; - else - return obj[key]; - }, - set: function (obj, key, value) { - if (typeof (key) === 'string' && (Number.isInteger(Number(key)))) // key is an index - return obj.elements[key] = value; - else - return obj[key] = value; - }, - ownKeys: function (obj) { - return Reflect.ownKeys(elements); - }, - getOwnPropertyDescriptor(target, key) { - if (typeof (key) === 'string' && (Number.isInteger(Number(key)))){ - // array index - return { enumerable: true, configurable: true}; - }else{ - return Reflect.getOwnPropertyDescriptor(target, key); - } - } - }); - } - /** + this.length = this.elements.length; + this.subtype = subtype; + return new Proxy(this, { + get: function (obj, key) { + if (key === 'IDENTITY'){ + return 'Vector'; + } else if (typeof (key) === 'string' && (Number.isInteger(Number(key)))) // key is an index + {return obj.elements[key];} + return obj[key]; + }, + set: function (obj, key, value) { + if (typeof (key) === 'string' && (Number.isInteger(Number(key)))) // key is an index + {return obj.elements[key] = value;} + return obj[key] = value; + }, + ownKeys: function (obj) { + return Reflect.ownKeys(elements); + }, + getOwnPropertyDescriptor(target, key) { + if (typeof (key) === 'string' && (Number.isInteger(Number(key)))){ + // array index + return { enumerable: true, configurable: true}; + } + return Reflect.getOwnPropertyDescriptor(target, key); + + } + }); + } + /** * Returns the string representation of the vector. * @returns {string} */ - toString() { - return "[".concat(this.elements.toString(), "]"); - } - /** + toString() { + return "[".concat(this.elements.toString(), "]"); + } + /** * * @param {number} index */ - at(index) { - return this.elements[index]; - }; - /** + at(index) { + return this.elements[index]; + } + /** * * @returns {ArrayIterator} */ - [Symbol.iterator]() { - return this.elements[Symbol.iterator](); - } + [Symbol.iterator]() { + return this.elements[Symbol.iterator](); + } - [Symbol.species] = { - /** - * instanceof - */ - get: function () { - return Vector; - }, - enumerable: false, - configurable: true - }; -}; + static get [Symbol.species]() { + return Vector; + } +} Object.defineProperty(Vector, Symbol.hasInstance, { - value: function (i) { - return (util.types.isProxy(i) && i.IDENTITY === 'Vector') || i instanceof Float32Array; } + value: function (i) { + return (util.types.isProxy(i) && i.IDENTITY === 'Vector') || i instanceof Float32Array; } }); module.exports = Vector; diff --git a/lib/utils.js b/lib/utils.js index b17f36b9..becbfb87 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1237,6 +1237,7 @@ const VIntCoding = (function () { * @param {Buffer} output * @returns {void} */ + // eslint-disable-next-line no-unused-vars function writeUnsignedVInt32(value, output) { writeUnsignedVInt(Long.fromNumber(value), output); } @@ -1252,6 +1253,7 @@ const VIntCoding = (function () { * @returns {Number} * @throws VIntOutOfRangeException If the vint doesn't fit into a 32-bit integer */ + // eslint-disable-next-line no-unused-vars function getUnsignedVInt32(input, readerIndex) { return checkedCast(getUnsignedVInt(input, readerIndex, input.length)); } @@ -1265,20 +1267,20 @@ const VIntCoding = (function () { */ function getUnsignedVInt(input, readerIndex, readerLimit) { if (readerIndex < 0) - throw new errors.ArgumentError( - "Reader index should be non-negative, but was " + readerIndex); + {throw new errors.ArgumentError( + "Reader index should be non-negative, but was " + readerIndex);} - if (readerIndex >= readerLimit) return Long.fromNumber(-1); + if (readerIndex >= readerLimit) {return Long.fromNumber(-1);} const firstByte = /** @type {Number} */ (input.at(readerIndex++)); // Bail out early if this is one byte, necessary or it fails later - if (firstByte >= 0) return Long.fromNumber(firstByte); + if (firstByte >= 0) {return Long.fromNumber(firstByte);} const size = numberOfExtraBytesToRead(firstByte); - if (readerIndex + size > readerLimit) return Long.fromNumber(-1); + if (readerIndex + size > readerLimit) {return Long.fromNumber(-1);} - let retval = Long.fromNumber(firstByte & firstByteValueMask(size)); + const retval = Long.fromNumber(firstByte & firstByteValueMask(size)); for (let ii = 0; ii < size; ii++) { const b = /** @type {Number} */ (input.at(readerIndex++)); retval.shiftLeft(8); @@ -1295,7 +1297,7 @@ const VIntCoding = (function () { */ function checkedCast(value) { const result = value.toInt(); - if (value.notEquals(result)) throw new errors.VIntOutOfRangeException(value); + if (value.notEquals(result)) {throw new errors.VIntOutOfRangeException(value);} return result; } diff --git a/test/integration/short/vector-tests-ts.js b/test/integration/short/vector-tests-ts.js deleted file mode 100644 index e49e3237..00000000 --- a/test/integration/short/vector-tests-ts.js +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -'use strict'; -var assert = require('assert'); -var util = require('util'); -var helper = require('../../test-helper.js'); -var Client = require('../../../lib/client.js'); -var ExecutionProfile = require('../../../lib/execution-profile.js').ExecutionProfile; -var types = require('../../../index.js').types; -var utils = require('../../../lib/utils.js'); -var errors = require('../../../lib/errors.js'); -var vit = helper.vit; -var vdescribe = helper.vdescribe; -var numericTests = require('./numeric-tests.js'); -var pagingTests = require('./paging-tests.js'); -vdescribe('5.0.0', 'Vector tests TypeScript', function () { - this.timeout(120000); - describe('#execute with vectors', function () { - var keyspace = helper.getRandomName('ks'); - var table = keyspace + '.' + helper.getRandomName('table'); - var createTableCql = "CREATE TABLE ".concat(table, " (id uuid PRIMARY KEY, v1 vector);"); - var setupInfo = helper.setup(1, { - keyspace: keyspace, - queries: [createTableCql] - }); - it('should insert and select vectors', function (done) { - var client = setupInfo.client; - // if client undefined, raise error - if (!client) - return done(new Error('client is not defined')); - var id = types.Uuid.random(); - var v1 = new Float32Array([1.1, 2.2, 3.3]); - var query = "INSERT INTO ".concat(table, " (id, v1) VALUES (?, ?)"); - client.execute(query, [id, v1], { prepare: true }, function (err) { - if (err) - return done(err); - client.execute("SELECT v1 FROM ".concat(table, " WHERE id = ?"), [id], { prepare: true }, function (err, result) { - if (err) - return done(err); - var v1 = result.rows[0].v1; - assert.strictEqual(result.rows.length, 1); - assert.strictEqual(v1.length, 3); - assert.strictEqual(v1[0], v1[0]); - assert.strictEqual(v1[1], v1[1]); - assert.strictEqual(v1[2], v1[2]); - done(); - }); - }); - }); - }); -}); diff --git a/test/integration/short/vector-tests-ts.ts b/test/integration/short/vector-tests-ts.ts index a857f5c0..3396fa6b 100644 --- a/test/integration/short/vector-tests-ts.ts +++ b/test/integration/short/vector-tests-ts.ts @@ -15,18 +15,10 @@ */ 'use strict'; const assert = require('assert'); -const util = require('util'); const helper = require('../../test-helper.js'); -const Client = require('../../../lib/client.js'); -const ExecutionProfile = require('../../../lib/execution-profile.js').ExecutionProfile; const { types } = require('../../../index.js'); -const utils = require('../../../lib/utils.js'); -const errors = require('../../../lib/errors.js'); -const vit = helper.vit; const vdescribe = helper.vdescribe; -const numericTests = require('./numeric-tests.js'); -const pagingTests = require('./paging-tests.js'); vdescribe('5.0.0', 'Vector tests TypeScript', function () { this.timeout(120000); diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index a418cf7d..7aba5a84 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -15,50 +15,41 @@ */ 'use strict'; const assert = require('assert'); -const util = require('util'); const helper = require('../../test-helper.js'); -const Client = require('../../../lib/client.js'); -const ExecutionProfile = require('../../../lib/execution-profile.js').ExecutionProfile; const { types } = require('../../../index.js'); -const utils = require('../../../lib/utils.js'); -const errors = require('../../../lib/errors.js'); -const vit = helper.vit; const vdescribe = helper.vdescribe; -const numericTests = require('./numeric-tests.js'); -const pagingTests = require('./paging-tests.js'); -const Vector = require('../../../lib/types/vector.js'); vdescribe('5.0.0', 'Vector tests', function () { - this.timeout(120000); - describe('#execute with vectors', function () { - const keyspace = helper.getRandomName('ks'); - const table = keyspace + '.' + helper.getRandomName('table'); - const createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY, v1 vector);`; + this.timeout(120000); + describe('#execute with vectors', function () { + const keyspace = helper.getRandomName('ks'); + const table = keyspace + '.' + helper.getRandomName('table'); + const createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY, v1 vector);`; - const setupInfo = helper.setup(1, { - keyspace: keyspace, - queries: [ createTableCql ] - }); - it('should insert and select vectors', function(done){ - const client = setupInfo.client; - // if client undefined, raise error - if(!client) return done(new Error('client is not defined')); - const id = types.Uuid.random(); - const v1 = new Float32Array([1.1, 2.2, 3.3]); - const query = `INSERT INTO ${table} (id, v1) VALUES (?, ?)`; - client.execute(query, [id, v1], {prepare : true}, function(err){ - if (err) return done(err); - client.execute(`SELECT v1 FROM ${table} WHERE id = ?`, [id], { prepare: true }, function(err, result){ - if (err) return done(err); - assert.strictEqual(result.rows.length, 1); - assert.strictEqual(result.rows[0].v1.length, 3); - assert.strictEqual(result.rows[0].v1[0], v1[0]); - assert.strictEqual(result.rows[0].v1[1], v1[1]); - assert.strictEqual(result.rows[0].v1[2], v1[2]); - done(); - }); - }); + const setupInfo = helper.setup(1, { + keyspace: keyspace, + queries: [ createTableCql ] + }); + it('should insert and select vectors', function(done){ + const client = setupInfo.client; + // if client undefined, raise error + if(!client) {return done(new Error('client is not defined'));} + const id = types.Uuid.random(); + const v1 = new Float32Array([1.1, 2.2, 3.3]); + const query = `INSERT INTO ${table} (id, v1) VALUES (?, ?)`; + client.execute(query, [id, v1], {prepare : true}, function(err){ + if (err) {return done(err);} + client.execute(`SELECT v1 FROM ${table} WHERE id = ?`, [id], { prepare: true }, function(err, result){ + if (err) {return done(err);} + assert.strictEqual(result.rows.length, 1); + assert.strictEqual(result.rows[0].v1.length, 3); + assert.strictEqual(result.rows[0].v1[0], v1[0]); + assert.strictEqual(result.rows[0].v1[1], v1[1]); + assert.strictEqual(result.rows[0].v1[2], v1[2]); + done(); }); + }); }); + }); }); \ No newline at end of file diff --git a/test/unit/encoder-tests.js b/test/unit/encoder-tests.js index 53c71225..9c2fc94f 100644 --- a/test/unit/encoder-tests.js +++ b/test/unit/encoder-tests.js @@ -1109,7 +1109,6 @@ describe('encoder', function () { type = encoder.parseFqTypeName('org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.FloatType,10)'); assert.strictEqual(dataTypes.custom, type.code); assert.ok(typeof type.info === 'object'); - assert.strictEqual(Object.keys(type.info).length, 1); // {code: dataTypes.float} assert.strictEqual(dataTypes.float, type.info.code); assert.strictEqual(10, type["dimension"]); }); From f1e167377433db32f47d993aed292bc6e4dda87b Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 10 Oct 2024 05:21:50 +0000 Subject: [PATCH 77/99] refactor pass vector test --- lib/encoder.js | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index d86a7325..0f6edf0f 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -16,7 +16,6 @@ 'use strict'; const util = require('util'); -// const { types } = require('../index.js'); const types = require('./types'); const dataTypes = types.dataTypes; const Long = types.Long; @@ -133,7 +132,7 @@ const zeroLengthTypesSupported = new Set([ * @typedef {(singleTypeNames[keyof singleTypeNames] | types.dataTypes.duration | types.dataTypes.text)} SingleTypeCodes * @typedef {('point' | 'polygon' | 'duration' | 'lineString' | 'dateRange')} CustomSimpleTypeCodes * @typedef {(customTypeNames[CustomSimpleTypeCodes]) | CustomSimpleTypeCodes | 'empty'} CustomSimpleTypeNames - * @typedef {{code : SingleTypeCodes, options? : {frozen?:boolean, reversed?:boolean} }} SingleColumnInfo + * @typedef {{code : SingleTypeCodes, info: null, options? : {frozen?:boolean, reversed?:boolean} }} SingleColumnInfo * @typedef {{code : (types.dataTypes.custom), info : CustomSimpleTypeNames, options? : {frozen?:boolean, reversed?:boolean}}} CustomSimpleColumnInfo * @typedef {{code : (types.dataTypes.map), info : [ColumnInfo, ColumnInfo], options?: {frozen?: Boolean, reversed?: Boolean}}} MapColumnInfo * @typedef {{code : (types.dataTypes.tuple), info : Array, options?: {frozen?: Boolean, reversed?: Boolean}}} TupleColumnInfo @@ -224,6 +223,11 @@ function defineInstanceMembers() { } // Special handling for vector custom types (since they have args) + if(typeName.startsWith(customTypeNames.vector)) { + const vectorColumnInfo = /** @type {VectorColumnInfo} */ (this.parseFqTypeName(typeName)); + return this.decodeVector(bytes, vectorColumnInfo); + } + if (columnInfo != null && 'customTypeName' in columnInfo && columnInfo.customTypeName === 'vector') { return this.decodeVector(bytes, columnInfo); } @@ -747,6 +751,10 @@ function defineInstanceMembers() { */ this.encodeCustom = function (value, customTypeName, columnInfo = null) { + if (customTypeName.startsWith(customTypeNames.vector)) { + const vectorColumnInfo = /** @type {VectorColumnInfo} */ (this.parseFqTypeName(customTypeName)); + return this.encodeVector(value, vectorColumnInfo); + } // Special handling for vector custom types (since they have args) if (columnInfo != null && 'customTypeName' in columnInfo && columnInfo.customTypeName === 'vector') { return this.encodeVector(value, columnInfo); @@ -1241,15 +1249,6 @@ function defineInstanceMembers() { length = typeName.length; } - // /** @type {ColumnInfo & {options : {frozen : boolean}}} */ - // const dataType = { - // code: 0, - // info: null, - // options: { - // frozen: false - // } - // }; - let innerTypes; let frozen = false; @@ -1258,7 +1257,7 @@ function defineInstanceMembers() { const info = typeName.substr(startIndex+1, length-2); return { code: dataTypes.custom, - info: info + info: info, }; } @@ -1347,7 +1346,8 @@ function defineInstanceMembers() { const info = await this._parseChildTypes(keyspace, innerTypes, udtResolver); return { code: dataTypes.tuple, - info: info + info: info, + options: {frozen: frozen} }; } @@ -1381,7 +1381,7 @@ function defineInstanceMembers() { const typeCode = dataTypes[typeName]; if (typeof typeCode === 'number') { - return {code : typeCode}; + return {code : typeCode, info: null}; } if (typeName === cqlNames.duration) { @@ -1468,7 +1468,7 @@ function defineInstanceMembers() { } const typeCode = singleTypeNames[typeName]; if (typeof typeCode === 'number') { - return {code : typeCode, options : options}; + return {code : typeCode, info: null, options : options}; } throw new TypeError('Not a valid type "' + typeName + '"'); } From 03a4432a9455348649426be214793949b8f9330a Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 11 Oct 2024 19:14:50 +0000 Subject: [PATCH 78/99] trying to add var size vint, not working yet --- lib/encoder.js | 15 +++--- lib/utils.js | 65 +++++++++++++++++++++++++- test/integration/short/vector-tests.js | 25 +++++++++- 3 files changed, 97 insertions(+), 8 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index 0f6edf0f..4c48cd82 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -745,7 +745,7 @@ function defineInstanceMembers() { /** * * @param {any} value - * @param {string} customTypeName + * @param {string | ColumnInfo} customTypeName when it's a vector, this will be the subtype columninfo, and value will be 0 * @param {ColumnInfo | null} columnInfo * @returns */ @@ -1055,13 +1055,16 @@ function defineInstanceMembers() { throw new TypeError("Cannot encode empty array as vector"); } - // TypedArrays are _not_ JS arrays so explicitly convert them here before trying to write them - // into a buffer - const elems = []; + const serializationSize = this.serializationSizeIfFixed(params.info); + const encoded = []; for (const elem of value) { - elems.push(this.encode(elem, params.info)); + const elemBuffer = this.encode(elem, params.info); + if (serializationSize === -1) { + encoded.push(utils.VIntCoding.uvintPack(elemBuffer.length)); + } + encoded.push(elemBuffer); } - return Buffer.concat(elems); + return Buffer.concat(encoded); }; /** diff --git a/lib/utils.js b/lib/utils.js index becbfb87..46da7572 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1163,6 +1163,7 @@ const VIntCoding = (function () { } /** + * uvint_unpack * @param {Buffer} input * @param {{ value: number}} offset * @returns {Long} @@ -1301,9 +1302,71 @@ const VIntCoding = (function () { return result; } + /** + * + * @param {Buffer} bytes + * @returns {[Number, Number]} + */ + function uvintUnpack(bytes) { + const firstByte = bytes[0]; + + if ((firstByte & 128) === 0) { + return [firstByte, 1]; + } + + const numExtraBytes = 8 - (~firstByte & 0xff).toString(2).length; + let rv = firstByte & (0xff >> numExtraBytes); + + for (let idx = 1; idx <= numExtraBytes; idx++) { + const newByte = bytes[idx]; + rv <<= 8; + rv |= newByte & 0xff; + } + + return [rv, numExtraBytes + 1]; +} + +/** + * + * @param {Number} val + * @returns {Buffer} + */ + function uvintPack(val) { + const rv = []; + if (val < 128) { + rv.push(val); + } else { + let v = val; + let numExtraBytes = 0; + let numBits = v.toString(2).length; + let reservedBits = numExtraBytes + 1; + + while (numBits > (8 - reservedBits)) { + numExtraBytes += 1; + numBits -= 8; + reservedBits = Math.min(numExtraBytes + 1, 8); + rv.push(v & 0xff); + v >>= 8; + } + + if (numExtraBytes > 8) { + throw new Error(`Value ${val} is too big and cannot be encoded as vint`); + } + + const n = 8 - numExtraBytes; + v |= (0xff >> n) << n; + rv.push(Math.abs(v)); + } + + rv.reverse(); + return Buffer.from(rv); +} + return { readVInt: readVInt, - writeVInt: writeVInt + writeVInt: writeVInt, + uvintPack: uvintPack, + uvintUnpack: uvintUnpack }; })(); diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index 7aba5a84..ddec650d 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -18,6 +18,7 @@ const assert = require('assert'); const helper = require('../../test-helper.js'); const { types } = require('../../../index.js'); +const Vector = require('../../../lib/types/vector.js'); const vdescribe = helper.vdescribe; vdescribe('5.0.0', 'Vector tests', function () { @@ -25,7 +26,7 @@ vdescribe('5.0.0', 'Vector tests', function () { describe('#execute with vectors', function () { const keyspace = helper.getRandomName('ks'); const table = keyspace + '.' + helper.getRandomName('table'); - const createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY, v1 vector);`; + const createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY, v1 vector, v2 vector);`; const setupInfo = helper.setup(1, { keyspace: keyspace, @@ -51,5 +52,27 @@ vdescribe('5.0.0', 'Vector tests', function () { }); }); }); + + + it('should insert and select vector of text', function(done){ + const client = setupInfo.client; + // if client undefined, raise error + if(!client) {return done(new Error('client is not defined'));} + const id = types.Uuid.random(); + const v1 = new Vector(['ab', 'b', 'cde'], 'text'); + const query = `INSERT INTO ${table} (id, v2) VALUES (?, ?)`; + client.execute(query, [id, v1], {prepare : true}, function(err){ + if (err) {return done(err);} + client.execute(`SELECT v2 FROM ${table} WHERE id = ?`, [id], { prepare: true }, function(err, result){ + if (err) {return done(err);} + assert.strictEqual(result.rows.length, 1); + assert.strictEqual(result.rows[0].v1.length, 3); + assert.strictEqual(result.rows[0].v1[0], v1[0]); + assert.strictEqual(result.rows[0].v1[1], v1[1]); + assert.strictEqual(result.rows[0].v1[2], v1[2]); + done(); + }); + }); + }); }); }); \ No newline at end of file From 482ed4cea9ad623f021968b56f5b11d812553f1d Mon Sep 17 00:00:00 2001 From: janehe Date: Sat, 12 Oct 2024 03:01:54 +0000 Subject: [PATCH 79/99] all unit tests passed --- lib/encoder.js | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index 4c48cd82..43991f2f 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -211,7 +211,7 @@ function defineInstanceMembers() { /** * * @param {Buffer} bytes - * @param {string} typeName + * @param {ColumnInfo | string} typeName * @param {ColumnInfo | null} columnInfo * @returns */ @@ -223,14 +223,15 @@ function defineInstanceMembers() { } // Special handling for vector custom types (since they have args) - if(typeName.startsWith(customTypeNames.vector)) { - const vectorColumnInfo = /** @type {VectorColumnInfo} */ (this.parseFqTypeName(typeName)); - return this.decodeVector(bytes, vectorColumnInfo); - } - if (columnInfo != null && 'customTypeName' in columnInfo && columnInfo.customTypeName === 'vector') { return this.decodeVector(bytes, columnInfo); } + + if(typeof typeName === 'string' && typeName.startsWith(customTypeNames.vector)) { + const vectorColumnInfo = /** @type {VectorColumnInfo} */ (this.parseFqTypeName(typeName)); + return this.decodeVector(bytes, vectorColumnInfo); + } + const handler = customDecoders[typeName]; if (handler) { return handler.call(this, bytes); @@ -751,14 +752,14 @@ function defineInstanceMembers() { */ this.encodeCustom = function (value, customTypeName, columnInfo = null) { - if (customTypeName.startsWith(customTypeNames.vector)) { - const vectorColumnInfo = /** @type {VectorColumnInfo} */ (this.parseFqTypeName(customTypeName)); - return this.encodeVector(value, vectorColumnInfo); - } // Special handling for vector custom types (since they have args) if (columnInfo != null && 'customTypeName' in columnInfo && columnInfo.customTypeName === 'vector') { return this.encodeVector(value, columnInfo); } + if (typeof customTypeName === 'string' && customTypeName.startsWith(customTypeNames.vector)) { + const vectorColumnInfo = /** @type {VectorColumnInfo} */ (this.parseFqTypeName(customTypeName)); + return this.encodeVector(value, vectorColumnInfo); + } const handler = customEncoders[customTypeName]; if (handler) { return handler.call(this, value); From 45c4ef734b8b58e684a830a2164972dc7d86827f Mon Sep 17 00:00:00 2001 From: janehe Date: Sat, 12 Oct 2024 05:03:57 +0000 Subject: [PATCH 80/99] float diff --- lib/encoder.js | 20 ++-- lib/types/vector.js | 8 ++ lib/utils.js | 2 +- test/unit/encoder-tests.js | 22 ++++- test/unit/encoder-vector-tests.js | 156 ++++++++++++++++++++++++++++++ 5 files changed, 196 insertions(+), 12 deletions(-) create mode 100644 test/unit/encoder-vector-tests.js diff --git a/lib/encoder.js b/lib/encoder.js index 43991f2f..8357f5b9 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -964,18 +964,20 @@ function defineInstanceMembers() { const subtype = params.info; const dimension = params.dimension; const elemLength = this.serializationSizeIfFixed(subtype); - if (elemLength === -1) { - throw new Error('Var size not implmented yet'); - } - const expectedLength = buffer.length / elemLength; - if ((elemLength * dimension) !== buffer.length) { - throw new TypeError(`Expected buffer of subtype ${subtype} with dimensions ${dimension} to be of size ${expectedLength}, observed size ${buffer.length}`); - } + const rv = []; let offset = 0; for (let i = 0; i < dimension; i++) { - offset = i * elemLength; - rv[i] = this.decode(buffer.slice(offset, offset + elemLength), subtype); + if (elemLength === -1) { + // var sized + const [size, bytesRead] = utils.VIntCoding.uvintUnpack(buffer.subarray(offset)); + offset += bytesRead; + rv[i] = this.decode(buffer.subarray(offset, offset + size), subtype); + offset += size; + }else{ + offset = i * elemLength; + rv[i] = this.decode(buffer.subarray(offset, offset + elemLength), subtype); + } } return new Vector(rv); }; diff --git a/lib/types/vector.js b/lib/types/vector.js index 6ae1824b..51610c87 100644 --- a/lib/types/vector.js +++ b/lib/types/vector.js @@ -96,6 +96,14 @@ class Vector { static get [Symbol.species]() { return Vector; } + + /** + * + * @param {(value: any, index: number, array: any[]) => void} callback + */ + forEach(callback) { + return this.elements.forEach(callback); + } } Object.defineProperty(Vector, Symbol.hasInstance, { diff --git a/lib/utils.js b/lib/utils.js index 46da7572..a77777b3 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1305,7 +1305,7 @@ const VIntCoding = (function () { /** * * @param {Buffer} bytes - * @returns {[Number, Number]} + * @returns {[number, number]} [size, bytes read] */ function uvintUnpack(bytes) { const firstByte = bytes[0]; diff --git a/test/unit/encoder-tests.js b/test/unit/encoder-tests.js index 9c2fc94f..c0c2c373 100644 --- a/test/unit/encoder-tests.js +++ b/test/unit/encoder-tests.js @@ -96,7 +96,7 @@ describe('encoder', function () { describe('#encode() and #decode()', function () { const typeEncoder = new Encoder(2, {}); it('should encode and decode a guessed double', function () { - const value = 1111; + const value = 1111.1; const encoded = typeEncoder.encode(value); const decoded = typeEncoder.decode(encoded, {code: dataTypes.double}); assert.strictEqual(decoded, value); @@ -685,7 +685,6 @@ describe('encoder', function () { const guessedTypeObj = Encoder.guessDataType(refVal); if (guessedTypeObj == null){ assert.fail(); - return; } const encoded = encoder.encode(refVal, guessedTypeObj); const decoded = encoder.decode(encoded, guessedTypeObj); @@ -718,6 +717,25 @@ describe('encoder', function () { } }); + it('should encode/decode Vector of texts as vector, encoder provided with full type', function () { + const encoder = new Encoder(4, {}); + const refVal = new Vector(['a', 'bc', 'de']); + /** @type {import('../../lib/encoder').VectorColumnInfo} */ + const typeObj = {code: dataTypes.custom, info: {code : dataTypes.ascii}, dimension : 3, customTypeName : 'vector'}; + const encoded = encoder.encode(refVal, typeObj); + const decoded = encoder.decode(encoded, typeObj); + helper.assertInstanceOf(decoded, Vector); + for (const k in decoded) { + if (decoded.hasOwnProperty(k)) { + assert.equal(decoded[k],refVal[k]); + } + else { + assert.fail(); + } + } + }); + + it('should fail to encode if full type provided and input vector fails to match dimensions of type', function () { const encoder = new Encoder(4, {}); const refVal = new Float32Array([1.2, 3.4, 5.6, 7.8]); diff --git a/test/unit/encoder-vector-tests.js b/test/unit/encoder-vector-tests.js new file mode 100644 index 00000000..79a3636b --- /dev/null +++ b/test/unit/encoder-vector-tests.js @@ -0,0 +1,156 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; +const { assert, expect } = require('chai'); +const Encoder = require('../../lib/encoder'); +const { types } = require('../../index'); +const Vector = require('../../lib/types/vector'); + +describe('Vector tests', function () { + const encoder = new Encoder(4, {}); + /** + * @type {Array<{subtypeString : string, typeInfo: import('../../lib/encoder').VectorColumnInfo, value: Array}>} + */ + const dataProvider = [ + { + subtypeString : 'float', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.float, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [1.1, 2.2, 3.3] + }, + { + subtypeString : 'double', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.double, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [1.1, 2.2, 3.3] + }, + { + subtypeString : 'text', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.text, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: ['a', 'bc', 'cde'] + }, + { + subtypeString : 'int', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.int, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [1, 2, 3] + }, + { + subtypeString : 'bigint', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.bigint, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [1, 2, 3] + }, + { + subtypeString : 'uuid', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.uuid, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [types.Uuid.random(), types.Uuid.random(), types.Uuid.random()] + }, + { + subtypeString : 'timeuuid', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.timeuuid, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [types.TimeUuid.now(), types.TimeUuid.now(), types.TimeUuid.now()] + }, + { + subtypeString : 'decimal', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.decimal, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [types.BigDecimal.fromString('1.1'), types.BigDecimal.fromString('2.2'), types.BigDecimal.fromString('3.3')] + } + ] + + dataProvider.forEach((data) => { + it(`should encode and decode vector of ${data.subtypeString}`, function () { + const vector = new Vector(data.value); + const encoded = encoder.encode(vector, data.typeInfo); + const decoded = encoder.decode(encoded, data.typeInfo); + for (const k in decoded) { + if (decoded.hasOwnProperty(k)) { + assert.equal(decoded[k],data.value[k]); + } + else { + assert.fail(); + } + } + }); + }); + + it('should encode and decode vector of float', function(){ + const vector = new Float32Array([1.1, 2.2, 3.3]); + const typeObj = { code: types.dataTypes.custom, info: {code: types.dataTypes.float}, dimension: 3, customTypeName: 'vector' }; + const encoded = encoder.encode(vector, typeObj); + const decoded = encoder.decode(encoded, typeObj); + for (const k in decoded) { + if (decoded.hasOwnProperty(k)) { + assert.equal(decoded[k],vector[k]); + } + else { + assert.fail(); + } + } + }) +}); \ No newline at end of file From bbdd976c303078e57b845061680c83ff5c4569d2 Mon Sep 17 00:00:00 2001 From: janehe Date: Mon, 14 Oct 2024 22:03:42 +0000 Subject: [PATCH 81/99] pass initial integration tests --- lib/utils.js | 52 +++--- test/integration/short/vector-tests.js | 106 ++++++----- test/unit/encoder-tests.js | 27 +-- test/unit/encoder-vector-tests.js | 233 ++++++++++++------------- 4 files changed, 209 insertions(+), 209 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index a77777b3..5f7ffea2 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1311,22 +1311,22 @@ const VIntCoding = (function () { const firstByte = bytes[0]; if ((firstByte & 128) === 0) { - return [firstByte, 1]; + return [firstByte, 1]; } const numExtraBytes = 8 - (~firstByte & 0xff).toString(2).length; let rv = firstByte & (0xff >> numExtraBytes); for (let idx = 1; idx <= numExtraBytes; idx++) { - const newByte = bytes[idx]; - rv <<= 8; - rv |= newByte & 0xff; + const newByte = bytes[idx]; + rv <<= 8; + rv |= newByte & 0xff; } return [rv, numExtraBytes + 1]; -} + } -/** + /** * * @param {Number} val * @returns {Buffer} @@ -1334,33 +1334,33 @@ const VIntCoding = (function () { function uvintPack(val) { const rv = []; if (val < 128) { - rv.push(val); + rv.push(val); } else { - let v = val; - let numExtraBytes = 0; - let numBits = v.toString(2).length; - let reservedBits = numExtraBytes + 1; - - while (numBits > (8 - reservedBits)) { - numExtraBytes += 1; - numBits -= 8; - reservedBits = Math.min(numExtraBytes + 1, 8); - rv.push(v & 0xff); - v >>= 8; - } + let v = val; + let numExtraBytes = 0; + let numBits = v.toString(2).length; + let reservedBits = numExtraBytes + 1; + + while (numBits > (8 - reservedBits)) { + numExtraBytes += 1; + numBits -= 8; + reservedBits = Math.min(numExtraBytes + 1, 8); + rv.push(v & 0xff); + v >>= 8; + } - if (numExtraBytes > 8) { - throw new Error(`Value ${val} is too big and cannot be encoded as vint`); - } + if (numExtraBytes > 8) { + throw new Error(`Value ${val} is too big and cannot be encoded as vint`); + } - const n = 8 - numExtraBytes; - v |= (0xff >> n) << n; - rv.push(Math.abs(v)); + const n = 8 - numExtraBytes; + v |= (0xff >> n) << n; + rv.push(Math.abs(v)); } rv.reverse(); return Buffer.from(rv); -} + } return { readVInt: readVInt, diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index ddec650d..879e9101 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -19,60 +19,86 @@ const helper = require('../../test-helper.js'); const { types } = require('../../../index.js'); const Vector = require('../../../lib/types/vector.js'); +const { util } = require('chai'); const vdescribe = helper.vdescribe; +const dataProvider = [ + { + subtypeString: 'float', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.float, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [1.1122000217437744, 2.212209939956665, 3.3999900817871094] + }, + { + subtypeString: 'double', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.double, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [1.1, 2.2, 3.3] + }, + { + subtypeString: 'text', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.text, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: ['ab', 'b', 'cde'] + } +]; + + vdescribe('5.0.0', 'Vector tests', function () { this.timeout(120000); + describe('#execute with vectors', function () { const keyspace = helper.getRandomName('ks'); const table = keyspace + '.' + helper.getRandomName('table'); - const createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY, v1 vector, v2 vector);`; - + let createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY`; + dataProvider.forEach(data => { + createTableCql += `, v${data.subtypeString} vector<${data.subtypeString}, 3>`; + }); + createTableCql += ');'; + const setupInfo = helper.setup(1, { keyspace: keyspace, - queries: [ createTableCql ] - }); - it('should insert and select vectors', function(done){ - const client = setupInfo.client; - // if client undefined, raise error - if(!client) {return done(new Error('client is not defined'));} - const id = types.Uuid.random(); - const v1 = new Float32Array([1.1, 2.2, 3.3]); - const query = `INSERT INTO ${table} (id, v1) VALUES (?, ?)`; - client.execute(query, [id, v1], {prepare : true}, function(err){ - if (err) {return done(err);} - client.execute(`SELECT v1 FROM ${table} WHERE id = ?`, [id], { prepare: true }, function(err, result){ - if (err) {return done(err);} - assert.strictEqual(result.rows.length, 1); - assert.strictEqual(result.rows[0].v1.length, 3); - assert.strictEqual(result.rows[0].v1[0], v1[0]); - assert.strictEqual(result.rows[0].v1[1], v1[1]); - assert.strictEqual(result.rows[0].v1[2], v1[2]); - done(); - }); - }); + queries: [createTableCql] }); + const client = setupInfo.client; + if (!client) { throw new Error('client setup failed'); } - it('should insert and select vector of text', function(done){ - const client = setupInfo.client; - // if client undefined, raise error - if(!client) {return done(new Error('client is not defined'));} - const id = types.Uuid.random(); - const v1 = new Vector(['ab', 'b', 'cde'], 'text'); - const query = `INSERT INTO ${table} (id, v2) VALUES (?, ?)`; - client.execute(query, [id, v1], {prepare : true}, function(err){ - if (err) {return done(err);} - client.execute(`SELECT v2 FROM ${table} WHERE id = ?`, [id], { prepare: true }, function(err, result){ - if (err) {return done(err);} - assert.strictEqual(result.rows.length, 1); - assert.strictEqual(result.rows[0].v1.length, 3); - assert.strictEqual(result.rows[0].v1[0], v1[0]); - assert.strictEqual(result.rows[0].v1[1], v1[1]); - assert.strictEqual(result.rows[0].v1[2], v1[2]); - done(); + dataProvider.forEach(data => { + it('should insert and select vector of subtype ' + data.subtypeString, function (done) { + const id = types.Uuid.random(); + const vector = new Vector(data.value, data.subtypeString); + const query = `INSERT INTO ${table} (id, v${data.subtypeString}) VALUES (?, ?)`; + client.execute(query, [id, vector], { prepare: true }, function (err) { + if (err) { return done(err); } + client.execute(`SELECT v${data.subtypeString} FROM ${table} WHERE id = ?`, [id], { prepare: true }, function (err, result) { + if (err) { return done(err); } + assert.strictEqual(result.rows.length, 1); + assert.strictEqual(util.inspect(result.rows[0][`v${data.subtypeString}`]), util.inspect(vector)); + done(); + }); }); }); }); + }); + }); \ No newline at end of file diff --git a/test/unit/encoder-tests.js b/test/unit/encoder-tests.js index c0c2c373..e8454963 100644 --- a/test/unit/encoder-tests.js +++ b/test/unit/encoder-tests.js @@ -689,13 +689,8 @@ describe('encoder', function () { const encoded = encoder.encode(refVal, guessedTypeObj); const decoded = encoder.decode(encoded, guessedTypeObj); helper.assertInstanceOf(decoded, Vector); - for (const k in decoded) { - if (decoded.hasOwnProperty(k)) { - assert.equal(decoded[k],refVal[k]); - } - else { - assert.fail(); - } + for (var i = 0; i < decoded.length; i++) { + assert.strictEqual(decoded[i],refVal[i]); } }); @@ -707,13 +702,8 @@ describe('encoder', function () { const encoded = encoder.encode(refVal, typeObj); const decoded = encoder.decode(encoded, typeObj); helper.assertInstanceOf(decoded, Vector); - for (const k in decoded) { - if (decoded.hasOwnProperty(k)) { - assert.equal(decoded[k],refVal[k]); - } - else { - assert.fail(); - } + for (let i = 0; i < decoded.length; i++) { + assert.strictEqual(decoded[i],refVal[i]); } }); @@ -725,13 +715,8 @@ describe('encoder', function () { const encoded = encoder.encode(refVal, typeObj); const decoded = encoder.decode(encoded, typeObj); helper.assertInstanceOf(decoded, Vector); - for (const k in decoded) { - if (decoded.hasOwnProperty(k)) { - assert.equal(decoded[k],refVal[k]); - } - else { - assert.fail(); - } + for (let k = 0; k < decoded.length; k++) { + assert.strictEqual(decoded[k],refVal[k]); } }); diff --git a/test/unit/encoder-vector-tests.js b/test/unit/encoder-vector-tests.js index 79a3636b..e7de91f3 100644 --- a/test/unit/encoder-vector-tests.js +++ b/test/unit/encoder-vector-tests.js @@ -14,143 +14,132 @@ * limitations under the License. */ 'use strict'; -const { assert, expect } = require('chai'); +const { assert, util } = require('chai'); const Encoder = require('../../lib/encoder'); const { types } = require('../../index'); const Vector = require('../../lib/types/vector'); +const Long = require('long'); describe('Vector tests', function () { - const encoder = new Encoder(4, {}); - /** + const encoder = new Encoder(4, {}); + /** * @type {Array<{subtypeString : string, typeInfo: import('../../lib/encoder').VectorColumnInfo, value: Array}>} */ - const dataProvider = [ - { - subtypeString : 'float', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.float, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [1.1, 2.2, 3.3] + const dataProvider = [ + { + subtypeString : 'float', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.float, }, - { - subtypeString : 'double', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.double, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [1.1, 2.2, 3.3] + customTypeName: 'vector', + dimension: 3 + }, + value: [1.1122000217437744, 2.212209939956665, 3.3999900817871094] + }, + { + subtypeString : 'double', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.double, }, - { - subtypeString : 'text', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.text, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: ['a', 'bc', 'cde'] + customTypeName: 'vector', + dimension: 3 + }, + value: [1.1, 2.2, 3.3] + }, + { + subtypeString : 'text', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.text, }, - { - subtypeString : 'int', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.int, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [1, 2, 3] + customTypeName: 'vector', + dimension: 3 + }, + value: ['a', 'bc', 'cde'] + }, + { + subtypeString : 'int', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.int, }, - { - subtypeString : 'bigint', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.bigint, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [1, 2, 3] + customTypeName: 'vector', + dimension: 3 + }, + value: [1, 2, 3] + }, + { + subtypeString : 'bigint', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.bigint, }, - { - subtypeString : 'uuid', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.uuid, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [types.Uuid.random(), types.Uuid.random(), types.Uuid.random()] + customTypeName: 'vector', + dimension: 3 + }, + value: [Long.fromInt(1), Long.fromInt(2), Long.fromInt(3)] + }, + { + subtypeString : 'uuid', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.uuid, }, - { - subtypeString : 'timeuuid', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.timeuuid, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [types.TimeUuid.now(), types.TimeUuid.now(), types.TimeUuid.now()] + customTypeName: 'vector', + dimension: 3 + }, + value: [types.Uuid.random(), types.Uuid.random(), types.Uuid.random()] + }, + { + subtypeString : 'timeuuid', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.timeuuid, }, - { - subtypeString : 'decimal', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.decimal, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [types.BigDecimal.fromString('1.1'), types.BigDecimal.fromString('2.2'), types.BigDecimal.fromString('3.3')] - } - ] + customTypeName: 'vector', + dimension: 3 + }, + value: [types.TimeUuid.now(), types.TimeUuid.now(), types.TimeUuid.now()] + }, + { + subtypeString : 'decimal', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.decimal, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [types.BigDecimal.fromString('1.1'), types.BigDecimal.fromString('2.2'), types.BigDecimal.fromString('3.3')] + } + ]; - dataProvider.forEach((data) => { - it(`should encode and decode vector of ${data.subtypeString}`, function () { - const vector = new Vector(data.value); - const encoded = encoder.encode(vector, data.typeInfo); - const decoded = encoder.decode(encoded, data.typeInfo); - for (const k in decoded) { - if (decoded.hasOwnProperty(k)) { - assert.equal(decoded[k],data.value[k]); - } - else { - assert.fail(); - } - } - }); + dataProvider.forEach((data) => { + it(`should encode and decode vector of ${data.subtypeString}`, function () { + const vector = new Vector(data.value); + const encoded = encoder.encode(vector, data.typeInfo); + const decoded = encoder.decode(encoded, data.typeInfo); + assert.strictEqual(util.inspect(decoded), util.inspect(vector)); }); + }); - it('should encode and decode vector of float', function(){ - const vector = new Float32Array([1.1, 2.2, 3.3]); - const typeObj = { code: types.dataTypes.custom, info: {code: types.dataTypes.float}, dimension: 3, customTypeName: 'vector' }; - const encoded = encoder.encode(vector, typeObj); - const decoded = encoder.decode(encoded, typeObj); - for (const k in decoded) { - if (decoded.hasOwnProperty(k)) { - assert.equal(decoded[k],vector[k]); - } - else { - assert.fail(); - } - } - }) + it('should encode and decode vector of float', function(){ + const vector = new Float32Array([1.1, 2.2, 3.3]); + const typeObj = { code: types.dataTypes.custom, info: {code: types.dataTypes.float}, dimension: 3, customTypeName: 'vector' }; + const encoded = encoder.encode(vector, typeObj); + const decoded = encoder.decode(encoded, typeObj); + for (let i = 0; i < vector.length; i++) { + assert.strictEqual(decoded[i], vector[i]); + } + }); }); \ No newline at end of file From ce8133813a98ff762ca0719c0b3b76ed31002b15 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 16 Oct 2024 18:35:39 +0000 Subject: [PATCH 82/99] guess data type pass --- lib/encoder.js | 24 ++++++++---- lib/types/index.js | 12 +++++- lib/types/vector.js | 9 ++++- test/integration/short/vector-tests.js | 52 ++++++++++++++++++++++---- test/unit/encoder-tests.js | 15 +++----- test/unit/encoder-vector-tests.js | 11 ++++++ 6 files changed, 96 insertions(+), 27 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index 8357f5b9..c18ce7ec 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -979,7 +979,8 @@ function defineInstanceMembers() { rv[i] = this.decode(buffer.subarray(offset, offset + elemLength), subtype); } } - return new Vector(rv); + const typeInfo = types.getDataTypeNameByCode(params.info); + return new Vector(rv, typeInfo); }; /** @@ -1086,9 +1087,10 @@ function defineInstanceMembers() { const params = parseParams(typeName, argsStartIndex, argsLength); if (params.length === 2) { /** @type {ColumnInfo} */ - const columnInfo = { code: dataTypes.custom, info: subtypeResolveFn(params[0].trim()), customTypeName : 'vector', dimension: parseInt(params[1].trim(), 10 )}; + const columnInfo = { code: dataTypes.custom, info: subtypeResolveFn.bind(this)(params[0].trim()), customTypeName : 'vector', dimension: parseInt(params[1].trim(), 10 )}; return columnInfo; } + throw new TypeError('Not a valid type ' + typeName); }; @@ -1362,7 +1364,7 @@ function defineInstanceMembers() { // parseVectorTypeArgs is not an async function but we are. To keep things simple let's ask the // function to just return whatever it finds for an arg and we'll eval it after the fact - const params = this.parseVectorTypeArgs(typeName, cqlNames.vector, (arg) => arg ); + const params = this.parseVectorTypeArgs.bind(this)(typeName, cqlNames.vector, (arg) => arg ); params["info"] = await this.parseTypeName(keyspace, params["info"]); return params; @@ -1557,7 +1559,7 @@ function defineInstanceMembers() { if (typeName.indexOf(customTypeNames.vector, startIndex) === startIndex) { // It's a vector, so record the subtype and dimension. - const params = this.parseVectorTypeArgs(typeName, customTypeNames.vector, this.parseFqTypeName); + const params = this.parseVectorTypeArgs.bind(this)(typeName, customTypeNames.vector, this.parseFqTypeName); params.options = options; return params; } @@ -1900,6 +1902,15 @@ Encoder.guessDataType = function (value) { dimension: value.length }; } + // try to fetch the subtype from the Vector, or else guess + if (value.subtype) { + return { + code: dataTypes.custom, + customTypeName: 'vector', + info: dataTypes.getByName(value.subtype), + dimension: value.length + }; + } const subtype = this.guessDataType(value[0]); if (subtype) { return { @@ -1908,10 +1919,9 @@ Encoder.guessDataType = function (value) { info: subtype, dimension: value.length }; - } - throw new TypeError("Cannot guess subtype from element " + value.get(0)); - + } + throw new TypeError("Cannot guess subtype from element " + value.get(0)); } else { throw new TypeError("Cannot guess subtype of empty vector"); } diff --git a/lib/types/index.js b/lib/types/index.js index 1542c6fa..93d7e496 100644 --- a/lib/types/index.js +++ b/lib/types/index.js @@ -183,8 +183,10 @@ const dataTypes = { * Map of Data types by code * @internal * @private + * @type {Record} */ const _dataTypesByCode = (function () { + /**@type {Record} */ const result = {}; for (const key in dataTypes) { if (!dataTypes.hasOwnProperty(key)) { @@ -411,10 +413,11 @@ function uuid(options, buffer, offset) { } /** - * Gets the data type name for a given type definition + * Gets the data type name for a given type definition, it will not work for udt or custom type * @internal * @ignore * @throws {ArgumentError} + * @param {import('../encoder').ColumnInfo} item */ function getDataTypeNameByCode(item) { if (!item || typeof item.code !== 'number') { @@ -424,7 +427,7 @@ function getDataTypeNameByCode(item) { if (!typeName) { throw new errors.ArgumentError(util.format('Type with code %d not found', item.code)); } - if (!item.info) { + if (!('info' in item) || !item.info) { return typeName; } if (Array.isArray(item.info)) { @@ -435,7 +438,12 @@ function getDataTypeNameByCode(item) { }).join(', ') + '>'); } + // special case for vector + if (item.code === dataTypes.custom && 'customTypeName' in item && item.customTypeName === 'vector') { + return 'vector<' + getDataTypeNameByCode(item.info) + ', ' + item.dimension + '>'; + } if (typeof item.info.code === 'number') { + // if udt, it will throw here return typeName + '<' + getDataTypeNameByCode(item.info) + '>'; } return typeName; diff --git a/lib/types/vector.js b/lib/types/vector.js index 51610c87..2a67e15b 100644 --- a/lib/types/vector.js +++ b/lib/types/vector.js @@ -24,7 +24,7 @@ class Vector { /** * * @param {Float32Array | Array} elements - * @param {String?} subtype + * @param {string?} subtype */ constructor (elements, subtype = null) { if (elements instanceof Float32Array) { @@ -104,6 +104,13 @@ class Vector { forEach(callback) { return this.elements.forEach(callback); } + + /** + * @returns {string | null} get the subtype string, e.g., "float", but it's optional so it can return null + */ + getSubtype(){ + return this.subtype; + } } Object.defineProperty(Vector, Symbol.hasInstance, { diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index 879e9101..38b20f0c 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -19,7 +19,7 @@ const helper = require('../../test-helper.js'); const { types } = require('../../../index.js'); const Vector = require('../../../lib/types/vector.js'); -const { util } = require('chai'); +const util = require('node:util'); const vdescribe = helper.vdescribe; const dataProvider = [ @@ -48,7 +48,7 @@ const dataProvider = [ value: [1.1, 2.2, 3.3] }, { - subtypeString: 'text', + subtypeString: 'varchar', typeInfo: { code: types.dataTypes.custom, info: { @@ -58,6 +58,19 @@ const dataProvider = [ dimension: 3 }, value: ['ab', 'b', 'cde'] + }, + { + subtypeString: 'list', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.list, + subTypes: [{ code: types.dataTypes.float }] + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [[1.1122000217437744, 2.212209939956665], [2.212209939956665, 2.212209939956665], [1.1122000217437744, 1.1122000217437744]] } ]; @@ -70,7 +83,7 @@ vdescribe('5.0.0', 'Vector tests', function () { const table = keyspace + '.' + helper.getRandomName('table'); let createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY`; dataProvider.forEach(data => { - createTableCql += `, v${data.subtypeString} vector<${data.subtypeString}, 3>`; + createTableCql += `, ${subtypeStringToColumnName(data.subtypeString)} vector<${data.subtypeString}, 3>`; }); createTableCql += ');'; @@ -86,19 +99,42 @@ vdescribe('5.0.0', 'Vector tests', function () { it('should insert and select vector of subtype ' + data.subtypeString, function (done) { const id = types.Uuid.random(); const vector = new Vector(data.value, data.subtypeString); - const query = `INSERT INTO ${table} (id, v${data.subtypeString}) VALUES (?, ?)`; + const query = `INSERT INTO ${table} (id, ${subtypeStringToColumnName(data.subtypeString)}) VALUES (?, ?)`; client.execute(query, [id, vector], { prepare: true }, function (err) { if (err) { return done(err); } - client.execute(`SELECT v${data.subtypeString} FROM ${table} WHERE id = ?`, [id], { prepare: true }, function (err, result) { + client.execute(`SELECT ${subtypeStringToColumnName(data.subtypeString)} FROM ${table} WHERE id = ?`, [id], { prepare: true }, function (err, result) { if (err) { return done(err); } assert.strictEqual(result.rows.length, 1); - assert.strictEqual(util.inspect(result.rows[0][`v${data.subtypeString}`]), util.inspect(vector)); + assert.strictEqual(util.inspect(result.rows[0][subtypeStringToColumnName(data.subtypeString)]), util.inspect(vector)); done(); }); }); }); - }); + it('should insert and select vector of subtype ' + data.subtypeString + ' while guessing data type', function (done) { + const id = types.Uuid.random(); + const vector = new Vector(data.value, data.subtypeString); + const query = `INSERT INTO ${table} (id, ${subtypeStringToColumnName(data.subtypeString)}) VALUES (?, ?)`; + client.execute(query, [id, vector], { prepare: true }, function (err) { + if (err) { return done(err); } + client.execute(`SELECT ${subtypeStringToColumnName(data.subtypeString)} FROM ${table} WHERE id = ?`, [id], { prepare: true }, function (err, result) { + if (err) { return done(err); } + assert.strictEqual(result.rows.length, 1); + assert.strictEqual(util.inspect(result.rows[0][subtypeStringToColumnName(data.subtypeString)]), util.inspect(vector)); + + done(); + }); + }); + }); + }); }); +}); -}); \ No newline at end of file +/** + * + * @param {string} subtypeString + * @returns + */ +function subtypeStringToColumnName(subtypeString) { + return "v" + subtypeString.replace('<', '_').replace('>', '_'); +} \ No newline at end of file diff --git a/test/unit/encoder-tests.js b/test/unit/encoder-tests.js index e8454963..64838492 100644 --- a/test/unit/encoder-tests.js +++ b/test/unit/encoder-tests.js @@ -689,7 +689,7 @@ describe('encoder', function () { const encoded = encoder.encode(refVal, guessedTypeObj); const decoded = encoder.decode(encoded, guessedTypeObj); helper.assertInstanceOf(decoded, Vector); - for (var i = 0; i < decoded.length; i++) { + for (let i = 0; i < decoded.length; i++) { assert.strictEqual(decoded[i],refVal[i]); } }); @@ -753,14 +753,11 @@ describe('encoder', function () { const encoded = encoder.encode(refVal, guessedTypeObj); const decoded = encoder.decode(encoded, guessedTypeObj); helper.assertInstanceOf(decoded, Vector); - // for (const k in decoded) { - // if (decoded.hasOwnProperty(k)) { - // assert.equal(decoded[k],refVal[k]); - // } - // else { - // assert.fail(); - // } - // } + for (let i = 0; i < decoded.length; i++) { + for (let j = 0; j < decoded[i].length; j++) { + assert.strictEqual(decoded[i][j],refVal[i][j]); + } + } }); }); diff --git a/test/unit/encoder-vector-tests.js b/test/unit/encoder-vector-tests.js index e7de91f3..daf02488 100644 --- a/test/unit/encoder-vector-tests.js +++ b/test/unit/encoder-vector-tests.js @@ -131,6 +131,17 @@ describe('Vector tests', function () { const decoded = encoder.decode(encoded, data.typeInfo); assert.strictEqual(util.inspect(decoded), util.inspect(vector)); }); + + it(`should encode and decode vector of ${data.subtypeString} while guessing data type`, function (){ + const vector = new Vector(data.value, data.subtypeString); + const guessedType = Encoder.guessDataType(vector); + if (!guessedType) { + throw new Error('Can not guess type'); + } + const encoded = encoder.encode(vector, guessedType); + const decoded = encoder.decode(encoded, guessedType); + assert.strictEqual(util.inspect(decoded), util.inspect(vector)); + }); }); it('should encode and decode vector of float', function(){ From aeafce684f2f16034b5e8ef155c8fbe1132f2703 Mon Sep 17 00:00:00 2001 From: janehe Date: Tue, 22 Oct 2024 19:53:05 +0000 Subject: [PATCH 83/99] add comprehensive integration tests --- lib/encoder.js | 14 +- lib/types/index.js | 4 +- test/integration/short/vector-tests.js | 284 ++++++++++++++++++++++++- 3 files changed, 289 insertions(+), 13 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index c18ce7ec..bb9cb263 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -17,6 +17,7 @@ const util = require('util'); const types = require('./types'); +// const { types } = require('./types'); const dataTypes = types.dataTypes; const Long = types.Long; const Integer = types.Integer; @@ -132,7 +133,7 @@ const zeroLengthTypesSupported = new Set([ * @typedef {(singleTypeNames[keyof singleTypeNames] | types.dataTypes.duration | types.dataTypes.text)} SingleTypeCodes * @typedef {('point' | 'polygon' | 'duration' | 'lineString' | 'dateRange')} CustomSimpleTypeCodes * @typedef {(customTypeNames[CustomSimpleTypeCodes]) | CustomSimpleTypeCodes | 'empty'} CustomSimpleTypeNames - * @typedef {{code : SingleTypeCodes, info: null, options? : {frozen?:boolean, reversed?:boolean} }} SingleColumnInfo + * @typedef {{code : SingleTypeCodes, info?: null, options? : {frozen?:boolean, reversed?:boolean} }} SingleColumnInfo * @typedef {{code : (types.dataTypes.custom), info : CustomSimpleTypeNames, options? : {frozen?:boolean, reversed?:boolean}}} CustomSimpleColumnInfo * @typedef {{code : (types.dataTypes.map), info : [ColumnInfo, ColumnInfo], options?: {frozen?: Boolean, reversed?: Boolean}}} MapColumnInfo * @typedef {{code : (types.dataTypes.tuple), info : Array, options?: {frozen?: Boolean, reversed?: Boolean}}} TupleColumnInfo @@ -972,13 +973,22 @@ function defineInstanceMembers() { // var sized const [size, bytesRead] = utils.VIntCoding.uvintUnpack(buffer.subarray(offset)); offset += bytesRead; + if (offset + size > buffer.length) { + throw new TypeError('Not enough bytes to decode the vector'); + } rv[i] = this.decode(buffer.subarray(offset, offset + size), subtype); offset += size; }else{ - offset = i * elemLength; + if (offset + elemLength > buffer.length) { + throw new TypeError('Not enough bytes to decode the vector'); + } rv[i] = this.decode(buffer.subarray(offset, offset + elemLength), subtype); + offset += elemLength; } } + if (offset !== buffer.length) { + throw new TypeError('Extra bytes found after decoding the vector'); + } const typeInfo = types.getDataTypeNameByCode(params.info); return new Vector(rv, typeInfo); }; diff --git a/lib/types/index.js b/lib/types/index.js index 93d7e496..a0144b8c 100644 --- a/lib/types/index.js +++ b/lib/types/index.js @@ -443,9 +443,11 @@ function getDataTypeNameByCode(item) { return 'vector<' + getDataTypeNameByCode(item.info) + ', ' + item.dimension + '>'; } if (typeof item.info.code === 'number') { - // if udt, it will throw here return typeName + '<' + getDataTypeNameByCode(item.info) + '>'; } + if (item.code === dataTypes.udt) { + return (/**@type {UdtColumnInfo}*/item).info.name; + } return typeName; } diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index 38b20f0c..b0b811f8 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -22,6 +22,9 @@ const Vector = require('../../../lib/types/vector.js'); const util = require('node:util'); const vdescribe = helper.vdescribe; +/** + * @type {Array<{subtypeString : string, typeInfo: import('../../../lib/encoder').VectorColumnInfo, value: Array}>} + */ const dataProvider = [ { subtypeString: 'float', @@ -60,20 +63,282 @@ const dataProvider = [ value: ['ab', 'b', 'cde'] }, { - subtypeString: 'list', + subtypeString: 'bigint', typeInfo: { code: types.dataTypes.custom, info: { - code: types.dataTypes.list, - subTypes: [{ code: types.dataTypes.float }] + code: types.dataTypes.bigint, + }, + customTypeName: 'vector', + dimension: 3, + }, + value: [new types.Long(1), new types.Long(2), new types.Long(3)] + }, + { + subtypeString: 'blob', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.blob, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [Buffer.from([1, 2, 3]), Buffer.from([4, 5, 6]), Buffer.from([7, 8, 9])] + }, + { + subtypeString: 'boolean', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.boolean, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [true, false, true] + }, + { + subtypeString: 'decimal', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.decimal, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [types.BigDecimal.fromString('1.1'), types.BigDecimal.fromString('2.2'), types.BigDecimal.fromString('3.3')] + }, + { + subtypeString: 'inet', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.inet, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [types.InetAddress.fromString('127.0.0.1'), types.InetAddress.fromString('0.0.0.0'), types.InetAddress.fromString('34.12.10.19')] + }, + { + subtypeString: 'tinyint', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.tinyint, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [1, 2, 3] + }, + { + subtypeString: 'smallint', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.smallint, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [1, 2, 3] + }, + { + subtypeString: 'int', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.int, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [-1, 0, -3] + }, + // TODO: what do we want to do with Duration? + // { + // subtypeString: 'duration', + // typeInfo: { + // code: types.dataTypes.custom, + // info: { + // code: types.dataTypes.custom, + // info: 'duration' + // }, + // customTypeName: 'vector', + // dimension: 3 + // }, + // value: [new types.Duration(1, 2, 3), new types.Duration(4, 5, 6), new types.Duration(7, 8, 9)] + // }, + { + subtypeString: 'date', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.date, }, customTypeName: 'vector', dimension: 3 }, - value: [[1.1122000217437744, 2.212209939956665], [2.212209939956665, 2.212209939956665], [1.1122000217437744, 1.1122000217437744]] + value: [new types.LocalDate(2020, 1, 1), new types.LocalDate(2020, 2, 1), new types.LocalDate(2020, 3, 1)] + }, + { + subtypeString: 'time', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.time, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [new types.LocalTime(types.Long.fromString('6331999999911')), new types.LocalTime(types.Long.fromString('6331999999911')), new types.LocalTime(types.Long.fromString('6331999999911'))] + }, + { + subtypeString: 'timestamp', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.timestamp, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [new Date(2020, 1, 1, 1, 1, 1, 1), new Date(2020, 2, 1, 1, 1, 1, 1), new Date(2020, 3, 1, 1, 1, 1, 1)] + }, + { + subtypeString: 'uuid', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.uuid, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [types.Uuid.random(), types.Uuid.random(), types.Uuid.random()] + }, + { + subtypeString: 'timeuuid', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.timeuuid, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [types.TimeUuid.now(), types.TimeUuid.now(), types.TimeUuid.now()] } ]; +const dataProviderWithCollections = dataProvider.flatMap(data => [ + data, + // vector, 3> + { + subtypeString: 'list<' + data.subtypeString + '>', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.list, + info: { + code: data.typeInfo.code, + info: data.typeInfo.info + } + }, + customTypeName: 'vector', + dimension: 3 + }, + value: data.value.map(value => [value, value, value]) + }, + // vector, 3> + { + subtypeString: 'map', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.map, + info: [ + { code: types.dataTypes.int }, + { code: data.typeInfo.code, info: data.typeInfo.info } + ] + }, + customTypeName: 'vector', + dimension: 3 + }, + value: data.value.map((value) => ({ 1: value, 2: value, 3: value })) + }, + // vector, 3> + { + subtypeString: 'set<' + data.subtypeString + '>', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.set, + info: { + code: data.typeInfo.code, + info: data.typeInfo.info + } + }, + customTypeName: 'vector', + dimension: 3 + }, + value: data.value.map(value => [value, value, value]) + }, + // vector, 3> + { + subtypeString: 'tuple<' + data.subtypeString + ', ' + data.subtypeString + '>', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.tuple, + info: [ + { code: data.typeInfo.code, info: data.typeInfo.info }, + { code: data.typeInfo.code, info: data.typeInfo.info } + ] + }, + customTypeName: 'vector', + dimension: 3 + }, + value: data.value.map(value => new types.Tuple(value, value)) + }, + // vector, 3> + { + subtypeString: 'vector<' + data.subtypeString + ', 3>', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.custom, + info: data.typeInfo.info, + customTypeName: 'vector', + dimension: 3 + }, + customTypeName: 'vector', + dimension: 3 + }, + value: data.value.map(value => new Vector([value, value, value], data.subtypeString)) + } +]).concat([ + // vector + { + subtypeString: 'my_udt', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.udt, + info: { + name: 'my_udt', + fields: [{ name: 'f1', type: { code: types.dataTypes.text } }], + } + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [{ f1: 'a' }, { f1: 'b' }, { f1: 'c' }] + } +]); vdescribe('5.0.0', 'Vector tests', function () { this.timeout(120000); @@ -82,20 +347,20 @@ vdescribe('5.0.0', 'Vector tests', function () { const keyspace = helper.getRandomName('ks'); const table = keyspace + '.' + helper.getRandomName('table'); let createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY`; - dataProvider.forEach(data => { + dataProviderWithCollections.forEach(data => { createTableCql += `, ${subtypeStringToColumnName(data.subtypeString)} vector<${data.subtypeString}, 3>`; }); createTableCql += ');'; - + const createUdtCql = `CREATE TYPE ${keyspace}.my_udt (f1 text);`; const setupInfo = helper.setup(1, { keyspace: keyspace, - queries: [createTableCql] + queries: [createUdtCql, createTableCql] }); const client = setupInfo.client; if (!client) { throw new Error('client setup failed'); } - dataProvider.forEach(data => { + dataProviderWithCollections.forEach(data => { it('should insert and select vector of subtype ' + data.subtypeString, function (done) { const id = types.Uuid.random(); const vector = new Vector(data.value, data.subtypeString); @@ -121,7 +386,6 @@ vdescribe('5.0.0', 'Vector tests', function () { if (err) { return done(err); } assert.strictEqual(result.rows.length, 1); assert.strictEqual(util.inspect(result.rows[0][subtypeStringToColumnName(data.subtypeString)]), util.inspect(vector)); - done(); }); }); @@ -136,5 +400,5 @@ vdescribe('5.0.0', 'Vector tests', function () { * @returns */ function subtypeStringToColumnName(subtypeString) { - return "v" + subtypeString.replace('<', '_').replace('>', '_'); + return "v" + subtypeString.replace(/<|>|,| /g, '_'); } \ No newline at end of file From ccb6257f7855fbfb185a529798fed620e8fccf39 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 23 Oct 2024 18:26:44 +0000 Subject: [PATCH 84/99] add type declarations --- index.d.ts | 4 ++++ lib/types/index.d.ts | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/index.d.ts b/index.d.ts index cf44d76e..90f2b6c3 100644 --- a/index.d.ts +++ b/index.d.ts @@ -349,6 +349,10 @@ export namespace errors { constructor(address: string, maxRequestsPerConnection: number, connectionLength: number); } + class VIntOutOfRangeException extends DriverError { + constructor(long: Long); + } + abstract class DriverError extends Error { info: string; diff --git a/lib/types/index.d.ts b/lib/types/index.d.ts index b03186cf..6664aef0 100644 --- a/lib/types/index.d.ts +++ b/lib/types/index.d.ts @@ -424,4 +424,45 @@ export namespace types { toJSON(): string; } + + class Vector { + static get [Symbol.species](): typeof Vector; + /** + * + * @param {Float32Array | Array} elements + * @param {string?} subtype + */ + constructor(elements: Float32Array | Array, subtype?: string | null); + elements: any[]; + /** + * Returns the number of the elements. + * @type Number + */ + length: number; + subtype: string; + /** + * Returns the string representation of the vector. + * @returns {string} + */ + toString(): string; + /** + * + * @param {number} index + */ + at(index: number): any; + /** + * + * @param {(value: any, index: number, array: any[]) => void} callback + */ + forEach(callback: (value: any, index: number, array: any[]) => void): void; + /** + * @returns {string | null} get the subtype string, e.g., "float", but it's optional so it can return null + */ + getSubtype(): string | null; + /** + * + * @returns {ArrayIterator} + */ + [Symbol.iterator](): ArrayIterator; + } } \ No newline at end of file From 203047d450e991130f38335374903a08d5074d53 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 24 Oct 2024 06:51:00 +0000 Subject: [PATCH 85/99] add more integration tests --- lib/encoder.js | 25 ++- test/integration/short/vector-tests.js | 19 +- test/unit/encoder-vector-tests.js | 277 +++++++++++++++++++++++-- 3 files changed, 282 insertions(+), 39 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index bb9cb263..dd2d2505 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -1912,26 +1912,29 @@ Encoder.guessDataType = function (value) { dimension: value.length }; } + + /** @type {ColumnInfo?} */ + let subtypeColumnInfo = null; // try to fetch the subtype from the Vector, or else guess if (value.subtype) { - return { - code: dataTypes.custom, - customTypeName: 'vector', - info: dataTypes.getByName(value.subtype), - dimension: value.length - }; + try { + subtypeColumnInfo = dataTypes.getByName(value.subtype); + } catch (TypeError) { + // ignore + } } - const subtype = this.guessDataType(value[0]); - if (subtype) { + if (value == null){ + subtypeColumnInfo = this.guessDataType(value[0]); + } + if (subtypeColumnInfo != null) { return { code: dataTypes.custom, customTypeName: 'vector', - info: subtype, + info: subtypeColumnInfo, dimension: value.length }; } - - throw new TypeError("Cannot guess subtype from element " + value.get(0)); + throw new TypeError("Cannot guess subtype from element " + value[0]); } else { throw new TypeError("Cannot guess subtype of empty vector"); } diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index b0b811f8..3fc1e787 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -244,8 +244,8 @@ const dataProviderWithCollections = dataProvider.flatMap(data => [ info: { code: types.dataTypes.list, info: { - code: data.typeInfo.code, - info: data.typeInfo.info + code: data.typeInfo.info.code, + info: data.typeInfo.info.info } }, customTypeName: 'vector', @@ -262,7 +262,7 @@ const dataProviderWithCollections = dataProvider.flatMap(data => [ code: types.dataTypes.map, info: [ { code: types.dataTypes.int }, - { code: data.typeInfo.code, info: data.typeInfo.info } + { code: data.typeInfo.info.code, info: data.typeInfo.info.info } ] }, customTypeName: 'vector', @@ -278,8 +278,8 @@ const dataProviderWithCollections = dataProvider.flatMap(data => [ info: { code: types.dataTypes.set, info: { - code: data.typeInfo.code, - info: data.typeInfo.info + code: data.typeInfo.info.code, + info: data.typeInfo.info.info } }, customTypeName: 'vector', @@ -295,8 +295,8 @@ const dataProviderWithCollections = dataProvider.flatMap(data => [ info: { code: types.dataTypes.tuple, info: [ - { code: data.typeInfo.code, info: data.typeInfo.info }, - { code: data.typeInfo.code, info: data.typeInfo.info } + { code: data.typeInfo.info.code, info: data.typeInfo.info.info }, + { code: data.typeInfo.info.code, info: data.typeInfo.info.info } ] }, customTypeName: 'vector', @@ -311,7 +311,10 @@ const dataProviderWithCollections = dataProvider.flatMap(data => [ code: types.dataTypes.custom, info: { code: types.dataTypes.custom, - info: data.typeInfo.info, + info: { + code: data.typeInfo.info.code, + info: data.typeInfo.info.info + }, customTypeName: 'vector', dimension: 3 }, diff --git a/test/unit/encoder-vector-tests.js b/test/unit/encoder-vector-tests.js index daf02488..3c6bcaf3 100644 --- a/test/unit/encoder-vector-tests.js +++ b/test/unit/encoder-vector-tests.js @@ -18,16 +18,15 @@ const { assert, util } = require('chai'); const Encoder = require('../../lib/encoder'); const { types } = require('../../index'); const Vector = require('../../lib/types/vector'); -const Long = require('long'); describe('Vector tests', function () { const encoder = new Encoder(4, {}); /** - * @type {Array<{subtypeString : string, typeInfo: import('../../lib/encoder').VectorColumnInfo, value: Array}>} - */ + * @type {Array<{subtypeString : string, typeInfo: import('../../../lib/encoder').VectorColumnInfo, value: Array}>} + */ const dataProvider = [ { - subtypeString : 'float', + subtypeString: 'float', typeInfo: { code: types.dataTypes.custom, info: { @@ -39,7 +38,7 @@ describe('Vector tests', function () { value: [1.1122000217437744, 2.212209939956665, 3.3999900817871094] }, { - subtypeString : 'double', + subtypeString: 'double', typeInfo: { code: types.dataTypes.custom, info: { @@ -51,7 +50,7 @@ describe('Vector tests', function () { value: [1.1, 2.2, 3.3] }, { - subtypeString : 'text', + subtypeString: 'varchar', typeInfo: { code: types.dataTypes.custom, info: { @@ -60,14 +59,74 @@ describe('Vector tests', function () { customTypeName: 'vector', dimension: 3 }, - value: ['a', 'bc', 'cde'] + value: ['ab', 'b', 'cde'] }, { - subtypeString : 'int', + subtypeString: 'bigint', typeInfo: { code: types.dataTypes.custom, info: { - code: types.dataTypes.int, + code: types.dataTypes.bigint, + }, + customTypeName: 'vector', + dimension: 3, + }, + value: [new types.Long(1), new types.Long(2), new types.Long(3)] + }, + { + subtypeString: 'blob', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.blob, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [Buffer.from([1, 2, 3]), Buffer.from([4, 5, 6]), Buffer.from([7, 8, 9])] + }, + { + subtypeString: 'boolean', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.boolean, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [true, false, true] + }, + { + subtypeString: 'decimal', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.decimal, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [types.BigDecimal.fromString('1.1'), types.BigDecimal.fromString('2.2'), types.BigDecimal.fromString('3.3')] + }, + { + subtypeString: 'inet', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.inet, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [types.InetAddress.fromString('127.0.0.1'), types.InetAddress.fromString('0.0.0.0'), types.InetAddress.fromString('34.12.10.19')] + }, + { + subtypeString: 'tinyint', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.tinyint, }, customTypeName: 'vector', dimension: 3 @@ -75,19 +134,81 @@ describe('Vector tests', function () { value: [1, 2, 3] }, { - subtypeString : 'bigint', + subtypeString: 'smallint', typeInfo: { code: types.dataTypes.custom, info: { - code: types.dataTypes.bigint, + code: types.dataTypes.smallint, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [1, 2, 3] + }, + { + subtypeString: 'int', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.int, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [-1, 0, -3] + }, + // TODO: what do we want to do with Duration? + // { + // subtypeString: 'duration', + // typeInfo: { + // code: types.dataTypes.custom, + // info: { + // code: types.dataTypes.custom, + // info: 'duration' + // }, + // customTypeName: 'vector', + // dimension: 3 + // }, + // value: [new types.Duration(1, 2, 3), new types.Duration(4, 5, 6), new types.Duration(7, 8, 9)] + // }, + { + subtypeString: 'date', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.date, }, customTypeName: 'vector', dimension: 3 }, - value: [Long.fromInt(1), Long.fromInt(2), Long.fromInt(3)] + value: [new types.LocalDate(2020, 1, 1), new types.LocalDate(2020, 2, 1), new types.LocalDate(2020, 3, 1)] }, { - subtypeString : 'uuid', + subtypeString: 'time', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.time, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [new types.LocalTime(types.Long.fromString('6331999999911')), new types.LocalTime(types.Long.fromString('6331999999911')), new types.LocalTime(types.Long.fromString('6331999999911'))] + }, + { + subtypeString: 'timestamp', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.timestamp, + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [new Date(2020, 1, 1, 1, 1, 1, 1), new Date(2020, 2, 1, 1, 1, 1, 1), new Date(2020, 3, 1, 1, 1, 1, 1)] + }, + { + subtypeString: 'uuid', typeInfo: { code: types.dataTypes.custom, info: { @@ -99,7 +220,7 @@ describe('Vector tests', function () { value: [types.Uuid.random(), types.Uuid.random(), types.Uuid.random()] }, { - subtypeString : 'timeuuid', + subtypeString: 'timeuuid', typeInfo: { code: types.dataTypes.custom, info: { @@ -109,30 +230,131 @@ describe('Vector tests', function () { dimension: 3 }, value: [types.TimeUuid.now(), types.TimeUuid.now(), types.TimeUuid.now()] + } + ]; + + const dataProviderWithCollections = dataProvider.flatMap(data => [ + data, + // vector, 3> + { + subtypeString: 'list<' + data.subtypeString + '>', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.list, + info: { + code: data.typeInfo.info.code, + info: data.typeInfo.info.info + } + }, + customTypeName: 'vector', + dimension: 3 + }, + value: data.value.map(value => [value, value, value]) }, + // vector, 3> { - subtypeString : 'decimal', + subtypeString: 'map', typeInfo: { code: types.dataTypes.custom, info: { - code: types.dataTypes.decimal, + code: types.dataTypes.map, + info: [ + { code: types.dataTypes.int }, + { code: data.typeInfo.info.code, info: data.typeInfo.info.info } + ] }, customTypeName: 'vector', dimension: 3 }, - value: [types.BigDecimal.fromString('1.1'), types.BigDecimal.fromString('2.2'), types.BigDecimal.fromString('3.3')] + value: data.value.map((value) => ({ 1: value, 2: value, 3: value })) + }, + // vector, 3> + { + subtypeString: 'set<' + data.subtypeString + '>', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.set, + info: { + code: data.typeInfo.info.code, + info: data.typeInfo.info.info + } + }, + customTypeName: 'vector', + dimension: 3 + }, + value: data.value.map(value => [value, value, value]) + }, + // vector, 3> + { + subtypeString: 'tuple<' + data.subtypeString + ', ' + data.subtypeString + '>', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.tuple, + info: [ + { code: data.typeInfo.info.code, info: data.typeInfo.info.info }, + { code: data.typeInfo.info.code, info: data.typeInfo.info.info } + ] + }, + customTypeName: 'vector', + dimension: 3 + }, + value: data.value.map(value => new types.Tuple(value, value)) + }, + // vector, 3> + { + subtypeString: 'vector<' + data.subtypeString + ', 3>', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.custom, + info: { + code: data.typeInfo.info.code, + info: data.typeInfo.info.info + }, + customTypeName: 'vector', + dimension: 3 + }, + customTypeName: 'vector', + dimension: 3 + }, + value: data.value.map(value => new Vector([value, value, value], data.subtypeString)) } - ]; + ]).concat([ + // vector + { + subtypeString: 'my_udt', + typeInfo: { + code: types.dataTypes.custom, + info: { + code: types.dataTypes.udt, + info: { + name: 'my_udt', + fields: [{ name: 'f1', type: { code: types.dataTypes.text } }], + } + }, + customTypeName: 'vector', + dimension: 3 + }, + value: [{ f1: 'a' }, { f1: 'b' }, { f1: 'c' }] + } + ]); - dataProvider.forEach((data) => { + dataProviderWithCollections.forEach((data) => { it(`should encode and decode vector of ${data.subtypeString}`, function () { - const vector = new Vector(data.value); + const vector = new Vector(data.value, data.subtypeString); const encoded = encoder.encode(vector, data.typeInfo); const decoded = encoder.decode(encoded, data.typeInfo); assert.strictEqual(util.inspect(decoded), util.inspect(vector)); }); it(`should encode and decode vector of ${data.subtypeString} while guessing data type`, function (){ + if (data.subtypeString === 'my_udt'){ + // cannot guess udt type + this.skip(); + } const vector = new Vector(data.value, data.subtypeString); const guessedType = Encoder.guessDataType(vector); if (!guessedType) { @@ -142,6 +364,21 @@ describe('Vector tests', function () { const decoded = encoder.decode(encoded, guessedType); assert.strictEqual(util.inspect(decoded), util.inspect(vector)); }); + + it(`should throw when providing less or more elements/bytes when encoding/decoding vector of ${data.subtypeString}`, function () { + const vector = new Vector(data.value, data.subtypeString); + const encoded = encoder.encode(vector, data.typeInfo); + const encodedBuffer = Buffer.from(encoded); + const encodedBufferShort = encodedBuffer.slice(0, encodedBuffer.length - 1); + const encodedBufferLong = Buffer.concat([encodedBuffer, Buffer.alloc(1)]); + assert.throws(() => encoder.decode(encodedBufferShort, data.typeInfo), 'Not enough bytes to decode the vector'); + assert.throws(() => encoder.decode(encodedBufferLong, data.typeInfo), 'Extra bytes found after decoding the vector'); + + const shortVector = new Vector(data.value.slice(0, data.value.length - 1), data.subtypeString); + const longVector = new Vector(data.value.concat(data.value), data.subtypeString); + assert.throws(() => encoder.encode(shortVector, data.typeInfo), 'Expected vector with 3 dimensions, observed size of 2'); + assert.throws(() => encoder.encode(longVector, data.typeInfo), 'Expected vector with 3 dimensions, observed size of 6'); + }); }); it('should encode and decode vector of float', function(){ From ad0682cf44e6207a918e18303b3bb17b666e00cf Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 24 Oct 2024 07:41:17 +0000 Subject: [PATCH 86/99] fix guessDataType --- lib/encoder.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/encoder.js b/lib/encoder.js index dd2d2505..34f7d0dd 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -1923,7 +1923,7 @@ Encoder.guessDataType = function (value) { // ignore } } - if (value == null){ + if (subtypeColumnInfo == null){ subtypeColumnInfo = this.guessDataType(value[0]); } if (subtypeColumnInfo != null) { From 9448898b1941ecd7a994ce36b9546f1b453ee315 Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 25 Oct 2024 18:39:51 +0000 Subject: [PATCH 87/99] add prepare to be false --- test/integration/short/vector-tests-ts.ts | 56 ----------------------- test/integration/short/vector-tests.js | 19 ++++++-- 2 files changed, 14 insertions(+), 61 deletions(-) delete mode 100644 test/integration/short/vector-tests-ts.ts diff --git a/test/integration/short/vector-tests-ts.ts b/test/integration/short/vector-tests-ts.ts deleted file mode 100644 index 3396fa6b..00000000 --- a/test/integration/short/vector-tests-ts.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -'use strict'; -const assert = require('assert'); -const helper = require('../../test-helper.js'); - -const { types } = require('../../../index.js'); -const vdescribe = helper.vdescribe; - -vdescribe('5.0.0', 'Vector tests TypeScript', function () { - this.timeout(120000); - describe('#execute with vectors', function () { - const keyspace = helper.getRandomName('ks'); - const table = keyspace + '.' + helper.getRandomName('table'); - const createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY, v1 vector);`; - - const setupInfo = helper.setup(1, { - keyspace: keyspace, - queries: [ createTableCql ] - }); - it('should insert and select vectors', function(done){ - const client = setupInfo.client; - // if client undefined, raise error - if(!client) return done(new Error('client is not defined')); - const id = types.Uuid.random(); - const v1 = new Float32Array([1.1, 2.2, 3.3]); - const query = `INSERT INTO ${table} (id, v1) VALUES (?, ?)`; - client.execute(query, [id, v1], {prepare : true}, function(err){ - if (err) return done(err); - client.execute(`SELECT v1 FROM ${table} WHERE id = ?`, [id], { prepare: true }, function(err, result){ - if (err) return done(err); - const v1 : Float32Array = result.rows[0].v1; - assert.strictEqual(result.rows.length, 1); - assert.strictEqual(v1.length, 3); - assert.strictEqual(v1[0], v1[0]); - assert.strictEqual(v1[1], v1[1]); - assert.strictEqual(v1[2], v1[2]); - done(); - }); - }); - }); - }); -}); \ No newline at end of file diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index 3fc1e787..28a4266f 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -364,7 +364,7 @@ vdescribe('5.0.0', 'Vector tests', function () { if (!client) { throw new Error('client setup failed'); } dataProviderWithCollections.forEach(data => { - it('should insert and select vector of subtype ' + data.subtypeString, function (done) { + it('should insert, select, and update vector of subtype ' + data.subtypeString, function (done) { const id = types.Uuid.random(); const vector = new Vector(data.value, data.subtypeString); const query = `INSERT INTO ${table} (id, ${subtypeStringToColumnName(data.subtypeString)}) VALUES (?, ?)`; @@ -374,16 +374,25 @@ vdescribe('5.0.0', 'Vector tests', function () { if (err) { return done(err); } assert.strictEqual(result.rows.length, 1); assert.strictEqual(util.inspect(result.rows[0][subtypeStringToColumnName(data.subtypeString)]), util.inspect(vector)); - done(); - }); + + const updatedValues = data.value.slice(); + updatedValues[0] = updatedValues[1]; + client.execute(`UPDATE ${table} SET ${subtypeStringToColumnName(data.subtypeString)} = ? WHERE id = ?`, [new Vector(updatedValues, data.subtypeString), id], { prepare: true }, function (err, result) { + if (err) { return done(err); } + done(); + } ); + }) }); }); - it('should insert and select vector of subtype ' + data.subtypeString + ' while guessing data type', function (done) { + it('should insert and select vector of subtype ' + data.subtypeString + ' with prepare to be false', function (done) { + if (data.subtypeString === 'my_udt') { + this.skip(); + } const id = types.Uuid.random(); const vector = new Vector(data.value, data.subtypeString); const query = `INSERT INTO ${table} (id, ${subtypeStringToColumnName(data.subtypeString)}) VALUES (?, ?)`; - client.execute(query, [id, vector], { prepare: true }, function (err) { + client.execute(query, [id, vector], { prepare: false }, function (err) { if (err) { return done(err); } client.execute(`SELECT ${subtypeStringToColumnName(data.subtypeString)} FROM ${table} WHERE id = ?`, [id], { prepare: true }, function (err, result) { if (err) { return done(err); } From 67a1233c22049b7fdf2a38e01056220f44c40906 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 12 Feb 2025 21:52:36 -0800 Subject: [PATCH 88/99] refactor vector data structure and test data provider --- lib/encoder.js | 57 +++-- lib/types/index.js | 17 +- lib/types/vector.js | 4 +- test/integration/short/vector-tests.js | 328 +------------------------ test/test-helper.js | 268 ++++++++++++++++++++ test/unit/encoder-tests.js | 12 +- test/unit/encoder-vector-tests.js | 325 +----------------------- 7 files changed, 318 insertions(+), 693 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index 34f7d0dd..59462124 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -140,7 +140,7 @@ const zeroLengthTypesSupported = new Set([ * @typedef {{code : (types.dataTypes.tuple | types.dataTypes.list)}} TupleListColumnInfoWithoutSubtype TODO: guessDataType can return null on tuple/list info, why? * @typedef {{code : (types.dataTypes.list | types.dataTypes.set), info : ColumnInfo, options?: {frozen?: Boolean, reversed?: Boolean}}} ListSetColumnInfo * @typedef {{code : (types.dataTypes.udt), info : {name : string, fields : Array<{name : string, type : ColumnInfo}>}, options? : {frozen?: Boolean, reversed?: Boolean}}} UdtColumnInfo - * @typedef {{code : (types.dataTypes.custom), customTypeName : ('vector'), info : ColumnInfo, dimension : number, options? : {frozen?:boolean, reversed?:boolean}}} VectorColumnInfo + * @typedef {{code : (types.dataTypes.custom), customTypeName : ('vector'), info : [ColumnInfo, number], options? : {frozen?:boolean, reversed?:boolean}}} VectorColumnInfo * @typedef {{code : (types.dataTypes.custom), info : string, options? : {frozen?:boolean, reversed?:boolean}}} OtherCustomColumnInfo * @typedef {SingleColumnInfo | CustomSimpleColumnInfo | MapColumnInfo | TupleColumnInfo | ListSetColumnInfo | VectorColumnInfo | OtherCustomColumnInfo | UdtColumnInfo | TupleListColumnInfoWithoutSubtype} ColumnInfo If this is a simple type, info is null; if this is a collection type with a simple subtype, info is a string, if this is a nested collection type, info is a ColumnInfo object */ @@ -212,11 +212,10 @@ function defineInstanceMembers() { /** * * @param {Buffer} bytes - * @param {ColumnInfo | string} typeName - * @param {ColumnInfo | null} columnInfo + * @param {ColumnInfo | string | [ColumnInfo, number]} typeName * @returns */ - this.decodeCustom = function (bytes, typeName, columnInfo = null) { + this.decodeCustom = function (bytes, typeName) { // Make sure we actually have something to process in typeName before we go any further if (!typeName || typeName.length === 0) { @@ -224,8 +223,8 @@ function defineInstanceMembers() { } // Special handling for vector custom types (since they have args) - if (columnInfo != null && 'customTypeName' in columnInfo && columnInfo.customTypeName === 'vector') { - return this.decodeVector(bytes, columnInfo); + if (Array.isArray(typeName) && typeof typeName[1] === 'number') { + return this.decodeVector(bytes, {code: dataTypes.vector, info: typeName, customTypeName: 'vector'}); } if(typeof typeName === 'string' && typeName.startsWith(customTypeNames.vector)) { @@ -747,15 +746,17 @@ function defineInstanceMembers() { /** * * @param {any} value - * @param {string | ColumnInfo} customTypeName when it's a vector, this will be the subtype columninfo, and value will be 0 - * @param {ColumnInfo | null} columnInfo + * @param {string | ColumnInfo | [ColumnInfo, number]} customTypeName when it's a vector, it will be [subtype, dimension] * @returns */ - this.encodeCustom = function (value, customTypeName, columnInfo = null) { + this.encodeCustom = function (value, customTypeName) { // Special handling for vector custom types (since they have args) - if (columnInfo != null && 'customTypeName' in columnInfo && columnInfo.customTypeName === 'vector') { - return this.encodeVector(value, columnInfo); + + //TODO: here we cannot tell whether it's a vector or not, because we don't have the customTypeName + // Unless we say as long as customTypeName is an array where the second item is a number, it's a vector? + if (Array.isArray(customTypeName) && typeof customTypeName[1] === 'number') { + return this.encodeVector(value, { code: dataTypes.vector, info: customTypeName , customTypeName: 'vector'}); } if (typeof customTypeName === 'string' && customTypeName.startsWith(customTypeNames.vector)) { const vectorColumnInfo = /** @type {VectorColumnInfo} */ (this.parseFqTypeName(customTypeName)); @@ -959,11 +960,11 @@ function defineInstanceMembers() { * * @param {Buffer} buffer * @param {VectorColumnInfo} params - * @returns {import('../index.js').types.Vector} + * @returns {Vector} */ this.decodeVector = function(buffer, params) { - const subtype = params.info; - const dimension = params.dimension; + const subtype = params.info[0]; + const dimension = params.info[1]; const elemLength = this.serializationSizeIfFixed(subtype); const rv = []; @@ -989,7 +990,7 @@ function defineInstanceMembers() { if (offset !== buffer.length) { throw new TypeError('Extra bytes found after decoding the vector'); } - const typeInfo = types.getDataTypeNameByCode(params.info); + const typeInfo = (types.getDataTypeNameByCode(subtype)); return new Vector(rv, typeInfo); }; @@ -1037,11 +1038,11 @@ function defineInstanceMembers() { return -1; case dataTypes.custom: if ('customTypeName' in cqlType && cqlType.customTypeName === 'vector'){ - const subtypeSerialSize = this.serializationSizeIfFixed(cqlType.info); + const subtypeSerialSize = this.serializationSizeIfFixed(cqlType.info[0]); if (subtypeSerialSize === -1){ return -1; } - return subtypeSerialSize * cqlType.dimension; + return subtypeSerialSize * cqlType.info[1]; } return -1; @@ -1060,7 +1061,7 @@ function defineInstanceMembers() { throw new TypeError("Driver only supports Vector type when encoding a vector"); } - const dimension = params.dimension; + const dimension = params.info[1]; if (value.length !== dimension) { throw new TypeError(`Expected vector with ${dimension} dimensions, observed size of ${value.length}`); } @@ -1069,10 +1070,10 @@ function defineInstanceMembers() { throw new TypeError("Cannot encode empty array as vector"); } - const serializationSize = this.serializationSizeIfFixed(params.info); + const serializationSize = this.serializationSizeIfFixed(params.info[0]); const encoded = []; for (const elem of value) { - const elemBuffer = this.encode(elem, params.info); + const elemBuffer = this.encode(elem, params.info[0]); if (serializationSize === -1) { encoded.push(utils.VIntCoding.uvintPack(elemBuffer.length)); } @@ -1096,8 +1097,8 @@ function defineInstanceMembers() { const argsLength = typeName.length - (stringToExclude.length + 2); const params = parseParams(typeName, argsStartIndex, argsLength); if (params.length === 2) { - /** @type {ColumnInfo} */ - const columnInfo = { code: dataTypes.custom, info: subtypeResolveFn.bind(this)(params[0].trim()), customTypeName : 'vector', dimension: parseInt(params[1].trim(), 10 )}; + /** @type {VectorColumnInfo} */ + const columnInfo = { code: dataTypes.custom, info: [subtypeResolveFn.bind(this)(params[0].trim()), parseInt(params[1].trim(), 10 )], customTypeName : 'vector'}; return columnInfo; } @@ -1375,7 +1376,7 @@ function defineInstanceMembers() { // parseVectorTypeArgs is not an async function but we are. To keep things simple let's ask the // function to just return whatever it finds for an arg and we'll eval it after the fact const params = this.parseVectorTypeArgs.bind(this)(typeName, cqlNames.vector, (arg) => arg ); - params["info"] = await this.parseTypeName(keyspace, params["info"]); + params["info"][0] = await this.parseTypeName(keyspace, params["info"][0]); return params; } @@ -1768,7 +1769,7 @@ Encoder.prototype.decode = function (buffer, type) { throw new Error('Unknown data type: ' + type.code); } - return decoder.call(this, buffer, 'info' in type? type.info : null, 'dimension' in type ? type : null); + return decoder.call(this, buffer, 'info' in type? type.info : null); }; /** @@ -1842,7 +1843,7 @@ Encoder.prototype.encode = function (value, typeInfo) { throw new Error('Type not supported ' + type.code); } - return encoder.call(this, value, 'info' in type ? type.info : null, 'dimension' in type ? type : null); + return encoder.call(this, value, 'info' in type ? type.info : null); }; /** @@ -1908,8 +1909,7 @@ Encoder.guessDataType = function (value) { return { code: dataTypes.custom, customTypeName: 'vector', - info: { code: dataTypes.float }, - dimension: value.length + info: [ {code: dataTypes.float}, value.length] }; } @@ -1930,8 +1930,7 @@ Encoder.guessDataType = function (value) { return { code: dataTypes.custom, customTypeName: 'vector', - info: subtypeColumnInfo, - dimension: value.length + info: [subtypeColumnInfo, value.length] }; } throw new TypeError("Cannot guess subtype from element " + value[0]); diff --git a/lib/types/index.js b/lib/types/index.js index a0144b8c..466d51e5 100644 --- a/lib/types/index.js +++ b/lib/types/index.js @@ -161,13 +161,12 @@ const dataTypes = { return this.getByName(x.trim()); }, this)}; } - const vectorMatches = /^(vector)<*(.+) *, *(\d+)>$/.exec(name); + const vectorMatches = /^(vector)<\s*(.+)\s*,\s*(\d+)\s*>$/.exec(name); if(vectorMatches){ return { code: this.custom, - customTypeName: 'vector', - info: this.getByName(vectorMatches[2]), - dimension: parseInt(vectorMatches[3], 10) + customTypeName:'vector', + info: [this.getByName(vectorMatches[2]), parseInt(vectorMatches[3], 10)] }; } } @@ -413,7 +412,7 @@ function uuid(options, buffer, offset) { } /** - * Gets the data type name for a given type definition, it will not work for udt or custom type + * Gets the data type name for a given type definition, it may not work for udt or custom type * @internal * @ignore * @throws {ArgumentError} @@ -430,6 +429,10 @@ function getDataTypeNameByCode(item) { if (!('info' in item) || !item.info) { return typeName; } + // special case for vector + if (item.code === dataTypes.custom && 'customTypeName' in item && item.customTypeName === 'vector') { + return 'vector<' + getDataTypeNameByCode(item.info[0]) + ', ' + item.info[1] + '>'; + } if (Array.isArray(item.info)) { return (typeName + '<' + @@ -438,10 +441,6 @@ function getDataTypeNameByCode(item) { }).join(', ') + '>'); } - // special case for vector - if (item.code === dataTypes.custom && 'customTypeName' in item && item.customTypeName === 'vector') { - return 'vector<' + getDataTypeNameByCode(item.info) + ', ' + item.dimension + '>'; - } if (typeof item.info.code === 'number') { return typeName + '<' + getDataTypeNameByCode(item.info) + '>'; } diff --git a/lib/types/vector.js b/lib/types/vector.js index 2a67e15b..0b2a52ea 100644 --- a/lib/types/vector.js +++ b/lib/types/vector.js @@ -24,9 +24,9 @@ class Vector { /** * * @param {Float32Array | Array} elements - * @param {string?} subtype + * @param {string} [subtype] */ - constructor (elements, subtype = null) { + constructor (elements, subtype) { if (elements instanceof Float32Array) { this.elements = Array.from(elements); } diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index 28a4266f..bb4d93e6 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -21,328 +21,6 @@ const { types } = require('../../../index.js'); const Vector = require('../../../lib/types/vector.js'); const util = require('node:util'); const vdescribe = helper.vdescribe; - -/** - * @type {Array<{subtypeString : string, typeInfo: import('../../../lib/encoder').VectorColumnInfo, value: Array}>} - */ -const dataProvider = [ - { - subtypeString: 'float', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.float, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [1.1122000217437744, 2.212209939956665, 3.3999900817871094] - }, - { - subtypeString: 'double', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.double, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [1.1, 2.2, 3.3] - }, - { - subtypeString: 'varchar', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.text, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: ['ab', 'b', 'cde'] - }, - { - subtypeString: 'bigint', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.bigint, - }, - customTypeName: 'vector', - dimension: 3, - }, - value: [new types.Long(1), new types.Long(2), new types.Long(3)] - }, - { - subtypeString: 'blob', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.blob, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [Buffer.from([1, 2, 3]), Buffer.from([4, 5, 6]), Buffer.from([7, 8, 9])] - }, - { - subtypeString: 'boolean', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.boolean, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [true, false, true] - }, - { - subtypeString: 'decimal', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.decimal, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [types.BigDecimal.fromString('1.1'), types.BigDecimal.fromString('2.2'), types.BigDecimal.fromString('3.3')] - }, - { - subtypeString: 'inet', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.inet, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [types.InetAddress.fromString('127.0.0.1'), types.InetAddress.fromString('0.0.0.0'), types.InetAddress.fromString('34.12.10.19')] - }, - { - subtypeString: 'tinyint', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.tinyint, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [1, 2, 3] - }, - { - subtypeString: 'smallint', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.smallint, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [1, 2, 3] - }, - { - subtypeString: 'int', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.int, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [-1, 0, -3] - }, - // TODO: what do we want to do with Duration? - // { - // subtypeString: 'duration', - // typeInfo: { - // code: types.dataTypes.custom, - // info: { - // code: types.dataTypes.custom, - // info: 'duration' - // }, - // customTypeName: 'vector', - // dimension: 3 - // }, - // value: [new types.Duration(1, 2, 3), new types.Duration(4, 5, 6), new types.Duration(7, 8, 9)] - // }, - { - subtypeString: 'date', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.date, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [new types.LocalDate(2020, 1, 1), new types.LocalDate(2020, 2, 1), new types.LocalDate(2020, 3, 1)] - }, - { - subtypeString: 'time', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.time, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [new types.LocalTime(types.Long.fromString('6331999999911')), new types.LocalTime(types.Long.fromString('6331999999911')), new types.LocalTime(types.Long.fromString('6331999999911'))] - }, - { - subtypeString: 'timestamp', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.timestamp, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [new Date(2020, 1, 1, 1, 1, 1, 1), new Date(2020, 2, 1, 1, 1, 1, 1), new Date(2020, 3, 1, 1, 1, 1, 1)] - }, - { - subtypeString: 'uuid', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.uuid, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [types.Uuid.random(), types.Uuid.random(), types.Uuid.random()] - }, - { - subtypeString: 'timeuuid', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.timeuuid, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [types.TimeUuid.now(), types.TimeUuid.now(), types.TimeUuid.now()] - } -]; - -const dataProviderWithCollections = dataProvider.flatMap(data => [ - data, - // vector, 3> - { - subtypeString: 'list<' + data.subtypeString + '>', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.list, - info: { - code: data.typeInfo.info.code, - info: data.typeInfo.info.info - } - }, - customTypeName: 'vector', - dimension: 3 - }, - value: data.value.map(value => [value, value, value]) - }, - // vector, 3> - { - subtypeString: 'map', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.map, - info: [ - { code: types.dataTypes.int }, - { code: data.typeInfo.info.code, info: data.typeInfo.info.info } - ] - }, - customTypeName: 'vector', - dimension: 3 - }, - value: data.value.map((value) => ({ 1: value, 2: value, 3: value })) - }, - // vector, 3> - { - subtypeString: 'set<' + data.subtypeString + '>', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.set, - info: { - code: data.typeInfo.info.code, - info: data.typeInfo.info.info - } - }, - customTypeName: 'vector', - dimension: 3 - }, - value: data.value.map(value => [value, value, value]) - }, - // vector, 3> - { - subtypeString: 'tuple<' + data.subtypeString + ', ' + data.subtypeString + '>', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.tuple, - info: [ - { code: data.typeInfo.info.code, info: data.typeInfo.info.info }, - { code: data.typeInfo.info.code, info: data.typeInfo.info.info } - ] - }, - customTypeName: 'vector', - dimension: 3 - }, - value: data.value.map(value => new types.Tuple(value, value)) - }, - // vector, 3> - { - subtypeString: 'vector<' + data.subtypeString + ', 3>', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.custom, - info: { - code: data.typeInfo.info.code, - info: data.typeInfo.info.info - }, - customTypeName: 'vector', - dimension: 3 - }, - customTypeName: 'vector', - dimension: 3 - }, - value: data.value.map(value => new Vector([value, value, value], data.subtypeString)) - } -]).concat([ - // vector - { - subtypeString: 'my_udt', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.udt, - info: { - name: 'my_udt', - fields: [{ name: 'f1', type: { code: types.dataTypes.text } }], - } - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [{ f1: 'a' }, { f1: 'b' }, { f1: 'c' }] - } -]); - vdescribe('5.0.0', 'Vector tests', function () { this.timeout(120000); @@ -350,7 +28,7 @@ vdescribe('5.0.0', 'Vector tests', function () { const keyspace = helper.getRandomName('ks'); const table = keyspace + '.' + helper.getRandomName('table'); let createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY`; - dataProviderWithCollections.forEach(data => { + helper.dataProviderWithCollections.forEach(data => { createTableCql += `, ${subtypeStringToColumnName(data.subtypeString)} vector<${data.subtypeString}, 3>`; }); createTableCql += ');'; @@ -363,7 +41,7 @@ vdescribe('5.0.0', 'Vector tests', function () { const client = setupInfo.client; if (!client) { throw new Error('client setup failed'); } - dataProviderWithCollections.forEach(data => { + helper.dataProviderWithCollections.forEach(data => { it('should insert, select, and update vector of subtype ' + data.subtypeString, function (done) { const id = types.Uuid.random(); const vector = new Vector(data.value, data.subtypeString); @@ -381,7 +59,7 @@ vdescribe('5.0.0', 'Vector tests', function () { if (err) { return done(err); } done(); } ); - }) + }); }); }); diff --git a/test/test-helper.js b/test/test-helper.js index c342d1e3..d1918a05 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -22,6 +22,7 @@ const util = require('util'); const path = require('path'); const policies = require('../lib/policies'); const types = require('../lib/types'); +// const { types } = require('../lib/types'); const utils = require('../lib/utils'); const spawn = require('child_process').spawn; const childProcessExec = require('child_process').exec; @@ -32,6 +33,7 @@ const defaultOptions = require('../lib/client-options').defaultOptions; const { Host, HostMap } = require('../lib/host'); const OperationState = require('../lib/operation-state'); const promiseUtils = require('../lib/promise-utils'); +const Vector = types.Vector; util.inherits(RetryMultipleTimes, policies.retry.RetryPolicy); @@ -1616,6 +1618,270 @@ helper.ads.getKrb5ConfigPath = function() { return path.join(this.dir, 'krb5.conf'); }; +/** + * @type {Array<{subtypeString : string, typeInfo: import('../lib/encoder').VectorColumnInfo, value: Array}>} + */ +const dataProvider = [ + { + subtypeString: 'float', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.float }, 3] + }, + value: [1.1122000217437744, 2.212209939956665, 3.3999900817871094] + }, + { + subtypeString: 'double', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.double }, 3] + }, + value: [1.1, 2.2, 3.3] + }, + { + subtypeString: 'varchar', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.text }, 3] + }, + value: ['ab', 'b', 'cde'] + }, + { + subtypeString: 'bigint', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.bigint }, 3] + }, + value: [new types.Long(1), new types.Long(2), new types.Long(3)] + }, + { + subtypeString: 'blob', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.blob }, 3] + }, + value: [Buffer.from([1, 2, 3]), Buffer.from([4, 5, 6]), Buffer.from([7, 8, 9])] + }, + { + subtypeString: 'boolean', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.boolean }, 3] + }, + value: [true, false, true] + }, + { + subtypeString: 'decimal', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.decimal }, 3] + }, + value: [types.BigDecimal.fromString('1.1'), types.BigDecimal.fromString('2.2'), types.BigDecimal.fromString('3.3')] + }, + { + subtypeString: 'inet', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.inet }, 3] + }, + value: [types.InetAddress.fromString('127.0.0.1'), types.InetAddress.fromString('0.0.0.0'), types.InetAddress.fromString('34.12.10.19')] + }, + { + subtypeString: 'tinyint', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.tinyint }, 3] + }, + value: [1, 2, 3] + }, + { + subtypeString: 'smallint', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.smallint }, 3] + }, + value: [1, 2, 3] + }, + { + subtypeString: 'int', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.int }, 3] + }, + value: [-1, 0, -3] + }, + { + subtypeString: 'date', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.date }, 3] + }, + value: [new types.LocalDate(2020, 1, 1), new types.LocalDate(2020, 2, 1), new types.LocalDate(2020, 3, 1)] + }, + { + subtypeString: 'time', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.time }, 3] + }, + value: [new types.LocalTime(types.Long.fromString('6331999999911')), new types.LocalTime(types.Long.fromString('6331999999911')), new types.LocalTime(types.Long.fromString('6331999999911'))] + }, + { + subtypeString: 'timestamp', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.timestamp }, 3] + }, + value: [new Date(2020, 1, 1, 1, 1, 1, 1), new Date(2020, 2, 1, 1, 1, 1, 1), new Date(2020, 3, 1, 1, 1, 1, 1)] + }, + { + subtypeString: 'uuid', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.uuid }, 3] + }, + value: [types.Uuid.random(), types.Uuid.random(), types.Uuid.random()] + }, + { + subtypeString: 'timeuuid', + typeInfo: { + code: types.dataTypes.custom, + customTypeName: 'vector', + info: [{ code: types.dataTypes.timeuuid }, 3] + }, + value: [types.TimeUuid.now(), types.TimeUuid.now(), types.TimeUuid.now()] + } +]; + +helper.dataProvider = dataProvider; + +const dataProviderWithCollections = dataProvider.flatMap(data => [ + data, + // vector, 3> + { + subtypeString: 'list<' + data.subtypeString + '>', + typeInfo: { + code: types.dataTypes.custom, + info: [{ + code: types.dataTypes.list, + info: { + code: data.typeInfo.info[0].code, + info: data.typeInfo.info[0]["info"] + } + },3], + customTypeName: 'vector', + }, + value: data.value.map(value => [value, value, value]) + }, + // vector, 3> + { + subtypeString: 'map', + typeInfo: { + code: types.dataTypes.custom, + info: [{ + code: types.dataTypes.map, + info: [ + { code: types.dataTypes.int }, + { + code: data.typeInfo.info[0].code, + info: data.typeInfo.info[0]["info"] + } + ] + },3], + customTypeName: 'vector' + }, + value: data.value.map((value) => ({ 1: value, 2: value, 3: value })) + }, + // vector, 3> + { + subtypeString: 'set<' + data.subtypeString + '>', + typeInfo: { + code: types.dataTypes.custom, + info: [{ + code: types.dataTypes.set, + info: { + code: data.typeInfo.info[0].code, + info: data.typeInfo.info[0]["info"] + } + },3], + customTypeName: 'vector', + }, + value: data.value.map(value => [value, value, value]) + }, + // vector, 3> + { + subtypeString: 'tuple<' + data.subtypeString + ', ' + data.subtypeString + '>', + typeInfo: { + code: types.dataTypes.custom, + info: [{ + code: types.dataTypes.tuple, + info: [ + { + code: data.typeInfo.info[0].code, + info: data.typeInfo.info[0]["info"] + }, + { + code: data.typeInfo.info[0].code, + info: data.typeInfo.info[0]["info"] + } + ] + },3], + customTypeName: 'vector', + }, + value: data.value.map(value => new types.Tuple(value, value)) + }, + // vector, 3> + { + subtypeString: 'vector<' + data.subtypeString + ', 3>', + typeInfo: { + code: types.dataTypes.custom, + info: [{ + code: types.dataTypes.custom, + info: [{ + code: data.typeInfo.info[0].code, + info: data.typeInfo.info[0]["info"] + },3], + customTypeName: 'vector', + },3], + customTypeName: 'vector', + }, + value: data.value.map(value => new Vector([value, value, value], data.subtypeString)) + } +]).concat([ +// vector + { + subtypeString: 'my_udt', + typeInfo: { + code: types.dataTypes.custom, + info: [{ + code: types.dataTypes.udt, + info: { + name: 'my_udt', + fields: [{ name: 'f1', type: { code: types.dataTypes.text } }], + } + }, 3], + customTypeName: 'vector', + }, + value: [{ f1: 'a' }, { f1: 'b' }, { f1: 'c' }] + } +]); + +helper.dataProviderWithCollections = dataProviderWithCollections; /** * A retry policy for testing purposes only, retries for a number of times * @param {Number} times @@ -1762,6 +2028,8 @@ class OrderedLoadBalancingPolicy extends policies.loadBalancing.RoundRobinPolicy } } + + module.exports = helper; module.exports.RetryMultipleTimes = RetryMultipleTimes; module.exports.OrderedLoadBalancingPolicy = OrderedLoadBalancingPolicy; diff --git a/test/unit/encoder-tests.js b/test/unit/encoder-tests.js index 64838492..354dcba9 100644 --- a/test/unit/encoder-tests.js +++ b/test/unit/encoder-tests.js @@ -698,7 +698,7 @@ describe('encoder', function () { const encoder = new Encoder(4, {}); const refVal = new Float32Array([1.2, 3.4, 5.6]); /** @type {import('../../lib/encoder').VectorColumnInfo} */ - const typeObj = {code: dataTypes.custom, info: {code : dataTypes.float}, dimension : 3, customTypeName : 'vector'}; + const typeObj = {code: dataTypes.custom, info: [{code : dataTypes.float},3], customTypeName : 'vector'}; const encoded = encoder.encode(refVal, typeObj); const decoded = encoder.decode(encoded, typeObj); helper.assertInstanceOf(decoded, Vector); @@ -711,7 +711,7 @@ describe('encoder', function () { const encoder = new Encoder(4, {}); const refVal = new Vector(['a', 'bc', 'de']); /** @type {import('../../lib/encoder').VectorColumnInfo} */ - const typeObj = {code: dataTypes.custom, info: {code : dataTypes.ascii}, dimension : 3, customTypeName : 'vector'}; + const typeObj = {code: dataTypes.custom, info: [{code : dataTypes.ascii},3], customTypeName : 'vector'}; const encoded = encoder.encode(refVal, typeObj); const decoded = encoder.decode(encoded, typeObj); helper.assertInstanceOf(decoded, Vector); @@ -1109,8 +1109,8 @@ describe('encoder', function () { type = encoder.parseFqTypeName('org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.FloatType,10)'); assert.strictEqual(dataTypes.custom, type.code); assert.ok(typeof type.info === 'object'); - assert.strictEqual(dataTypes.float, type.info.code); - assert.strictEqual(10, type["dimension"]); + assert.strictEqual(dataTypes.float, type.info[0].code); + assert.strictEqual(10, type.info[1]); }); it('should parse frozen types', function () { @@ -1242,8 +1242,8 @@ describe('encoder', function () { }); } else if (typeof item[2] === 'object') { - assert.strictEqual(dataType.info.code, item[2].code); - assert.strictEqual(dataType.dimension, item[2].dimension); + assert.strictEqual(dataType.info[0].code, item[2].code); + assert.strictEqual(dataType.info[1], item[2].dimension); } else { assert.strictEqual(dataType.info.code, item[2]); diff --git a/test/unit/encoder-vector-tests.js b/test/unit/encoder-vector-tests.js index 3c6bcaf3..4b3cc356 100644 --- a/test/unit/encoder-vector-tests.js +++ b/test/unit/encoder-vector-tests.js @@ -18,331 +18,12 @@ const { assert, util } = require('chai'); const Encoder = require('../../lib/encoder'); const { types } = require('../../index'); const Vector = require('../../lib/types/vector'); +const helper = require('../test-helper'); describe('Vector tests', function () { const encoder = new Encoder(4, {}); - /** - * @type {Array<{subtypeString : string, typeInfo: import('../../../lib/encoder').VectorColumnInfo, value: Array}>} - */ - const dataProvider = [ - { - subtypeString: 'float', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.float, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [1.1122000217437744, 2.212209939956665, 3.3999900817871094] - }, - { - subtypeString: 'double', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.double, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [1.1, 2.2, 3.3] - }, - { - subtypeString: 'varchar', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.text, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: ['ab', 'b', 'cde'] - }, - { - subtypeString: 'bigint', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.bigint, - }, - customTypeName: 'vector', - dimension: 3, - }, - value: [new types.Long(1), new types.Long(2), new types.Long(3)] - }, - { - subtypeString: 'blob', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.blob, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [Buffer.from([1, 2, 3]), Buffer.from([4, 5, 6]), Buffer.from([7, 8, 9])] - }, - { - subtypeString: 'boolean', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.boolean, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [true, false, true] - }, - { - subtypeString: 'decimal', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.decimal, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [types.BigDecimal.fromString('1.1'), types.BigDecimal.fromString('2.2'), types.BigDecimal.fromString('3.3')] - }, - { - subtypeString: 'inet', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.inet, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [types.InetAddress.fromString('127.0.0.1'), types.InetAddress.fromString('0.0.0.0'), types.InetAddress.fromString('34.12.10.19')] - }, - { - subtypeString: 'tinyint', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.tinyint, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [1, 2, 3] - }, - { - subtypeString: 'smallint', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.smallint, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [1, 2, 3] - }, - { - subtypeString: 'int', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.int, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [-1, 0, -3] - }, - // TODO: what do we want to do with Duration? - // { - // subtypeString: 'duration', - // typeInfo: { - // code: types.dataTypes.custom, - // info: { - // code: types.dataTypes.custom, - // info: 'duration' - // }, - // customTypeName: 'vector', - // dimension: 3 - // }, - // value: [new types.Duration(1, 2, 3), new types.Duration(4, 5, 6), new types.Duration(7, 8, 9)] - // }, - { - subtypeString: 'date', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.date, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [new types.LocalDate(2020, 1, 1), new types.LocalDate(2020, 2, 1), new types.LocalDate(2020, 3, 1)] - }, - { - subtypeString: 'time', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.time, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [new types.LocalTime(types.Long.fromString('6331999999911')), new types.LocalTime(types.Long.fromString('6331999999911')), new types.LocalTime(types.Long.fromString('6331999999911'))] - }, - { - subtypeString: 'timestamp', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.timestamp, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [new Date(2020, 1, 1, 1, 1, 1, 1), new Date(2020, 2, 1, 1, 1, 1, 1), new Date(2020, 3, 1, 1, 1, 1, 1)] - }, - { - subtypeString: 'uuid', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.uuid, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [types.Uuid.random(), types.Uuid.random(), types.Uuid.random()] - }, - { - subtypeString: 'timeuuid', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.timeuuid, - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [types.TimeUuid.now(), types.TimeUuid.now(), types.TimeUuid.now()] - } - ]; - - const dataProviderWithCollections = dataProvider.flatMap(data => [ - data, - // vector, 3> - { - subtypeString: 'list<' + data.subtypeString + '>', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.list, - info: { - code: data.typeInfo.info.code, - info: data.typeInfo.info.info - } - }, - customTypeName: 'vector', - dimension: 3 - }, - value: data.value.map(value => [value, value, value]) - }, - // vector, 3> - { - subtypeString: 'map', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.map, - info: [ - { code: types.dataTypes.int }, - { code: data.typeInfo.info.code, info: data.typeInfo.info.info } - ] - }, - customTypeName: 'vector', - dimension: 3 - }, - value: data.value.map((value) => ({ 1: value, 2: value, 3: value })) - }, - // vector, 3> - { - subtypeString: 'set<' + data.subtypeString + '>', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.set, - info: { - code: data.typeInfo.info.code, - info: data.typeInfo.info.info - } - }, - customTypeName: 'vector', - dimension: 3 - }, - value: data.value.map(value => [value, value, value]) - }, - // vector, 3> - { - subtypeString: 'tuple<' + data.subtypeString + ', ' + data.subtypeString + '>', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.tuple, - info: [ - { code: data.typeInfo.info.code, info: data.typeInfo.info.info }, - { code: data.typeInfo.info.code, info: data.typeInfo.info.info } - ] - }, - customTypeName: 'vector', - dimension: 3 - }, - value: data.value.map(value => new types.Tuple(value, value)) - }, - // vector, 3> - { - subtypeString: 'vector<' + data.subtypeString + ', 3>', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.custom, - info: { - code: data.typeInfo.info.code, - info: data.typeInfo.info.info - }, - customTypeName: 'vector', - dimension: 3 - }, - customTypeName: 'vector', - dimension: 3 - }, - value: data.value.map(value => new Vector([value, value, value], data.subtypeString)) - } - ]).concat([ - // vector - { - subtypeString: 'my_udt', - typeInfo: { - code: types.dataTypes.custom, - info: { - code: types.dataTypes.udt, - info: { - name: 'my_udt', - fields: [{ name: 'f1', type: { code: types.dataTypes.text } }], - } - }, - customTypeName: 'vector', - dimension: 3 - }, - value: [{ f1: 'a' }, { f1: 'b' }, { f1: 'c' }] - } - ]); - dataProviderWithCollections.forEach((data) => { + helper.dataProviderWithCollections.forEach((data) => { it(`should encode and decode vector of ${data.subtypeString}`, function () { const vector = new Vector(data.value, data.subtypeString); const encoded = encoder.encode(vector, data.typeInfo); @@ -383,7 +64,7 @@ describe('Vector tests', function () { it('should encode and decode vector of float', function(){ const vector = new Float32Array([1.1, 2.2, 3.3]); - const typeObj = { code: types.dataTypes.custom, info: {code: types.dataTypes.float}, dimension: 3, customTypeName: 'vector' }; + const typeObj = { code: types.dataTypes.custom, info: [{code: types.dataTypes.float}, 3], customTypeName: 'vector' }; const encoded = encoder.encode(vector, typeObj); const decoded = encoder.decode(encoded, typeObj); for (let i = 0; i < vector.length; i++) { From f218183230eb5116157190cd432b186087db50a3 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 13 Feb 2025 00:20:33 -0800 Subject: [PATCH 89/99] revert package-lock.json --- examples/package-lock.json | 50 -------------------------------------- lib/encoder.js | 28 +++------------------ lib/types/index.js | 6 ++--- test/test-helper.js | 12 +++++++++ 4 files changed, 18 insertions(+), 78 deletions(-) delete mode 100644 examples/package-lock.json diff --git a/examples/package-lock.json b/examples/package-lock.json deleted file mode 100644 index 6659ea47..00000000 --- a/examples/package-lock.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "name": "cassandra-driver-examples", - "version": "0.0.1", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "cassandra-driver-examples", - "version": "0.0.1", - "license": "Apache-2.0", - "dependencies": { - "async": "^1.5.2", - "cassandra-driver": "file:../" - } - }, - "..": { - "name": "cassandra-driver", - "version": "4.7.2", - "license": "Apache-2.0", - "dependencies": { - "@types/long": "~5.0.0", - "@types/node": "^18.11.18", - "adm-zip": "~0.5.10", - "long": "~5.2.3" - }, - "devDependencies": { - "@types/mocha": "^10.0.8", - "chai": "~4.3.8", - "kerberos": "~2.0.3", - "mocha": "~10.2.0", - "mocha-jenkins-reporter": "~0.4.8", - "proxyquire": "~2.1.3", - "sinon": "~15.2.0", - "temp": ">= 0.8.3" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/async": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", - "integrity": "sha512-nSVgobk4rv61R9PUSDtYt7mPVB2olxNR5RWJcAsH676/ef11bUZwvu7+RGYrYauVdDPcO519v68wRhXQtxsV9w==" - }, - "node_modules/cassandra-driver": { - "resolved": "..", - "link": true - } - } -} diff --git a/lib/encoder.js b/lib/encoder.js index 59462124..e0c659c3 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -1016,26 +1016,6 @@ function defineInstanceMembers() { return 16; case dataTypes.uuid: return 16; - - case dataTypes.ascii: - case dataTypes.tinyint: - case dataTypes.blob: - case dataTypes.counter: - case dataTypes.decimal: - case dataTypes.duration: - case dataTypes.inet: - case dataTypes.varint: - case dataTypes.smallint: - case dataTypes.date: - case dataTypes.time: - case dataTypes.text: - case dataTypes.varchar: - case dataTypes.list: - case dataTypes.map: - case dataTypes.set: - case dataTypes.udt: - case dataTypes.tuple: - return -1; case dataTypes.custom: if ('customTypeName' in cqlType && cqlType.customTypeName === 'vector'){ const subtypeSerialSize = this.serializationSizeIfFixed(cqlType.info[0]); @@ -1047,7 +1027,7 @@ function defineInstanceMembers() { } return -1; default: - throw new TypeError('Cannot calculate size'); + return -1; } }; /** @@ -1829,11 +1809,9 @@ Encoder.prototype.encode = function (value, typeInfo) { } else { //Lets guess - const guessedType = Encoder.guessDataType(value); - if (!guessedType) { + type = Encoder.guessDataType(value); + if (!type) { throw new TypeError('Target data type could not be guessed, you should use prepared statements for accurate type mapping. Value: ' + util.inspect(value)); - }else{ - type = guessedType; } } diff --git a/lib/types/index.js b/lib/types/index.js index 466d51e5..f28dbb55 100644 --- a/lib/types/index.js +++ b/lib/types/index.js @@ -161,12 +161,12 @@ const dataTypes = { return this.getByName(x.trim()); }, this)}; } - const vectorMatches = /^(vector)<\s*(.+)\s*,\s*(\d+)\s*>$/.exec(name); + const vectorMatches = /^vector<\s*(.+)\s*,\s*(\d+)\s*>$/.exec(name); if(vectorMatches){ return { code: this.custom, - customTypeName:'vector', - info: [this.getByName(vectorMatches[2]), parseInt(vectorMatches[3], 10)] + customTypeName: 'vector', + info: [this.getByName(vectorMatches[1]), parseInt(vectorMatches[2], 10)] }; } } diff --git a/test/test-helper.js b/test/test-helper.js index d1918a05..2a01576b 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -1721,6 +1721,18 @@ const dataProvider = [ }, value: [-1, 0, -3] }, + { + subtypeString: 'duration', + typeInfo: { + code: types.dataTypes.custom, + info: [{ + code: types.dataTypes.custom, + info: 'org.apache.cassandra.db.marshal.DurationType' + },3], + customTypeName: 'vector', + }, + value: [new types.Duration(1, 2, 3), new types.Duration(4, 5, 6), new types.Duration(7, 8, 9)] + }, { subtypeString: 'date', typeInfo: { From a515250ff42021aeddf069c5838ec8a51f74f73a Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 13 Feb 2025 00:29:15 -0800 Subject: [PATCH 90/99] revert @types/mocha --- package-lock.json | 7 ------- package.json | 1 - 2 files changed, 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index edcc5042..192ef19d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,6 @@ "long": "~5.2.3" }, "devDependencies": { - "@types/mocha": "^10.0.8", "chai": "~4.3.8", "kerberos": "~2.0.3", "mocha": "~10.2.0", @@ -71,12 +70,6 @@ "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", "dev": true }, - "node_modules/@types/mocha": { - "version": "10.0.8", - "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.8.tgz", - "integrity": "sha512-HfMcUmy9hTMJh66VNcmeC9iVErIZJli2bszuXc6julh5YGuRb/W5OnkHjwLNYdFlMis0sY3If5SEAp+PktdJjw==", - "dev": true - }, "node_modules/@types/node": { "version": "18.19.54", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.54.tgz", diff --git a/package.json b/package.json index 67d3655a..3ec7eea5 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,6 @@ "long": "~5.2.3" }, "devDependencies": { - "@types/mocha": "^10.0.8", "chai": "~4.3.8", "kerberos": "~2.0.3", "mocha": "~10.2.0", From 9ba3fa99a9763ffa4819573c1fa509375d9ac298 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 13 Feb 2025 00:30:49 -0800 Subject: [PATCH 91/99] revert package-lock.json --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 192ef19d..4983ffc8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -71,9 +71,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.19.54", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.54.tgz", - "integrity": "sha512-+BRgt0G5gYjTvdLac9sIeE0iZcJxi4Jc4PV5EUzqi+88jmQLr+fRZdv2tCTV7IHKSGxM6SaLoOXQWWUiLUItMw==", + "version": "18.19.53", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.53.tgz", + "integrity": "sha512-GLxgUgHhDKO1Edw9Q0lvMbiO/IQXJwJlMaqxSGBXMpPy8uhkCs2iiPFaB2Q/gmobnFkckD3rqTBMVjXdwq+nKg==", "dependencies": { "undici-types": "~5.26.4" } From 82b10dc62bc524f53a98f580b2a9bb5d1a0cb6aa Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 13 Feb 2025 00:45:25 -0800 Subject: [PATCH 92/99] delete // const { types } = require('./types'); --- lib/encoder.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/encoder.js b/lib/encoder.js index e0c659c3..11a34103 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -17,7 +17,6 @@ const util = require('util'); const types = require('./types'); -// const { types } = require('./types'); const dataTypes = types.dataTypes; const Long = types.Long; const Integer = types.Integer; From fa561a4a992fddd7f3ba104e73bb5ba2afcd7347 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 13 Feb 2025 17:05:10 -0800 Subject: [PATCH 93/99] refactor API from (value, type.info) to (value, type) --- lib/encoder.js | 92 +++++++++++++++++++++++++++----------------------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/lib/encoder.js b/lib/encoder.js index 11a34103..dedc6d4e 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -211,27 +211,22 @@ function defineInstanceMembers() { /** * * @param {Buffer} bytes - * @param {ColumnInfo | string | [ColumnInfo, number]} typeName + * @param {OtherCustomColumnInfo | VectorColumnInfo} columnInfo * @returns */ - this.decodeCustom = function (bytes, typeName) { + this.decodeCustom = function (bytes, columnInfo) { // Make sure we actually have something to process in typeName before we go any further - if (!typeName || typeName.length === 0) { + if (!columnInfo) { return this.handleBuffer(bytes); } // Special handling for vector custom types (since they have args) - if (Array.isArray(typeName) && typeof typeName[1] === 'number') { - return this.decodeVector(bytes, {code: dataTypes.vector, info: typeName, customTypeName: 'vector'}); + if ('customTypeName' in columnInfo && columnInfo.customTypeName === 'vector') { + return this.decodeVector(bytes, columnInfo); } - if(typeof typeName === 'string' && typeName.startsWith(customTypeNames.vector)) { - const vectorColumnInfo = /** @type {VectorColumnInfo} */ (this.parseFqTypeName(typeName)); - return this.decodeVector(bytes, vectorColumnInfo); - } - - const handler = customDecoders[typeName]; + const handler = customDecoders[columnInfo.info]; if (handler) { return handler.call(this, bytes); } @@ -316,7 +311,8 @@ function defineInstanceMembers() { /* * Reads a list from bytes */ - this.decodeList = function (bytes, subtype) { + this.decodeList = function (bytes, columnInfo) { + const subtype = columnInfo.info; const totalItems = this.decodeCollectionLength(bytes, 0); let offset = this.collectionLengthSize; const list = new Array(totalItems); @@ -333,8 +329,8 @@ function defineInstanceMembers() { /* * Reads a Set from bytes */ - this.decodeSet = function (bytes, subtype) { - const arr = this.decodeList(bytes, subtype); + this.decodeSet = function (bytes, columnInfo) { + const arr = this.decodeList(bytes, columnInfo); if (this.encodingOptions.set) { const setConstructor = this.encodingOptions.set; return new setConstructor(arr); @@ -344,7 +340,8 @@ function defineInstanceMembers() { /* * Reads a map (key / value) from bytes */ - this.decodeMap = function (bytes, subtypes) { + this.decodeMap = function (bytes, columnInfo) { + const subtypes = columnInfo.info; let map; const totalItems = this.decodeCollectionLength(bytes, 0); let offset = this.collectionLengthSize; @@ -391,10 +388,11 @@ function defineInstanceMembers() { /** * Decodes a user defined type into an object * @param {Buffer} bytes - * @param {{fields: Array}} udtInfo + * @param {UdtColumnInfo} columnInfo * @private */ - this.decodeUdt = function (bytes, udtInfo) { + this.decodeUdt = function (bytes, columnInfo) { + const udtInfo = columnInfo.info; const result = {}; let offset = 0; for (let i = 0; i < udtInfo.fields.length && offset < bytes.length; i++) { @@ -413,7 +411,8 @@ function defineInstanceMembers() { return result; }; - this.decodeTuple = function (bytes, tupleInfo) { + this.decodeTuple = function (bytes, columnInfo) { + const tupleInfo = columnInfo.info; const elements = new Array(tupleInfo.length); let offset = 0; @@ -745,27 +744,20 @@ function defineInstanceMembers() { /** * * @param {any} value - * @param {string | ColumnInfo | [ColumnInfo, number]} customTypeName when it's a vector, it will be [subtype, dimension] + * @param {OtherCustomColumnInfo | VectorColumnInfo} columnInfo * @returns */ - this.encodeCustom = function (value, customTypeName) { - - // Special handling for vector custom types (since they have args) + this.encodeCustom = function (value, columnInfo) { - //TODO: here we cannot tell whether it's a vector or not, because we don't have the customTypeName - // Unless we say as long as customTypeName is an array where the second item is a number, it's a vector? - if (Array.isArray(customTypeName) && typeof customTypeName[1] === 'number') { - return this.encodeVector(value, { code: dataTypes.vector, info: customTypeName , customTypeName: 'vector'}); + if ('customTypeName' in columnInfo && columnInfo.customTypeName === 'vector') { + return this.encodeVector(value, columnInfo); } - if (typeof customTypeName === 'string' && customTypeName.startsWith(customTypeNames.vector)) { - const vectorColumnInfo = /** @type {VectorColumnInfo} */ (this.parseFqTypeName(customTypeName)); - return this.encodeVector(value, vectorColumnInfo); - } - const handler = customEncoders[customTypeName]; + + const handler = customEncoders[columnInfo.info]; if (handler) { return handler.call(this, value); } - throw new TypeError('No encoding handler found for type ' + customTypeName); + throw new TypeError('No encoding handler found for type ' + columnInfo); }; /** * @param {Boolean} value @@ -811,7 +803,8 @@ function defineInstanceMembers() { buf.writeInt8(value, 0); return buf; }; - this.encodeList = function (value, subtype) { + this.encodeList = function (value, columnInfo) { + const subtype = columnInfo.info; if (!Array.isArray(value)) { throw new TypeError('Not a valid list value, expected Array obtained ' + util.inspect(value)); } @@ -833,24 +826,25 @@ function defineInstanceMembers() { } return Buffer.concat(parts); }; - this.encodeSet = function (value, subtype) { + this.encodeSet = function (value, columnInfo) { if (this.encodingOptions.set && value instanceof this.encodingOptions.set) { const arr = []; value.forEach(function (x) { arr.push(x); }); - return this.encodeList(arr, subtype); + return this.encodeList(arr, columnInfo); } - return this.encodeList(value, subtype); + return this.encodeList(value, columnInfo); }; /** * Serializes a map into a Buffer * @param value - * @param {Array} [subtypes] + * @param {MapColumnInfo} columnInfo * @returns {Buffer} * @private */ - this.encodeMap = function (value, subtypes) { + this.encodeMap = function (value, columnInfo) { + const subtypes = columnInfo.info; const parts = []; let propCounter = 0; let keySubtype = null; @@ -900,7 +894,14 @@ function defineInstanceMembers() { parts.unshift(this.getLengthBuffer(propCounter)); return Buffer.concat(parts); }; - this.encodeUdt = function (value, udtInfo) { + /** + * + * @param {any} value + * @param {UdtColumnInfo} columnInfo + * @returns + */ + this.encodeUdt = function (value, columnInfo) { + const udtInfo = columnInfo.info; const parts = []; let totalLength = 0; for (let i = 0; i < udtInfo.fields.length; i++) { @@ -924,7 +925,14 @@ function defineInstanceMembers() { } return Buffer.concat(parts, totalLength); }; - this.encodeTuple = function (value, tupleInfo) { + /** + * + * @param {any} value + * @param {TupleColumnInfo} columnInfo + * @returns + */ + this.encodeTuple = function (value, columnInfo) { + const tupleInfo = columnInfo.info; const parts = []; let totalLength = 0; const length = Math.min(tupleInfo.length, value.length); @@ -1748,7 +1756,7 @@ Encoder.prototype.decode = function (buffer, type) { throw new Error('Unknown data type: ' + type.code); } - return decoder.call(this, buffer, 'info' in type? type.info : null); + return decoder.call(this, buffer, type); }; /** @@ -1820,7 +1828,7 @@ Encoder.prototype.encode = function (value, typeInfo) { throw new Error('Type not supported ' + type.code); } - return encoder.call(this, value, 'info' in type ? type.info : null); + return encoder.call(this, value, type); }; /** From 6827dea90fc5f08d4c6791729d6dca4053cabab7 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 13 Feb 2025 17:30:53 -0800 Subject: [PATCH 94/99] Fixing ArrayIterator --- lib/types/index.d.ts | 4 ++-- lib/types/vector.js | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/types/index.d.ts b/lib/types/index.d.ts index 6664aef0..3826ed80 100644 --- a/lib/types/index.d.ts +++ b/lib/types/index.d.ts @@ -461,8 +461,8 @@ export namespace types { getSubtype(): string | null; /** * - * @returns {ArrayIterator} + * @returns {IterableIterator} */ - [Symbol.iterator](): ArrayIterator; + [Symbol.iterator](): IterableIterator; } } \ No newline at end of file diff --git a/lib/types/vector.js b/lib/types/vector.js index 0b2a52ea..308511ee 100644 --- a/lib/types/vector.js +++ b/lib/types/vector.js @@ -85,10 +85,11 @@ class Vector { at(index) { return this.elements[index]; } + /** - * - * @returns {ArrayIterator} - */ + * + * @returns {IterableIterator} an iterator over the elements of the vector + */ [Symbol.iterator]() { return this.elements[Symbol.iterator](); } @@ -106,7 +107,7 @@ class Vector { } /** - * @returns {string | null} get the subtype string, e.g., "float", but it's optional so it can return null + * @returns {string | undefined} get the subtype string, e.g., "float", but it's optional so it can return null */ getSubtype(){ return this.subtype; From f94a29e49dd77bd3baa7a044c9eba0f8eaaca388 Mon Sep 17 00:00:00 2001 From: janehe Date: Thu, 13 Feb 2025 17:42:04 -0800 Subject: [PATCH 95/99] skip set --- test/integration/short/vector-tests.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index bb4d93e6..e0548874 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -29,6 +29,10 @@ vdescribe('5.0.0', 'Vector tests', function () { const table = keyspace + '.' + helper.getRandomName('table'); let createTableCql = `CREATE TABLE ${table} (id uuid PRIMARY KEY`; helper.dataProviderWithCollections.forEach(data => { + // skip set because it's not allowed in C* 5.0 + if (data.subtypeString === 'set') { + return; + } createTableCql += `, ${subtypeStringToColumnName(data.subtypeString)} vector<${data.subtypeString}, 3>`; }); createTableCql += ');'; @@ -42,6 +46,10 @@ vdescribe('5.0.0', 'Vector tests', function () { if (!client) { throw new Error('client setup failed'); } helper.dataProviderWithCollections.forEach(data => { + // skip set because it's not allowed in C* 5.0 + if (data.subtypeString === 'set') { + return; + } it('should insert, select, and update vector of subtype ' + data.subtypeString, function (done) { const id = types.Uuid.random(); const vector = new Vector(data.value, data.subtypeString); @@ -58,7 +66,7 @@ vdescribe('5.0.0', 'Vector tests', function () { client.execute(`UPDATE ${table} SET ${subtypeStringToColumnName(data.subtypeString)} = ? WHERE id = ?`, [new Vector(updatedValues, data.subtypeString), id], { prepare: true }, function (err, result) { if (err) { return done(err); } done(); - } ); + }); }); }); }); From f4560439b2dbb6b8b6dc71db8b884b1694b7e5da Mon Sep 17 00:00:00 2001 From: janehe Date: Fri, 14 Feb 2025 12:56:03 -0800 Subject: [PATCH 96/99] Deal with duration --- lib/encoder.js | 14 ++++++++++++++ test/integration/short/vector-tests.js | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/encoder.js b/lib/encoder.js index dedc6d4e..7a78754d 100644 --- a/lib/encoder.js +++ b/lib/encoder.js @@ -226,6 +226,11 @@ function defineInstanceMembers() { return this.decodeVector(bytes, columnInfo); } + if(typeof columnInfo.info === 'string' && columnInfo.info.startsWith(customTypeNames.vector)) { + const vectorColumnInfo = /** @type {VectorColumnInfo} */ (this.parseFqTypeName(columnInfo.info)); + return this.decodeVector(bytes, vectorColumnInfo); + } + const handler = customDecoders[columnInfo.info]; if (handler) { return handler.call(this, bytes); @@ -753,6 +758,11 @@ function defineInstanceMembers() { return this.encodeVector(value, columnInfo); } + if(typeof columnInfo.info === 'string' && columnInfo.info.startsWith(customTypeNames.vector)) { + const vectorColumnInfo = /** @type {VectorColumnInfo} */ (this.parseFqTypeName(columnInfo.info)); + return this.encodeVector(value, vectorColumnInfo); + } + const handler = customEncoders[columnInfo.info]; if (handler) { return handler.call(this, value); @@ -1476,6 +1486,10 @@ function defineInstanceMembers() { if (typeof typeCode === 'number') { return {code : typeCode, info: null, options : options}; } + // special handling for duration + if (typeName === customTypeNames.duration) { + return {code: dataTypes.duration, options: options}; + } throw new TypeError('Not a valid type "' + typeName + '"'); } if (typeName.indexOf(complexTypeNames.list, startIndex) === startIndex) { diff --git a/test/integration/short/vector-tests.js b/test/integration/short/vector-tests.js index e0548874..77a3b522 100644 --- a/test/integration/short/vector-tests.js +++ b/test/integration/short/vector-tests.js @@ -80,7 +80,7 @@ vdescribe('5.0.0', 'Vector tests', function () { const query = `INSERT INTO ${table} (id, ${subtypeStringToColumnName(data.subtypeString)}) VALUES (?, ?)`; client.execute(query, [id, vector], { prepare: false }, function (err) { if (err) { return done(err); } - client.execute(`SELECT ${subtypeStringToColumnName(data.subtypeString)} FROM ${table} WHERE id = ?`, [id], { prepare: true }, function (err, result) { + client.execute(`SELECT ${subtypeStringToColumnName(data.subtypeString)} FROM ${table} WHERE id = ?`, [id], { prepare: false }, function (err, result) { if (err) { return done(err); } assert.strictEqual(result.rows.length, 1); assert.strictEqual(util.inspect(result.rows[0][subtypeStringToColumnName(data.subtypeString)]), util.inspect(vector)); From eb1957c5fc31fbeb20c8a409bc39083597ed24f7 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 19 Feb 2025 14:54:58 -0800 Subject: [PATCH 97/99] doc change --- doc/features/datatypes/README.md | 2 +- doc/features/datatypes/collections/README.md | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/features/datatypes/README.md b/doc/features/datatypes/README.md index da554ac0..fec28e5d 100644 --- a/doc/features/datatypes/README.md +++ b/doc/features/datatypes/README.md @@ -29,7 +29,7 @@ tuple|[Tuple](tuples) uuid|[Uuid](uuids) varchar|String varint|[Integer](numerical) -vector|[Float32Array](collections) +vector|[Vector](collections) ## Encoding data diff --git a/doc/features/datatypes/collections/README.md b/doc/features/datatypes/collections/README.md index 879d7dc2..ab21f136 100644 --- a/doc/features/datatypes/collections/README.md +++ b/doc/features/datatypes/collections/README.md @@ -73,8 +73,7 @@ client.execute('SELECT map_val FROM tbl') ### Vector -As of version 4.7.0 the driver also includes support for the vector type available in Cassandra 5.0. Vectors are represented as instances of -the [Float32Array] class. For example, to create and write to a vector with three dimensions you can do the following: +The driver supports vectors of arbitrary subtypes available in Apache Cassandra 5.0. Vectors are represented as instances of `cassandra.types.Vector` class. For example, to create and write to a vector with three dimensions you can do the following: ```javascript await c.connect() @@ -84,8 +83,11 @@ await c.connect() .then(() => c.execute("create custom index ann_index on test.foo(j) using 'StorageAttachedIndex'")) // Base inserts using simple and prepared statements - .then(() => c.execute(`insert into test.foo (i, j) values (?, ?)`, [cassandra.types.Integer.fromInt(1), new Float32Array([8, 2.3, 58])])) - .then(() => c.execute(`insert into test.foo (i, j) values (?, ?)`, [cassandra.types.Integer.fromInt(5), new Float32Array([23, 18, 3.9])], {prepare: true})); + .then(() => c.execute(`insert into test.foo (i, j) values (?, ?)`, [cassandra.types.Integer.fromInt(1), new cassandra.types.Vector([8, 2.3, 58])])) + .then(() => c.execute(`insert into test.foo (i, j) values (?, ?)`, [cassandra.types.Integer.fromInt(5), new cassandra.types.Vector([23, 18, 3.9])], {prepare: true})); ``` +In driver versions 4.7.0 - 4.7.2, the vector of floats was represented as a [Float32Array]. This is still supported for backward compatibility, but it is deprecated, and it is recommended to use the `cassandra.types.Vector` class instead. + +```javascript [Float32Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Float32Array From d202034a99ef533089a6fd847da7ef85078c8ae2 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Wed, 19 Feb 2025 15:52:03 -0800 Subject: [PATCH 98/99] Update README.md --- doc/features/datatypes/collections/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/features/datatypes/collections/README.md b/doc/features/datatypes/collections/README.md index ab21f136..2f8c25d6 100644 --- a/doc/features/datatypes/collections/README.md +++ b/doc/features/datatypes/collections/README.md @@ -89,5 +89,4 @@ await c.connect() In driver versions 4.7.0 - 4.7.2, the vector of floats was represented as a [Float32Array]. This is still supported for backward compatibility, but it is deprecated, and it is recommended to use the `cassandra.types.Vector` class instead. -```javascript [Float32Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Float32Array From 0c641c787307324f0e3d591c432f7387fc0a423f Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 19 Feb 2025 18:27:29 -0800 Subject: [PATCH 99/99] add subtype hint in README --- doc/features/datatypes/collections/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/features/datatypes/collections/README.md b/doc/features/datatypes/collections/README.md index ab21f136..ddc082ab 100644 --- a/doc/features/datatypes/collections/README.md +++ b/doc/features/datatypes/collections/README.md @@ -83,7 +83,7 @@ await c.connect() .then(() => c.execute("create custom index ann_index on test.foo(j) using 'StorageAttachedIndex'")) // Base inserts using simple and prepared statements - .then(() => c.execute(`insert into test.foo (i, j) values (?, ?)`, [cassandra.types.Integer.fromInt(1), new cassandra.types.Vector([8, 2.3, 58])])) + .then(() => c.execute(`insert into test.foo (i, j) values (?, ?)`, [cassandra.types.Integer.fromInt(1), new cassandra.types.Vector([8, 2.3, 58], 'float')])) .then(() => c.execute(`insert into test.foo (i, j) values (?, ?)`, [cassandra.types.Integer.fromInt(5), new cassandra.types.Vector([23, 18, 3.9])], {prepare: true})); ```