webmachinelearning · huningxin · Jan 11, 2021 · Nov 12, 2020 · Nov 17, 2020 · Nov 19, 2020
diff --git a/README.md b/README.md
@@ -44,6 +44,20 @@ A JavaScript implementation of the [Web Neural Network API](https://webmachinele
 
 Open the web browser and navigate to http://localhost:8080/test
 
+#### Run only CTS tests in node.js.
+
+```sh
+> npm run cts
+```
+
+#### Run only CTS tests in web browser.
+
+```sh
+> npm start
+```
+
+Open the web browser and navigate to http://localhost:8080/test/cts.html
+
 ## Other scripts
 ### Build docs
 

diff --git a/node_setup.js b/node_setup.js
@@ -0,0 +1,3 @@
+global.navigator = {};
+require('./dist/webnn-polyfill.js');
+global.chai = require('chai');
diff --git a/package.json b/package.json
@@ -18,7 +18,8 @@
     "lintjs": "eslint . --no-eslintrc --config .eslintrcforjs.js --ext .js",
     "lint": "npm run lintts && npm run lintjs",
     "format": "clang-format --glob=\"{{src,test}/**/*.ts,{src,test}/**/*.js}\" --style=file -i",
-    "test": "cross-env NODE_ENV=test mocha --require ./test/node_setup.js --exit --recursive"
+    "test": "cross-env NODE_ENV=test mocha --require ./node_setup.js --exit test/*/*.js test/cts/from_nnapi/tests/cts.js",
+    "cts": "cross-env NODE_ENV=test mocha --require ./node_setup.js --exit test/cts/from_nnapi/tests/cts.js"
   },
   "repository": {
     "type": "git",
@@ -45,9 +46,9 @@
     "@babel/polyfill": "^7.11.5",
     "@babel/preset-env": "^7.11.0",
     "@babel/register": "^7.11.5",
-    "@tensorflow/tfjs-backend-cpu": "^2.7.0",
-    "@tensorflow/tfjs-backend-webgl": "^2.7.0",
-    "@tensorflow/tfjs-core": "^2.7.0",
+    "@tensorflow/tfjs-backend-cpu": "^2.8.3",
+    "@tensorflow/tfjs-backend-webgl": "^2.8.3",
+    "@tensorflow/tfjs-core": "^2.8.3",
     "@typescript-eslint/eslint-plugin": "^4.0.1",
     "@typescript-eslint/parser": "^4.0.1",
     "babel-loader": "^8.1.0",

diff --git a/src/nn/ops/conv2d.ts b/src/nn/ops/conv2d.ts
@@ -66,8 +66,8 @@ export class Conv2d extends SingleOutputOperation {
     let inputChannels: number;
     if (this.layout_ === OperandLayout.nchw) {
       // nchw -> nhwc
-      input = input.transpose([0, 2, 3, 1]);
       inputChannels = input.shape[1];
+      input = input.transpose([0, 2, 3, 1]);
       // nchw filter: [output_channels, input_channels/groups, height, width]
       // nhwc filter: [height, width, input_channels/groups, output_channels]
       filter = filter.transpose([2, 3, 1, 0]);
@@ -99,4 +99,4 @@ export class Conv2d extends SingleOutputOperation {
     }
     return output;
   }
-}
+}
diff --git a/test/cts.html b/test/cts.html
@@ -0,0 +1,23 @@
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>WebNN API Polyfill | Mocha Tests</title>
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <link rel="stylesheet" href="https://unpkg.com/mocha/mocha.css" />
+</head>
+<body>
+  <div id="mocha"></div>
+  <script src="https://unpkg.com/chai/chai.js"></script>
+  <script src="https://unpkg.com/mocha/mocha.js"></script>
+  <script src="../dist/webnn-polyfill.js"></script>
+  <script type="module" src="./utils.js"></script>
+  <script class="mocha-init">
+    mocha.setup('bdd');
+    mocha.checkLeaks();
+  </script>
+  <script type="module" src="./cts/from_nnapi/tests/cts.js"></script>
+  <script class="mocha-exec" type="module">
+    mocha.run();
+  </script>
+</body>
+</html>
diff --git a/test/cts/README.md b/test/cts/README.md
@@ -0,0 +1,5 @@
+# Compatibility Test Suite (CTS)
+
+This [cts](./) folder is for holding converted tools and tests from existed native tests of Machine Learning for WebNN API.
+
+[./from_nnapi](./from_nnapi): holds converted tests from [Android Neural Networks API (NNAPI) CTS](https://android.googlesource.com/platform/frameworks/ml/+/refs/tags/android-cts-10.0_r5/nn/runtime/test/specs/).
diff --git a/test/cts/from_nnapi/COPYING b/test/cts/from_nnapi/COPYING
@@ -0,0 +1,18 @@
+These two Python scripts cts_generator.py and test_generator.py in
+test/generator/src folder come from those scripts in nn/tools/test_generator of
+https://android.googlesource.com/platform/frameworks/ml/
+(tags: android-cts-10.0_r5) with some modification below:
+
+1. Modify cts_generator.py to generate CTS for WebNN API.
+2. Modify test_generator.py to generate CTS for WebNN API.
+
+Copyright 2017 The Android Open Source Project
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/test/cts/from_nnapi/README.md b/test/cts/from_nnapi/README.md
@@ -0,0 +1,101 @@
+# Converted CTS from NNAPI CTS
+
+## Test Generator Tool
+This [test_generator](./test_generator) tool is for converting existed native 
+[NNAPI CTS](https://android.googlesource.com/platform/frameworks/ml/+/refs/tags/android-cts-10.0_r5/nn/runtime/test/specs/) of V1_0, V1_1 and V1_2 versions to these tests for WebNN API.
+
+### NNAPI Operations being map to WebNN API Operations Tables
+* Part I 
+
+The `test_generator` tool has supported to convert those tests of following
+NNAPI operations to the tests for such WebNN API operations of first wave.
+
+| NNAPI                         | WebNN API (first wave ops)    |
+|:------------------------------|:------------------------------|
+| RELU1                         | clamp                         |
+| RELU6                         | clamp                         |
+| CONCATENATION                 | concat                        |
+| ADD                           | add [+ relu/clamp]            |
+| SUB                           | sub [+ relu/clamp]            |
+| MUL                           | mul [+ relu/clamp]            |
+| DIV                           | div [+ relu/clamp]            |
+| MAXIMUM                       | max                           |
+| MINIMUM                       | min                           |
+| EXP                           | exp                           |
+| LOGISTIC                      | sigmoid                       |
+| SQRT                          | sqrt                          |
+| TANH                          | tanh                          |
+| FULLY_CONNECTED               | matmul [+ add+ relu/clamp]    |
+| AVERAGE_POOL_2D               | averagePool2d [+ relu/clamp]  |
+| MAX_POOL_2D                   | maxPool2d  [+ relu/clamp]     |
+| CONV_2D                       | conv2d [+ add + relu/clamp]   |
+| DEPTHWISE_CONV_2D             | conv2d [+ add + relu/clamp]   |
+| RELU                          | relu                          |
+| RESHAPE                       | reshape                       |
+| SLICE                         | slice                         |
+| SOFTMAX                       | softmax                       |
+| SPLIT                         | split                         |
+| SQUEEZE                       | squeeze                       |
+| TRANSPOSE                     | transpose                     |
+
+* Part II
+
+And there're these following NNAPI operations which could be map to others
+WebNN API operations of next waves.
+
+| NNAPI                         | WebNN API (next waves ops)     |
+|:------------------------------|:------------------------------|
+| ABS                           | abs                           |
+| FLOOR                         | floor                         |
+| LOG                           | log                           |
+| NEG                           | neg                           |
+| SIN                           | sin                           |
+| L2_POOL_2D                    | l2Pool2d  [+ relu/clamp]      |
+
+* Note: 
+
+1. Current WebNN Polyfill API supports Float32 and Int32 two types, so these
+NNAPI CTS using Float32 and Int32 types were able to be converted, while those
+NNAPI CTS using Uint8 and Int8 types would be converted until WebNN Polyfill API
+supports Uint8 and Int8 types.
+
+2. Native NNAPI supports Float16, while there's lack of Float16 in JavaScript
+environments, so such NNAPI CTS with Float16 wouldn't be convertted.
+
+### Usage
+* Prerequisites
+  * Python3
+  * Numpy
+  * Download and unzip NNAPI CTS Specs tarball files locally by below commands
+
+    ```shell
+    cd test_generator
+    ./ready_nnapi_cts_specs.sh
+    ```
+* Generate Tests
+
+  ```shell
+  npm start
+  ```
+
+ Generated tests would be in following three folders  
+[./tests/V1_0](./test/V1_0)  
+[./tests/V1_1](./test/V1_1)  
+[./tests/V1_2](./test/V1_2)  
+and these tests could also be in all-in-one 
+[./tests/cts.js](./tests/cts.js) file.
+
+
+### Accuracy for Generated Tests
+The converted tests follow these [reference accuracy](https://android.googlesource.com/platform/frameworks/ml/+/refs/tags/android-cts-10.0_r5/nn/runtime/test/TestGenerated.cpp#117):
+```cpp
+  float fpAtol = 1e-5f;
+  float fpRtol = 5.0f * 1.1920928955078125e-7f;
+```
+And for relaxed tests
+```cpp
+  // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
+  fpAtol = 5.0f * 0.0009765625f;
+  // Set the relative tolerance to be 5ULP of the corresponding FP precision.
+  fpRtol = 5.0f * 0.0009765625f;
+```
diff --git a/test/cts/from_nnapi/test_generator/package.json b/test/cts/from_nnapi/test_generator/package.json
@@ -0,0 +1,18 @@
+{
+  "name": "test_generator",
+  "version": "0.0.1",
+  "description": "Convert native NNAPI CTS tests to WebNN API tests",
+  "license": "Apache 2.0",
+  "main": "./src/main.py",
+  "directories": {
+    "src": "src files"
+  },
+  "scripts": {
+    "start": "python3 ./src/main.py ./specs -t ../tests -c ../tests/cts.js"
+  },
+  "dependencies": {},
+  "devDependencies": {},
+  "authors": [
+    "Bruce Dai <[email protected]>"
+  ]
+}
diff --git a/test/cts/from_nnapi/test_generator/ready_nnapi_cts_specs.sh b/test/cts/from_nnapi/test_generator/ready_nnapi_cts_specs.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+TAG=android-cts-10.0_r5
+NNAPI_VERSION="
+V1_0
+V1_1
+V1_2
+"
+
+# download and unzip Specs tarball files
+for version in ${NNAPI_VERSION}
+do
+  wget https://android.googlesource.com/platform/frameworks/ml/+archive/refs/tags/${TAG}/nn/runtime/test/specs/${version}.tar.gz
+  mkdir -p specs/${version}
+  tar -xvzf ${version}.tar.gz -C specs/${version}
+done