diff --git a/.github/scripts/test_java_demo.py b/.github/scripts/test_java_demo.py index 432cfd4f7d..f7bc847bda 100644 --- a/.github/scripts/test_java_demo.py +++ b/.github/scripts/test_java_demo.py @@ -47,6 +47,21 @@ 'configs': [ 'https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/litehrnet.tar' # noqa: E501 ] + }, + { + 'task': + 'PoseTracker', + 'configs': [ + 'https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/rtmdet-nano.tar', # noqa: E501 + 'https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/rtmpose-tiny.tar' # noqa: E501 + ] + }, + { + 'task': + 'RotatedDetection', + 'configs': [ + 'https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/gliding-vertex.tar' # noqa: E501 + ] } ] @@ -60,24 +75,34 @@ def main(): for params in PARAMS: task = params['task'] configs = params['configs'] - java_demo_cmd = [ - 'java', '-cp', 'csrc/mmdeploy/apis/java:demo/java', - 'demo/java/' + task + '.java', 'cpu' - ] + java_command = '\"cpu' for config in configs: model_url = config os.system('wget {} && tar xvf {}'.format(model_url, model_url.split('/')[-1])) model_dir = model_url.split('/')[-1].split('.')[0] - java_demo_cmd.append(model_dir) - java_demo_cmd.append('/home/runner/work/mmdeploy/mmdeploy/demo' + - '/resources/human-pose.jpg') - java_demo_cmd_str = ' '.join(java_demo_cmd) - os.system('export JAVA_HOME=/home/runner/work/mmdeploy/mmdeploy/' + - 'jdk-18 && export PATH=${JAVA_HOME}/bin:${PATH} && java' + - ' --version && export LD_LIBRARY_PATH=/home/runner/work/' + - 'mmdeploy/mmdeploy/build/lib:${LD_LIBRARY_PATH} && ' + - java_demo_cmd_str) + java_command += (' ' + model_dir) + if task in [ + 'ImageClassification', 'ObjectDetection', 'ImageSegmentation', + 'ImageRestorer', 'PoseDetection', 'RotatedDetection' + ]: + java_command += (' $GITHUB_WORKSPACE/demo' + + '/resources/human-pose.jpg\"') + elif task in ['Ocr']: + java_command += (' $GITHUB_WORKSPACE/demo' + + '/resources/text_det.jpg\"') + elif task in ['PoseTracker']: + os.system( + 'wget https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/dance.mp4' # noqa: E501 + ) + java_command += ' dance.mp4\"' + else: + java_command += '\"' + print(f'java_command: {java_command}') + os.system('ant -DtaskName=' + task + + ' -DjarDir=${OPENCV_DIR}/build/bin ' + + '-DlibDir=${OPENCV_DIR}/build/lib:$GITHUB_WORKSPACE/' + + 'build/lib -Dcommand=' + java_command) if __name__ == '__main__': diff --git a/.github/workflows/java_api.yml b/.github/workflows/java_api.yml index 296ac3b5cc..984e27c0f8 100644 --- a/.github/workflows/java_api.yml +++ b/.github/workflows/java_api.yml @@ -35,38 +35,45 @@ jobs: run: | wget https://download.java.net/java/GA/jdk18/43f95e8614114aeaa8e8a5fcf20a682d/36/GPL/openjdk-18_linux-x64_bin.tar.gz tar xvf openjdk-18_linux-x64_bin.tar.gz - - name: gcc-multilib - run: sudo apt install gcc-multilib g++-multilib wget libprotobuf-dev protobuf-compiler + export JAVA_HOME=${PWD}/jdk-18 + export PATH=${JAVA_HOME}/bin:${PATH} - name: Install onnxruntime run: | wget https://github.com/microsoft/onnxruntime/releases/download/v1.8.1/onnxruntime-linux-x64-1.8.1.tgz tar -zxvf onnxruntime-linux-x64-1.8.1.tgz - pushd onnxruntime-linux-x64-1.8.1 - export ONNXRUNTIME_DIR=${PWD} + export ONNXRUNTIME_DIR=${PWD}/onnxruntime-linux-x64-1.8.1 export LD_LIBRARY_PATH=$ONNXRUNTIME_DIR/lib:$LD_LIBRARY_PATH - popd - - name: Install opencv + - name: Install OpenCV run: | sudo apt-get install libopencv-dev - - name: Build java class + sudo apt-get install ffmpeg libgstreamer1.0-0 libgstreamer-plugins-base1.0-0 libavcodec-dev libavformat-dev libswscale-dev + - name: Install ant + run: | + sudo apt-get install ant + - name: Build Java class run: | pushd csrc/mmdeploy/apis/java javac mmdeploy/*.java popd - pushd demo/java - javac -classpath ../../csrc/mmdeploy/apis/java/ Utils.java - popd - - name: Install mmdeploy with onnxruntime backend and java api + - name: Install mmdeploy with onnxruntime backend and Java API run: | mkdir -p build && pushd build - export LD_LIBRARY_PATH=/home/runner/work/mmdeploy/mmdeploy/ncnn/install/lib/:$LD_LIBRARY_PATH - cmake -DMMDEPLOY_BUILD_SDK=ON -DMMDEPLOY_BUILD_SDK_JAVA_API=ON -DMMDEPLOY_TARGET_BACKENDS=ort -DMMDEPLOY_CODEBASES=all -DONNXRUNTIME_DIR=~/work/mmdeploy/mmdeploy/onnxruntime-linux-x64-1.8.1 .. - make install + export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/onnxruntime-linux-x64-1.8.1/lib/:$LD_LIBRARY_PATH + cmake -DMMDEPLOY_BUILD_SDK=ON -DMMDEPLOY_BUILD_SDK_JAVA_API=ON -DMMDEPLOY_TARGET_BACKENDS=ort -DMMDEPLOY_CODEBASES=all -DONNXRUNTIME_DIR=$GITHUB_WORKSPACE/onnxruntime-linux-x64-1.8.1 .. + make -j$(nproc) && make install popd - - name: Test javademo + - name: Prepare OpenCV Java and test Java demo run: | - export JAVA_HOME=${PWD}/jdk-18 - export PATH=${JAVA_HOME}/bin:${PATH} - export LD_LIBRARY_PATH=/build/lib:${LD_LIBRARY_PATH} - java --version + export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME} + export JAVA_AWT_LIBRARY=${JAVA_HOME} + export JAVA_INCLUDE_PATH=${JAVA_HOME}/include + export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/darwin + export JAVA_JVM_LIBRARY=${JAVA_HOME} + + mkdir -p opencv/build/bin + wget https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/opencv-470.jar -P opencv/build/bin + wget https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/opencv470-lib.tar -P opencv/build + tar -xf opencv/build/opencv470-lib.tar -C opencv/build + export OPENCV_DIR=$GITHUB_WORKSPACE/opencv + export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/build/lib:${OPENCV_DIR}/build/lib:$LD_LIBRARY_PATH python .github/scripts/test_java_demo.py diff --git a/csrc/mmdeploy/apis/java/CMakeLists.txt b/csrc/mmdeploy/apis/java/CMakeLists.txt index 07725613a3..04313f1934 100644 --- a/csrc/mmdeploy/apis/java/CMakeLists.txt +++ b/csrc/mmdeploy/apis/java/CMakeLists.txt @@ -23,5 +23,12 @@ add_jar(${PROJECT_NAME} SOURCES mmdeploy/TextRecognizer.java mmdeploy/Restorer.java mmdeploy/PoseDetector.java + mmdeploy/Context.java + mmdeploy/Device.java + mmdeploy/Model.java + mmdeploy/Profiler.java + mmdeploy/Scheduler.java + mmdeploy/PoseTracker.java + mmdeploy/RotatedDetector.java OUTPUT_NAME mmdeploy OUTPUT_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) diff --git a/csrc/mmdeploy/apis/java/README.md b/csrc/mmdeploy/apis/java/README.md index d1fbdee176..834552300f 100644 --- a/csrc/mmdeploy/apis/java/README.md +++ b/csrc/mmdeploy/apis/java/README.md @@ -20,14 +20,14 @@ export JAVA_HOME=${PWD}/jdk-18 export PATH=${JAVA_HOME}/bin:${PATH} ``` -**Step 3.** Switch default Java version: +**Step 3.** Switch default Java version(optional): ```bash sudo update-alternatives --config java sudo update-alternatives --config javac ``` -You should select the version you will use. +If you have multiple Java versions, you should select the version you will use. ### Installation diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Classifier.java b/csrc/mmdeploy/apis/java/mmdeploy/Classifier.java index 3919881fcc..9767cb4d30 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/Classifier.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/Classifier.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: the Java API class of Classifier. */ public class Classifier { static { System.loadLibrary("mmdeploy_java"); @@ -7,22 +8,49 @@ public class Classifier { private final long handle; + /** @description: Single classification result of a picture. */ public static class Result { + + /** Class id. */ public int label_id; + + /** Class score. */ public float score; + + /** Initializes a new instance of the Result class. + * @param label_id: class id. + * @param score: class score. + */ public Result(int label_id, float score) { this.label_id = label_id; this.score = score; } } - public Classifier(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the Classifier class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create Classifier failed exception. + */ + public Classifier(String modelPath, String deviceName, int deviceId) throws Exception{ handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create Classifier failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get label information of each image in a batch. + * @param images: input mats. + * @return: results of each input mat. + * @exception Exception: apply Classifier failed exception. + */ + public Result[][] apply(Mat[] images) throws Exception{ int[] counts = new int[images.length]; Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply Classifier failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -36,12 +64,22 @@ public Result[][] apply(Mat[] images) { return rets; } - public Result[] apply(Mat image) { + /** Get label information of one image. + * @param image: input mat. + * @return: result of input mat. + * @exception Exception: apply Classifier failed exception. + */ + public Result[] apply(Mat image) throws Exception{ int[] counts = new int[1]; Mat[] images = new Mat[]{image}; - return apply(handle, images, counts); + Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply Classifier failed!"); + } + return results; } + /** Release the instance of Classifier. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Context.java b/csrc/mmdeploy/apis/java/mmdeploy/Context.java new file mode 100644 index 0000000000..1322773102 --- /dev/null +++ b/csrc/mmdeploy/apis/java/mmdeploy/Context.java @@ -0,0 +1,74 @@ + +package mmdeploy; + +/** @description: the Context class. */ +public class Context { + static { + System.loadLibrary("mmdeploy_java"); + } + + private final long contextHandle; + + /** @description: ContextType. */ + public enum ContextType { + DEVICE, + STREAM, + MODEL, + SCHEDULER, + MAT, + PROFILER + } + + /** Initializes a new instance of the Context class. */ + public Context() { + contextHandle = create(); + } + + /** Add Model to the Context. + * @param name: name. + * @param model: model. + */ + public void add(String name, Model model) { + add(contextHandle, ContextType.MODEL.ordinal(), name, model.handle()); + } + + /** Add Scheduler to the Context. + * @param name: name. + * @param scheduler: scheduler. + */ + public void add(String name, Scheduler scheduler) { + add(contextHandle, ContextType.SCHEDULER.ordinal(), name, scheduler.handle()); + } + + /** Add Device to the Context. + * @param device: device. + */ + public void add(Device device) { + add(contextHandle, ContextType.DEVICE.ordinal(), "", device.handle()); + } + + /** Add Profiler to the Context. + * @param profiler: profiler. + */ + public void add(Profiler profiler) { + add(contextHandle, ContextType.PROFILER.ordinal(), "", profiler.handle()); + } + + /** Release the instance of Context. */ + public void release() { + destroy(contextHandle); + } + + /** Get the handle of Context + * @return: the handle of context. + */ + public long handle() { + return contextHandle; + } + + private native long create(); + + public native int add(long context, int contextType, String name, long handle); + + private native void destroy(long context); +} diff --git a/csrc/mmdeploy/apis/java/mmdeploy/DataType.java b/csrc/mmdeploy/apis/java/mmdeploy/DataType.java index ea52d01348..4a0ce41730 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/DataType.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/DataType.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: DataType. */ public enum DataType { FLOAT(0), HALF(1), @@ -7,6 +8,9 @@ public enum DataType { INT32(3); final int value; + /** Initializes a new instance of the DataType class. + * @param value: the value. + */ DataType(int value) { this.value = value; } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Detector.java b/csrc/mmdeploy/apis/java/mmdeploy/Detector.java index 0692d3cf42..ce85403fc2 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/Detector.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/Detector.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: the Java API class of Detector. */ public class Detector { static { System.loadLibrary("mmdeploy_java"); @@ -7,11 +8,27 @@ public class Detector { private final long handle; + /** @description: Single detection result of a picture. */ public static class Result { + + /** Bbox class id. */ public int label_id; + + /** Bbox score. */ public float score; + + /** Bbox coordinates. */ public Rect bbox; + + /** Bbox mask. */ public InstanceMask mask; + + /** Initializes a new instance of the Result class. + * @param label_id: bbox class id. + * @param score: bbox score. + * @param bbox: bbox coordinates. + * @param mask: bbox mask. + */ public Result(int label_id, float score, Rect bbox, InstanceMask mask) { this.label_id = label_id; this.score = score; @@ -20,13 +37,30 @@ public Result(int label_id, float score, Rect bbox, InstanceMask mask) { } } - public Detector(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the Detector class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create Detector failed exception. + */ + public Detector(String modelPath, String deviceName, int deviceId) throws Exception { handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create Detector failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get information of each image in a batch. + * @param images: input mats. + * @return: results of each input mat. + * @exception Exception: apply Detector failed exception. + */ + public Result[][] apply(Mat[] images) throws Exception { int[] counts = new int[images.length]; Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply Detector failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -40,12 +74,22 @@ public Result[][] apply(Mat[] images) { return rets; } - public Result[] apply(Mat image) { + /** Get information of one image. + * @param image: input mat. + * @return: result of input mat. + * @exception Exception: apply Detector failed exception. + */ + public Result[] apply(Mat image) throws Exception{ int[] counts = new int[1]; Mat[] images = new Mat[]{image}; - return apply(handle, images, counts); + Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply Detector failed!"); + } + return results; } + /** Release the instance of Detector. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Device.java b/csrc/mmdeploy/apis/java/mmdeploy/Device.java new file mode 100644 index 0000000000..a54a8ed149 --- /dev/null +++ b/csrc/mmdeploy/apis/java/mmdeploy/Device.java @@ -0,0 +1,52 @@ +package mmdeploy; + +/** @description: the Device class. */ +public class Device { + static { + System.loadLibrary("mmdeploy_java"); + } + + private final long deviceHandle; + private String deviceName; + private int deviceIndex; + + /** Initialize a new instance of the Device class. + * @param name: device name. + * @param index: device index. + */ + public Device(String name, int index) { + deviceName = name; + deviceIndex = index; + deviceHandle = create(deviceName, deviceIndex); + } + + /** Get device name. + * @return: device name. + */ + public String name() { + return deviceName; + } + + /** Get device index. + * @return: device index. + */ + public int index() { + return deviceIndex; + } + + /** Get device handle. + * @return: device handle. + */ + public long handle() { + return deviceHandle; + } + + /** Release the instance of Device. */ + public void release() { + destroy(deviceHandle); + } + + private native long create(String name, int index); + + private native void destroy(long deviceHandle); +} diff --git a/csrc/mmdeploy/apis/java/mmdeploy/InstanceMask.java b/csrc/mmdeploy/apis/java/mmdeploy/InstanceMask.java index 9958da6e8a..55645ba379 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/InstanceMask.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/InstanceMask.java @@ -1,10 +1,19 @@ package mmdeploy; +/** @description: InstanceMask. */ public class InstanceMask { + + /** Mask shape. */ public int[] shape; - public char[] data; + /** Mask data. */ + public char[] data; + /** Initialize a new instance of the InstanceMask class. + * @param height: height. + * @param width: width. + * @param data: mask data. + */ public InstanceMask(int height, int width, char[] data) { shape = new int[]{height, width}; this.data = data; diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Mat.java b/csrc/mmdeploy/apis/java/mmdeploy/Mat.java index 1bfe6c5d36..09fd647e04 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/Mat.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/Mat.java @@ -1,12 +1,28 @@ package mmdeploy; +/** @description: Mat. */ public class Mat { + + /** Shape. */ public int[] shape; + + /** Pixel format. */ public int format; + + /** Data type. */ public int type; - public byte[] data; + /** Mat data. */ + public byte[] data; + /** Initialize a new instance of the Mat class. + * @param height: height. + * @param width: width. + * @param channel: channel. + * @param format: pixel format. + * @param type: data type. + * @param data: mat data. + */ public Mat(int height, int width, int channel, PixelFormat format, DataType type, byte[] data) { shape = new int[]{height, width, channel}; diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Model.java b/csrc/mmdeploy/apis/java/mmdeploy/Model.java new file mode 100644 index 0000000000..892de9f02e --- /dev/null +++ b/csrc/mmdeploy/apis/java/mmdeploy/Model.java @@ -0,0 +1,33 @@ +package mmdeploy; + +/** @description: the Model class. */ +public class Model { + static { + System.loadLibrary("mmdeploy_java"); + } + + private final long modelHandle; + + /** Initialize a new instance of the Model class. + * @param path: model path. + */ + public Model(String path) { + modelHandle = create(path); + } + + /** Release the instance of Model. */ + public void release() { + destroy(modelHandle); + } + + /** Get model handle. + * @return: model handle. + */ + public long handle() { + return modelHandle; + } + + private native long create(String path); + + private native void destroy(long modelHandle); +} diff --git a/csrc/mmdeploy/apis/java/mmdeploy/PixelFormat.java b/csrc/mmdeploy/apis/java/mmdeploy/PixelFormat.java index bb1167ada6..0bf3324bb2 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/PixelFormat.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/PixelFormat.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: PixelFormat. */ public enum PixelFormat { BGR(0), RGB(1), @@ -9,6 +10,9 @@ public enum PixelFormat { BGRA(5); final int value; + /** Initialize a new instance of the PixelFormat class. + * @param value: the value. + */ PixelFormat(int value) { this.value = value; } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/PointF.java b/csrc/mmdeploy/apis/java/mmdeploy/PointF.java index 564e5e9b9f..215755a389 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/PointF.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/PointF.java @@ -1,10 +1,18 @@ package mmdeploy; +/** @description: the PointF class. */ public class PointF { + + /** x coordinate. */ public float x; - public float y; + /** y coordinate. */ + public float y; + /** Initialize a new instance of the PointF class. + * @param x: x coordinate. + * @param y: y coordinate. + */ public PointF(float x, float y) { this.x = x; this.y = y; diff --git a/csrc/mmdeploy/apis/java/mmdeploy/PoseDetector.java b/csrc/mmdeploy/apis/java/mmdeploy/PoseDetector.java index c8f8f3e094..03cdd66a06 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/PoseDetector.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/PoseDetector.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: the Java API class of PoseDetector. */ public class PoseDetector { static { System.loadLibrary("mmdeploy_java"); @@ -7,21 +8,48 @@ public class PoseDetector { private final long handle; + /** @description: Single pose estimation result of a picture. */ public static class Result { + + /** Points. */ public PointF[] point; + + /** Scores of points */ public float[] score; + + /** Initializes a new instance of the Result class. + * @param point: points. + * @param score: scores of points. + */ public Result(PointF[] point, float [] score) { this.point = point; this.score = score; } } - public PoseDetector(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the PoseDetector class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create PoseDetector failed exception. + */ + public PoseDetector(String modelPath, String deviceName, int deviceId) throws Exception{ handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create PoseDetector failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get information of each image in a batch. + * @param images: input mats. + * @return: results of each input mat. + * @exception Exception: apply PoseDetector failed exception. + */ + public Result[][] apply(Mat[] images) throws Exception{ Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply PoseDetector failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -33,11 +61,21 @@ public Result[][] apply(Mat[] images) { return rets; } - public Result[] apply(Mat image) { + /** Get information of one image. + * @param image: input mat. + * @return: result of input mat. + * @exception Exception: apply PoseDetector failed exception. + */ + public Result[] apply(Mat image) throws Exception{ Mat[] images = new Mat[]{image}; - return apply(handle, images); + Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply PoseDetector failed!"); + } + return results; } + /** Release the instance of PoseDetector. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/PoseTracker.java b/csrc/mmdeploy/apis/java/mmdeploy/PoseTracker.java new file mode 100644 index 0000000000..82aa3a5baa --- /dev/null +++ b/csrc/mmdeploy/apis/java/mmdeploy/PoseTracker.java @@ -0,0 +1,244 @@ +package mmdeploy; + +/** @description: the Java API class of PoseTracker. */ +public class PoseTracker { + static { + System.loadLibrary("mmdeploy_java"); + } + + private final long handle; + private long stateHandle; + + /** @description: Single tracking result of a bbox. */ + public static class Result { + + /** Keypoints. */ + public PointF[] keypoints; + + /** Scores. */ + public float[] scores; + + /** Bbox */ + public Rect bbox; + + /** Target ID. */ + public int targetID; + + /** Initializes a new instance of the Result class. + * @param keypoints: keypoints. + * @param scores: scores. + * @param bbox: bbox. + * @param targetID: target ID. + */ + public Result(PointF[] keypoints, float[] scores, Rect bbox, int targetID) { + this.keypoints = keypoints; + this.scores = scores; + this.bbox = bbox; + this.targetID = targetID; + } + } + + /** @description: PoseTracker parameters. */ + public static class Params { + + /** Det interval. */ + public int detInterval; + + /** Det label. */ + public int detLabel; + + /** Det threshold. */ + public float detThr; + + /** Det min bbox size. */ + public float detMinBboxSize; + + /** Det nms threshold. */ + public float detNmsThr; + + /** Pose max number of bboxes. */ + public int poseMaxNumBboxes; + + /** Pose keypoint threshold. */ + public float poseKptThr; + + /** Pose min keypoints. */ + public int poseMinKeypoints; + + /** Pose bbox scale. */ + public float poseBboxScale; + + /** Pose min bbox size. */ + public float poseMinBboxSize; + + /** Pose nms threshold. */ + public float poseNmsThr; + + /** Keypoint sigmas */ + public float[] keypointSigmas; + + /** Keypoint sigmas size. */ + public int keypointSigmasSize; + + /** Track iou threshold. */ + public float trackIouThr; + + /** Track max missing. */ + public int trackMaxMissing; + + /** Track history size. */ + public int trackHistorySize; + + /** std weight position. */ + public float stdWeightPosition; + + /** std weight velocity. */ + public float stdWeightVelocity; + + /** Smooth params. */ + public float[] smoothParams; + + /** Initializes a new instance of the Params class. + * @param detInterval: det interval. + * @param detLabel: det label. + * @param detThr: det threshold. + * @param detMinBboxSize: det min bbox size. + * @param detNmsThr: det nms threshold. + * @param poseMaxNumBboxes: pose max number of bboxes. + * @param poseKptThr: pose keypoint threshold. + * @param poseMinKeypoints: pose min keypoints. + * @param poseBboxScale: pose bbox scale. + * @param poseMinBboxSize: pose min bbox size. + * @param poseNmsThr: pose nms threshold. + * @param keypointSigmas: keypoint sigmas. + * @param keypointSigmasSize: keypoint sigmas size. + * @param trackIouThr: track iou threshold. + * @param trackMaxMissing: track max missing. + * @param trackHistorySize: track history size. + * @param stdWeightPosition: std weight position. + * @param stdWeightVelocity: std weight velocity. + * @param smoothParams: smooth params. + */ + public Params(int detInterval, int detLabel, float detThr, float detMinBboxSize, float detNmsThr, int poseMaxNumBboxes, + float poseKptThr, int poseMinKeypoints, float poseBboxScale, float poseMinBboxSize, float poseNmsThr, float[] keypointSigmas, + int keypointSigmasSize, float trackIouThr, int trackMaxMissing, int trackHistorySize, float stdWeightPosition, float stdWeightVelocity, + float[] smoothParams) { + this.detInterval = detInterval; + this.detLabel = detLabel; + this.detThr = detThr; + this.detMinBboxSize = detMinBboxSize; + this.detNmsThr = detNmsThr; + this.poseMaxNumBboxes = poseMaxNumBboxes; + this.poseKptThr = poseKptThr; + this.poseMinKeypoints = poseMinKeypoints; + this.poseBboxScale = poseBboxScale; + this.poseMinBboxSize = poseMinBboxSize; + this.poseNmsThr = poseNmsThr; + this.keypointSigmas = keypointSigmas.clone(); + this.keypointSigmasSize = keypointSigmasSize; + this.trackIouThr = trackIouThr; + this.trackMaxMissing = trackMaxMissing; + this.trackHistorySize = trackHistorySize; + this.stdWeightPosition = stdWeightPosition; + this.stdWeightVelocity = stdWeightVelocity; + this.smoothParams = smoothParams.clone(); + } + } + + /** Initializes a new instance of the PoseTracker class. + * @param detect: detect model. + * @param pose: pose model. + * @param context: context. + * @exception Exception: create PoseTracker failed exception. + */ + public PoseTracker(Model detect, Model pose, Context context) throws Exception{ + handle = create(detect.handle(), pose.handle(), context.handle()); + if (handle == -1) { + throw new Exception("Create PoseDetector failed!"); + } + } + + /** Initializes a new instance of the Params class. + * @return: default value of params. + */ + public Params initParams() { + Params params = setDefaultParams(); + return params; + } + + /** Initializes a new instance of the State class. + * @param params: params. + * @return: the handle of State. + */ + public long createState(Params params) { + stateHandle = createState(handle, params); + return stateHandle; + } + + /** Get information of each frame in a batch. + * @param states: states of each frame in a batch. + * @param frames: input mats. + * @param detects: use detects result or not. + * @exception Exception: apply PoseTracker failed exception. + * @return: results of each input mat. + */ + public Result[][] apply(long[] states, Mat[] frames, int[] detects) throws Exception{ + int[] counts = new int[frames.length]; + Result[] results = apply(handle, states, frames, detects, counts); + if (results == null) { + throw new Exception("Apply PoseTracker failed!"); + } + Result[][] rets = new Result[frames.length][]; + int offset = 0; + for (int i = 0; i < frames.length; ++i) { + Result[] row = new Result[counts[i]]; + if (counts[i] >= 0) { + System.arraycopy(results, offset, row, 0, counts[i]); + } + offset += counts[i]; + rets[i] = row; + } + return rets; + } + + /** Get information of one frame. + * @param state: state of frame. + * @param frame: input mat. + * @param detect: use detect result or not. + * @exception Exception: apply PoseTracker failed exception. + * @return: result of input mat. + */ + public Result[] apply(long state, Mat frame, int detect) throws Exception{ + long[] states = new long[]{state}; + Mat[] frames = new Mat[]{frame}; + int[] detects = new int[]{detect}; + int[] counts = new int[1]; + Result[] results = apply(handle, states, frames, detects, counts); + if (results == null) { + throw new Exception("Apply PoseTracker failed!"); + } + return results; + } + + /** Release the instance of PoseTracker. */ + public void release() { + destroy(handle); + } + + /** Release the instance of State. */ + public void releaseState(long state) { + destroyState(state); + } + + private native long create(long detect, long pose, long context); + + private native void destroy(long handle); + + private native long createState(long pipeline, Params params); + + private native void destroyState(long state); + + public native Params setDefaultParams(); + + private native Result[] apply(long handle, long[] states, Mat[] frames, int[] detects, int[] counts); +} diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Profiler.java b/csrc/mmdeploy/apis/java/mmdeploy/Profiler.java new file mode 100644 index 0000000000..de26c73d1e --- /dev/null +++ b/csrc/mmdeploy/apis/java/mmdeploy/Profiler.java @@ -0,0 +1,33 @@ +package mmdeploy; + +/** @description: the Profiler class. */ +public class Profiler { + static { + System.loadLibrary("mmdeploy_java"); + } + + private final long profilerHandle; + + /** Initialize a new instance of the Profiler class. + * @param path: profiler path. + */ + public Profiler(String path) { + profilerHandle = create(path); + } + + /** Release the instance of Profiler. */ + public void release() { + destroy(profilerHandle); + } + + /** Get profiler handle. + * @return: profiler handle. + */ + public long handle() { + return profilerHandle; + } + + private native long create(String path); + + private native void destroy(long profilerHandle); +} diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Rect.java b/csrc/mmdeploy/apis/java/mmdeploy/Rect.java index 28ce707600..e3556f9415 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/Rect.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/Rect.java @@ -1,12 +1,26 @@ package mmdeploy; +/** @description: the Rect class. */ public class Rect { + + /** left coordinate. */ public float left; + + /** top coordinate. */ public float top; + + /** right coordinate. */ public float right; - public float bottom; + /** bottom coordinate. */ + public float bottom; + /** Initialize a new instance of the Rect class. + * @param left: left coordinate. + * @param top: top coordinate. + * @param right: right coordinate. + * @param bottom: bottom coordinate. + */ public Rect(float left, float top, float right, float bottom) { this.left = left; this.top = top; diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Restorer.java b/csrc/mmdeploy/apis/java/mmdeploy/Restorer.java index 0485a5642a..a8cd80dc5f 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/Restorer.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/Restorer.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: the Java API class of Restorer. */ public class Restorer { static { System.loadLibrary("mmdeploy_java"); @@ -7,19 +8,43 @@ public class Restorer { private final long handle; + /** @description: Single image restore result of a picture. */ public static class Result { + + /** Result mat. */ public Mat res; + + /** Initializes a new instance of the Result class. + * @param res: result mat. + */ public Result(Mat res) { this.res = res; } } - public Restorer(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the Restorer class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create Restorer failed exception. + */ + public Restorer(String modelPath, String deviceName, int deviceId) throws Exception { handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create Restorer failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get information of each image in a batch. + * @param images: input mats. + * @exception Exception: apply Restorer failed exception. + * @return: results of each input mat. + */ + public Result[][] apply(Mat[] images) throws Exception { Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply Restorer failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -31,11 +56,21 @@ public Result[][] apply(Mat[] images) { return rets; } - public Result[] apply(Mat image) { + /** Get information of one image. + * @param image: input mat. + * @exception Exception: apply Restorer failed exception. + * @return: result of input mat. + */ + public Result[] apply(Mat image) throws Exception{ Mat[] images = new Mat[]{image}; - return apply(handle, images); + Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply Restorer failed!"); + } + return results; } + /** Release the instance of Restorer. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/RotatedDetector.java b/csrc/mmdeploy/apis/java/mmdeploy/RotatedDetector.java new file mode 100644 index 0000000000..16891a5911 --- /dev/null +++ b/csrc/mmdeploy/apis/java/mmdeploy/RotatedDetector.java @@ -0,0 +1,97 @@ +package mmdeploy; + +/** @description: the Java API class of RotatedDetector. */ +public class RotatedDetector { + static { + System.loadLibrary("mmdeploy_java"); + } + + private final long handle; + + /** @description: Single rotated detection result of a picture. */ + public static class Result { + + /** Label ID. */ + public int label_id; + + /** Score. */ + public float score; + + /** Rotated bbox. */ + public float[] rbbox; + + /** Initializes a new instance of the Result class. + * @param label_id: label ID. + * @param score: score. + * @param rbbox: rotated bbox. + */ + public Result(int label_id, float score, float[] rbbox) { + this.label_id = label_id; + this.score = score; + this.rbbox = rbbox; + } + } + + /** Initializes a new instance of the RotatedDetector class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create RotatedDetector failed exception. + */ + public RotatedDetector(String modelPath, String deviceName, int deviceId) throws Exception{ + handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create RotatedDetector failed!"); + } + } + + /** Get information of each image in a batch. + * @param images: input mats. + * @exception Exception: apply RotatedDetector failed exception. + * @return: results of each input mat. + */ + public Result[][] apply(Mat[] images) throws Exception{ + int[] counts = new int[images.length]; + Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply RotatedDetector failed!"); + } + Result[][] rets = new Result[images.length][]; + int offset = 0; + for (int i = 0; i < images.length; ++i) { + Result[] row = new Result[counts[i]]; + if (counts[i] >= 0) { + System.arraycopy(results, offset, row, 0, counts[i]); + } + offset += counts[i]; + rets[i] = row; + } + return rets; + } + + /** Get information of one image. + * @param image: input mat. + * @exception Exception: apply RotatedDetector failed exception. + * @return: result of input mat. + */ + public Result[] apply(Mat image) throws Exception{ + int[] counts = new int[1]; + Mat[] images = new Mat[]{image}; + Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply RotatedDetector failed!"); + } + return results; + } + + /** Release the instance of Rotated Detector. */ + public void release() { + destroy(handle); + } + + private native long create(String modelPath, String deviceName, int deviceId); + + private native void destroy(long handle); + + private native Result[] apply(long handle, Mat[] images, int[] count); +} diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Scheduler.java b/csrc/mmdeploy/apis/java/mmdeploy/Scheduler.java new file mode 100644 index 0000000000..58a70ad795 --- /dev/null +++ b/csrc/mmdeploy/apis/java/mmdeploy/Scheduler.java @@ -0,0 +1,52 @@ +package mmdeploy; + +/** @description: the Scheduler class. */ +public class Scheduler { + static { + System.loadLibrary("mmdeploy_java"); + } + + private static long schedulerHandle; + + /** Initialize a new instance of the Scheduler class. + * @param scheduler: scheduler handle. + */ + public Scheduler(long scheduler) { + schedulerHandle = scheduler; + } + + /** Create thread pool scheduler. + * @param numThreads: thread number. + * @return: scheduler handle. + */ + public static long threadPool(int numThreads) { + schedulerHandle = createThreadPool(numThreads); + return schedulerHandle; + } + + /** Create single thread scheduler. + * @return: scheduler handle. + */ + public static long thread() { + schedulerHandle = createThread(); + return schedulerHandle; + } + + /** Get scheduler handle. + * @return: scheduler handle. + */ + public long handle() { + return schedulerHandle; + } + + /** Release the instance of Scheduler. */ + public void release() { + destroy(schedulerHandle); + } + + private static native long createThreadPool(int numThreads); + + private static native long createThread(); + + private native void destroy(long schedulerHandle); +} diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Segmentor.java b/csrc/mmdeploy/apis/java/mmdeploy/Segmentor.java index e5f593efd9..b854f6aa77 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/Segmentor.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/Segmentor.java @@ -7,25 +7,63 @@ public class Segmentor { private final long handle; + /** @description: Single image segmentation result of a picture. */ public static class Result { + + /** Height. */ public int height; + + /** Width. */ public int width; + + /** Number of classes. */ public int classes; + + /** Segmentation mask. */ public int[] mask; - public Result(int height, int width, int classes, int [] mask) { + + /** Segmentation score. */ + public float[] score; + + /** Initializes a new instance of the Result class. + * @param height: height. + * @param width: width. + * @param classes: number of classes. + * @param mask: segmentation mask. + * @param score: segmentation score. + */ + public Result(int height, int width, int classes, int [] mask, float [] score) { this.height = height; this.width = width; this.classes = classes; this.mask = mask; + this.score = score; } } - public Segmentor(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the Segmentor class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create Segmentator failed exception. + */ + public Segmentor(String modelPath, String deviceName, int deviceId) throws Exception{ handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create Segmentor failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get information of each image in a batch. + * @param images: input mats. + * @exception Exception: apply Segmentor failed exception. + * @return: results of each input mat. + */ + public Result[][] apply(Mat[] images) throws Exception{ Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply Segmentor failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -37,11 +75,21 @@ public Result[][] apply(Mat[] images) { return rets; } - public Result[] apply(Mat image) { + /** Get information of one image. + * @param image: input mat. + * @exception Exception: apply Segmentor failed exception. + * @return: result of input mat. + */ + public Result[] apply(Mat image) throws Exception{ Mat[] images = new Mat[]{image}; - return apply(handle, images); + Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply Segmentor failed!"); + } + return results; } + /** Release the instance of Segmentor. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/TextDetector.java b/csrc/mmdeploy/apis/java/mmdeploy/TextDetector.java index 3a1862a3f6..689c13a823 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/TextDetector.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/TextDetector.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: the Java API class of TextDetector. */ public class TextDetector { static { System.loadLibrary("mmdeploy_java"); @@ -7,22 +8,49 @@ public class TextDetector { private final long handle; + /** @description: Single text detection result of a picture. */ public static class Result { + + /** Bbox. */ public PointF[] bbox; + + /** Score. */ public float score; + + /** Initializes a new instance of the Result class. + * @param bbox: bbox. + * @param score: score. + */ public Result(PointF[] bbox, float score) { this.bbox = bbox; this.score = score; } } - public TextDetector(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the TextDetector class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create TextDetector failed exception. + */ + public TextDetector(String modelPath, String deviceName, int deviceId) throws Exception{ handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create TextDetector failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get information of each image in a batch. + * @param images: input mats. + * @exception Exception: apply TextDetector failed exception. + * @return: results of each input mat. + */ + public Result[][] apply(Mat[] images) throws Exception{ int[] counts = new int[images.length]; Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply TextDetector failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -36,12 +64,22 @@ public Result[][] apply(Mat[] images) { return rets; } - public Result[] apply(Mat image) { + /** Get information of one image. + * @param image: input mat. + * @exception Exception: apply TextDetector failed exception. + * @return: result of input mat. + */ + public Result[] apply(Mat image) throws Exception{ int[] counts = new int[1]; Mat[] images = new Mat[]{image}; - return apply(handle, images, counts); + Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply TextDetector failed!"); + } + return results; } + /** Release the instance of TextDetector. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/TextRecognizer.java b/csrc/mmdeploy/apis/java/mmdeploy/TextRecognizer.java index dd5f662ff8..fff67972f1 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/TextRecognizer.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/TextRecognizer.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: the Java API class of TextRecognizer. */ public class TextRecognizer { static { System.loadLibrary("mmdeploy_java"); @@ -7,21 +8,48 @@ public class TextRecognizer { private final long handle; + /** @description: Single text recognition result of a picture. */ public static class Result { + + /** Text. */ public byte [] text; + + /** Score. */ public float [] score; + + /** Initializes a new instance of the Result class. + * @param text: text. + * @param score: score. + */ public Result(byte [] text, float [] score) { this.text = text; this.score = score; } } - public TextRecognizer(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the TextRecognizer class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create TextRecognizer failed exception. + */ + public TextRecognizer(String modelPath, String deviceName, int deviceId) throws Exception{ handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create TextRecognizer failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get information of each image in a batch. + * @param images: input mats. + * @exception Exception: apply TextRecognizer failed exception. + * @return: results of each input mat. + */ + public Result[][] apply(Mat[] images) throws Exception{ Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply TextRecognizer failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -33,16 +61,32 @@ public Result[][] apply(Mat[] images) { return rets; } - public Result[] apply(Mat image) { + /** Get information of one image. + * @param image: input mat. + * @exception Exception: apply TextDetector failed exception. + * @return: result of input mat. + */ + public Result[] apply(Mat image) throws Exception{ Mat[] images = new Mat[]{image}; - return apply(handle, images); + Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply TextRecognizer failed!"); + } + return results; } + /** Get information of one image from bboxes. + * @param image: input mat. + * @param bbox: bboxes information. + * @param bbox_count: numter of bboxes + * @return: result of input mat. + */ public Result[] applyBbox(Mat image, TextDetector.Result[] bbox, int[] bbox_count) { Mat[] images = new Mat[]{image}; return applyBbox(handle, images, bbox, bbox_count); } + /** Release the instance of TextRecognizer. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/native/CMakeLists.txt b/csrc/mmdeploy/apis/java/native/CMakeLists.txt index aad606d74a..6324cd21a1 100644 --- a/csrc/mmdeploy/apis/java/native/CMakeLists.txt +++ b/csrc/mmdeploy/apis/java/native/CMakeLists.txt @@ -14,7 +14,14 @@ mmdeploy_add_library(${PROJECT_NAME} SHARED EXCLUDE mmdeploy_Restorer.cpp mmdeploy_PoseDetector.cpp mmdeploy_TextDetector.cpp - mmdeploy_TextRecognizer.cpp) + mmdeploy_TextRecognizer.cpp + mmdeploy_PoseTracker.cpp + mmdeploy_Context.cpp + mmdeploy_Device.cpp + mmdeploy_Model.cpp + mmdeploy_Profiler.cpp + mmdeploy_Scheduler.cpp + mmdeploy_RotatedDetector.cpp) target_include_directories(${PROJECT_NAME} PRIVATE ${JNI_INCLUDE_DIRS}) diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Classifier.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_Classifier.cpp index 55e1acdf3b..2a3309361e 100644 --- a/csrc/mmdeploy/apis/java/native/mmdeploy_Classifier.cpp +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Classifier.cpp @@ -17,6 +17,7 @@ jlong Java_mmdeploy_Classifier_create(JNIEnv *env, jobject, jstring modelPath, j env->ReleaseStringUTFChars(deviceName, device_name); if (ec) { MMDEPLOY_ERROR("failed to create classifier, code = {}", ec); + return -1; } return (jlong)classifier; } @@ -28,13 +29,14 @@ void Java_mmdeploy_Classifier_destroy(JNIEnv *, jobject, jlong handle) { jobjectArray Java_mmdeploy_Classifier_apply(JNIEnv *env, jobject thiz, jlong handle, jobjectArray images, jintArray counts) { - return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) { + return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) -> jobjectArray { mmdeploy_classification_t *results{}; int *result_count{}; auto ec = mmdeploy_classifier_apply((mmdeploy_classifier_t)handle, imgs, size, &results, &result_count); if (ec) { MMDEPLOY_ERROR("failed to apply classifier, code = {}", ec); + return NULL; } auto result_cls = env->FindClass("mmdeploy/Classifier$Result"); diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Context.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_Context.cpp new file mode 100644 index 0000000000..dbd401724e --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Context.cpp @@ -0,0 +1,43 @@ +#include "mmdeploy_Context.h" + +#include + +#include "mmdeploy/apis/c/mmdeploy/common.h" +#include "mmdeploy/apis/c/mmdeploy/executor.h" +#include "mmdeploy/apis/c/mmdeploy/model.h" +#include "mmdeploy/apis/java/native/common.h" +#include "mmdeploy/core/logger.h" + +jlong Java_mmdeploy_Context_create(JNIEnv *env, jobject) { + mmdeploy_context_t context{}; + mmdeploy_context_create(&context); + return (jlong)context; +} + +jint Java_mmdeploy_Context_add(JNIEnv *env, jobject, jlong context_, jint contextType, jstring name, + jlong handle) { + auto object_name = env->GetStringUTFChars(name, nullptr); + if ((int)contextType == MMDEPLOY_TYPE_SCHEDULER) { + mmdeploy_context_add((mmdeploy_context_t)context_, (mmdeploy_context_type_t)contextType, + object_name, (mmdeploy_scheduler_t)handle); + } else if ((int)contextType == MMDEPLOY_TYPE_MODEL) { + mmdeploy_context_add((mmdeploy_context_t)context_, (mmdeploy_context_type_t)contextType, + object_name, (mmdeploy_model_t)handle); + } else if ((int)contextType == MMDEPLOY_TYPE_DEVICE) { + mmdeploy_context_add((mmdeploy_context_t)context_, (mmdeploy_context_type_t)contextType, + nullptr, (mmdeploy_device_t)handle); + } else if ((int)contextType == MMDEPLOY_TYPE_PROFILER) { + mmdeploy_context_add((mmdeploy_context_t)context_, (mmdeploy_context_type_t)contextType, + nullptr, (mmdeploy_profiler_t)handle); + } else { + MMDEPLOY_ERROR("wrong context type, got {}", (int)contextType); + return MMDEPLOY_E_NOT_SUPPORTED; + } + env->ReleaseStringUTFChars(name, object_name); + return 0; +} + +void Java_mmdeploy_Context_destroy(JNIEnv *, jobject, jlong context_) { + MMDEPLOY_DEBUG("Java_mmdeploy_Context_destroy"); + mmdeploy_context_destroy((mmdeploy_context_t)context_); +} diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Context.h b/csrc/mmdeploy/apis/java/native/mmdeploy_Context.h new file mode 100644 index 0000000000..42df819580 --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Context.h @@ -0,0 +1,34 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class mmdeploy_Context */ + +#ifndef _Included_mmdeploy_Context +#define _Included_mmdeploy_Context +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: mmdeploy_Context + * Method: create + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_mmdeploy_Context_create(JNIEnv *, jobject); + +/* + * Class: mmdeploy_Context + * Method: add + * Signature: (JILjava/lang/String;J)I + */ +JNIEXPORT jint JNICALL Java_mmdeploy_Context_add(JNIEnv *, jobject, jlong, jint, jstring, jlong); + +/* + * Class: mmdeploy_Context + * Method: destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_mmdeploy_Context_destroy(JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Detector.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_Detector.cpp index f61f0aed45..c03ff1a1ff 100644 --- a/csrc/mmdeploy/apis/java/native/mmdeploy_Detector.cpp +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Detector.cpp @@ -16,6 +16,7 @@ jlong Java_mmdeploy_Detector_create(JNIEnv *env, jobject, jstring modelPath, jst env->ReleaseStringUTFChars(deviceName, device_name); if (ec) { MMDEPLOY_ERROR("failed to create detector, code = {}", ec); + return -1; } return (jlong)detector; } @@ -27,13 +28,14 @@ void Java_mmdeploy_Detector_destroy(JNIEnv *, jobject, jlong handle) { jobjectArray Java_mmdeploy_Detector_apply(JNIEnv *env, jobject thiz, jlong handle, jobjectArray images, jintArray counts) { - return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) { + return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) -> jobjectArray { mmdeploy_detection_t *results{}; int *result_count{}; auto ec = mmdeploy_detector_apply((mmdeploy_detector_t)handle, imgs, size, &results, &result_count); if (ec) { MMDEPLOY_ERROR("failed to apply detector, code = {}", ec); + return NULL; } auto result_cls = env->FindClass("mmdeploy/Detector$Result"); auto result_ctor = diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Device.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_Device.cpp new file mode 100644 index 0000000000..8dbec9285b --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Device.cpp @@ -0,0 +1,24 @@ +#include "mmdeploy_Device.h" + +#include + +#include "mmdeploy/apis/c/mmdeploy/common.h" +#include "mmdeploy/apis/java/native/common.h" +#include "mmdeploy/core/logger.h" + +jlong Java_mmdeploy_Device_create(JNIEnv *env, jobject, jstring name, jint index) { + auto device_name = env->GetStringUTFChars(name, nullptr); + mmdeploy_device_t device{}; + auto ec = mmdeploy_device_create(device_name, (int)index, &device); + env->ReleaseStringUTFChars(name, device_name); + if (ec) { + MMDEPLOY_ERROR("failed to create device, code = {}", ec); + return -1; + } + return (jlong)device; +} + +void Java_mmdeploy_Device_destroy(JNIEnv *, jobject, jlong device_) { + MMDEPLOY_DEBUG("Java_mmdeploy_Device_destroy"); + mmdeploy_device_destroy((mmdeploy_device_t)device_); +} diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Device.h b/csrc/mmdeploy/apis/java/native/mmdeploy_Device.h new file mode 100644 index 0000000000..7d7ee9dee7 --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Device.h @@ -0,0 +1,27 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class mmdeploy_Device */ + +#ifndef _Included_mmdeploy_Device +#define _Included_mmdeploy_Device +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: mmdeploy_Device + * Method: create + * Signature: (Ljava/lang/String;I)J + */ +JNIEXPORT jlong JNICALL Java_mmdeploy_Device_create(JNIEnv *, jobject, jstring, jint); + +/* + * Class: mmdeploy_Device + * Method: destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_mmdeploy_Device_destroy(JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Model.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_Model.cpp new file mode 100644 index 0000000000..2bbc9a6920 --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Model.cpp @@ -0,0 +1,24 @@ +#include "mmdeploy_Model.h" + +#include + +#include "mmdeploy/apis/c/mmdeploy/model.h" +#include "mmdeploy/apis/java/native/common.h" +#include "mmdeploy/core/logger.h" + +jlong Java_mmdeploy_Model_create(JNIEnv *env, jobject, jstring path) { + auto model_path = env->GetStringUTFChars(path, nullptr); + mmdeploy_model_t model{}; + auto ec = mmdeploy_model_create_by_path(model_path, &model); + env->ReleaseStringUTFChars(path, model_path); + if (ec) { + MMDEPLOY_ERROR("failed to create model, code = {}", ec); + return -1; + } + return (jlong)model; +} + +void Java_mmdeploy_Model_destroy(JNIEnv *, jobject, jlong model_) { + MMDEPLOY_DEBUG("Java_mmdeploy_Model_destroy"); + mmdeploy_model_destroy((mmdeploy_model_t)model_); +} diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Model.h b/csrc/mmdeploy/apis/java/native/mmdeploy_Model.h new file mode 100644 index 0000000000..11e23a1a81 --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Model.h @@ -0,0 +1,27 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class mmdeploy_Model */ + +#ifndef _Included_mmdeploy_Model +#define _Included_mmdeploy_Model +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: mmdeploy_Model + * Method: create + * Signature: (Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_mmdeploy_Model_create(JNIEnv *, jobject, jstring); + +/* + * Class: mmdeploy_Model + * Method: destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_mmdeploy_Model_destroy(JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_PoseDetector.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_PoseDetector.cpp index fcbbb863d5..4956555a6e 100644 --- a/csrc/mmdeploy/apis/java/native/mmdeploy_PoseDetector.cpp +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_PoseDetector.cpp @@ -17,6 +17,7 @@ jlong Java_mmdeploy_PoseDetector_create(JNIEnv *env, jobject, jstring modelPath, env->ReleaseStringUTFChars(deviceName, device_name); if (ec) { MMDEPLOY_ERROR("failed to create pose estimator, code = {}", ec); + return -1; } return (jlong)pose_estimator; } @@ -28,11 +29,12 @@ void Java_mmdeploy_PoseDetector_destroy(JNIEnv *, jobject, jlong handle) { jobjectArray Java_mmdeploy_PoseDetector_apply(JNIEnv *env, jobject thiz, jlong handle, jobjectArray images) { - return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) { + return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) -> jobjectArray { mmdeploy_pose_detection_t *results{}; auto ec = mmdeploy_pose_detector_apply((mmdeploy_pose_detector_t)handle, imgs, size, &results); if (ec) { MMDEPLOY_ERROR("failed to apply pose estimator, code = {}", ec); + return NULL; } auto result_cls = env->FindClass("mmdeploy/PoseDetector$Result"); auto result_ctor = env->GetMethodID(result_cls, "", "([Lmmdeploy/PointF;[F)V"); diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_PoseTracker.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_PoseTracker.cpp new file mode 100644 index 0000000000..c0d1685729 --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_PoseTracker.cpp @@ -0,0 +1,194 @@ +#include "mmdeploy_PoseTracker.h" + +#include + +#include "mmdeploy/apis/c/mmdeploy/pose_tracker.h" +#include "mmdeploy/apis/java/native/common.h" +#include "mmdeploy/core/logger.h" + +jlong Java_mmdeploy_PoseTracker_create(JNIEnv *env, jobject, jlong detModel, jlong poseModel, + jlong context) { + mmdeploy_pose_tracker_t pose_tracker{}; + auto ec = mmdeploy_pose_tracker_create((mmdeploy_model_t)detModel, (mmdeploy_model_t)poseModel, + (mmdeploy_context_t)context, &pose_tracker); + if (ec) { + MMDEPLOY_ERROR("failed to create pose tracker, code = {}", ec); + return -1; + } + return (jlong)pose_tracker; +} + +void Java_mmdeploy_PoseTracker_destroy(JNIEnv *, jobject, jlong handle) { + MMDEPLOY_DEBUG("Java_mmdeploy_PoseTracker_destroy"); + mmdeploy_pose_tracker_destroy((mmdeploy_pose_tracker_t)handle); +} + +jobject param_cpp_to_java(JNIEnv *env, mmdeploy_pose_tracker_param_t *params) { + auto param_cls = env->FindClass("mmdeploy/PoseTracker$Params"); + auto param_ctor = env->GetMethodID(param_cls, "", "(IIFFFIFIFFF[FIFIIFF[F)V"); + + jfloatArray keypointSigmas = env->NewFloatArray(params->keypoint_sigmas_size); + env->SetFloatArrayRegion(keypointSigmas, 0, params->keypoint_sigmas_size, + (jfloat *)params->keypoint_sigmas); + jfloatArray smoothParams = env->NewFloatArray(3); + env->SetFloatArrayRegion(smoothParams, 0, 3, (jfloat *)params->smooth_params); + + auto param = env->NewObject( + param_cls, param_ctor, (jint)params->det_interval, (jint)params->det_label, + (jfloat)params->det_thr, (jfloat)params->det_min_bbox_size, (jfloat)params->det_nms_thr, + (jint)params->pose_max_num_bboxes, (jfloat)params->pose_kpt_thr, + (jint)params->pose_min_keypoints, (jfloat)params->pose_bbox_scale, + (jfloat)params->pose_min_bbox_size, (jfloat)params->pose_nms_thr, keypointSigmas, + (jint)params->keypoint_sigmas_size, (jfloat)params->track_iou_thr, + (jint)params->track_max_missing, (jint)params->track_history_size, + (jfloat)params->std_weight_position, (jfloat)params->std_weight_velocity, smoothParams); + return param; +} + +void param_java_to_cpp(JNIEnv *env, mmdeploy_pose_tracker_param_t *params, jobject customParam) { + auto param_cls = env->FindClass("mmdeploy/PoseTracker$Params"); + auto param_ctor = env->GetMethodID(param_cls, "", "(IIFFFIFIFFF[FIFIIFF[F)V"); + + jfieldID fieldID_detInterval = env->GetFieldID(param_cls, "detInterval", "I"); + jint detInterval = env->GetIntField(customParam, fieldID_detInterval); + params->det_interval = (int)detInterval; + jfieldID fieldID_detLabel = env->GetFieldID(param_cls, "detLabel", "I"); + jint detLabel = env->GetIntField(customParam, fieldID_detLabel); + params->det_label = (int)detLabel; + jfieldID fieldID_detThr = env->GetFieldID(param_cls, "detThr", "F"); + jfloat detThr = env->GetFloatField(customParam, fieldID_detThr); + params->det_thr = (float)detThr; + jfieldID fieldID_detMinBboxSize = env->GetFieldID(param_cls, "detMinBboxSize", "F"); + jfloat detMinBboxSize = env->GetFloatField(customParam, fieldID_detMinBboxSize); + params->det_min_bbox_size = (float)detMinBboxSize; + jfieldID fieldID_detNmsThr = env->GetFieldID(param_cls, "detNmsThr", "F"); + jfloat detNmsThr = env->GetFloatField(customParam, fieldID_detNmsThr); + params->det_nms_thr = (float)detNmsThr; + jfieldID fieldID_poseMaxNumBboxes = env->GetFieldID(param_cls, "poseMaxNumBboxes", "I"); + jint poseMaxNumBboxes = env->GetIntField(customParam, fieldID_poseMaxNumBboxes); + params->pose_max_num_bboxes = (int)poseMaxNumBboxes; + jfieldID fieldID_poseKptThr = env->GetFieldID(param_cls, "poseKptThr", "F"); + jfloat poseKptThr = env->GetFloatField(customParam, fieldID_poseKptThr); + params->pose_kpt_thr = (float)poseKptThr; + jfieldID fieldID_poseMinKeypoints = env->GetFieldID(param_cls, "poseMinKeypoints", "I"); + jint poseMinKeypoints = env->GetIntField(customParam, fieldID_poseMinKeypoints); + params->pose_min_keypoints = (int)poseMinKeypoints; + jfieldID fieldID_poseBboxScale = env->GetFieldID(param_cls, "poseBboxScale", "F"); + jfloat poseBboxScale = env->GetFloatField(customParam, fieldID_poseBboxScale); + params->pose_bbox_scale = (float)poseBboxScale; + jfieldID fieldID_poseMinBboxSize = env->GetFieldID(param_cls, "poseMinBboxSize", "F"); + jfloat poseMinBboxSize = env->GetFloatField(customParam, fieldID_poseMinBboxSize); + params->pose_min_bbox_size = (float)poseMinBboxSize; + jfieldID fieldID_poseNmsThr = env->GetFieldID(param_cls, "poseNmsThr", "F"); + jfloat poseNmsThr = env->GetFloatField(customParam, fieldID_poseNmsThr); + params->pose_nms_thr = (float)poseNmsThr; + jfieldID fieldID_keypointSigmas = env->GetFieldID(param_cls, "keypointSigmas", "[F"); + auto keypointSigmasObj = env->GetObjectField(customParam, fieldID_keypointSigmas); + float *keypointSigmas = + (float *)env->GetFloatArrayElements((jfloatArray)keypointSigmasObj, nullptr); + params->keypoint_sigmas = keypointSigmas; + env->ReleaseFloatArrayElements((jfloatArray)keypointSigmasObj, keypointSigmas, JNI_ABORT); + jfieldID fieldID_keypointSigmasSize = env->GetFieldID(param_cls, "keypointSigmasSize", "I"); + jint keypointSigmasSize = env->GetIntField(customParam, fieldID_keypointSigmasSize); + params->keypoint_sigmas_size = keypointSigmasSize; + jfieldID fieldID_trackIouThr = env->GetFieldID(param_cls, "trackIouThr", "F"); + jfloat trackIouThr = env->GetFloatField(customParam, fieldID_trackIouThr); + params->track_iou_thr = trackIouThr; + jfieldID fieldID_trackMaxMissing = env->GetFieldID(param_cls, "trackMaxMissing", "I"); + jint trackMaxMissing = env->GetIntField(customParam, fieldID_trackMaxMissing); + params->track_max_missing = trackMaxMissing; + jfieldID fieldID_trackHistorySize = env->GetFieldID(param_cls, "trackHistorySize", "I"); + jint trackHistorySize = env->GetIntField(customParam, fieldID_trackHistorySize); + params->track_history_size = trackHistorySize; + jfieldID fieldID_stdWeightPosition = env->GetFieldID(param_cls, "stdWeightPosition", "F"); + jfloat stdWeightPosition = env->GetFloatField(customParam, fieldID_stdWeightPosition); + params->std_weight_position = stdWeightPosition; + jfieldID fieldID_stdWeightVelocity = env->GetFieldID(param_cls, "stdWeightVelocity", "F"); + jfloat stdWeightVelocity = env->GetFloatField(customParam, fieldID_stdWeightVelocity); + params->std_weight_velocity = stdWeightVelocity; + jfieldID fieldID_smoothParams = env->GetFieldID(param_cls, "smoothParams", "[F"); + auto smoothParamsObj = env->GetObjectField(customParam, fieldID_smoothParams); + float *smoothParams = (float *)env->GetFloatArrayElements((jfloatArray)smoothParamsObj, nullptr); + params->smooth_params[0] = smoothParams[0]; + params->smooth_params[1] = smoothParams[1]; + params->smooth_params[2] = smoothParams[2]; + env->ReleaseFloatArrayElements((jfloatArray)smoothParamsObj, smoothParams, JNI_ABORT); +} + +jobject Java_mmdeploy_PoseTracker_setDefaultParams(JNIEnv *env, jobject) { + mmdeploy_pose_tracker_param_t params{}; + mmdeploy_pose_tracker_default_params(¶ms); + return param_cpp_to_java(env, ¶ms); +} + +jlong Java_mmdeploy_PoseTracker_createState(JNIEnv *env, jobject, jlong pipeline, + jobject paramsObject) { + mmdeploy_pose_tracker_state_t state{}; + mmdeploy_pose_tracker_param_t params{}; + param_java_to_cpp(env, ¶ms, paramsObject); + auto ec = mmdeploy_pose_tracker_create_state((mmdeploy_pose_tracker_t)pipeline, ¶ms, &state); + if (ec) { + MMDEPLOY_ERROR("failed to create pose tracker state, code = {}", ec); + return -1; + } + return (jlong)state; +} + +void Java_mmdeploy_PoseTracker_destroyState(JNIEnv *, jobject, jlong state) { + MMDEPLOY_DEBUG("Java_mmdeploy_PoseTracker_destroy"); + mmdeploy_pose_tracker_destroy_state((mmdeploy_pose_tracker_state_t)state); +} + +jobjectArray Java_mmdeploy_PoseTracker_apply(JNIEnv *env, jobject thiz, jlong handle, + jlongArray states, jobjectArray frames, + jintArray detects, jintArray counts) { + return With(env, frames, [&](const mmdeploy_mat_t imgs[], int size) -> jobjectArray { + mmdeploy_pose_tracker_target_t *results{}; + int *result_count{}; + auto states_array = env->GetLongArrayElements(states, nullptr); + auto detects_array = env->GetIntArrayElements(detects, nullptr); + auto ec = mmdeploy_pose_tracker_apply((mmdeploy_pose_tracker_t)handle, + (mmdeploy_pose_tracker_state_t *)states_array, imgs, + (int32_t *)detects_array, size, &results, &result_count); + if (ec) { + MMDEPLOY_ERROR("failed to apply pose tracker, code = {}", ec); + return NULL; + } + auto result_cls = env->FindClass("mmdeploy/PoseTracker$Result"); + auto result_ctor = + env->GetMethodID(result_cls, "", "([Lmmdeploy/PointF;[FLmmdeploy/Rect;I)V"); + auto total = std::accumulate(result_count, result_count + size, 0); + auto array = env->NewObjectArray(total, result_cls, nullptr); + auto pointf_cls = env->FindClass("mmdeploy/PointF"); + auto pointf_ctor = env->GetMethodID(pointf_cls, "", "(FF)V"); + auto rect_cls = env->FindClass("mmdeploy/Rect"); + auto rect_ctor = env->GetMethodID(rect_cls, "", "(FFFF)V"); + for (int i = 0; i < total; ++i) { + auto keypoint_array = env->NewObjectArray(results[i].keypoint_count, pointf_cls, nullptr); + for (int j = 0; j < results[i].keypoint_count; ++j) { + auto keypointj = env->NewObject(pointf_cls, pointf_ctor, (jfloat)results[i].keypoints[j].x, + (jfloat)results[i].keypoints[j].y); + env->SetObjectArrayElement(keypoint_array, j, keypointj); + } + auto score_array = env->NewFloatArray(results[i].keypoint_count); + env->SetFloatArrayRegion(score_array, 0, results[i].keypoint_count, + (jfloat *)results[i].scores); + auto rect = env->NewObject(rect_cls, rect_ctor, (jfloat)results[i].bbox.left, + (jfloat)results[i].bbox.top, (jfloat)results[i].bbox.right, + (jfloat)results[i].bbox.bottom); + auto target_id = results[i].target_id; + auto res = env->NewObject(result_cls, result_ctor, keypoint_array, score_array, rect, + (int)target_id); + env->SetObjectArrayElement(array, i, res); + } + auto counts_array = env->GetIntArrayElements(counts, nullptr); + for (int i = 0; i < size; ++i) { + counts_array[i] = result_count[i]; + } + env->ReleaseIntArrayElements(counts, counts_array, 0); + env->ReleaseLongArrayElements(states, states_array, 0); + env->ReleaseIntArrayElements(detects, detects_array, 0); + mmdeploy_pose_tracker_release_result(results, result_count, size); + return array; + }); +} diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_PoseTracker.h b/csrc/mmdeploy/apis/java/native/mmdeploy_PoseTracker.h new file mode 100644 index 0000000000..8e8d3905c8 --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_PoseTracker.h @@ -0,0 +1,56 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class mmdeploy_PoseTracker */ + +#ifndef _Included_mmdeploy_PoseTracker +#define _Included_mmdeploy_PoseTracker +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: mmdeploy_PoseTracker + * Method: create + * Signature: (JJJ)J + */ +JNIEXPORT jlong JNICALL Java_mmdeploy_PoseTracker_create(JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: mmdeploy_PoseTracker + * Method: destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_mmdeploy_PoseTracker_destroy(JNIEnv *, jobject, jlong); + +/* + * Class: mmdeploy_PoseTracker + * Method: createState + * Signature: (JLmmdeploy/PoseTracker/Params;)J + */ +JNIEXPORT jlong JNICALL Java_mmdeploy_PoseTracker_createState(JNIEnv *, jobject, jlong, jobject); + +/* + * Class: mmdeploy_PoseTracker + * Method: destroyState + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_mmdeploy_PoseTracker_destroyState(JNIEnv *, jobject, jlong); + +/* + * Class: mmdeploy_PoseTracker + * Method: setDefaultParams + * Signature: ()Lmmdeploy/PoseTracker/Params; + */ +JNIEXPORT jobject JNICALL Java_mmdeploy_PoseTracker_setDefaultParams(JNIEnv *, jobject); + +/* + * Class: mmdeploy_PoseTracker + * Method: apply + * Signature: (J[J[Lmmdeploy/Mat;[I[I)[Lmmdeploy/PoseTracker/Result; + */ +JNIEXPORT jobjectArray JNICALL Java_mmdeploy_PoseTracker_apply(JNIEnv *, jobject, jlong, jlongArray, + jobjectArray, jintArray, jintArray); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Profiler.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_Profiler.cpp new file mode 100644 index 0000000000..2c63233c5c --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Profiler.cpp @@ -0,0 +1,24 @@ +#include "mmdeploy_Profiler.h" + +#include + +#include "mmdeploy/apis/c/mmdeploy/common.h" +#include "mmdeploy/apis/java/native/common.h" +#include "mmdeploy/core/logger.h" + +jlong Java_mmdeploy_Profiler_create(JNIEnv *env, jobject, jstring path) { + auto profiler_path = env->GetStringUTFChars(path, nullptr); + mmdeploy_profiler_t profiler{}; + auto ec = mmdeploy_profiler_create(profiler_path, &profiler); + env->ReleaseStringUTFChars(path, profiler_path); + if (ec) { + MMDEPLOY_ERROR("failed to create profiler, code = {}", ec); + return -1; + } + return (jlong)profiler; +} + +void Java_mmdeploy_Profiler_destroy(JNIEnv *, jobject, jlong profiler_) { + MMDEPLOY_DEBUG("Java_mmdeploy_Profiler_destroy"); + mmdeploy_profiler_destroy((mmdeploy_profiler_t)profiler_); +} diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Profiler.h b/csrc/mmdeploy/apis/java/native/mmdeploy_Profiler.h new file mode 100644 index 0000000000..2bcdbc42cc --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Profiler.h @@ -0,0 +1,27 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class mmdeploy_Profiler */ + +#ifndef _Included_mmdeploy_Profiler +#define _Included_mmdeploy_Profiler +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: mmdeploy_Profiler + * Method: create + * Signature: (Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_mmdeploy_Profiler_create(JNIEnv *, jobject, jstring); + +/* + * Class: mmdeploy_Profiler + * Method: destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_mmdeploy_Profiler_destroy(JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Restorer.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_Restorer.cpp index 37bf954baa..f124d5edae 100644 --- a/csrc/mmdeploy/apis/java/native/mmdeploy_Restorer.cpp +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Restorer.cpp @@ -16,6 +16,7 @@ jlong Java_mmdeploy_Restorer_create(JNIEnv *env, jobject, jstring modelPath, jst env->ReleaseStringUTFChars(deviceName, device_name); if (ec) { MMDEPLOY_ERROR("failed to create restorer, code = {}", ec); + return -1; } return (jlong)restorer; } @@ -27,11 +28,12 @@ void Java_mmdeploy_Restorer_destroy(JNIEnv *, jobject, jlong handle) { jobjectArray Java_mmdeploy_Restorer_apply(JNIEnv *env, jobject thiz, jlong handle, jobjectArray images) { - return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) { + return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) -> jobjectArray { mmdeploy_mat_t *results{}; auto ec = mmdeploy_restorer_apply((mmdeploy_restorer_t)handle, imgs, size, &results); if (ec) { MMDEPLOY_ERROR("failed to apply restorer, code = {}", ec); + return NULL; } const char *java_enum_format[] = {"BGR", "RGB", "GRAYSCALE", "NV12", "NV21", "BGRA"}; const char *java_enum_type[] = {"FLOAT", "HALF", "INT8", "INT32"}; diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.cpp new file mode 100644 index 0000000000..3872e7e158 --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.cpp @@ -0,0 +1,61 @@ +#include "mmdeploy_RotatedDetector.h" + +#include + +#include "mmdeploy/apis/c/mmdeploy/rotated_detector.h" +#include "mmdeploy/apis/java/native/common.h" +#include "mmdeploy/core/logger.h" + +jlong Java_mmdeploy_RotatedDetector_create(JNIEnv *env, jobject, jstring modelPath, + jstring deviceName, jint device_id) { + auto model_path = env->GetStringUTFChars(modelPath, nullptr); + auto device_name = env->GetStringUTFChars(deviceName, nullptr); + mmdeploy_rotated_detector_t rotated_detector{}; + auto ec = mmdeploy_rotated_detector_create_by_path(model_path, device_name, (int)device_id, + &rotated_detector); + env->ReleaseStringUTFChars(modelPath, model_path); + env->ReleaseStringUTFChars(deviceName, device_name); + if (ec) { + MMDEPLOY_ERROR("failed to create rotated detector, code = {}", ec); + return -1; + } + return (jlong)rotated_detector; +} + +void Java_mmdeploy_RotatedDetector_destroy(JNIEnv *, jobject, jlong handle) { + MMDEPLOY_DEBUG("Java_mmdeploy_RotatedDetector_destroy"); + mmdeploy_rotated_detector_destroy((mmdeploy_rotated_detector_t)handle); +} + +jobjectArray Java_mmdeploy_RotatedDetector_apply(JNIEnv *env, jobject thiz, jlong handle, + jobjectArray images, jintArray counts) { + return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) -> jobjectArray { + mmdeploy_rotated_detection_t *results{}; + int *result_count{}; + auto ec = mmdeploy_rotated_detector_apply((mmdeploy_rotated_detector_t)handle, imgs, size, + &results, &result_count); + if (ec) { + MMDEPLOY_ERROR("failed to apply rotated detector, code = {}", ec); + return NULL; + } + auto result_cls = env->FindClass("mmdeploy/RotatedDetector$Result"); + auto result_ctor = env->GetMethodID(result_cls, "", "(IF[F)V"); + auto total = std::accumulate(result_count, result_count + size, 0); + auto array = env->NewObjectArray(total, result_cls, nullptr); + + for (int i = 0; i < total; ++i) { + jfloatArray rbbox = env->NewFloatArray(5); + env->SetFloatArrayRegion(rbbox, 0, 5, (jfloat *)results[i].rbbox); + auto res = env->NewObject(result_cls, result_ctor, (jint)results[i].label_id, + (jfloat)results[i].score, rbbox); + env->SetObjectArrayElement(array, i, res); + } + auto counts_array = env->GetIntArrayElements(counts, nullptr); + for (int i = 0; i < size; ++i) { + counts_array[i] = result_count[i]; + } + env->ReleaseIntArrayElements(counts, counts_array, 0); + mmdeploy_rotated_detector_release_result(results, result_count); + return array; + }); +} diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.h b/csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.h new file mode 100644 index 0000000000..6de527ec40 --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.h @@ -0,0 +1,36 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class mmdeploy_RotatedDetector */ + +#ifndef _Included_mmdeploy_RotatedDetector +#define _Included_mmdeploy_RotatedDetector +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: mmdeploy_RotatedDetector + * Method: create + * Signature: (Ljava/lang/String;Ljava/lang/String;I)J + */ +JNIEXPORT jlong JNICALL Java_mmdeploy_RotatedDetector_create(JNIEnv *, jobject, jstring, jstring, + jint); + +/* + * Class: mmdeploy_RotatedDetector + * Method: destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_mmdeploy_RotatedDetector_destroy(JNIEnv *, jobject, jlong); + +/* + * Class: mmdeploy_RotatedDetector + * Method: apply + * Signature: (J[Lmmdeploy/Mat;[I)[Lmmdeploy/RotatedDetector/Result; + */ +JNIEXPORT jobjectArray JNICALL Java_mmdeploy_RotatedDetector_apply(JNIEnv *, jobject, jlong, + jobjectArray, jintArray); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Scheduler.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_Scheduler.cpp new file mode 100644 index 0000000000..2c1f1c42c0 --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Scheduler.cpp @@ -0,0 +1,23 @@ +#include "mmdeploy_Scheduler.h" + +#include + +#include "mmdeploy/apis/c/mmdeploy/common.h" +#include "mmdeploy/apis/c/mmdeploy/executor.h" +#include "mmdeploy/apis/java/native/common.h" +#include "mmdeploy/core/logger.h" + +jlong Java_mmdeploy_Scheduler_createThreadPool(JNIEnv *env, jobject, jint numThreads) { + mmdeploy_scheduler_t scheduler = mmdeploy_executor_create_thread_pool((int)numThreads); + return (jlong)scheduler; +} + +jlong Java_mmdeploy_Scheduler_createThread(JNIEnv *env, jobject) { + mmdeploy_scheduler_t scheduler = mmdeploy_executor_create_thread(); + return (jlong)scheduler; +} + +void Java_mmdeploy_Scheduler_destroy(JNIEnv *, jobject, jlong scheduler_) { + MMDEPLOY_DEBUG("Java_mmdeploy_Scheduler_destroy"); + mmdeploy_scheduler_destroy((mmdeploy_scheduler_t)scheduler_); +} diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Scheduler.h b/csrc/mmdeploy/apis/java/native/mmdeploy_Scheduler.h new file mode 100644 index 0000000000..363015cf95 --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Scheduler.h @@ -0,0 +1,34 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class mmdeploy_Scheduler */ + +#ifndef _Included_mmdeploy_Scheduler +#define _Included_mmdeploy_Scheduler +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: mmdeploy_Scheduler + * Method: createThreadPool + * Signature: (I)J + */ +JNIEXPORT jlong JNICALL Java_mmdeploy_Scheduler_createThreadPool(JNIEnv *, jclass, jint); + +/* + * Class: mmdeploy_Scheduler + * Method: createThread + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_mmdeploy_Scheduler_createThread(JNIEnv *, jclass); + +/* + * Class: mmdeploy_Scheduler + * Method: destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_mmdeploy_Scheduler_destroy(JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Segmentor.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_Segmentor.cpp index f3c466cd08..12df31a49e 100644 --- a/csrc/mmdeploy/apis/java/native/mmdeploy_Segmentor.cpp +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Segmentor.cpp @@ -16,6 +16,7 @@ jlong Java_mmdeploy_Segmentor_create(JNIEnv *env, jobject, jstring modelPath, js env->ReleaseStringUTFChars(deviceName, device_name); if (ec) { MMDEPLOY_ERROR("failed to create segmentor, code = {}", ec); + return -1; } return (jlong)segmentor; } @@ -27,23 +28,40 @@ void Java_mmdeploy_Segmentor_destroy(JNIEnv *, jobject, jlong handle) { jobjectArray Java_mmdeploy_Segmentor_apply(JNIEnv *env, jobject thiz, jlong handle, jobjectArray images) { - return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) { + return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) -> jobjectArray { mmdeploy_segmentation_t *results{}; auto ec = mmdeploy_segmentor_apply((mmdeploy_segmentor_t)handle, imgs, size, &results); if (ec) { MMDEPLOY_ERROR("failed to apply segmentor, code = {}", ec); + return NULL; } auto result_cls = env->FindClass("mmdeploy/Segmentor$Result"); - auto result_ctor = env->GetMethodID(result_cls, "", "(III[I)V"); + auto result_ctor = env->GetMethodID(result_cls, "", "(III[I[F)V"); auto array = env->NewObjectArray(size, result_cls, nullptr); - + jintArray jmask; + jfloatArray jscore; for (int i = 0; i < size; ++i) { int *mask = results[i].mask; - jintArray jmask = env->NewIntArray(results[i].height * results[i].width); - env->SetIntArrayRegion(jmask, 0, results[i].width * results[i].height, (const jint *)mask); + float *score = results[i].score; + if (results[i].mask) { + jmask = env->NewIntArray(results[i].height * results[i].width); + env->SetIntArrayRegion(jmask, 0, results[i].width * results[i].height, (const jint *)mask); + } else { + jmask = env->NewIntArray(0); + env->SetIntArrayRegion(jmask, 0, 0, nullptr); + } + if (results[i].score) { + jscore = env->NewFloatArray(results[i].classes * results[i].height * results[i].width); + env->SetFloatArrayRegion(jscore, 0, + results[i].classes * results[i].height * results[i].width, + (const jfloat *)score); + } else { + jscore = env->NewFloatArray(0); + env->SetFloatArrayRegion(jscore, 0, 0, nullptr); + } auto res = env->NewObject(result_cls, result_ctor, (jint)results[i].height, - (jint)results[i].width, (jint)results[i].classes, jmask); + (jint)results[i].width, (jint)results[i].classes, jmask, jscore); env->SetObjectArrayElement(array, i, res); } mmdeploy_segmentor_release_result(results, size); diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_TextDetector.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_TextDetector.cpp index 90c08b21bc..943d1e625b 100644 --- a/csrc/mmdeploy/apis/java/native/mmdeploy_TextDetector.cpp +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_TextDetector.cpp @@ -17,6 +17,7 @@ jlong Java_mmdeploy_TextDetector_create(JNIEnv *env, jobject, jstring modelPath, env->ReleaseStringUTFChars(deviceName, device_name); if (ec) { MMDEPLOY_ERROR("failed to create text_detector, code = {}", ec); + return -1; } return (jlong)text_detector; } @@ -28,13 +29,14 @@ void Java_mmdeploy_TextDetector_destroy(JNIEnv *, jobject, jlong handle) { jobjectArray Java_mmdeploy_TextDetector_apply(JNIEnv *env, jobject thiz, jlong handle, jobjectArray images, jintArray counts) { - return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) { + return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) -> jobjectArray { mmdeploy_text_detection_t *results{}; int *result_count{}; auto ec = mmdeploy_text_detector_apply((mmdeploy_text_detector_t)handle, imgs, size, &results, &result_count); if (ec) { MMDEPLOY_ERROR("failed to apply detector, code = {}", ec); + return NULL; } auto result_cls = env->FindClass("mmdeploy/TextDetector$Result"); auto result_ctor = env->GetMethodID(result_cls, "", "([Lmmdeploy/PointF;F)V"); diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_TextRecognizer.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_TextRecognizer.cpp index 54da958cff..06987fb623 100644 --- a/csrc/mmdeploy/apis/java/native/mmdeploy_TextRecognizer.cpp +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_TextRecognizer.cpp @@ -17,6 +17,7 @@ jlong Java_mmdeploy_TextRecognizer_create(JNIEnv *env, jobject, jstring modelPat env->ReleaseStringUTFChars(deviceName, device_name); if (ec) { MMDEPLOY_ERROR("failed to create text recognizer, code = {}", ec); + return -1; } return (jlong)text_recognizer; } @@ -28,12 +29,13 @@ void Java_mmdeploy_TextRecognizer_destroy(JNIEnv *, jobject, jlong handle) { jobjectArray Java_mmdeploy_TextRecognizer_apply(JNIEnv *env, jobject thiz, jlong handle, jobjectArray images) { - return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) { + return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) -> jobjectArray { mmdeploy_text_recognition_t *results{}; auto ec = mmdeploy_text_recognizer_apply((mmdeploy_text_recognizer_t)handle, imgs, size, &results); if (ec) { MMDEPLOY_ERROR("failed to apply text recognizer, code = {}", ec); + return NULL; } auto result_cls = env->FindClass("mmdeploy/TextRecognizer$Result"); auto result_ctor = env->GetMethodID(result_cls, "", "([C[F)V"); diff --git a/demo/java/README.md b/demo/java/README.md index b1e3dc42b6..eb9b60aead 100644 --- a/demo/java/README.md +++ b/demo/java/README.md @@ -1,22 +1,68 @@ # Usage -**step 1.** Compile Utils with Java APIs. +**step 1.** Install Apache Ant. + +Please check the Apache Ant existence using `ant -h` in the command line. + +If there is no Apache Ant installed, please follow the command below. ``` -cd demo/java -javac --class-path ../../csrc/mmdeploy/apis/java/ Utils.java -cd ../.. +sudo apt-get update +sudo apt-get install ant +``` + +Set environment variable + +``` +export ANT_HOME=/usr/share/ant +export PATH=${ANT_HOME}/bin:${PATH} +``` + +**step 2.** Build OpenCV jar package (PoseTracker only). + +PoseTracker demo needs OpenCV Java, if you are interested in PoseTracker demo, you need to build OpenCV jar package first. + +Using OpenCV-4.7.0 as example: + +``` +export OPENCV_VERSION=4.7.0 +export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME} +export JAVA_AWT_LIBRARY=${JAVA_HOME} +export JAVA_INCLUDE_PATH=${JAVA_HOME}/include +export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/darwin +export JAVA_JVM_LIBRARY=${JAVA_HOME} + +wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip +unzip ${OPENCV_VERSION}.zip && rm -rf ${OPENCV_VERSION}.zip && cd opencv-${OPENCV_VERSION} +mkdir -p build && cd build +cmake -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_JAVA=ON .. +make -j8 && make install ``` -**step 2.** Run the demo in the console. +**step 3.** Build MMDeploy Java API. + +Please check the existence of `*.class` in `${MMDEPLOY_DIR}/csrc/mmdeploy/apis/java/mmdeploy`. + +If there is no existence of `*.class`, please follow this [tutorial](../../csrc/mmdeploy/apis/java/README.md) to build Java class. + +**step 4.** Build demo. Use **ImageClassification** as example. -First, you should set your model path and image path to `${MODEL_PATH}` and `${IMAGE_PATH}`. And then follow the bash codes. +First, you should set your mmdeploy path, opencv path, model path and image path to `${MMDEPLOY_DIR}`, `${OPENCV_DIR}`, `${MODEL_PATH}` and `${IMAGE_PATH}`. And then follow the bash codes. ```bash +cd demo/java export TASK=ImageClassification -export LD_LIBRARY_PATH=${PWD}/build/lib:${LD_LIBRARY_PATH} +export ARGS=cpu\ ${MODEL_PATH}\ ${IMAGE_PATH} +ant -DtaskName=${TASK} -DjarDir=${OPENCV_DIR}/build/bin -DlibDir=${OPENCV_DIR}/build/lib:${MMDEPLOY_DIR}/build/lib -Dcommand=${ARGS} +``` + +As for **PoseTracker**, you should execute: + +```bash cd demo/java -java -cp ../../csrc/mmdeploy/apis/java:./ ${TASK}.java cpu ${MODEL_PATH} ${IMAGE_PATH} +export TASK=PoseTracker +export ARGS=cpu\ ${DET_MODEL_PATH}\ ${POSE_MODEL_PATH}\ ${VIDEO_PATH} +ant -DtaskName=${TASK} -DjarDir=${OPENCV_DIR}/build/bin -DlibDir=${OPENCV_DIR}/build/lib:${MMDEPLOY_DIR}/build/lib -Dcommand="${ARGS}" ``` diff --git a/demo/java/Utils.java b/demo/java/Utils.java deleted file mode 100644 index daf0607f1e..0000000000 --- a/demo/java/Utils.java +++ /dev/null @@ -1,21 +0,0 @@ -import mmdeploy.PixelFormat; -import mmdeploy.DataType; -import mmdeploy.Mat; - -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.awt.image.DataBufferByte; -import java.io.File; -import java.io.IOException; - -public class Utils { - public static Mat loadImage(String path) throws IOException { - BufferedImage img = ImageIO.read(new File(path)); - return bufferedImage2Mat(img); - } - public static Mat bufferedImage2Mat(BufferedImage img) { - byte[] data = ((DataBufferByte) img.getData().getDataBuffer()).getData(); - return new Mat(img.getHeight(), img.getWidth(), img.getColorModel().getNumComponents(), - PixelFormat.BGR, DataType.INT8, data); - } -} diff --git a/demo/java/build.xml b/demo/java/build.xml new file mode 100644 index 0000000000..919214171c --- /dev/null +++ b/demo/java/build.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/demo/java/ImageClassification.java b/demo/java/src/ImageClassification.java similarity index 82% rename from demo/java/ImageClassification.java rename to demo/java/src/ImageClassification.java index 6016f53b7d..b8ff53aa57 100644 --- a/demo/java/ImageClassification.java +++ b/demo/java/src/ImageClassification.java @@ -9,8 +9,14 @@ import java.io.File; import java.io.IOException; +/** @description: this is a class for ImageClassification java demo. */ public class ImageClassification { + /** The main function for ImageClassification Java demo. + * @param deviceName: the device name of the demo. + * @param modelPath: the image classification model path. + * @param imagePath: the image path. + */ public static void main(String[] args) { // Parse arguments if (args.length != 3) { diff --git a/demo/java/ImageRestorer.java b/demo/java/src/ImageRestorer.java similarity index 83% rename from demo/java/ImageRestorer.java rename to demo/java/src/ImageRestorer.java index 957a494b05..719169db74 100644 --- a/demo/java/ImageRestorer.java +++ b/demo/java/src/ImageRestorer.java @@ -9,8 +9,14 @@ import java.io.File; import java.io.IOException; +/** @description: this is a class for ImageRestorer java demo. */ public class ImageRestorer { + /** The main function for ImageRestorer Java demo. + * @param deviceName: the device name of the demo. + * @param modelPath: the image restorer model path. + * @param imagePath: the image path. + */ public static void main(String[] args) { // Parse arguments if (args.length != 3) { @@ -26,6 +32,7 @@ public static void main(String[] args) { try { restorer = new Restorer(modelPath, deviceName, 0); + // load image Mat img = Utils.loadImage(imagePath); diff --git a/demo/java/ImageSegmentation.java b/demo/java/src/ImageSegmentation.java similarity index 82% rename from demo/java/ImageSegmentation.java rename to demo/java/src/ImageSegmentation.java index 86953fb0ed..7a64b9a11e 100644 --- a/demo/java/ImageSegmentation.java +++ b/demo/java/src/ImageSegmentation.java @@ -9,8 +9,14 @@ import java.io.File; import java.io.IOException; +/** @description: this is a class for ImageSegmentation java demo. */ public class ImageSegmentation { + /** The main function for ImageSegmentation Java demo. + * @param deviceName: the device name of the demo. + * @param modelPath: the image segmentation model path. + * @param imagePath: the image path. + */ public static void main(String[] args) { // Parse arguments if (args.length != 3) { @@ -26,6 +32,7 @@ public static void main(String[] args) { try { segmentor = new Segmentor(modelPath, deviceName, 0); + // load image Mat img = Utils.loadImage(imagePath); diff --git a/demo/java/ObjectDetection.java b/demo/java/src/ObjectDetection.java similarity index 90% rename from demo/java/ObjectDetection.java rename to demo/java/src/ObjectDetection.java index 01eee41b4f..3f68c25755 100644 --- a/demo/java/ObjectDetection.java +++ b/demo/java/src/ObjectDetection.java @@ -12,8 +12,14 @@ import java.io.File; import java.io.IOException; +/** @description: this is a class for ObjectDetection java demo. */ public class ObjectDetection { + /** The main function for ObjectDetection Java demo. + * @param deviceName: the device name of the demo. + * @param modelPath: the object detection model path. + * @param imagePath: the image path. + */ public static void main(String[] args) { // Parse arguments if (args.length != 3) { @@ -28,12 +34,14 @@ public static void main(String[] args) { Detector detector = null; try { detector = new Detector(modelPath, deviceName, 0); + // load image BufferedImage srcImg = ImageIO.read(new File(imagePath)); Mat img = Utils.bufferedImage2Mat(srcImg); // apply detector Detector.Result[] result = detector.apply(img); + // print results Graphics ghandle = srcImg.createGraphics(); for (int i = 0; i < result.length; i++) { diff --git a/demo/java/Ocr.java b/demo/java/src/Ocr.java similarity index 77% rename from demo/java/Ocr.java rename to demo/java/src/Ocr.java index 1edc04d77d..8d9a718523 100644 --- a/demo/java/Ocr.java +++ b/demo/java/src/Ocr.java @@ -10,12 +10,19 @@ import java.io.File; import java.io.IOException; +/** @description: this is a class for Ocr java demo. */ public class Ocr { + /** The main function for Ocr Java demo. + * @param deviceName: the device name of the demo. + * @param detModelPath: the text detection model path. + * @param recModelPath: the text recognition model path. + * @param imagePath: the image path. + */ public static void main(String[] args) { // Parse arguments if (args.length != 4) { - System.out.println("usage:\njava TextDetection deviceName detModelPath recModelPath imagePath"); + System.out.println("usage:\njava Ocr deviceName detModelPath recModelPath imagePath"); return; } String deviceName = args[0]; @@ -30,13 +37,19 @@ public static void main(String[] args) { try { text_detector = new TextDetector(detModelPath, deviceName, 0); text_recognizer = new TextRecognizer(recModelPath, deviceName, 0); + // load image Mat img = Utils.loadImage(imagePath); // apply text detector TextDetector.Result[] detResult = text_detector.apply(img); + int [] detResultCount = {detResult.length}; TextRecognizer.Result[] recResult = text_recognizer.applyBbox(img, detResult, detResultCount); + if (recResult == null) { + System.out.println("Apply TextRecognizer failed."); + System.exit(1); + } // print results for (int i = 0; i < detResultCount[0]; ++i) { System.out.printf("box[%d]: %s\n", i, new String(recResult[i].text)); diff --git a/demo/java/PoseDetection.java b/demo/java/src/PoseDetection.java similarity index 69% rename from demo/java/PoseDetection.java rename to demo/java/src/PoseDetection.java index e2c381ac34..6c6bba4f7b 100644 --- a/demo/java/PoseDetection.java +++ b/demo/java/src/PoseDetection.java @@ -9,8 +9,14 @@ import java.io.File; import java.io.IOException; +/** @description: this is a class for PoseDetection java demo. */ public class PoseDetection { + /** The main function for PoseDetection Java demo. + * @param deviceName: the device name of the demo. + * @param modelPath: the pose detection model path. + * @param imagePath: the image path. + */ public static void main(String[] args) { // Parse arguments if (args.length != 3) { @@ -22,15 +28,16 @@ public static void main(String[] args) { String imagePath = args[2]; // create pose estimator - PoseDetector pose_estimator = null; + PoseDetector poseEstimator = null; try { - pose_estimator = new PoseDetector(modelPath, deviceName, 0); + poseEstimator = new PoseDetector(modelPath, deviceName, 0); + // load image Mat img = Utils.loadImage(imagePath); // apply pose estimator - PoseDetector.Result[] result = pose_estimator.apply(img); + PoseDetector.Result[] result = poseEstimator.apply(img); // print results for (PoseDetector.Result value : result) { @@ -42,8 +49,8 @@ public static void main(String[] args) { System.out.println("exception: " + e.getMessage()); } finally { // release pose estimator - if (pose_estimator != null) { - pose_estimator.release(); + if (poseEstimator != null) { + poseEstimator.release(); } } } diff --git a/demo/java/src/PoseTracker.java b/demo/java/src/PoseTracker.java new file mode 100644 index 0000000000..4525476dad --- /dev/null +++ b/demo/java/src/PoseTracker.java @@ -0,0 +1,170 @@ +import mmdeploy.PixelFormat; +import mmdeploy.PointF; +import mmdeploy.DataType; +import mmdeploy.Mat; +import mmdeploy.Model; +import mmdeploy.Device; +import mmdeploy.Context; +import mmdeploy.Profiler; +import mmdeploy.PoseTracker.*; + +import org.opencv.videoio.*; +import org.opencv.core.*; +import org.opencv.imgproc.*; +import org.opencv.imgcodecs.*; +import org.opencv.highgui.*; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.awt.image.DataBufferByte; +import java.io.File; +import java.io.IOException; +import java.lang.Math; + +/** @description: this is a class for PoseTracker java demo. */ +public class PoseTracker { + static { + System.loadLibrary(Core.NATIVE_LIBRARY_NAME); + } + + /** This function visualize the PoseTracker results. + * @param frame: a video frame + * @param results: results of PoseTracker + * @param size: image size + * @param frameID: the frame index in the video + * @param withBbox: draw the person bbox or not + * @return: whether the quit keyboard input received + */ + public static boolean Visualize(org.opencv.core.Mat frame, mmdeploy.PoseTracker.Result[] results, int size, + int frameID, boolean withBbox) { + int skeleton[][] = {{15, 13}, {13, 11}, {16, 14}, {14, 12}, {11, 12}, {5, 11}, {6, 12}, + {5, 6}, {5, 7}, {6, 8}, {7, 9}, {8, 10}, {1, 2}, {0, 1}, + {0, 2}, {1, 3}, {2, 4}, {3, 5}, {4, 6}}; + Scalar palette[] = {new Scalar(255, 128, 0), new Scalar(255, 153, 51), new Scalar(255, 178, 102), + new Scalar(230, 230, 0), new Scalar(255, 153, 255), new Scalar(153, 204, 255), + new Scalar(255, 102, 255), new Scalar(255, 51, 255), new Scalar(102, 178, 255), + new Scalar(51, 153, 255), new Scalar(255, 153, 153), new Scalar(255, 102, 102), + new Scalar(255, 51, 51), new Scalar(153, 255, 153), new Scalar(102, 255, 102), + new Scalar(51, 255, 51), new Scalar(0, 255, 0), new Scalar(0, 0, 255), + new Scalar(255, 0, 0), new Scalar(255, 255, 255)}; + int linkColor[] = { + 0, 0, 0, 0, 7, 7, 7, 9, 9, 9, 9, 9, 16, 16, 16, 16, 16, 16, 16 + }; + int pointColor[] = {16, 16, 16, 16, 16, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0}; + + float scale = (float)size / (float)Math.max(frame.cols(), frame.rows()); + if (scale != 1) { + Imgproc.resize(frame, frame, new Size(), scale, scale); + } + else { + frame = frame.clone(); + } + for (int i = 0; i < results.length; i++) { + mmdeploy.PoseTracker.Result pt = results[i]; + for (int j = 0; j < pt.keypoints.length; j++) { + PointF p = pt.keypoints[j]; + p.x *= scale; + p.y *= scale; + pt.keypoints[j] = p; + } + float scoreThr = 0.5f; + int used[] = new int[pt.keypoints.length * 2]; + for (int j = 0; j < skeleton.length; j++) { + int u = skeleton[j][0]; + int v = skeleton[j][1]; + if (pt.scores[u] > scoreThr && pt.scores[v] > scoreThr) { + used[u] = used[v] = 1; + Point pointU = new Point(pt.keypoints[u].x, pt.keypoints[u].y); + Point pointV = new Point(pt.keypoints[v].x, pt.keypoints[v].y); + Imgproc.line(frame, pointU, pointV, palette[linkColor[j]], 1); + } + } + for (int j = 0; j < pt.keypoints.length; j++) { + if (used[j] == 1) { + Point p = new Point(pt.keypoints[j].x, pt.keypoints[j].y); + Imgproc.circle(frame, p, 1, palette[pointColor[j]], 2); + } + } + if (withBbox) { + float bbox[] = {pt.bbox.left, pt.bbox.top, pt.bbox.right, pt.bbox.bottom}; + for (int j = 0; j < 4; j++) { + bbox[j] *= scale; + } + Imgproc.rectangle(frame, new Point(bbox[0], bbox[1]), + new Point(bbox[2], bbox[3]), new Scalar(0, 255, 0)); + } + } + + HighGui.imshow("Pose Tracker", frame); + // Press any key to quit. + return HighGui.waitKey(5) == -1; + } + + /** The main function for PoseTracker Java demo. + * @param deviceName: the device name of PoseTracker + * @param detModelPath: the person detection model path + * @param poseModelPath: the pose estimation model path + * @param videoPath: the video path + */ + public static void main(String[] args) { + // Parse arguments + if (args.length != 4) { + System.out.println("usage:\n-Dcommand needs deviceName detModel poseModel videoPath"); + return; + } + String deviceName = args[0]; + String detModelPath = args[1]; + String poseModelPath = args[2]; + String videoPath = args[3]; + + // create pose tracker + mmdeploy.PoseTracker poseTracker = null; + Model detModel = new Model(detModelPath); + Model poseModel = new Model(poseModelPath); + Device device = new Device(deviceName, 0); + if (detModel.handle() == -1 || poseModel.handle() == -1 || device.handle() == -1) { + System.out.println("failed to create model or device"); + System.exit(1); + } + Context context = new Context(); + context.add(device); + try { + poseTracker = new mmdeploy.PoseTracker(detModel, poseModel, context); + + mmdeploy.PoseTracker.Params params = poseTracker.initParams(); + params.detInterval = 5; + params.poseMaxNumBboxes = 6; + long stateHandle = poseTracker.createState(params); + VideoCapture cap = new VideoCapture(videoPath); + if (!cap.isOpened()) { + System.out.printf("failed to open video: %s", videoPath); + System.exit(1); + } + int frameID = 0; + org.opencv.core.Mat frame = new org.opencv.core.Mat(); + while (true) { + cap.read(frame); + System.out.printf("processing frame %d\n", frameID); + if (frame.empty()) { + break; + } + Mat mat = Utils.cvMatToMat(frame); + // process + mmdeploy.PoseTracker.Result[] result = poseTracker.apply(stateHandle, mat, -1); + + // visualize + if (!Visualize(frame, result, 1280, frameID++, true)) { + break; + } + } + } catch (Exception e) { + System.out.println("exception: " + e.getMessage()); + } finally { + // release pose tracker + if (poseTracker != null) { + poseTracker.release(); + } + System.exit(0); + } + } +} diff --git a/demo/java/src/RotatedDetection.java b/demo/java/src/RotatedDetection.java new file mode 100644 index 0000000000..ed27966cba --- /dev/null +++ b/demo/java/src/RotatedDetection.java @@ -0,0 +1,83 @@ +import mmdeploy.RotatedDetector; +import mmdeploy.PixelFormat; +import mmdeploy.DataType; +import mmdeploy.Mat; + +import javax.imageio.ImageIO; + +import java.awt.Color; +import java.awt.image.BufferedImage; +import java.awt.image.DataBufferByte; +import java.awt.Graphics; +import java.io.File; +import java.io.IOException; +import java.lang.Math; + +/** @description: this is a class for RotatedDetection java demo. */ +public class RotatedDetection { + + /** The main function for RotatedDetection Java demo. + * @param deviceName: the device name of the demo. + * @param modelPath: the rotated detection model path. + * @param imagePath: the image path. + */ + public static void main(String[] args) { + // Parse arguments + if (args.length != 3) { + System.out.println("usage:\njava RotatedDetection deviceName modelPath imagePath"); + return; + } + String deviceName = args[0]; + String modelPath = args[1]; + String imagePath = args[2]; + + // create rotated detector + RotatedDetector rotatedDetector = null; + try { + rotatedDetector = new RotatedDetector(modelPath, deviceName, 0); + + // load image + BufferedImage srcImg = ImageIO.read(new File(imagePath)); + Mat img = Utils.bufferedImage2Mat(srcImg); + + // apply rotated detector + RotatedDetector.Result[] result = rotatedDetector.apply(img); + + // print results + Graphics ghandle = srcImg.createGraphics(); + for (int i = 0; i < result.length; i++) { + RotatedDetector.Result value = result[i]; + float cx, cy, w, h, angle; + cx = value.rbbox[0]; + cy = value.rbbox[1]; + w = value.rbbox[2]; + h = value.rbbox[3]; + angle = value.rbbox[4]; + float wx = w / 2 * (float)Math.cos(angle); + float wy = w / 2 * (float)Math.sin(angle); + float hx = -h / 2 * (float)Math.sin(angle); + float hy = h / 2 * (float)Math.cos(angle); + System.out.printf("box %d, score %.2f, point1: (%.2f, %.2f), point2: (%.2f, %.2f), point3: (%.2f, %.2f), point4: (%.2f, %.2f)\n", + i, value.score, cx - wx - hx, cy - wy - hy, cx + wx - hx, cy + wy - hy, cx + wx + hx, cy + wy + hy, cx - wx + hx, cy - wy + hy); + + // skip rotated detections less than specified score threshold + if (value.score < 0.1) { + continue; + } + ghandle.setColor(new Color(0, 255, 0)); + int[] polygonX = new int[] {(int)(cx - wx - hx), (int)(cx + wx - hx), (int)(cx + wx + hx), (int)(cx - wx + hx)}; + int[] polygonY = new int[] {(int)(cy - wy - hy), (int)(cy + wy - hy), (int)(cy + wy + hy), (int)(cy - wy + hy)}; + ghandle.drawPolygon(polygonX, polygonY, 4); + } + ghandle.dispose(); + ImageIO.write(srcImg, "png", new File("output_rotated_detection.png")); + } catch (Exception e) { + System.out.println("exception: " + e.getMessage()); + } finally { + // release rotated detector + if (rotatedDetector != null) { + rotatedDetector.release(); + } + } + } +} diff --git a/demo/java/src/Utils.java b/demo/java/src/Utils.java new file mode 100644 index 0000000000..92de836b2e --- /dev/null +++ b/demo/java/src/Utils.java @@ -0,0 +1,49 @@ +import mmdeploy.PixelFormat; +import mmdeploy.DataType; +import mmdeploy.Mat; + +import org.opencv.core.*; +import org.opencv.imgcodecs.*; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.awt.image.DataBufferByte; +import java.io.File; +import java.io.IOException; +import java.lang.*; + +/** @description: this is a util class for java demo. */ +public class Utils { + + /** This function loads the image by path. + * @param path: the image path. + * @return: the image with Mat format. + * @exception IOException: throws an IO exception when load failed. + */ + public static Mat loadImage(String path) throws IOException { + BufferedImage img = ImageIO.read(new File(path)); + return bufferedImage2Mat(img); + } + + /** This function changes bufferedImage to Mat. + * @param img: the bufferedImage. + * @return: the image with Mat format. + */ + public static Mat bufferedImage2Mat(BufferedImage img) { + byte[] data = ((DataBufferByte) img.getData().getDataBuffer()).getData(); + return new Mat(img.getHeight(), img.getWidth(), img.getColorModel().getNumComponents(), + PixelFormat.BGR, DataType.INT8, data); + } + + /** This function changes cvMat to Mat. + * @param cvMat: the image with opencv Mat format. + * @return: the image with Mat format. + */ + public static Mat cvMatToMat(org.opencv.core.Mat cvMat) + { + byte[] dataPointer = new byte[cvMat.rows() * cvMat.cols() * cvMat.channels() * (int)cvMat.elemSize()]; + cvMat.get(0, 0, dataPointer); + return new Mat(cvMat.rows(), cvMat.cols(), cvMat.channels(), + PixelFormat.BGR, DataType.INT8, dataPointer); + } +}