Skip to content

Commit b59d83b

Browse files
[Feature] Add Java APIs and demos for pose tracker, segmentor and rotated-object detector (#1795)
* move to dev-1.x * fix wrong class name * redesign Params argument mechanisms * change demo build to ant * remove unused debugging file * fix demo * remove hardcode * a general build.xml * update README * fix clang-format * rename follow java format * update docs * fix lint * fix java_api build * fix ci * fix java api ci * fix lint * fix opencv build privilege * fix CI ant build * remove opencv build because is too slow * fix lib path * fix wrong dir * fix run demo location * fix ant command * fix opencv * mv opencv to java demo * fix CI * fix comments * add absolute lib path in ci * find lib * find lib * fix mmdeploy lib path in ci * fix pose ocr pose-tracker * support RotatedDetection and Segmentation score * add cmake * fix lint * fix yapf * fix clang-format * fix clang-format * fix java api ci * fix ocr src pic * add rotate * add opencv lib * fix lib * fix libgstreamer missing * add libgstreamer-plugin * fix comments * fix comments * add avcodec for posetracker * fix CI * fix java ci comments * fix test_java_demo format * fix lint and ffmpeg missing * fix comments * a copy of array for java * fix demo stuck * fix test_java_demo.py * fix popd and pushd * fix java_api * fix java api handle * update for api * add java docstrings for demo * add docstring for posetracker java and fix handle * add some java api docstrings * return ec * add error code for java * add all java docs * fix clang-format * fix PoseTracker apply api for batch inference * fix missing error code * remove author * remove author * remove destroy window * fix wrong code * fix Context * fix java docs * fix comments * fix compile failed * fix for and if format * fix error code * fix bracket
1 parent 91a0a9a commit b59d83b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+2177
-115
lines changed

.github/scripts/test_java_demo.py

Lines changed: 38 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,21 @@
4747
'configs': [
4848
'https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/litehrnet.tar' # noqa: E501
4949
]
50+
},
51+
{
52+
'task':
53+
'PoseTracker',
54+
'configs': [
55+
'https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/rtmdet-nano.tar', # noqa: E501
56+
'https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/rtmpose-tiny.tar' # noqa: E501
57+
]
58+
},
59+
{
60+
'task':
61+
'RotatedDetection',
62+
'configs': [
63+
'https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/gliding-vertex.tar' # noqa: E501
64+
]
5065
}
5166
]
5267

@@ -60,24 +75,34 @@ def main():
6075
for params in PARAMS:
6176
task = params['task']
6277
configs = params['configs']
63-
java_demo_cmd = [
64-
'java', '-cp', 'csrc/mmdeploy/apis/java:demo/java',
65-
'demo/java/' + task + '.java', 'cpu'
66-
]
78+
java_command = '\"cpu'
6779
for config in configs:
6880
model_url = config
6981
os.system('wget {} && tar xvf {}'.format(model_url,
7082
model_url.split('/')[-1]))
7183
model_dir = model_url.split('/')[-1].split('.')[0]
72-
java_demo_cmd.append(model_dir)
73-
java_demo_cmd.append('/home/runner/work/mmdeploy/mmdeploy/demo' +
74-
'/resources/human-pose.jpg')
75-
java_demo_cmd_str = ' '.join(java_demo_cmd)
76-
os.system('export JAVA_HOME=/home/runner/work/mmdeploy/mmdeploy/' +
77-
'jdk-18 && export PATH=${JAVA_HOME}/bin:${PATH} && java' +
78-
' --version && export LD_LIBRARY_PATH=/home/runner/work/' +
79-
'mmdeploy/mmdeploy/build/lib:${LD_LIBRARY_PATH} && ' +
80-
java_demo_cmd_str)
84+
java_command += (' ' + model_dir)
85+
if task in [
86+
'ImageClassification', 'ObjectDetection', 'ImageSegmentation',
87+
'ImageRestorer', 'PoseDetection', 'RotatedDetection'
88+
]:
89+
java_command += (' $GITHUB_WORKSPACE/demo' +
90+
'/resources/human-pose.jpg\"')
91+
elif task in ['Ocr']:
92+
java_command += (' $GITHUB_WORKSPACE/demo' +
93+
'/resources/text_det.jpg\"')
94+
elif task in ['PoseTracker']:
95+
os.system(
96+
'wget https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/dance.mp4' # noqa: E501
97+
)
98+
java_command += ' dance.mp4\"'
99+
else:
100+
java_command += '\"'
101+
print(f'java_command: {java_command}')
102+
os.system('ant -DtaskName=' + task +
103+
' -DjarDir=${OPENCV_DIR}/build/bin ' +
104+
'-DlibDir=${OPENCV_DIR}/build/lib:$GITHUB_WORKSPACE/' +
105+
'build/lib -Dcommand=' + java_command)
81106

82107

83108
if __name__ == '__main__':

.github/workflows/java_api.yml

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -35,38 +35,45 @@ jobs:
3535
run: |
3636
wget https://download.java.net/java/GA/jdk18/43f95e8614114aeaa8e8a5fcf20a682d/36/GPL/openjdk-18_linux-x64_bin.tar.gz
3737
tar xvf openjdk-18_linux-x64_bin.tar.gz
38-
- name: gcc-multilib
39-
run: sudo apt install gcc-multilib g++-multilib wget libprotobuf-dev protobuf-compiler
38+
export JAVA_HOME=${PWD}/jdk-18
39+
export PATH=${JAVA_HOME}/bin:${PATH}
4040
- name: Install onnxruntime
4141
run: |
4242
wget https://github.com/microsoft/onnxruntime/releases/download/v1.8.1/onnxruntime-linux-x64-1.8.1.tgz
4343
tar -zxvf onnxruntime-linux-x64-1.8.1.tgz
44-
pushd onnxruntime-linux-x64-1.8.1
45-
export ONNXRUNTIME_DIR=${PWD}
44+
export ONNXRUNTIME_DIR=${PWD}/onnxruntime-linux-x64-1.8.1
4645
export LD_LIBRARY_PATH=$ONNXRUNTIME_DIR/lib:$LD_LIBRARY_PATH
47-
popd
48-
- name: Install opencv
46+
- name: Install OpenCV
4947
run: |
5048
sudo apt-get install libopencv-dev
51-
- name: Build java class
49+
sudo apt-get install ffmpeg libgstreamer1.0-0 libgstreamer-plugins-base1.0-0 libavcodec-dev libavformat-dev libswscale-dev
50+
- name: Install ant
51+
run: |
52+
sudo apt-get install ant
53+
- name: Build Java class
5254
run: |
5355
pushd csrc/mmdeploy/apis/java
5456
javac mmdeploy/*.java
5557
popd
56-
pushd demo/java
57-
javac -classpath ../../csrc/mmdeploy/apis/java/ Utils.java
58-
popd
59-
- name: Install mmdeploy with onnxruntime backend and java api
58+
- name: Install mmdeploy with onnxruntime backend and Java API
6059
run: |
6160
mkdir -p build && pushd build
62-
export LD_LIBRARY_PATH=/home/runner/work/mmdeploy/mmdeploy/ncnn/install/lib/:$LD_LIBRARY_PATH
63-
cmake -DMMDEPLOY_BUILD_SDK=ON -DMMDEPLOY_BUILD_SDK_JAVA_API=ON -DMMDEPLOY_TARGET_BACKENDS=ort -DMMDEPLOY_CODEBASES=all -DONNXRUNTIME_DIR=~/work/mmdeploy/mmdeploy/onnxruntime-linux-x64-1.8.1 ..
64-
make install
61+
export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/onnxruntime-linux-x64-1.8.1/lib/:$LD_LIBRARY_PATH
62+
cmake -DMMDEPLOY_BUILD_SDK=ON -DMMDEPLOY_BUILD_SDK_JAVA_API=ON -DMMDEPLOY_TARGET_BACKENDS=ort -DMMDEPLOY_CODEBASES=all -DONNXRUNTIME_DIR=$GITHUB_WORKSPACE/onnxruntime-linux-x64-1.8.1 ..
63+
make -j$(nproc) && make install
6564
popd
66-
- name: Test javademo
65+
- name: Prepare OpenCV Java and test Java demo
6766
run: |
68-
export JAVA_HOME=${PWD}/jdk-18
69-
export PATH=${JAVA_HOME}/bin:${PATH}
70-
export LD_LIBRARY_PATH=/build/lib:${LD_LIBRARY_PATH}
71-
java --version
67+
export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}
68+
export JAVA_AWT_LIBRARY=${JAVA_HOME}
69+
export JAVA_INCLUDE_PATH=${JAVA_HOME}/include
70+
export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/darwin
71+
export JAVA_JVM_LIBRARY=${JAVA_HOME}
72+
73+
mkdir -p opencv/build/bin
74+
wget https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/opencv-470.jar -P opencv/build/bin
75+
wget https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/opencv470-lib.tar -P opencv/build
76+
tar -xf opencv/build/opencv470-lib.tar -C opencv/build
77+
export OPENCV_DIR=$GITHUB_WORKSPACE/opencv
78+
export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/build/lib:${OPENCV_DIR}/build/lib:$LD_LIBRARY_PATH
7279
python .github/scripts/test_java_demo.py

csrc/mmdeploy/apis/java/CMakeLists.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,12 @@ add_jar(${PROJECT_NAME} SOURCES
2323
mmdeploy/TextRecognizer.java
2424
mmdeploy/Restorer.java
2525
mmdeploy/PoseDetector.java
26+
mmdeploy/Context.java
27+
mmdeploy/Device.java
28+
mmdeploy/Model.java
29+
mmdeploy/Profiler.java
30+
mmdeploy/Scheduler.java
31+
mmdeploy/PoseTracker.java
32+
mmdeploy/RotatedDetector.java
2633
OUTPUT_NAME mmdeploy
2734
OUTPUT_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})

csrc/mmdeploy/apis/java/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,14 @@ export JAVA_HOME=${PWD}/jdk-18
2020
export PATH=${JAVA_HOME}/bin:${PATH}
2121
```
2222

23-
**Step 3.** Switch default Java version:
23+
**Step 3.** Switch default Java version(optional):
2424

2525
```bash
2626
sudo update-alternatives --config java
2727
sudo update-alternatives --config javac
2828
```
2929

30-
You should select the version you will use.
30+
If you have multiple Java versions, you should select the version you will use.
3131

3232
### Installation
3333

csrc/mmdeploy/apis/java/mmdeploy/Classifier.java

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,56 @@
11
package mmdeploy;
22

3+
/** @description: the Java API class of Classifier. */
34
public class Classifier {
45
static {
56
System.loadLibrary("mmdeploy_java");
67
}
78

89
private final long handle;
910

11+
/** @description: Single classification result of a picture. */
1012
public static class Result {
13+
14+
/** Class id. */
1115
public int label_id;
16+
17+
/** Class score. */
1218
public float score;
19+
20+
/** Initializes a new instance of the Result class.
21+
* @param label_id: class id.
22+
* @param score: class score.
23+
*/
1324
public Result(int label_id, float score) {
1425
this.label_id = label_id;
1526
this.score = score;
1627
}
1728
}
1829

19-
public Classifier(String modelPath, String deviceName, int deviceId) {
30+
/** Initializes a new instance of the Classifier class.
31+
* @param modelPath: model path.
32+
* @param deviceName: device name.
33+
* @param deviceId: device ID.
34+
* @exception Exception: create Classifier failed exception.
35+
*/
36+
public Classifier(String modelPath, String deviceName, int deviceId) throws Exception{
2037
handle = create(modelPath, deviceName, deviceId);
38+
if (handle == -1) {
39+
throw new Exception("Create Classifier failed!");
40+
}
2141
}
2242

23-
public Result[][] apply(Mat[] images) {
43+
/** Get label information of each image in a batch.
44+
* @param images: input mats.
45+
* @return: results of each input mat.
46+
* @exception Exception: apply Classifier failed exception.
47+
*/
48+
public Result[][] apply(Mat[] images) throws Exception{
2449
int[] counts = new int[images.length];
2550
Result[] results = apply(handle, images, counts);
51+
if (results == null) {
52+
throw new Exception("Apply Classifier failed!");
53+
}
2654
Result[][] rets = new Result[images.length][];
2755
int offset = 0;
2856
for (int i = 0; i < images.length; ++i) {
@@ -36,12 +64,22 @@ public Result[][] apply(Mat[] images) {
3664
return rets;
3765
}
3866

39-
public Result[] apply(Mat image) {
67+
/** Get label information of one image.
68+
* @param image: input mat.
69+
* @return: result of input mat.
70+
* @exception Exception: apply Classifier failed exception.
71+
*/
72+
public Result[] apply(Mat image) throws Exception{
4073
int[] counts = new int[1];
4174
Mat[] images = new Mat[]{image};
42-
return apply(handle, images, counts);
75+
Result[] results = apply(handle, images, counts);
76+
if (results == null) {
77+
throw new Exception("Apply Classifier failed!");
78+
}
79+
return results;
4380
}
4481

82+
/** Release the instance of Classifier. */
4583
public void release() {
4684
destroy(handle);
4785
}
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
2+
package mmdeploy;
3+
4+
/** @description: the Context class. */
5+
public class Context {
6+
static {
7+
System.loadLibrary("mmdeploy_java");
8+
}
9+
10+
private final long contextHandle;
11+
12+
/** @description: ContextType. */
13+
public enum ContextType {
14+
DEVICE,
15+
STREAM,
16+
MODEL,
17+
SCHEDULER,
18+
MAT,
19+
PROFILER
20+
}
21+
22+
/** Initializes a new instance of the Context class. */
23+
public Context() {
24+
contextHandle = create();
25+
}
26+
27+
/** Add Model to the Context.
28+
* @param name: name.
29+
* @param model: model.
30+
*/
31+
public void add(String name, Model model) {
32+
add(contextHandle, ContextType.MODEL.ordinal(), name, model.handle());
33+
}
34+
35+
/** Add Scheduler to the Context.
36+
* @param name: name.
37+
* @param scheduler: scheduler.
38+
*/
39+
public void add(String name, Scheduler scheduler) {
40+
add(contextHandle, ContextType.SCHEDULER.ordinal(), name, scheduler.handle());
41+
}
42+
43+
/** Add Device to the Context.
44+
* @param device: device.
45+
*/
46+
public void add(Device device) {
47+
add(contextHandle, ContextType.DEVICE.ordinal(), "", device.handle());
48+
}
49+
50+
/** Add Profiler to the Context.
51+
* @param profiler: profiler.
52+
*/
53+
public void add(Profiler profiler) {
54+
add(contextHandle, ContextType.PROFILER.ordinal(), "", profiler.handle());
55+
}
56+
57+
/** Release the instance of Context. */
58+
public void release() {
59+
destroy(contextHandle);
60+
}
61+
62+
/** Get the handle of Context
63+
* @return: the handle of context.
64+
*/
65+
public long handle() {
66+
return contextHandle;
67+
}
68+
69+
private native long create();
70+
71+
public native int add(long context, int contextType, String name, long handle);
72+
73+
private native void destroy(long context);
74+
}

csrc/mmdeploy/apis/java/mmdeploy/DataType.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
package mmdeploy;
22

3+
/** @description: DataType. */
34
public enum DataType {
45
FLOAT(0),
56
HALF(1),
67
INT8(2),
78
INT32(3);
89
final int value;
910

11+
/** Initializes a new instance of the DataType class.
12+
* @param value: the value.
13+
*/
1014
DataType(int value) {
1115
this.value = value;
1216
}

0 commit comments

Comments
 (0)