(Java) Add support for resolution alignment during encoding (#25)

kanat · web-flow · commit c3b9942b585b · 2024-03-26T13:35:47.000-07:00
* (Java) Add support for resolution alignment during encoding

* rename native to delegate

* extract HardwareVideoEncoderWrapperFactory

* change class order

* add missing import

* compile fixes

* fix logging

* fix unreachable return statement

* fix dependencies

* move ResolutionAdjustment to video_java
diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn
@@ -232,6 +232,7 @@ if (is_android) {
       "api/org/webrtc/WrappedNativeVideoEncoder.java",
       "api/org/webrtc/YuvConverter.java",
       "api/org/webrtc/YuvHelper.java",
+      "api/org/webrtc/ResolutionAdjustment.java",
       "src/java/org/webrtc/EglBase10Impl.java",
       "src/java/org/webrtc/EglBase14Impl.java",
       "src/java/org/webrtc/GlGenericDrawer.java",
@@ -363,6 +364,8 @@ if (is_android) {
       "api/org/webrtc/DefaultVideoDecoderFactory.java",
       "api/org/webrtc/DefaultVideoEncoderFactory.java",
       "api/org/webrtc/WrappedVideoDecoderFactory.java",
+      "api/org/webrtc/DefaultAlignedVideoEncoderFactory.java",
+      "api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java",
     ]
 
     deps = [
@@ -394,6 +397,8 @@ if (is_android) {
     sources = [
       "api/org/webrtc/HardwareVideoDecoderFactory.java",
       "api/org/webrtc/HardwareVideoEncoderFactory.java",
+      "api/org/webrtc/HardwareVideoEncoderWrapper.java",
+      "api/org/webrtc/HardwareVideoEncoderWrapperFactory.java",
       "api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java",
       "src/java/org/webrtc/AndroidVideoDecoder.java",
       "src/java/org/webrtc/BaseBitrateAdjuster.java",
diff --git a/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.webrtc;
+
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+
+/**
+ * The main difference with the standard [DefaultAlignedVideoEncoderFactory] is that this fixes
+ * issues with resolutions that are not aligned (e.g. VP8 requires 16x16 alignment). You can
+ * set the alignment by setting [resolutionAdjustment]. Internally the resolution during streaming
+ * will be cropped to comply with the adjustment. Fallback behaviour is the same as with the
+ * standard [DefaultVideoEncoderFactory] and it will use the SW encoder if HW fails
+ * or is not available.
+ *
+ * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf3042072
+ * e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/SimulcastVideoEnco
+ * derFactoryWrapper.kt#L18
+ */
+public class DefaultAlignedVideoEncoderFactory implements VideoEncoderFactory {
+    private final VideoEncoderFactory hardwareVideoEncoderFactory;
+    private final VideoEncoderFactory softwareVideoEncoderFactory;
+
+    public DefaultAlignedVideoEncoderFactory(
+            EglBase.Context eglContext,
+            boolean enableIntelVp8Encoder,
+            boolean enableH264HighProfile,
+            ResolutionAdjustment resolutionAdjustment
+    ) {
+        HardwareVideoEncoderFactory defaultFactory =
+                new HardwareVideoEncoderFactory(eglContext, enableIntelVp8Encoder, enableH264HighProfile);
+        hardwareVideoEncoderFactory = (resolutionAdjustment == ResolutionAdjustment.NONE) ?
+                defaultFactory :
+                new HardwareVideoEncoderWrapperFactory(defaultFactory, resolutionAdjustment.getValue());
+        softwareVideoEncoderFactory = new SoftwareVideoEncoderFactory();
+    }
+
+    @Override
+    public VideoEncoder createEncoder(VideoCodecInfo info) {
+        VideoEncoder softwareEncoder = softwareVideoEncoderFactory.createEncoder(info);
+        VideoEncoder hardwareEncoder = hardwareVideoEncoderFactory.createEncoder(info);
+        if (hardwareEncoder != null && softwareEncoder != null) {
+            return new VideoEncoderFallback(softwareEncoder, hardwareEncoder);
+        }
+        return hardwareEncoder != null ? hardwareEncoder : softwareEncoder;
+    }
+
+    @Override
+    public VideoCodecInfo[] getSupportedCodecs() {
+        LinkedHashSet<VideoCodecInfo> supportedCodecInfos = new LinkedHashSet<>();
+        supportedCodecInfos.addAll(Arrays.asList(softwareVideoEncoderFactory.getSupportedCodecs()));
+        supportedCodecInfos.addAll(Arrays.asList(hardwareVideoEncoderFactory.getSupportedCodecs()));
+        return supportedCodecInfos.toArray(new VideoCodecInfo[0]);
+    }
+}
+
diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java
@@ -0,0 +1,219 @@
+/*
+ * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.webrtc;
+
+/**
+ * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf304207
+ * 2e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/HardwareVideoEnco
+ * derWrapperFactory.kt
+ */
+class HardwareVideoEncoderWrapper implements VideoEncoder {
+
+    private static final String TAG = "HardwareVideoEncoderWrapper";
+
+    private final VideoEncoder internalEncoder;
+    private final int alignment;
+
+    public HardwareVideoEncoderWrapper(VideoEncoder internalEncoder, int alignment) {
+        this.internalEncoder = internalEncoder;
+        this.alignment = alignment;
+    }
+
+    private static class CropSizeCalculator {
+
+        private static final String TAG = "CropSizeCalculator";
+
+        private final int alignment;
+        private final int originalWidth;
+        private final int originalHeight;
+        private final int cropX;
+        private final int cropY;
+
+        public CropSizeCalculator(int alignment, int originalWidth, int originalHeight) {
+            this.alignment = alignment;
+            this.originalWidth = originalWidth;
+            this.originalHeight = originalHeight;
+            this.cropX = originalWidth % alignment;
+            this.cropY = originalHeight % alignment;
+            if (originalWidth != 0 && originalHeight != 0) {
+                Logging.v(TAG, "init(): alignment=" + alignment +
+                        " size=" + originalWidth + "x" + originalHeight + " => " + getCroppedWidth() + "x" + getCroppedHeight());
+            }
+        }
+
+        public int getCroppedWidth() {
+            return originalWidth - cropX;
+        }
+
+        public int getCroppedHeight() {
+            return originalHeight - cropY;
+        }
+
+        public boolean isCropRequired() {
+            return cropX != 0 || cropY != 0;
+        }
+
+        public boolean hasFrameSizeChanged(int nextWidth, int nextHeight) {
+            if (originalWidth == nextWidth && originalHeight == nextHeight) {
+                return false;
+            } else {
+                Logging.v(TAG, "frame size has changed: " +
+                        originalWidth + "x" + originalHeight + " => " + nextWidth + "x" + nextHeight);
+                return true;
+            }
+        }
+    }
+
+    private CropSizeCalculator calculator = new CropSizeCalculator(1, 0, 0);
+
+    private VideoCodecStatus retryWithoutCropping(int width, int height, Runnable retryFunc) {
+        Logging.v(TAG, "retrying without resolution adjustment");
+        calculator = new CropSizeCalculator(1, width, height);
+        retryFunc.run();
+        return VideoCodecStatus.OK;
+    }
+
+    @Override
+    public VideoCodecStatus initEncode(VideoEncoder.Settings originalSettings, VideoEncoder.Callback callback) {
+        calculator = new CropSizeCalculator(alignment, originalSettings.width, originalSettings.height);
+        if (!calculator.isCropRequired()) {
+            return internalEncoder.initEncode(originalSettings, callback);
+        } else {
+            VideoEncoder.Settings croppedSettings = new VideoEncoder.Settings(
+                    originalSettings.numberOfCores,
+                    calculator.getCroppedWidth(),
+                    calculator.getCroppedHeight(),
+                    originalSettings.startBitrate,
+                    originalSettings.maxFramerate,
+                    originalSettings.numberOfSimulcastStreams,
+                    originalSettings.automaticResizeOn,
+                    originalSettings.capabilities
+            );
+            try {
+                VideoCodecStatus result = internalEncoder.initEncode(croppedSettings, callback);
+                if (result == VideoCodecStatus.FALLBACK_SOFTWARE) {
+                    Logging.e(TAG, "internalEncoder.initEncode() returned FALLBACK_SOFTWARE: " +
+                            "croppedSettings " + croppedSettings);
+                    return retryWithoutCropping(
+                            originalSettings.width,
+                            originalSettings.height,
+                            () -> internalEncoder.initEncode(originalSettings, callback)
+                    );
+                } else {
+                    return result;
+                }
+            } catch (Exception e) {
+                Logging.e(TAG, "internalEncoder.initEncode() failed", e);
+                return retryWithoutCropping(
+                        originalSettings.width,
+                        originalSettings.height,
+                        () -> internalEncoder.initEncode(originalSettings, callback)
+                );
+            }
+        }
+    }
+
+    @Override
+    public VideoCodecStatus release() {
+        return internalEncoder.release();
+    }
+
+    @Override
+    public VideoCodecStatus encode(VideoFrame frame, VideoEncoder.EncodeInfo encodeInfo) {
+        if (calculator.hasFrameSizeChanged(frame.getBuffer().getWidth(), frame.getBuffer().getHeight())) {
+            calculator = new CropSizeCalculator(alignment, frame.getBuffer().getWidth(), frame.getBuffer().getHeight());
+        }
+        if (!calculator.isCropRequired()) {
+            return internalEncoder.encode(frame, encodeInfo);
+        } else {
+            int croppedWidth = calculator.getCroppedWidth();
+            int croppedHeight = calculator.getCroppedHeight();
+            VideoFrame.Buffer croppedBuffer = frame.getBuffer().cropAndScale(
+                    calculator.cropX / 2,
+                    calculator.cropY / 2,
+                    croppedWidth,
+                    croppedHeight,
+                    croppedWidth,
+                    croppedHeight
+            );
+            VideoFrame croppedFrame = new VideoFrame(croppedBuffer, frame.getRotation(), frame.getTimestampNs());
+            try {
+                VideoCodecStatus result = internalEncoder.encode(croppedFrame, encodeInfo);
+                if (result == VideoCodecStatus.FALLBACK_SOFTWARE) {
+                    Logging.e(TAG, "internalEncoder.encode() returned FALLBACK_SOFTWARE");
+                    return retryWithoutCropping(
+                            frame.getBuffer().getWidth(),
+                            frame.getBuffer().getHeight(),
+                            () -> internalEncoder.encode(frame, encodeInfo)
+                    );
+                } else {
+                    return result;
+                }
+            } catch (Exception e) {
+                Logging.e(TAG, "internalEncoder.encode() failed", e);
+                return retryWithoutCropping(
+                        frame.getBuffer().getWidth(),
+                        frame.getBuffer().getHeight(),
+                        () -> internalEncoder.encode(frame, encodeInfo)
+                );
+            } finally {
+                croppedBuffer.release();
+            }
+        }
+    }
+
+    @Override
+    public VideoCodecStatus setRateAllocation(VideoEncoder.BitrateAllocation allocation, int frameRate) {
+        return internalEncoder.setRateAllocation(allocation, frameRate);
+    }
+
+    @Override
+    public VideoEncoder.ScalingSettings getScalingSettings() {
+        return internalEncoder.getScalingSettings();
+    }
+
+    @Override
+    public String getImplementationName() {
+        return internalEncoder.getImplementationName();
+    }
+
+    @Override
+    public long createNativeVideoEncoder() {
+        return internalEncoder.createNativeVideoEncoder();
+    }
+
+    @Override
+    public boolean isHardwareEncoder() {
+        return internalEncoder.isHardwareEncoder();
+    }
+
+    @Override
+    public VideoCodecStatus setRates(VideoEncoder.RateControlParameters rcParameters) {
+        return internalEncoder.setRates(rcParameters);
+    }
+
+    @Override
+    public VideoEncoder.ResolutionBitrateLimits[] getResolutionBitrateLimits() {
+        return internalEncoder.getResolutionBitrateLimits();
+    }
+
+    @Override
+    public VideoEncoder.EncoderInfo getEncoderInfo() {
+        return internalEncoder.getEncoderInfo();
+    }
+}
+
+
diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.webrtc;
+
+/**
+ * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf304207
+ * 2e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/HardwareVideoEnco
+ * derWrapperFactory.kt
+ */
+class HardwareVideoEncoderWrapperFactory implements VideoEncoderFactory {
+
+    private static final String TAG = "HardwareVideoEncoderWrapperFactory";
+
+    private final HardwareVideoEncoderFactory factory;
+    private final int resolutionPixelAlignment;
+
+    public HardwareVideoEncoderWrapperFactory(HardwareVideoEncoderFactory factory, int resolutionPixelAlignment) {
+        this.factory = factory;
+        this.resolutionPixelAlignment = resolutionPixelAlignment;
+        if (resolutionPixelAlignment == 0) {
+            throw new IllegalArgumentException("resolutionPixelAlignment should not be 0");
+        }
+    }
+
+    @Override
+    public VideoEncoder createEncoder(VideoCodecInfo videoCodecInfo) {
+        try {
+            VideoEncoder encoder = factory.createEncoder(videoCodecInfo);
+            if (encoder == null) {
+                return null;
+            }
+            return new HardwareVideoEncoderWrapper(encoder, resolutionPixelAlignment);
+        } catch (Exception e) {
+            Logging.e(TAG, "createEncoder failed", e);
+            return null;
+        }
+    }
+
+    @Override
+    public VideoCodecInfo[] getSupportedCodecs() {
+        return factory.getSupportedCodecs();
+    }
+}
diff --git a/sdk/android/api/org/webrtc/ResolutionAdjustment.java b/sdk/android/api/org/webrtc/ResolutionAdjustment.java
diff --git a/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java
