[camera] Reland implementations of flip/change camera while recording (#3272)

[camera] Reland implementations of flip/change camera while recording

Author: BradenBagby

packages/camera/camera_android/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.10.5
+
+* Allows camera to be switched while video recording.
+
 ## 0.10.4+2
 
 * Aligns Dart and Flutter SDK constraints.

packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java

@@ -96,13 +96,28 @@ class Camera
    * Holds all of the camera features/settings and will be used to update the request builder when
    * one changes.
    */
-  private final CameraFeatures cameraFeatures;
+  private CameraFeatures cameraFeatures;
+
+  private String imageFormatGroup;
+
+  /**
+   * Takes an input/output surface and orients the recording correctly. This is needed because
+   * switching cameras while recording causes the wrong orientation.
+   */
+  private VideoRenderer videoRenderer;
+
+  /**
+   * Whether or not the camera aligns with the initial way the camera was facing if the camera was
+   * flipped.
+   */
+  private int initialCameraFacing;
 
   private final SurfaceTextureEntry flutterTexture;
+  private final ResolutionPreset resolutionPreset;
   private final boolean enableAudio;
   private final Context applicationContext;
   private final DartMessenger dartMessenger;
-  private final CameraProperties cameraProperties;
+  private CameraProperties cameraProperties;
   private final CameraFeatureFactory cameraFeatureFactory;
   private final Activity activity;
   /** A {@link CameraCaptureSession.CaptureCallback} that handles events related to JPEG capture. */
@@ -192,6 +207,7 @@ public Camera(
     this.applicationContext = activity.getApplicationContext();
     this.cameraProperties = cameraProperties;
     this.cameraFeatureFactory = cameraFeatureFactory;
+    this.resolutionPreset = resolutionPreset;
     this.cameraFeatures =
         CameraFeatures.init(
             cameraFeatureFactory, cameraProperties, activity, dartMessenger, resolutionPreset);
@@ -232,6 +248,7 @@ private void prepareMediaRecorder(String outputFilePath) throws IOException {
     if (mediaRecorder != null) {
       mediaRecorder.release();
     }
+    closeRenderer();
 
     final PlatformChannel.DeviceOrientation lockedOrientation =
         cameraFeatures.getSensorOrientation().getLockedCaptureOrientation();
@@ -259,6 +276,7 @@ private void prepareMediaRecorder(String outputFilePath) throws IOException {
 
   @SuppressLint("MissingPermission")
   public void open(String imageFormatGroup) throws CameraAccessException {
+    this.imageFormatGroup = imageFormatGroup;
     final ResolutionFeature resolutionFeature = cameraFeatures.getResolution();
 
     if (!resolutionFeature.checkIsSupported()) {
@@ -303,14 +321,16 @@ public void onOpened(@NonNull CameraDevice device) {
             cameraDevice = new DefaultCameraDeviceWrapper(device);
             try {
               startPreview();
+              if (!recordingVideo) // only send initialization if we werent already recording and switching cameras
               dartMessenger.sendCameraInitializedEvent(
-                  resolutionFeature.getPreviewSize().getWidth(),
-                  resolutionFeature.getPreviewSize().getHeight(),
-                  cameraFeatures.getExposureLock().getValue(),
-                  cameraFeatures.getAutoFocus().getValue(),
-                  cameraFeatures.getExposurePoint().checkIsSupported(),
-                  cameraFeatures.getFocusPoint().checkIsSupported());
-            } catch (CameraAccessException e) {
+                    resolutionFeature.getPreviewSize().getWidth(),
+                    resolutionFeature.getPreviewSize().getHeight(),
+                    cameraFeatures.getExposureLock().getValue(),
+                    cameraFeatures.getAutoFocus().getValue(),
+                    cameraFeatures.getExposurePoint().checkIsSupported(),
+                    cameraFeatures.getFocusPoint().checkIsSupported());
+
+            } catch (CameraAccessException | InterruptedException e) {
               dartMessenger.sendCameraErrorEvent(e.getMessage());
               close();
             }
@@ -320,7 +340,8 @@ public void onOpened(@NonNull CameraDevice device) {
           public void onClosed(@NonNull CameraDevice camera) {
             Log.i(TAG, "open | onClosed");
 
-            // Prevents calls to methods that would otherwise result in IllegalStateException exceptions.
+            // Prevents calls to methods that would otherwise result in IllegalStateException
+            // exceptions.
             cameraDevice = null;
             closeCaptureSession();
             dartMessenger.sendCameraClosingEvent();
@@ -735,7 +756,7 @@ public void startVideoRecording(
     if (imageStreamChannel != null) {
       setStreamHandler(imageStreamChannel);
     }
-
+    initialCameraFacing = cameraProperties.getLensFacing();
     recordingVideo = true;
     try {
       startCapture(true, imageStreamChannel != null);
@@ -747,6 +768,13 @@ public void startVideoRecording(
     }
   }
 
+  private void closeRenderer() {
+    if (videoRenderer != null) {
+      videoRenderer.close();
+      videoRenderer = null;
+    }
+  }
+
   public void stopVideoRecording(@NonNull final Result result) {
     if (!recordingVideo) {
       result.success(null);
@@ -757,6 +785,7 @@ public void stopVideoRecording(@NonNull final Result result) {
         cameraFeatureFactory.createAutoFocusFeature(cameraProperties, false));
     recordingVideo = false;
     try {
+      closeRenderer();
       captureSession.abortCaptures();
       mediaRecorder.stop();
     } catch (CameraAccessException | IllegalStateException e) {
@@ -765,7 +794,7 @@ public void stopVideoRecording(@NonNull final Result result) {
     mediaRecorder.reset();
     try {
       startPreview();
-    } catch (CameraAccessException | IllegalStateException e) {
+    } catch (CameraAccessException | IllegalStateException | InterruptedException e) {
       result.error("videoRecordingFailed", e.getMessage(), null);
       return;
     }
@@ -1049,13 +1078,50 @@ public void resumePreview() {
         null, (code, message) -> dartMessenger.sendCameraErrorEvent(message));
   }
 
-  public void startPreview() throws CameraAccessException {
+  public void startPreview() throws CameraAccessException, InterruptedException {
+    // If recording is already in progress, the camera is being flipped, so send it through the VideoRenderer to keep the correct orientation.
+    if (recordingVideo) {
+      startPreviewWithVideoRendererStream();
+    } else {
+      startRegularPreview();
+    }
+  }
+
+  private void startRegularPreview() throws CameraAccessException {
     if (pictureImageReader == null || pictureImageReader.getSurface() == null) return;
     Log.i(TAG, "startPreview");
-
     createCaptureSession(CameraDevice.TEMPLATE_PREVIEW, pictureImageReader.getSurface());
   }
 
+  private void startPreviewWithVideoRendererStream()
+      throws CameraAccessException, InterruptedException {
+    if (videoRenderer == null) return;
+
+    // get rotation for rendered video
+    final PlatformChannel.DeviceOrientation lockedOrientation =
+        cameraFeatures.getSensorOrientation().getLockedCaptureOrientation();
+    DeviceOrientationManager orientationManager =
+        cameraFeatures.getSensorOrientation().getDeviceOrientationManager();
+
+    int rotation = 0;
+    if (orientationManager != null) {
+      rotation =
+          lockedOrientation == null
+              ? orientationManager.getVideoOrientation()
+              : orientationManager.getVideoOrientation(lockedOrientation);
+    }
+
+    if (cameraProperties.getLensFacing() != initialCameraFacing) {
+
+      // If the new camera is facing the opposite way than the initial recording,
+      // the rotation should be flipped 180 degrees.
+      rotation = (rotation + 180) % 360;
+    }
+    videoRenderer.setRotation(rotation);
+
+    createCaptureSession(CameraDevice.TEMPLATE_RECORD, videoRenderer.getInputSurface());
+  }
+
   public void startPreviewWithImageStream(EventChannel imageStreamChannel)
       throws CameraAccessException {
     setStreamHandler(imageStreamChannel);
@@ -1179,17 +1245,7 @@ private void closeCaptureSession() {
   public void close() {
     Log.i(TAG, "close");
 
-    if (cameraDevice != null) {
-      cameraDevice.close();
-      cameraDevice = null;
-
-      // Closing the CameraDevice without closing the CameraCaptureSession is recommended
-      // for quickly closing the camera:
-      // https://developer.android.com/reference/android/hardware/camera2/CameraCaptureSession#close()
-      captureSession = null;
-    } else {
-      closeCaptureSession();
-    }
+    stopAndReleaseCamera();
 
     if (pictureImageReader != null) {
       pictureImageReader.close();
@@ -1208,6 +1264,75 @@ public void close() {
     stopBackgroundThread();
   }
 
+  private void stopAndReleaseCamera() {
+    if (cameraDevice != null) {
+      cameraDevice.close();
+      cameraDevice = null;
+
+      // Closing the CameraDevice without closing the CameraCaptureSession is recommended
+      // for quickly closing the camera:
+      // https://developer.android.com/reference/android/hardware/camera2/CameraCaptureSession#close()
+      captureSession = null;
+    } else {
+      closeCaptureSession();
+    }
+  }
+
+  private void prepareVideoRenderer() {
+    if (videoRenderer != null) return;
+    final ResolutionFeature resolutionFeature = cameraFeatures.getResolution();
+
+    // handle videoRenderer errors
+    Thread.UncaughtExceptionHandler videoRendererUncaughtExceptionHandler =
+        new Thread.UncaughtExceptionHandler() {
+          @Override
+          public void uncaughtException(Thread thread, Throwable ex) {
+            dartMessenger.sendCameraErrorEvent(
+                "Failed to process frames after camera was flipped.");
+          }
+        };
+
+    videoRenderer =
+        new VideoRenderer(
+            mediaRecorder.getSurface(),
+            resolutionFeature.getCaptureSize().getWidth(),
+            resolutionFeature.getCaptureSize().getHeight(),
+            videoRendererUncaughtExceptionHandler);
+  }
+
+  public void setDescriptionWhileRecording(
+      @NonNull final Result result, CameraProperties properties) {
+
+    if (!recordingVideo) {
+      result.error("setDescriptionWhileRecordingFailed", "Device was not recording", null);
+      return;
+    }
+
+    // See VideoRenderer.java requires API 26 to switch camera while recording
+    if (android.os.Build.VERSION.SDK_INT < android.os.Build.VERSION_CODES.O) {
+      result.error(
+          "setDescriptionWhileRecordingFailed",
+          "Device does not support switching the camera while recording",
+          null);
+      return;
+    }
+
+    stopAndReleaseCamera();
+    prepareVideoRenderer();
+    cameraProperties = properties;
+    cameraFeatures =
+        CameraFeatures.init(
+            cameraFeatureFactory, cameraProperties, activity, dartMessenger, resolutionPreset);
+    cameraFeatures.setAutoFocus(
+        cameraFeatureFactory.createAutoFocusFeature(cameraProperties, true));
+    try {
+      open(imageFormatGroup);
+    } catch (CameraAccessException e) {
+      result.error("setDescriptionWhileRecordingFailed", e.getMessage(), null);
+    }
+    result.success(null);
+  }
+
   public void dispose() {
     Log.i(TAG, "dispose");
 

packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java

@@ -354,6 +354,18 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull final Result result)
           result.success(null);
           break;
         }
+      case "setDescriptionWhileRecording":
+        {
+          try {
+            String cameraName = call.argument("cameraName");
+            CameraProperties cameraProperties =
+                new CameraPropertiesImpl(cameraName, CameraUtils.getCameraManager(activity));
+            camera.setDescriptionWhileRecording(result, cameraProperties);
+          } catch (Exception e) {
+            handleException(e, result);
+          }
+          break;
+        }
       case "dispose":
         {
           if (camera != null) {