diff --git a/play-services-base/src/main/java/com/google/android/gms/common/GooglePlayServicesUtil.java b/play-services-base/src/main/java/com/google/android/gms/common/GooglePlayServicesUtil.java index 4850b81658..fd168573b2 100644 --- a/play-services-base/src/main/java/com/google/android/gms/common/GooglePlayServicesUtil.java +++ b/play-services-base/src/main/java/com/google/android/gms/common/GooglePlayServicesUtil.java @@ -136,7 +136,11 @@ public static String getOpenSourceSoftwareLicenseInfo(Context context) { * @return The Context object of the Buddy APK or null if the Buddy APK is not installed on the device. */ public static Context getRemoteContext(Context context) { - return null; // TODO + try { + return context.createPackageContext(Constants.GMS_PACKAGE_NAME, Context.CONTEXT_INCLUDE_CODE | Context.CONTEXT_IGNORE_SECURITY); + } catch (PackageManager.NameNotFoundException unused) { + return null; + } } /** @@ -145,7 +149,11 @@ public static Context getRemoteContext(Context context) { * @return The Resources object of the Buddy APK or null if the Buddy APK is not installed on the device. */ public static Resources getRemoteResources(Context context) { - return null; // TODO + try { + return context.getPackageManager().getResourcesForApplication(Constants.GMS_PACKAGE_NAME); + } catch (PackageManager.NameNotFoundException unused) { + return null; + } } /** diff --git a/play-services-core/src/main/AndroidManifest.xml b/play-services-core/src/main/AndroidManifest.xml index b9957108eb..4d8ffaddee 100644 --- a/play-services-core/src/main/AndroidManifest.xml +++ b/play-services-core/src/main/AndroidManifest.xml @@ -188,7 +188,8 @@ androidx.compose.ui.graphics, androidx.compose.ui.geometry, androidx.compose.ui.tooling.preview, - androidx.compose.runtime.saveable" + androidx.compose.runtime.saveable, + org.opencv" /> sContextCache = new WeakHashMap<>(); + // WeakHashMap cannot be used, and there is a high probability that it will be recycled, causing ClassLoader to be rebuilt + private static final Map sClassLoaderCache = new HashMap<>(); + + public static DynamiteContext createDynamiteContext(String moduleId, Context originalContext) { + if (originalContext == null) { + Log.w(TAG, "create Original context is null"); + return null; + } + String cacheKey = moduleId + "-" + originalContext.getPackageName(); + synchronized (sContextCache) { + DynamiteContext cached = sContextCache.get(cacheKey); + if (cached != null) { + Log.d(TAG, "Using cached DynamiteContext for cacheKey: " + cacheKey); + return cached; + } + } + try { + DynamiteModuleInfo moduleInfo = new DynamiteModuleInfo(moduleId); + Context gmsContext = originalContext.createPackageContext(Constants.GMS_PACKAGE_NAME, 0); + Context originalAppContext = originalContext.getApplicationContext(); + + DynamiteContext dynamiteContext; + if (originalAppContext == null || originalAppContext == originalContext) { + dynamiteContext = new DynamiteContext(moduleInfo, originalContext, gmsContext, null); + } else { + dynamiteContext = new DynamiteContext(moduleInfo, originalContext, gmsContext, new DynamiteContext(moduleInfo, originalAppContext, gmsContext, null)); + } + moduleInfo.init(dynamiteContext); + + synchronized (sContextCache) { + sContextCache.put(cacheKey, dynamiteContext); + } + Log.d(TAG, "Created and cached a new DynamiteContext for cacheKey: " + cacheKey); + return dynamiteContext; + } catch (PackageManager.NameNotFoundException e) { + Log.w(TAG, e); + return null; + } + } + + public static ClassLoader createClassLoader(DynamiteModuleInfo moduleInfo, Context gmsContext, Context originalContext) { + String cacheKey = moduleInfo.getModuleId() + "-" + originalContext.getPackageName(); + synchronized (sClassLoaderCache) { + ClassLoader cached = sClassLoaderCache.get(cacheKey); + if (cached != null) { + Log.d(TAG, "Using cached ClassLoader for cacheKey: " + cacheKey + " cached: " + cached.hashCode()); + return cached; + } + } + StringBuilder nativeLoaderDirs = new StringBuilder(gmsContext.getApplicationInfo().nativeLibraryDir); + if (SDK_INT >= 23 && Process.is64Bit()) { + for (String abi : SUPPORTED_64_BIT_ABIS) { + nativeLoaderDirs.append(File.pathSeparator).append(gmsContext.getApplicationInfo().sourceDir).append("!/lib/").append(abi); + } + } else if (SDK_INT >= 21) { + for (String abi : SUPPORTED_32_BIT_ABIS) { + nativeLoaderDirs.append(File.pathSeparator).append(gmsContext.getApplicationInfo().sourceDir).append("!/lib/").append(abi); + } + } else { + nativeLoaderDirs.append(File.pathSeparator).append(gmsContext.getApplicationInfo().sourceDir).append("!/lib/").append(CPU_ABI); + } + ClassLoader classLoader = new PathClassLoader(gmsContext.getApplicationInfo().sourceDir, nativeLoaderDirs.toString(), new FilteredClassLoader(originalContext.getClassLoader(), moduleInfo.getMergedClasses(), moduleInfo.getMergedPackages())); + synchronized (sClassLoaderCache) { + sClassLoaderCache.put(cacheKey, classLoader); + } + Log.d(TAG, "Created and cached a new ClassLoader for cacheKey: " + cacheKey + " ClassLoader: " + classLoader.hashCode()); + return classLoader; + } +} + diff --git a/play-services-core/src/main/java/com/google/android/gms/chimera/container/DynamiteContext.java b/play-services-core/src/main/java/com/google/android/gms/chimera/container/DynamiteContext.java index f93fdae0d3..608cc3c920 100644 --- a/play-services-core/src/main/java/com/google/android/gms/chimera/container/DynamiteContext.java +++ b/play-services-core/src/main/java/com/google/android/gms/chimera/container/DynamiteContext.java @@ -8,22 +8,10 @@ import android.content.Context; import android.content.ContextWrapper; import android.content.pm.ApplicationInfo; -import android.content.pm.PackageManager; -import android.os.Process; -import android.util.Log; import androidx.annotation.RequiresApi; -import org.microg.gms.common.Constants; - -import java.io.File; - -import dalvik.system.PathClassLoader; - -import static android.os.Build.CPU_ABI; -import static android.os.Build.SUPPORTED_32_BIT_ABIS; -import static android.os.Build.SUPPORTED_64_BIT_ABIS; -import static android.os.Build.VERSION.SDK_INT; +import com.google.android.gms.chimera.DynamiteContextFactory; public class DynamiteContext extends ContextWrapper { private static final String TAG = "DynamiteContext"; @@ -45,19 +33,7 @@ public DynamiteContext(DynamiteModuleInfo moduleInfo, Context base, Context gmsC @Override public ClassLoader getClassLoader() { if (classLoader == null) { - StringBuilder nativeLoaderDirs = new StringBuilder(gmsContext.getApplicationInfo().nativeLibraryDir); - if (SDK_INT >= 23 && Process.is64Bit()) { - for (String abi : SUPPORTED_64_BIT_ABIS) { - nativeLoaderDirs.append(File.pathSeparator).append(gmsContext.getApplicationInfo().sourceDir).append("!/lib/").append(abi); - } - } else if (SDK_INT >= 21) { - for (String abi : SUPPORTED_32_BIT_ABIS) { - nativeLoaderDirs.append(File.pathSeparator).append(gmsContext.getApplicationInfo().sourceDir).append("!/lib/").append(abi); - } - } else { - nativeLoaderDirs.append(File.pathSeparator).append(gmsContext.getApplicationInfo().sourceDir).append("!/lib/").append(CPU_ABI); - } - classLoader = new PathClassLoader(gmsContext.getApplicationInfo().sourceDir, nativeLoaderDirs.toString(), new FilteredClassLoader(originalContext.getClassLoader(), moduleInfo.getMergedClasses(), moduleInfo.getMergedPackages())); + classLoader = DynamiteContextFactory.createClassLoader(moduleInfo, gmsContext, originalContext); } return classLoader; } @@ -82,23 +58,4 @@ public Context getApplicationContext() { public Context createDeviceProtectedStorageContext() { return new DynamiteContext(moduleInfo, originalContext.createDeviceProtectedStorageContext(), gmsContext.createDeviceProtectedStorageContext(), appContext); } - - public static DynamiteContext create(String moduleId, Context originalContext) { - try { - DynamiteModuleInfo moduleInfo = new DynamiteModuleInfo(moduleId); - Context gmsContext = originalContext.createPackageContext(Constants.GMS_PACKAGE_NAME, 0); - Context originalAppContext = originalContext.getApplicationContext(); - DynamiteContext dynamiteContext; - if (originalAppContext == null || originalAppContext == originalContext) { - dynamiteContext = new DynamiteContext(moduleInfo, originalContext, gmsContext, null); - } else { - dynamiteContext = new DynamiteContext(moduleInfo, originalContext, gmsContext, new DynamiteContext(moduleInfo, originalAppContext, gmsContext, null)); - } - moduleInfo.init(dynamiteContext); - return dynamiteContext; - } catch (PackageManager.NameNotFoundException e) { - Log.w(TAG, e); - return null; - } - } } diff --git a/play-services-core/src/main/java/com/google/android/gms/chimera/container/DynamiteLoaderImpl.java b/play-services-core/src/main/java/com/google/android/gms/chimera/container/DynamiteLoaderImpl.java index b60bc263cb..07be4d28ec 100644 --- a/play-services-core/src/main/java/com/google/android/gms/chimera/container/DynamiteLoaderImpl.java +++ b/play-services-core/src/main/java/com/google/android/gms/chimera/container/DynamiteLoaderImpl.java @@ -17,19 +17,14 @@ package com.google.android.gms.chimera.container; import android.content.Context; -import android.content.ContextWrapper; -import android.content.pm.PackageManager; import android.os.RemoteException; import android.util.Log; +import com.google.android.gms.chimera.DynamiteContextFactory; import com.google.android.gms.dynamic.IObjectWrapper; import com.google.android.gms.dynamic.ObjectWrapper; import com.google.android.gms.dynamite.IDynamiteLoader; -import org.microg.gms.common.Constants; - -import java.lang.reflect.Field; - public class DynamiteLoaderImpl extends IDynamiteLoader.Stub { private static final String TAG = "GmsDynamiteLoaderImpl"; @@ -43,7 +38,7 @@ public IObjectWrapper createModuleContext(IObjectWrapper wrappedContext, String public IObjectWrapper createModuleContextV2(IObjectWrapper wrappedContext, String moduleId, int minVersion) throws RemoteException { Log.d(TAG, "createModuleContext for " + moduleId + " at version " + minVersion); final Context originalContext = (Context) ObjectWrapper.unwrap(wrappedContext); - return ObjectWrapper.wrap(DynamiteContext.create(moduleId, originalContext)); + return ObjectWrapper.wrap(DynamiteContextFactory.createDynamiteContext(moduleId, originalContext)); } @Override diff --git a/play-services-mlkit/face-detection/build.gradle b/play-services-mlkit/face-detection/build.gradle new file mode 100644 index 0000000000..b372d0cd75 --- /dev/null +++ b/play-services-mlkit/face-detection/build.gradle @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +apply plugin: 'com.android.library' +apply plugin: 'maven-publish' +apply plugin: 'signing' + +android { + namespace "com.google.mlkit.vision.face" + + compileSdkVersion androidCompileSdk + buildToolsVersion "$androidBuildVersionTools" + + buildFeatures { + aidl = true + } + + defaultConfig { + versionName version + minSdkVersion androidMinSdk + targetSdkVersion androidTargetSdk + } + + compileOptions { + sourceCompatibility = 1.8 + targetCompatibility = 1.8 + } +} + +apply from: '../../gradle/publish-android.gradle' + +description = 'microG implementation of play-services-mlkit-face-detection' + +dependencies { + // Dependencies from play-services-mlkit-face-detection:17.1.0 + api project(':play-services-base') + api project(':play-services-basement') + api project(':play-services-tasks') + + annotationProcessor project(":safe-parcel-processor") +} diff --git a/play-services-mlkit/face-detection/src/main/AndroidManifest.xml b/play-services-mlkit/face-detection/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..7918bbe131 --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/AndroidManifest.xml @@ -0,0 +1,9 @@ + + + + + + \ No newline at end of file diff --git a/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/FaceDetectionOptions.aidl b/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/FaceDetectionOptions.aidl new file mode 100644 index 0000000000..e031d3d504 --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/FaceDetectionOptions.aidl @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face; + +parcelable FaceDetectionOptions; \ No newline at end of file diff --git a/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/FrameMetadataParcel.aidl b/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/FrameMetadataParcel.aidl new file mode 100644 index 0000000000..38b2434bca --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/FrameMetadataParcel.aidl @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face; + +parcelable FrameMetadataParcel; \ No newline at end of file diff --git a/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/aidls/FaceParcel.aidl b/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/aidls/FaceParcel.aidl new file mode 100644 index 0000000000..f4017bee75 --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/aidls/FaceParcel.aidl @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face.aidls; + +parcelable FaceParcel; \ No newline at end of file diff --git a/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/aidls/IFaceDetector.aidl b/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/aidls/IFaceDetector.aidl new file mode 100644 index 0000000000..04fcaeb0fa --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/aidls/IFaceDetector.aidl @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face.aidls; + +import com.google.android.gms.dynamic.IObjectWrapper; +import java.util.List; +import com.google.mlkit.vision.face.FrameMetadataParcel; +import com.google.mlkit.vision.face.aidls.FaceParcel; + +interface IFaceDetector { + void initDetector() = 0; + void close() = 1; + List detectFaces(IObjectWrapper wrapper, in FrameMetadataParcel metadata) = 2; +} \ No newline at end of file diff --git a/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/aidls/IFaceDetectorCreator.aidl b/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/aidls/IFaceDetectorCreator.aidl new file mode 100644 index 0000000000..0fd043792e --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/aidl/com/google/mlkit/vision/face/aidls/IFaceDetectorCreator.aidl @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face.aidls; + +import com.google.mlkit.vision.face.aidls.IFaceDetector; +import com.google.android.gms.dynamic.IObjectWrapper; +import com.google.mlkit.vision.face.FaceDetectionOptions; + +interface IFaceDetectorCreator { + IFaceDetector newFaceDetector(IObjectWrapper context, in FaceDetectionOptions faceDetectionOptions) = 0; +} \ No newline at end of file diff --git a/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/Face.java b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/Face.java new file mode 100644 index 0000000000..a6f0e2a668 --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/Face.java @@ -0,0 +1,204 @@ +/** + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face; + +import android.graphics.Rect; + +import android.util.SparseArray; +import androidx.annotation.NonNull; + +import androidx.annotation.Nullable; +import com.google.mlkit.vision.face.aidls.ContourParcel; +import com.google.mlkit.vision.face.aidls.FaceParcel; +import com.google.mlkit.vision.face.aidls.LandmarkParcel; +import org.microg.gms.common.Hide; +import org.microg.gms.utils.ToStringHelper; + +import java.util.ArrayList; +import java.util.List; + +public class Face { + @NonNull + private final Rect boundingBox; + private final int trackingId; + private final float rightEyeOpenProbability; + private final float leftEyeOpenProbability; + private final float smileProbability; + private final float eulerX; + private final float eulerY; + private final float eulerZ; + @NonNull + private final SparseArray landmarks = new SparseArray<>(); + @NonNull + private final SparseArray contours = new SparseArray<>(); + + private static boolean isValidLandmarkType(int landmarkType) { + return landmarkType == 0 || landmarkType == 1 || (landmarkType >= 3 && landmarkType <= 7) || (landmarkType >= 9 && landmarkType <= 11); + } + + private static boolean isValidContourType(int contourType) { + return contourType >= 1 && contourType <= 15; + } + + @Hide + public Face(FaceParcel faceParcel) { + boundingBox = faceParcel.boundingBox; + trackingId = faceParcel.id; + for (LandmarkParcel landmarkParcel : faceParcel.landmarkParcelList) { + if (isValidLandmarkType(landmarkParcel.type)) { + landmarks.put(landmarkParcel.type, new FaceLandmark(landmarkParcel.type, landmarkParcel.position)); + } + } + for (ContourParcel contourParcel : faceParcel.contourParcelList) { + if (isValidContourType(contourParcel.type)) { + contours.put(contourParcel.type, new FaceContour(contourParcel.type, contourParcel.pointsList)); + } + } + eulerX = faceParcel.tiltAngle; + eulerY = faceParcel.panAngle; + eulerZ = faceParcel.rollAngle; + smileProbability = faceParcel.smileProbability; + leftEyeOpenProbability = faceParcel.leftEyeOpenProbability; + rightEyeOpenProbability = faceParcel.rightEyeOpenProbability; + } + + /** + * Gets a list of all available {@link FaceContour}s. All {@link FaceContour}s are defined in {@link FaceContour.ContourType}. If no contours are available, an + * empty list is returned. + */ + @NonNull + public List getAllContours() { + List list = new ArrayList<>(); + for (int i = 0; i < contours.size(); i++) { + list.add(contours.valueAt(i)); + } + return list; + } + + /** + * Gets a list of all available {@link FaceLandmark}s. All possible {@link FaceLandmark}s are defined in {@link FaceLandmark.LandmarkType}. If no landmarks are + * available, an empty list is returned. + */ + @NonNull + public List getAllLandmarks() { + List list = new ArrayList<>(); + for (int i = 0; i < landmarks.size(); i++) { + list.add(landmarks.valueAt(i)); + } + return list; + } + + /** + * Returns the {@code NonNull} axis-aligned bounding rectangle of the detected face. + */ + @NonNull + public Rect getBoundingBox() { + return boundingBox; + } + + /** + * Gets contour based on the provided {@link FaceContour.ContourType}. It returns {@code null} if the contour is not available. + */ + @Nullable + public FaceContour getContour(@FaceContour.ContourType int contourType) { + return contours.get(contourType); + } + + /** + * Returns the rotation of the face about the horizontal axis of the image, in degrees. Positive euler X is the face is looking up. + * + * @return the rotation of the face about the horizontal axis of the image + */ + public Float getHeadEulerAngleX() { + return eulerX; + } + + /** + * Returns the rotation of the face about the vertical axis of the image, in degrees. Positive euler y is when the face turns toward the right side + * of the image that is being processed. + * + * @return the rotation of the face about the vertical axis of the image + */ + public Float getHeadEulerAngleY() { + return eulerY; + } + + /** + * Returns the rotation of the face about the axis pointing out of the image, in degrees. Positive euler z is a counter-clockwise rotation within the image plane. + */ + public Float getHeadEulerAngleZ() { + return eulerZ; + } + + /** + * Gets a {@link FaceLandmark} based on the provided {@link FaceLandmark.LandmarkType}. It returns {@code null} if the landmark type is not available. + */ + public FaceLandmark getLandmark(@FaceLandmark.LandmarkType int landmarkType) { + return landmarks.get(landmarkType); + } + + /** + * Returns a value between 0.0 and 1.0 giving a probability that the face's left eye is open. This returns {@code null} if the probability was not + * computed. The probability is not computed if classification is not enabled via + * {@link FaceDetectorOptions.Builder#setClassificationMode(int)} or the feature is not available. + */ + public Float getLeftEyeOpenProbability() { + if (leftEyeOpenProbability < 0.0f || leftEyeOpenProbability > 1.0f) { + return null; + } + return leftEyeOpenProbability; + } + + /** + * Returns a value between 0.0 and 1.0 giving a probability that the face's right eye is open. This returns {@code null} if the probability was not + * computed. The probability is not computed if classification is not enabled via + * {@link FaceDetectorOptions.Builder#setClassificationMode(int)} or the feature is not available. + */ + public Float getRightEyeOpenProbability() { + if (rightEyeOpenProbability < 0.0f || rightEyeOpenProbability > 1.0f) { + return null; + } + return rightEyeOpenProbability; + } + + /** + * Returns a value between 0.0 and 1.0 giving a probability that the face is smiling. This returns {@code null} if the probability was not computed. + * The probability is not computed if classification is not enabled via {@link FaceDetectorOptions.Builder#setClassificationMode(int)} or the + * required landmarks are not found. + */ + public Float getSmilingProbability() { + if (smileProbability < 0.0f || smileProbability > 1.0f) { + return null; + } + return smileProbability; + } + + /** + * Returns the tracking ID if the tracking is enabled. Otherwise, returns {@code null}. + */ + public Integer getTrackingId() { + if (trackingId == -1) { + return null; + } + return trackingId; + } + + @Override + public String toString() { + return ToStringHelper.name("Face") + .field("boundingBox", boundingBox) + .field("trackingId", trackingId) + .field("rightEyeOpenProbability", rightEyeOpenProbability) + .field("leftEyeOpenProbability", leftEyeOpenProbability) + .field("smileProbability", smileProbability) + .field("eulerX", eulerX) + .field("eulerY", eulerY) + .field("eulerZ", eulerZ) + .field("landmarks", landmarks) + .field("contours", contours) + .toString(); + } +} diff --git a/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/FaceContour.java b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/FaceContour.java new file mode 100644 index 0000000000..3eb4e5940c --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/FaceContour.java @@ -0,0 +1,124 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face; + +import android.graphics.PointF; + +import androidx.annotation.IntDef; +import androidx.annotation.NonNull; + +import org.microg.gms.utils.ToStringHelper; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.util.List; + +/** + * Represent a face contour. A contour is a list of points on a detected face, such as the mouth. + *

+ * When 'left' and 'right' are used, they are relative to the subject in the image. For example, the {@link #LEFT_EYE} landmark is the subject's left eye, + * not the eye that is on the left when viewing the image. + */ +public class FaceContour { + + /** + * The outline of the subject's face. + */ + public static final int FACE = 1; + /** + * The top outline of the subject's left eyebrow. + */ + public static final int LEFT_EYEBROW_TOP = 2; + /** + * The bottom outline of the subject's left eyebrow. + */ + public static final int LEFT_EYEBROW_BOTTOM = 3; + /** + * The top outline of the subject's right eyebrow. + */ + public static final int RIGHT_EYEBROW_TOP = 4; + /** + * The bottom outline of the subject's right eyebrow. + */ + public static final int RIGHT_EYEBROW_BOTTOM = 5; + /** + * The outline of the subject's left eye. + */ + public static final int LEFT_EYE = 6; + /** + * The outline of the subject's right eye. + */ + public static final int RIGHT_EYE = 7; + /** + * The top outline of the subject's upper lip. + */ + public static final int UPPER_LIP_TOP = 8; + /** + * The bottom outline of the subject's upper lip. + */ + public static final int UPPER_LIP_BOTTOM = 9; + /** + * The top outline of the subject's lower lip. + */ + public static final int LOWER_LIP_TOP = 10; + /** + * The bottom outline of the subject's lower lip. + */ + public static final int LOWER_LIP_BOTTOM = 11; + /** + * the outline of the subject's nose bridge. + */ + public static final int NOSE_BRIDGE = 12; + /** + * The outline of the subject's nose bridge. + */ + public static final int NOSE_BOTTOM = 13; + /** + * The center of the left cheek. + */ + public static final int LEFT_CHEEK = 14; + /** + * The center of the right cheek. + */ + public static final int RIGHT_CHEEK = 15; + + @Retention(RetentionPolicy.CLASS) + @IntDef(value = {FACE, LEFT_EYEBROW_TOP, LEFT_EYEBROW_BOTTOM, RIGHT_EYEBROW_TOP, RIGHT_EYEBROW_BOTTOM, LEFT_EYE, RIGHT_EYE, UPPER_LIP_TOP, UPPER_LIP_BOTTOM, LOWER_LIP_TOP, LOWER_LIP_BOTTOM, NOSE_BRIDGE, NOSE_BOTTOM, LEFT_CHEEK, RIGHT_CHEEK}) + public @interface ContourType { + } + + private final @ContourType int type; + @NonNull + private final List points; + + FaceContour(@ContourType int type, @NonNull List points) { + this.type = type; + this.points = points; + } + + /** + * Gets the {@link FaceContour.ContourType} type. + */ + @FaceContour.ContourType + public int getFaceContourType() { + return type; + } + + /** + * Gets a list of 2D points for this face contour, where (0, 0) is the upper-left corner of the image. The point is guaranteed to be within the + * bounds of the image. + */ + @NonNull + public List getPoints() { + return points; + } + + @NonNull + @Override + public String toString() { + return ToStringHelper.name("FaceContour").field("type", type).field("points", points.toArray()).toString(); + } +} diff --git a/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/FaceDetectionOptions.java b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/FaceDetectionOptions.java new file mode 100644 index 0000000000..5ab5bba71c --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/FaceDetectionOptions.java @@ -0,0 +1,145 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face; + +import android.os.Parcel; + +import androidx.annotation.IntDef; +import androidx.annotation.NonNull; + +import com.google.android.gms.common.internal.safeparcel.AbstractSafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelableCreatorAndWriter; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.util.concurrent.Executor; + +@SafeParcelable.Class +public class FaceDetectionOptions extends AbstractSafeParcelable { + + public static final int CLASSIFICATION_MODE_ALL = 2; + public static final int CLASSIFICATION_MODE_NONE = 1; + public static final int CONTOUR_MODE_ALL = 2; + public static final int CONTOUR_MODE_NONE = 1; + public static final int LANDMARK_MODE_ALL = 2; + public static final int LANDMARK_MODE_NONE = 1; + public static final int PERFORMANCE_MODE_ACCURATE = 2; + public static final int PERFORMANCE_MODE_FAST = 1; + + @Retention(RetentionPolicy.CLASS) + @IntDef(value = {CLASSIFICATION_MODE_NONE, CLASSIFICATION_MODE_ALL}) + public @interface ClassificationMode { + } + + @Retention(RetentionPolicy.CLASS) + @IntDef(value = {CONTOUR_MODE_NONE, CONTOUR_MODE_ALL}) + public @interface ContourMode { + } + + @Retention(RetentionPolicy.CLASS) + @IntDef(value = {LANDMARK_MODE_NONE, LANDMARK_MODE_ALL}) + public @interface LandmarkMode { + } + + @Retention(RetentionPolicy.CLASS) + @IntDef(value = {PERFORMANCE_MODE_FAST, PERFORMANCE_MODE_ACCURATE}) + public @interface PerformanceMode { + } + + @Field(1) + private final int landmarkMode; + @Field(2) + private final int contourMode; + @Field(3) + private final int classificationMode; + @Field(4) + private final int performanceMode; + @Field(5) + private final boolean trackingEnabled; + @Field(6) + private final float minFaceSize; + private Executor executor; + + @Constructor + FaceDetectionOptions(@Param(1) int landmarkMode, @Param(2) int contourMode, @Param(3) int classificationMode, @Param(4) int performanceMode, @Param(5) boolean trackingEnabled, @Param(6) float minFaceSize) { + this.landmarkMode = landmarkMode; + this.contourMode = contourMode; + this.classificationMode = classificationMode; + this.performanceMode = performanceMode; + this.trackingEnabled = trackingEnabled; + this.minFaceSize = minFaceSize; + } + + public Executor getExecutor() { + return executor; + } + + public void setExecutor(Executor executor) { + this.executor = executor; + } + + public static class Builder { + private int landmarkMode; + private int contourMode; + private int classificationMode; + private int performanceMode; + private boolean trackingEnabled; + private float minFaceSize; + private Executor executor; + + public Builder enableTracking(boolean enable) { + this.trackingEnabled = enable; + return this; + } + + public Builder setClassificationMode(@ClassificationMode int mode) { + this.classificationMode = mode; + return this; + } + + public Builder setContourMode(@ContourMode int mode) { + this.contourMode = mode; + return this; + } + + public Builder setExecutor(Executor executor) { + this.executor = executor; + return this; + } + + public Builder setLandmarkMode(@LandmarkMode int mode) { + this.landmarkMode = mode; + return this; + } + + public Builder setMinFaceSize(float minFaceSize) { + this.minFaceSize = minFaceSize; + return this; + } + + public Builder setPerformanceMode(@PerformanceMode int mode) { + this.performanceMode = mode; + return this; + } + + public FaceDetectionOptions build() { + FaceDetectionOptions faceDetectionOptions = new FaceDetectionOptions(landmarkMode, contourMode, classificationMode, performanceMode, trackingEnabled, minFaceSize); + if (executor != null) { + faceDetectionOptions.setExecutor(executor); + } + return faceDetectionOptions; + } + } + + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + CREATOR.writeToParcel(this, dest, flags); + } + + public static final SafeParcelableCreatorAndWriter CREATOR = findCreator(FaceDetectionOptions.class); + +} diff --git a/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/FaceLandmark.java b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/FaceLandmark.java new file mode 100644 index 0000000000..b92d267189 --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/FaceLandmark.java @@ -0,0 +1,104 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face; + +import android.graphics.PointF; + +import androidx.annotation.IntDef; +import androidx.annotation.NonNull; + +import org.microg.gms.utils.ToStringHelper; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Represent a face landmark. A landmark is a point on a detected face, such as an eye, nose, or mouth. + *

+ * When 'left' and 'right' are used, they are relative to the subject in the image. For example, the {@link #LEFT_EYE} landmark is the subject's left eye, + * not the eye that is on the left when viewing the image. + */ +public class FaceLandmark { + /** + * The center of the subject's bottom lip. + */ + public static final int MOUTH_BOTTOM = 0; + /** + * The midpoint between the subject's left mouth corner and the outer corner of the subject's left eye. For full profile faces, this becomes the + * centroid of the nose base, nose tip, left ear lobe and left ear tip. + */ + public static final int LEFT_CHEEK = 1; + /** + * The midpoint of the subject's left ear tip and left ear lobe. + */ + public static final int LEFT_EAR = 3; + /** + * The center of the subject's left eye cavity. + */ + public static final int LEFT_EYE = 4; + /** + * The subject's left mouth corner where the lips meet. + */ + public static final int MOUTH_LEFT = 5; + /** + * The midpoint between the subject's nostrils where the nose meets the face. + */ + public static final int NOSE_BASE = 6; + /** + * The midpoint between the subject's right mouth corner and the outer corner of the subject's right eye. For full profile faces, this becomes the + * centroid of the nose base, nose tip, right ear lobe and right ear tip. + */ + public static final int RIGHT_CHEEK = 7; + /** + * The midpoint of the subject's right ear tip and right ear lobe. + */ + public static final int RIGHT_EAR = 9; + /** + * The center of the subject's right eye cavity. + */ + public static final int RIGHT_EYE = 10; + /** + * The subject's right mouth corner where the lips meet. + */ + public static final int MOUTH_RIGHT = 11; + + @Retention(RetentionPolicy.CLASS) + @IntDef(value = {MOUTH_BOTTOM, LEFT_CHEEK, LEFT_EAR, LEFT_EYE, MOUTH_LEFT, NOSE_BASE, RIGHT_CHEEK, RIGHT_EAR, RIGHT_EYE, MOUTH_RIGHT}) + public @interface LandmarkType { + } + + private final @LandmarkType int type; + @NonNull + private final PointF position; + + FaceLandmark(@LandmarkType int type, @NonNull PointF position) { + this.type = type; + this.position = position; + } + + /** + * Gets the {@link FaceLandmark.LandmarkType} type. + */ + @LandmarkType + public int getLandmarkType() { + return type; + } + + /** + * Gets a 2D point for landmark position, where (0, 0) is the upper-left corner of the image. The point is guaranteed to be within the bounds of + * the image. + */ + @NonNull + public PointF getPosition() { + return position; + } + + @NonNull + @Override + public String toString() { + return ToStringHelper.name("FaceLandmark").field("type", type).field("position", position).toString(); + } +} diff --git a/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/FrameMetadataParcel.java b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/FrameMetadataParcel.java new file mode 100644 index 0000000000..5407df3f3e --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/FrameMetadataParcel.java @@ -0,0 +1,37 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face; + +import android.os.Parcel; + +import androidx.annotation.NonNull; + +import com.google.android.gms.common.internal.safeparcel.AbstractSafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelableCreatorAndWriter; + +@SafeParcelable.Class +public class FrameMetadataParcel extends AbstractSafeParcelable { + + @Field(1) + public int format; + @Field(2) + public int width; + @Field(3) + public int height; + @Field(4) + public int rotation; + @Field(5) + public long timestampMillis; + + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + CREATOR.writeToParcel(this, dest, flags); + } + + public static final SafeParcelableCreatorAndWriter CREATOR = findCreator(FrameMetadataParcel.class); + +} diff --git a/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/aidls/ContourParcel.java b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/aidls/ContourParcel.java new file mode 100644 index 0000000000..6429bdbb6a --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/aidls/ContourParcel.java @@ -0,0 +1,36 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face.aidls; + +import android.graphics.PointF; +import android.os.Parcel; +import androidx.annotation.NonNull; +import com.google.android.gms.common.internal.safeparcel.AbstractSafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelableCreatorAndWriter; + +import java.util.List; + +@SafeParcelable.Class +public class ContourParcel extends AbstractSafeParcelable { + @Field(1) + public final int type; + @Field(2) + public final List pointsList; + + @Constructor + public ContourParcel(@Param(1) int type, @Param(2) List pointsList) { + this.type = type; + this.pointsList = pointsList; + } + + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + CREATOR.writeToParcel(this, dest, flags); + } + + public static final SafeParcelableCreatorAndWriter CREATOR = findCreator(ContourParcel.class); +} diff --git a/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/aidls/FaceParcel.java b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/aidls/FaceParcel.java new file mode 100644 index 0000000000..9b69b7e5c9 --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/aidls/FaceParcel.java @@ -0,0 +1,74 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face.aidls; + +import android.graphics.Rect; +import android.os.Parcel; +import androidx.annotation.NonNull; +import com.google.android.gms.common.internal.safeparcel.AbstractSafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelableCreatorAndWriter; + +import java.util.List; + +@SafeParcelable.Class +public class FaceParcel extends AbstractSafeParcelable { + @Field(1) + public final int id; + + @Field(2) + public final Rect boundingBox; + + @Field(3) + public final float rollAngle; + + @Field(4) + public final float panAngle; + + @Field(5) + public final float tiltAngle; + + @Field(6) + public final float leftEyeOpenProbability; + + @Field(7) + public final float rightEyeOpenProbability; + + @Field(8) + public final float smileProbability; + + @Field(9) + public final float confidenceScore; + + @Field(10) + public final List landmarkParcelList; + + @Field(11) + public final List contourParcelList; + + @Constructor + public FaceParcel(@Param(1) int id, @Param(2) Rect boundingBox, @Param(3) float rollAngle, @Param(4) float panAngle, @Param(5) float tiltAngle, @Param(6) float leftEyeOpenProbability, @Param(7) float rightEyeOpenProbability, @Param(8) float smileProbability, @Param(9) float confidenceScore, @Param(10) List landmarkParcelList, @Param(11) List contourParcelList) { + this.id = id; + this.boundingBox = boundingBox; + this.rollAngle = rollAngle; + this.panAngle = panAngle; + this.tiltAngle = tiltAngle; + this.leftEyeOpenProbability = leftEyeOpenProbability; + this.rightEyeOpenProbability = rightEyeOpenProbability; + this.smileProbability = smileProbability; + this.confidenceScore = confidenceScore; + this.landmarkParcelList = landmarkParcelList; + this.contourParcelList = contourParcelList; + } + + + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + CREATOR.writeToParcel(this, dest, flags); + } + + public static final SafeParcelableCreatorAndWriter CREATOR = findCreator(FaceParcel.class); +} diff --git a/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/aidls/LandmarkParcel.java b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/aidls/LandmarkParcel.java new file mode 100644 index 0000000000..bffb4cb71a --- /dev/null +++ b/play-services-mlkit/face-detection/src/main/java/com/google/mlkit/vision/face/aidls/LandmarkParcel.java @@ -0,0 +1,34 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face.aidls; + +import android.graphics.PointF; +import android.os.Parcel; +import androidx.annotation.NonNull; +import com.google.android.gms.common.internal.safeparcel.AbstractSafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelableCreatorAndWriter; + +@SafeParcelable.Class +public class LandmarkParcel extends AbstractSafeParcelable { + @Field(1) + public final int type; + @Field(2) + public final PointF position; + + @Constructor + public LandmarkParcel(@Param(1) int type, @Param(2) PointF position) { + this.type = type; + this.position = position; + } + + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + CREATOR.writeToParcel(this, dest, flags); + } + + public static final SafeParcelableCreatorAndWriter CREATOR = findCreator(LandmarkParcel.class); +} diff --git a/play-services-vision/build.gradle b/play-services-vision/build.gradle index 8fea34919f..30332f9435 100644 --- a/play-services-vision/build.gradle +++ b/play-services-vision/build.gradle @@ -38,4 +38,6 @@ dependencies { api project(':play-services-base') api project(':play-services-basement') api project(':play-services-vision-common') + + annotationProcessor project(":safe-parcel-processor") } diff --git a/play-services-vision/core/build.gradle b/play-services-vision/core/build.gradle index 1a8dffdead..5db3d5c277 100644 --- a/play-services-vision/core/build.gradle +++ b/play-services-vision/core/build.gradle @@ -10,11 +10,15 @@ apply plugin: 'signing' dependencies { api project(':play-services-mlkit-barcode-scanning') + api project(':play-services-mlkit-face-detection') api project(':play-services-vision') implementation project(':play-services-base-core') implementation "androidx.annotation:annotation:$annotationVersion" implementation "com.google.zxing:core:3.5.2" + + implementation 'org.opencv:opencv:4.11.0' + implementation "androidx.camera:camera-core:1.3.0" implementation "androidx.camera:camera-camera2:1.3.0" implementation "androidx.camera:camera-lifecycle:1.3.0" diff --git a/play-services-vision/core/src/main/assets/face_detection_yunet_2023mar.onnx b/play-services-vision/core/src/main/assets/face_detection_yunet_2023mar.onnx new file mode 100644 index 0000000000..f9beb3044f Binary files /dev/null and b/play-services-vision/core/src/main/assets/face_detection_yunet_2023mar.onnx differ diff --git a/play-services-vision/core/src/main/java/com/google/android/gms/dynamite/descriptors/com/google/android/gms/vision/dynamite/face/ModuleDescriptor.java b/play-services-vision/core/src/main/java/com/google/android/gms/dynamite/descriptors/com/google/android/gms/vision/dynamite/face/ModuleDescriptor.java new file mode 100644 index 0000000000..7b0b13c8e8 --- /dev/null +++ b/play-services-vision/core/src/main/java/com/google/android/gms/dynamite/descriptors/com/google/android/gms/vision/dynamite/face/ModuleDescriptor.java @@ -0,0 +1,11 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.dynamite.descriptors.com.google.android.gms.vision.dynamite.face; + +public class ModuleDescriptor { + public static final String MODULE_ID = "com.google.android.gms.vision.dynamite.face"; + public static final int MODULE_VERSION = 1; +} diff --git a/play-services-vision/core/src/main/java/com/google/android/gms/dynamite/descriptors/com/google/android/gms/vision/face/ModuleDescriptor.java b/play-services-vision/core/src/main/java/com/google/android/gms/dynamite/descriptors/com/google/android/gms/vision/face/ModuleDescriptor.java new file mode 100644 index 0000000000..37d42a0a12 --- /dev/null +++ b/play-services-vision/core/src/main/java/com/google/android/gms/dynamite/descriptors/com/google/android/gms/vision/face/ModuleDescriptor.java @@ -0,0 +1,11 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.dynamite.descriptors.com.google.android.gms.vision.face; + +public class ModuleDescriptor { + public static final String MODULE_ID = "com.google.android.gms.vision.face"; + public static final int MODULE_VERSION = 1; +} diff --git a/play-services-vision/core/src/main/java/com/google/android/gms/dynamite/descriptors/com/google/mlkit/dynamite/face/ModuleDescriptor.java b/play-services-vision/core/src/main/java/com/google/android/gms/dynamite/descriptors/com/google/mlkit/dynamite/face/ModuleDescriptor.java new file mode 100644 index 0000000000..3adc2806b2 --- /dev/null +++ b/play-services-vision/core/src/main/java/com/google/android/gms/dynamite/descriptors/com/google/mlkit/dynamite/face/ModuleDescriptor.java @@ -0,0 +1,11 @@ +/** + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.dynamite.descriptors.com.google.mlkit.dynamite.face; + +public class ModuleDescriptor { + public static final String MODULE_ID = "com.google.mlkit.dynamite.face"; + public static final int MODULE_VERSION = 1; +} diff --git a/play-services-vision/core/src/main/kotlin/com/google/android/gms/vision/client/DynamiteNativeFaceDetectorCreator.kt b/play-services-vision/core/src/main/kotlin/com/google/android/gms/vision/client/DynamiteNativeFaceDetectorCreator.kt new file mode 100644 index 0000000000..6406f69652 --- /dev/null +++ b/play-services-vision/core/src/main/kotlin/com/google/android/gms/vision/client/DynamiteNativeFaceDetectorCreator.kt @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: 2025, microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.client + +import android.content.Context +import android.os.SystemClock +import android.util.Log +import androidx.annotation.Keep +import com.google.android.gms.common.GooglePlayServicesUtil +import com.google.android.gms.dynamic.IObjectWrapper +import com.google.android.gms.dynamic.unwrap +import com.google.android.gms.vision.face.internal.client.DetectionOptions +import com.google.android.gms.vision.face.internal.client.INativeFaceDetector +import com.google.android.gms.vision.face.internal.client.INativeFaceDetectorCreator +import org.microg.gms.vision.face.TAG +import org.microg.gms.vision.face.FaceDetector +import org.opencv.android.OpenCVLoader + +@Keep +class DynamiteNativeFaceDetectorCreator : INativeFaceDetectorCreator.Stub() { + + override fun newFaceDetector(context: IObjectWrapper?, faceDetectionOptions: DetectionOptions?): INativeFaceDetector? { + Log.d(TAG, "DynamiteNativeFaceDetectorCreator newFaceDetector faceDetectionOptions:${faceDetectionOptions.toString()}") + try { + val elapsedRealtime = SystemClock.elapsedRealtime() + val context = context.unwrap() ?: throw RuntimeException("Context is null") + val remoteContext = GooglePlayServicesUtil.getRemoteContext(context) ?: throw RuntimeException("remoteContext is null") + Log.d(TAG, "newFaceDetector: context: ${context.packageName} remoteContext: ${remoteContext.packageName}") + if (!OpenCVLoader.initLocal()) { + throw RuntimeException("Unable to load OpenCV") + } + Log.d(TAG, "DynamiteNativeFaceDetectorCreator newFaceDetector: load library in ${SystemClock.elapsedRealtime() - elapsedRealtime}ms") + return FaceDetector(remoteContext, faceDetectionOptions) + } catch (e: Throwable) { + Log.w(TAG, "DynamiteNativeFaceDetectorCreator newFaceDetector load failed ", e) + return null + } + } +} \ No newline at end of file diff --git a/play-services-vision/core/src/main/kotlin/com/google/android/gms/vision/face/ChimeraNativeFaceDetectorCreator.kt b/play-services-vision/core/src/main/kotlin/com/google/android/gms/vision/face/ChimeraNativeFaceDetectorCreator.kt new file mode 100644 index 0000000000..5ece33ee89 --- /dev/null +++ b/play-services-vision/core/src/main/kotlin/com/google/android/gms/vision/face/ChimeraNativeFaceDetectorCreator.kt @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: 2025, microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.face + +import android.content.Context +import android.os.SystemClock +import android.util.Log +import androidx.annotation.Keep +import com.google.android.gms.common.GooglePlayServicesUtil +import com.google.android.gms.dynamic.IObjectWrapper +import com.google.android.gms.dynamic.unwrap +import com.google.android.gms.vision.face.internal.client.DetectionOptions +import com.google.android.gms.vision.face.internal.client.INativeFaceDetector +import com.google.android.gms.vision.face.internal.client.INativeFaceDetectorCreator +import org.microg.gms.vision.face.FaceDetector +import org.microg.gms.vision.face.TAG +import org.opencv.android.OpenCVLoader + +@Keep +class ChimeraNativeFaceDetectorCreator : INativeFaceDetectorCreator.Stub() { + override fun newFaceDetector(context: IObjectWrapper?, faceDetectionOptions: DetectionOptions?): INativeFaceDetector? { + Log.d(TAG, "ChimeraNativeFaceDetectorCreator newFaceDetector faceDetectionOptions:${faceDetectionOptions.toString()}") + try { + val elapsedRealtime = SystemClock.elapsedRealtime() + val context = context.unwrap() ?: throw RuntimeException("Context is null") + val remoteContext = GooglePlayServicesUtil.getRemoteContext(context) ?: throw RuntimeException("remoteContext is null") + Log.d(TAG, "newFaceDetector: context: ${context.packageName} remoteContext: ${remoteContext.packageName}") + if (!OpenCVLoader.initLocal()) { + throw RuntimeException("Unable to load OpenCV") + } + Log.d(TAG, "ChimeraNativeFaceDetectorCreator newFaceDetector: load library in ${SystemClock.elapsedRealtime() - elapsedRealtime}ms") + return FaceDetector(remoteContext, faceDetectionOptions) + } catch (e: Throwable) { + Log.w(TAG, "ChimeraNativeFaceDetectorCreator newFaceDetector load failed ", e) + return null + } + } +} \ No newline at end of file diff --git a/play-services-vision/core/src/main/kotlin/com/google/android/gms/vision/face/mlkit/FaceDetectorCreator.kt b/play-services-vision/core/src/main/kotlin/com/google/android/gms/vision/face/mlkit/FaceDetectorCreator.kt new file mode 100644 index 0000000000..a0948771a2 --- /dev/null +++ b/play-services-vision/core/src/main/kotlin/com/google/android/gms/vision/face/mlkit/FaceDetectorCreator.kt @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: 2025, microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.face.mlkit + +import android.content.Context +import android.os.SystemClock +import android.util.Log +import androidx.annotation.Keep +import com.google.android.gms.common.GooglePlayServicesUtil +import com.google.android.gms.dynamic.IObjectWrapper +import com.google.android.gms.dynamic.unwrap +import com.google.mlkit.vision.face.FaceDetectionOptions +import com.google.mlkit.vision.face.aidls.IFaceDetector +import com.google.mlkit.vision.face.aidls.IFaceDetectorCreator +import org.microg.gms.vision.face.TAG +import org.microg.gms.vision.face.mlkit.FaceDetector +import org.opencv.android.OpenCVLoader + +@Keep +class FaceDetectorCreator : IFaceDetectorCreator.Stub() { + + override fun newFaceDetector(context: IObjectWrapper?, faceDetectionOptions: FaceDetectionOptions?): IFaceDetector? { + Log.d(TAG, "MLKit newFaceDetector options:${faceDetectionOptions}") + try { + val elapsedRealtime = SystemClock.elapsedRealtime() + val context = context.unwrap() ?: throw RuntimeException("Context is null") + val remoteContext = GooglePlayServicesUtil.getRemoteContext(context) ?: throw RuntimeException("remoteContext is null") + Log.d(TAG, "newFaceDetector: context: ${context.packageName} remoteContext: ${remoteContext.packageName}") + if (!OpenCVLoader.initLocal()) { + throw RuntimeException("Unable to load OpenCV") + } + Log.d(TAG, "FaceDetectorCreator newFaceDetector: load library in ${SystemClock.elapsedRealtime() - elapsedRealtime}ms") + return FaceDetector(remoteContext, faceDetectionOptions) + } catch (e: Throwable) { + Log.w(TAG, "FaceDetectorCreator newFaceDetector load failed ", e) + return null + } + } +} \ No newline at end of file diff --git a/play-services-vision/core/src/main/kotlin/com/google/mlkit/vision/face/bundled/internal/ThickFaceDetectorCreator.kt b/play-services-vision/core/src/main/kotlin/com/google/mlkit/vision/face/bundled/internal/ThickFaceDetectorCreator.kt new file mode 100644 index 0000000000..9ebcd2078d --- /dev/null +++ b/play-services-vision/core/src/main/kotlin/com/google/mlkit/vision/face/bundled/internal/ThickFaceDetectorCreator.kt @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.mlkit.vision.face.bundled.internal + +import android.content.Context +import android.os.SystemClock +import android.util.Log +import androidx.annotation.Keep +import com.google.android.gms.common.GooglePlayServicesUtil +import com.google.android.gms.dynamic.IObjectWrapper +import com.google.android.gms.dynamic.unwrap +import com.google.mlkit.vision.face.FaceDetectionOptions +import com.google.mlkit.vision.face.aidls.IFaceDetector +import com.google.mlkit.vision.face.aidls.IFaceDetectorCreator +import org.microg.gms.vision.face.TAG +import org.microg.gms.vision.face.mlkit.FaceDetector +import org.opencv.android.OpenCVLoader + +@Keep +class ThickFaceDetectorCreator : IFaceDetectorCreator.Stub() { + + override fun newFaceDetector(context: IObjectWrapper?, faceDetectionOptions: FaceDetectionOptions?): IFaceDetector? { + Log.d(TAG, "MLKit newFaceDetector options:${faceDetectionOptions}") + try { + val elapsedRealtime = SystemClock.elapsedRealtime() + val context = context.unwrap() ?: throw RuntimeException("Context is null") + val remoteContext = GooglePlayServicesUtil.getRemoteContext(context) ?: throw RuntimeException("remoteContext is null") + Log.d(TAG, "ThickFaceDetectorCreator newFaceDetector: context: ${context.packageName} remoteContext: ${remoteContext.packageName}") + if (!OpenCVLoader.initLocal()) { + throw RuntimeException("Unable to load OpenCV") + } + Log.d(TAG, "ThickFaceDetectorCreator newFaceDetector: load library in ${SystemClock.elapsedRealtime() - elapsedRealtime}ms") + return FaceDetector(remoteContext, faceDetectionOptions) + } catch (e: Throwable) { + Log.w(TAG, "ThickFaceDetectorCreator newFaceDetector load failed ", e) + return null + } + } +} \ No newline at end of file diff --git a/play-services-vision/core/src/main/kotlin/org/microg/gms/vision/face/FaceDetector.kt b/play-services-vision/core/src/main/kotlin/org/microg/gms/vision/face/FaceDetector.kt new file mode 100644 index 0000000000..e954dd5284 --- /dev/null +++ b/play-services-vision/core/src/main/kotlin/org/microg/gms/vision/face/FaceDetector.kt @@ -0,0 +1,109 @@ +/* + * SPDX-FileCopyrightText: 2025, microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.microg.gms.vision.face + +import android.content.Context +import android.util.Log +import com.google.android.gms.dynamic.IObjectWrapper +import com.google.android.gms.dynamic.unwrap +import com.google.android.gms.vision.face.Contour +import com.google.android.gms.vision.face.Landmark +import com.google.android.gms.vision.face.internal.client.DetectionOptions +import com.google.android.gms.vision.face.internal.client.FaceParcel +import com.google.android.gms.vision.face.internal.client.INativeFaceDetector +import com.google.android.gms.vision.internal.FrameMetadataParcel +import com.google.mlkit.vision.face.Face +import java.nio.ByteBuffer + +class FaceDetector(val context: Context, private val options: DetectionOptions?) : INativeFaceDetector.Stub() { + + private val mFaceDetector by lazy { FaceDetectorHelper(context) } + + override fun closeDetectorJni() { + Log.d(TAG, "closeDetectorJni") + mFaceDetector.release() + } + + override fun isNativeFaceDetectorAvailable(i: Int): Boolean { + Log.d(TAG, "isNativeFaceDetectorAvailable type:${i}") + return true + } + + override fun detectFacesFromPlanes( + planeFirst: IObjectWrapper?, + planeSencond: IObjectWrapper?, + planeThird: IObjectWrapper?, + firstPixelStride: Int, + secondPixelStride: Int, + thirdPixelStride: Int, + firstRowStride: Int, + secondRowStride: Int, + thirdRowStride: Int, + metadataParcel: FrameMetadataParcel? + ): Array { + Log.d( + TAG, + "detectFacesFromPlanes planeFirst:${planeFirst} ,planeSecond:${planeSencond} ,planeThird:${planeThird}," + "firstPixelStride:${firstPixelStride} ,secondPixelStride:${secondPixelStride} ,thirdPixelStride:${thirdPixelStride} ," + "firstRowStride:${firstRowStride} ,secondRowStride:${secondRowStride} ,thirdRowStride:${thirdRowStride}," + "metadataParcel:${metadataParcel}" + ) + val yBuffer = planeFirst?.unwrap() ?: return emptyArray() + val uBuffer = planeSencond?.unwrap() ?: return emptyArray() + val vBuffer = planeThird?.unwrap() ?: return emptyArray() + val width = metadataParcel?.width ?: return emptyArray() + val height = metadataParcel?.height ?: return emptyArray() + val rotation = metadataParcel.rotation + val nv21 = ByteArray(width * height * 3 / 2) + var offset = 0 + for (row in 0 until height) { + yBuffer.position(row * firstRowStride) + yBuffer.get(nv21, offset, width) + offset += width + } + val chromaWidth = width / 2 + val chromaHeight = height / 2 + for (row in 0 until chromaHeight) { + for (col in 0 until chromaWidth) { + val uIndex = row * secondRowStride + col * secondPixelStride + val vIndex = row * thirdRowStride + col * thirdPixelStride + nv21[offset++] = vBuffer.get(vIndex) + nv21[offset++] = uBuffer.get(uIndex) + } + } + return mFaceDetector.detectFaces(nv21, width, height, rotation).map { + it.toFaceParcel() + }.toTypedArray().also { + it.forEach { Log.d(TAG, "detectFacesFromPlanes: $it") } + } + } + + override fun detectFaceParcels(wrapper: IObjectWrapper?, metadata: FrameMetadataParcel?): Array { + Log.d(TAG, "detectFaceParcels byteBuffer:${wrapper} ,metadataParcel:${metadata}") + if (wrapper == null || metadata == null) return emptyArray() + val buffer = wrapper.unwrap() ?: return emptyArray() + return mFaceDetector.detectFaces(buffer.array(), metadata.width, metadata.height, metadata.rotation).map { + it.toFaceParcel() + }.toTypedArray().also { + it.forEach { Log.d(TAG, "detectFaceParcels: $it") } + } + } +} + +private fun com.google.mlkit.vision.face.aidls.FaceParcel.toFaceParcel() = FaceParcel( + 1, + id, + (boundingBox.left + boundingBox.width() / 2).toFloat(), + (boundingBox.top + boundingBox.height() / 2).toFloat(), + boundingBox.width().toFloat(), + boundingBox.height().toFloat(), + panAngle, + rollAngle, + tiltAngle, + landmarkParcelList.map { landmark -> Landmark(landmark.type, landmark.position.x, landmark.position.y, landmark.type) }.toTypedArray(), + leftEyeOpenProbability, + rightEyeOpenProbability, + smileProbability, + contourParcelList.map { contour -> Contour(contour.type, contour.pointsList) }.toTypedArray(), + confidenceScore +) \ No newline at end of file diff --git a/play-services-vision/core/src/main/kotlin/org/microg/gms/vision/face/FaceDetectorHelper.kt b/play-services-vision/core/src/main/kotlin/org/microg/gms/vision/face/FaceDetectorHelper.kt new file mode 100644 index 0000000000..383685a9f3 --- /dev/null +++ b/play-services-vision/core/src/main/kotlin/org/microg/gms/vision/face/FaceDetectorHelper.kt @@ -0,0 +1,278 @@ +/** + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.microg.gms.vision.face + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.ImageFormat +import android.graphics.PointF +import android.graphics.Rect +import android.graphics.YuvImage +import android.media.Image +import android.util.Log +import com.google.mlkit.vision.face.FaceContour +import com.google.mlkit.vision.face.FaceLandmark +import com.google.mlkit.vision.face.aidls.ContourParcel +import com.google.mlkit.vision.face.aidls.FaceParcel +import com.google.mlkit.vision.face.aidls.LandmarkParcel +import org.opencv.android.Utils +import org.opencv.core.Core +import org.opencv.core.CvType +import org.opencv.core.Mat +import org.opencv.core.MatOfByte +import org.opencv.core.Size +import org.opencv.imgproc.Imgproc +import org.opencv.objdetect.FaceDetectorYN +import java.io.ByteArrayOutputStream +import kotlin.math.hypot + +const val TAG = "FaceDetection" + +class FaceDetectorHelper(context: Context) { + + private var faceDetectorYN: FaceDetectorYN? = null + private var inputSize = Size(320.0, 320.0) + + init { + try { + val buffer: ByteArray + context.assets.open("face_detection_yunet_2023mar.onnx").use { + val size = it.available() + buffer = ByteArray(size) + it.read(buffer) + } + faceDetectorYN = FaceDetectorYN.create("onnx", MatOfByte(*buffer), MatOfByte(), inputSize, 0.7f, 0.3f, 5000) + } catch (e: Exception) { + throw RuntimeException("faceDetectorYN initialization failed") + } + } + + fun detectFaces(bitmap: Bitmap, rotation: Int): List { + Log.d(TAG, "detectFaces: source is bitmap") + val rootMat = bitmapToMat(bitmap) ?: return emptyList() + return processMat(rootMat, rotation) + } + + fun detectFaces(nv21ByteArray: ByteArray, width: Int, height: Int, rotation: Int): List { + Log.d(TAG, "detectFaces: source is nv21Buffer") + val rootMat = nv21ToMat(nv21ByteArray, width, height) ?: return emptyList() + return processMat(rootMat, rotation) + } + + fun detectFaces(image: Image, rotation: Int): List { + Log.d(TAG, "detectFaces: source is image") + val rootMat = imageToMat(image) ?: return emptyList() + return processMat(rootMat, rotation) + } + + fun release() { + try { + faceDetectorYN = null + } catch (e: Exception) { + Log.d(TAG, "release failed", e) + } + } + + private fun processMat(mat: Mat, rotation: Int): List { + val faceDetector = faceDetectorYN ?: return emptyList() + val facesMat = Mat() + val degree = degree(rotation) + Log.d(TAG, "processMat: degree: $degree") + when (degree) { + 2 -> Core.rotate(mat, facesMat, Core.ROTATE_90_COUNTERCLOCKWISE) + 3 -> Core.rotate(mat, facesMat, Core.ROTATE_180) + 4 -> Core.rotate(mat, facesMat, Core.ROTATE_90_CLOCKWISE) + else -> mat.copyTo(facesMat) + } + val matSize = Size(facesMat.cols().toDouble(), facesMat.rows().toDouble()) + Log.d(TAG, "processMat: inputSize: $inputSize") + if (inputSize != matSize) { + inputSize = matSize + faceDetector.inputSize = matSize + } + Log.d(TAG, "processMat: matSize: $matSize") + val result = Mat() + val status = faceDetectorYN!!.detect(facesMat, result) + Log.d(TAG, "processMat: detect: $status facesMat: ${result.size()}") + return parseDetections(result) + } + + /** + * faces: detection results stored in a 2D cv::Mat of shape [num_faces, 15] + * 0-1: x, y of bbox top left corner + * 2-3: width, height of bbox + * 4-5: x, y of right eye (blue point in the example image) + * 6-7: x, y of left eye (red point in the example image) + * 8-9: x, y of nose tip (green point in the example image) + * 10-11: x, y of right corner of mouth (pink point in the example image) + * 12-13: x, y of left corner of mouth (yellow point in the example image) + * 14: face score + */ + private fun parseDetections(detections: Mat): List { + val faces = mutableListOf() + val faceData = FloatArray(detections.cols() * detections.channels()) + for (i in 0 until detections.rows()) { + detections.get(i, 0, faceData) + val confidence = faceData[14] + val boundingBox = Rect(faceData[0].toInt(), faceData[1].toInt(), (faceData[0] + faceData[2]).toInt(), (faceData[1] + faceData[3]).toInt()) + + val leftEyeMark = LandmarkParcel(FaceLandmark.LEFT_EYE, PointF(faceData[4], faceData[5])) + val mouthLeftMark = LandmarkParcel(FaceLandmark.MOUTH_LEFT, PointF(faceData[10], faceData[11])) + val noseBaseMark = LandmarkParcel(FaceLandmark.NOSE_BASE, PointF(faceData[8], faceData[9])) + val rightEyeMark = LandmarkParcel(FaceLandmark.RIGHT_EYE, PointF(faceData[6], faceData[7])) + val mouthRightMark = LandmarkParcel(FaceLandmark.MOUTH_RIGHT, PointF(faceData[12], faceData[13])) + + // These are calculated for better compatibility, the model doesn't actually provide proper values here + val mouthBottomMark = LandmarkParcel(FaceLandmark.MOUTH_BOTTOM, calculateMidPoint(mouthLeftMark, mouthRightMark)) + val leftCheekMark = LandmarkParcel(FaceLandmark.LEFT_CHEEK, calculateMidPoint(leftEyeMark, mouthLeftMark)) + val leftEarMark = LandmarkParcel(FaceLandmark.LEFT_EAR, PointF(boundingBox.right.toFloat(), noseBaseMark.position.y)) + val rightCheekMark = LandmarkParcel(FaceLandmark.RIGHT_CHEEK, calculateMidPoint(rightEyeMark, mouthRightMark)) + val rightEarMark = LandmarkParcel(FaceLandmark.RIGHT_EAR, PointF(boundingBox.left.toFloat(), noseBaseMark.position.y)) + + val smilingProbability = calculateSmilingProbability(mouthLeftMark, mouthRightMark) + val leftEyeOpenProbability = calculateEyeOpenProbability(rightEyeMark, mouthRightMark) + val rightEyeOpenProbability = calculateEyeOpenProbability(leftEyeMark, mouthLeftMark) + + val faceContour = ContourParcel(FaceContour.FACE, arrayListOf( + PointF(boundingBox.left.toFloat(), boundingBox.top.toFloat()), + PointF(boundingBox.left.toFloat(), boundingBox.bottom.toFloat()), + PointF(boundingBox.right.toFloat(), boundingBox.bottom.toFloat()), + PointF(boundingBox.right.toFloat(), boundingBox.top.toFloat()), + )) + val leftEyebrowTopContour = ContourParcel(FaceContour.LEFT_EYEBROW_TOP, arrayListOf(leftEyeMark.position)) + val leftEyebrowBottomContour = ContourParcel(FaceContour.LEFT_EYEBROW_BOTTOM, arrayListOf(leftEyeMark.position)) + val rightEyebrowTopContour = ContourParcel(FaceContour.RIGHT_EYEBROW_TOP, arrayListOf(rightEyeMark.position)) + val rightEyebrowBottomContour = ContourParcel(FaceContour.RIGHT_EYEBROW_BOTTOM, arrayListOf(rightEyeMark.position)) + val leftEyeContour = ContourParcel(FaceContour.LEFT_EYE, arrayListOf(leftEyeMark.position)) + val rightEyeContour = ContourParcel(FaceContour.RIGHT_EYE, arrayListOf(rightEyeMark.position)) + val upperLipTopContour = ContourParcel(FaceContour.UPPER_LIP_TOP, arrayListOf(mouthLeftMark.position, mouthBottomMark.position, mouthRightMark.position, mouthBottomMark.position)) + val upperLipBottomContour = ContourParcel(FaceContour.UPPER_LIP_BOTTOM, arrayListOf(mouthLeftMark.position, mouthBottomMark.position, mouthRightMark.position, mouthBottomMark.position)) + val lowerLipTopContour = ContourParcel(FaceContour.LOWER_LIP_TOP, arrayListOf(mouthLeftMark.position, mouthBottomMark.position, mouthRightMark.position, mouthBottomMark.position)) + val lowerLipBottomContour = ContourParcel(FaceContour.LOWER_LIP_BOTTOM, arrayListOf(mouthLeftMark.position, mouthBottomMark.position, mouthRightMark.position, mouthBottomMark.position)) + val noseBridgeContour = ContourParcel(FaceContour.NOSE_BRIDGE, arrayListOf(noseBaseMark.position)) + val noseBottomContour = ContourParcel(FaceContour.NOSE_BOTTOM, arrayListOf(noseBaseMark.position)) + val leftCheekContour = ContourParcel(FaceContour.LEFT_CHEEK, arrayListOf(leftCheekMark.position)) + val rightCheekContour = ContourParcel(FaceContour.RIGHT_CHEEK, arrayListOf(rightCheekMark.position)) + + faces.add(FaceParcel( + i, + boundingBox, + 0f, + 0f, + 0f, + leftEyeOpenProbability, + rightEyeOpenProbability, + smilingProbability, + confidence, + arrayListOf(mouthBottomMark, leftCheekMark, leftEarMark, leftEyeMark, mouthLeftMark, noseBaseMark, rightCheekMark, rightEarMark, rightEyeMark, mouthRightMark), + arrayListOf(faceContour, leftEyebrowTopContour, leftEyebrowBottomContour, rightEyebrowTopContour, rightEyebrowBottomContour, leftEyeContour, rightEyeContour, upperLipTopContour, upperLipBottomContour, lowerLipTopContour, lowerLipBottomContour, noseBridgeContour, noseBottomContour, leftCheekContour, rightCheekContour) + ).also { + Log.d(TAG, "parseDetections: face->$it") + }) + } + Log.d(TAG, "parseDetections: faces->${faces.size}") + return faces + } + + private fun calculateSmilingProbability(rightMouthCorner: LandmarkParcel, leftMouthCorner: LandmarkParcel): Float { + val mouthWidth = hypot( + (rightMouthCorner.position.x - leftMouthCorner.position.x).toDouble(), (rightMouthCorner.position.y - leftMouthCorner.position.y).toDouble() + ).toFloat() + return (mouthWidth / 100).coerceIn(0f, 1f) + } + + private fun calculateEyeOpenProbability(eye: LandmarkParcel, mouthCorner: LandmarkParcel): Float { + val eyeMouthDistance = hypot( + (eye.position.x - mouthCorner.position.x).toDouble(), (eye.position.y - mouthCorner.position.y).toDouble() + ).toFloat() + return (eyeMouthDistance / 50).coerceIn(0f, 1f) + } + + private fun calculateMidPoint(eye: LandmarkParcel, mouth: LandmarkParcel): PointF { + return PointF((eye.position.x + mouth.position.x) / 2, (eye.position.y + mouth.position.y) / 2) + } + + private fun yuv420ToBitmap(image: Image): Bitmap? { + val width = image.width + val height = image.height + + val yBuffer = image.planes[0].buffer + val uBuffer = image.planes[1].buffer + val vBuffer = image.planes[2].buffer + + val ySize = yBuffer.remaining() + val uSize = uBuffer.remaining() + val vSize = vBuffer.remaining() + + val nv21 = ByteArray(ySize + uSize + vSize) + yBuffer.get(nv21, 0, ySize) + vBuffer.get(nv21, ySize, vSize) + uBuffer.get(nv21, ySize + vSize, uSize) + + return nv21toBitmap(nv21, width, height) + } + + private fun nv21toBitmap(byteArray: ByteArray, width: Int, height: Int): Bitmap? { + try { + val yuvImage = YuvImage(byteArray, ImageFormat.NV21, width, height, null) + val out = ByteArrayOutputStream() + yuvImage.compressToJpeg(Rect(0, 0, width, height), 100, out) + val jpegBytes = out.toByteArray() + return BitmapFactory.decodeByteArray(jpegBytes, 0, jpegBytes.size) + } catch (e: Exception) { + Log.w(TAG, "nv21toBitmap: failed ", e) + return null + } + } + + private fun bitmapToMat(bitmap: Bitmap): Mat? { + try { + val mat = Mat(bitmap.height, bitmap.width, CvType.CV_8UC4) + Utils.bitmapToMat(bitmap, mat) + Imgproc.cvtColor(mat, mat, Imgproc.COLOR_RGBA2BGR) + return mat + } catch (e: Exception) { + Log.w(TAG, "bitmapToMat: failed", e) + return null + } + } + + private fun imageToMat(image: Image): Mat? { + val bitmap = when (image.format) { + ImageFormat.JPEG -> { + val buffer = image.planes[0].buffer + val bytes = ByteArray(buffer.remaining()) + buffer.get(bytes) + BitmapFactory.decodeByteArray(bytes, 0, bytes.size) + } + + ImageFormat.YUV_420_888 -> { + yuv420ToBitmap(image) + } + + else -> { + null + } + } + return bitmap?.let { bitmapToMat(it) } + } + + private fun nv21ToMat(nv21ByteArray: ByteArray, width: Int, height: Int): Mat? { + val bitmap = nv21toBitmap(nv21ByteArray, width, height) + return bitmap?.let { bitmapToMat(it) } + } + + private fun degree(rotation: Int): Int { + if (rotation == 0) return 1 + if (rotation == 1) return 4 + if (rotation == 2) return 3 + if (rotation == 3) return 2 + return 1 + } + +} + diff --git a/play-services-vision/core/src/main/kotlin/org/microg/gms/vision/face/mlkit/FaceDetector.kt b/play-services-vision/core/src/main/kotlin/org/microg/gms/vision/face/mlkit/FaceDetector.kt new file mode 100644 index 0000000000..256a62ab01 --- /dev/null +++ b/play-services-vision/core/src/main/kotlin/org/microg/gms/vision/face/mlkit/FaceDetector.kt @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: 2025, microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.microg.gms.vision.face.mlkit + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.ImageFormat +import android.media.Image +import android.util.Log +import com.google.android.gms.dynamic.IObjectWrapper +import com.google.android.gms.dynamic.unwrap +import com.google.mlkit.vision.face.FaceDetectionOptions +import com.google.mlkit.vision.face.FrameMetadataParcel +import com.google.mlkit.vision.face.aidls.FaceParcel +import com.google.mlkit.vision.face.aidls.IFaceDetector +import org.microg.gms.vision.face.TAG +import org.microg.gms.vision.face.FaceDetectorHelper +import java.nio.ByteBuffer + +class FaceDetector(val context: Context, val options: FaceDetectionOptions?) : IFaceDetector.Stub() { + + private var mFaceDetector: FaceDetectorHelper? = null + + override fun detectFaces(wrapper: IObjectWrapper?, metadata: FrameMetadataParcel?): List { + Log.d(TAG, "MLKit detectFaces method: metadata:${metadata}") + if (wrapper == null || metadata == null || mFaceDetector == null) return arrayListOf() + val format = metadata.format + val rotation = metadata.rotation + if (format == -1) { + val bitmap = wrapper.unwrap() ?: return arrayListOf() + return mFaceDetector?.detectFaces(bitmap, rotation) ?: arrayListOf() + } + if (format == ImageFormat.NV21) { + val byteBuffer = wrapper.unwrap() ?: return arrayListOf() + return mFaceDetector?.detectFaces(byteBuffer.array(), metadata.width, metadata.height, rotation) ?: arrayListOf() + } + if (format == ImageFormat.YUV_420_888) { + val image = wrapper.unwrap() ?: return arrayListOf() + return mFaceDetector?.detectFaces(image, rotation) ?: arrayListOf() + } + return arrayListOf() + } + + override fun initDetector() { + Log.d(TAG, "MLKit initDetector method isInitialized") + if (mFaceDetector == null) { + try { + mFaceDetector = FaceDetectorHelper(context) + } catch (e: Exception) { + Log.d(TAG, "initDetector: failed", e) + } + } + } + + override fun close() { + Log.d(TAG, "MLKit close") + mFaceDetector?.release() + mFaceDetector = null + } +} \ No newline at end of file diff --git a/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/Contour.aidl b/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/Contour.aidl new file mode 100644 index 0000000000..f78f86c5eb --- /dev/null +++ b/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/Contour.aidl @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.face; + +parcelable Contour; \ No newline at end of file diff --git a/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/Landmark.aidl b/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/Landmark.aidl new file mode 100644 index 0000000000..a422a5b5a5 --- /dev/null +++ b/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/Landmark.aidl @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.face; + +parcelable Landmark; \ No newline at end of file diff --git a/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/internal/client/DetectionOptions.aidl b/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/internal/client/DetectionOptions.aidl new file mode 100644 index 0000000000..1c32dab938 --- /dev/null +++ b/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/internal/client/DetectionOptions.aidl @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.face.internal.client; + +parcelable DetectionOptions; \ No newline at end of file diff --git a/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/internal/client/FaceParcel.aidl b/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/internal/client/FaceParcel.aidl new file mode 100644 index 0000000000..e7da6ddd4c --- /dev/null +++ b/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/internal/client/FaceParcel.aidl @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.face.internal.client; + +parcelable FaceParcel; \ No newline at end of file diff --git a/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/internal/client/INativeFaceDetector.aidl b/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/internal/client/INativeFaceDetector.aidl new file mode 100644 index 0000000000..72241ad645 --- /dev/null +++ b/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/internal/client/INativeFaceDetector.aidl @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.face.internal.client; + +import com.google.android.gms.vision.face.internal.client.FaceParcel; +import com.google.android.gms.dynamic.IObjectWrapper; +import com.google.android.gms.vision.internal.FrameMetadataParcel; + +interface INativeFaceDetector { + FaceParcel[] detectFaceParcels(IObjectWrapper byteBuffer, in FrameMetadataParcel metadata) = 0; + boolean isNativeFaceDetectorAvailable(int type) = 1; + void closeDetectorJni() = 2; + FaceParcel[] detectFacesFromPlanes(IObjectWrapper planeFirst, IObjectWrapper planeSecond, IObjectWrapper planeThird, int firstPixelStride, int secondPixelStride, int thirdPixelStride, int firstRowStride, int secondRowStride, int thirdRowStride, in FrameMetadataParcel metadata) = 3; +} \ No newline at end of file diff --git a/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/internal/client/INativeFaceDetectorCreator.aidl b/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/internal/client/INativeFaceDetectorCreator.aidl new file mode 100644 index 0000000000..3fc5bc64c6 --- /dev/null +++ b/play-services-vision/src/main/aidl/com/google/android/gms/vision/face/internal/client/INativeFaceDetectorCreator.aidl @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.face.internal.client; + +import com.google.android.gms.vision.face.internal.client.DetectionOptions; +import com.google.android.gms.dynamic.IObjectWrapper; +import com.google.android.gms.vision.face.internal.client.INativeFaceDetector; + +interface INativeFaceDetectorCreator { + INativeFaceDetector newFaceDetector(IObjectWrapper context, in DetectionOptions detectionOptions) = 0; +} \ No newline at end of file diff --git a/play-services-vision/src/main/java/com/google/android/gms/vision/face/Contour.java b/play-services-vision/src/main/java/com/google/android/gms/vision/face/Contour.java new file mode 100644 index 0000000000..79296dde02 --- /dev/null +++ b/play-services-vision/src/main/java/com/google/android/gms/vision/face/Contour.java @@ -0,0 +1,40 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.face; + +import android.graphics.PointF; +import android.os.Parcel; + +import androidx.annotation.NonNull; + +import com.google.android.gms.common.internal.safeparcel.AbstractSafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelableCreatorAndWriter; + +import java.util.List; + +@SafeParcelable.Class +public class Contour extends AbstractSafeParcelable { + + @Field(1) + public int type; + @Field(2) + public List points; + + @Constructor + public Contour(@Param(1) int type, @Param(2) List points) { + this.type = type; + this.points = points; + } + + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + CREATOR.writeToParcel(this, dest, flags); + } + + public static final SafeParcelableCreatorAndWriter CREATOR = findCreator(Contour.class); + +} diff --git a/play-services-vision/src/main/java/com/google/android/gms/vision/face/Landmark.java b/play-services-vision/src/main/java/com/google/android/gms/vision/face/Landmark.java new file mode 100644 index 0000000000..d0376c35dd --- /dev/null +++ b/play-services-vision/src/main/java/com/google/android/gms/vision/face/Landmark.java @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.face; + +import android.os.Parcel; + +import androidx.annotation.NonNull; + +import com.google.android.gms.common.internal.safeparcel.AbstractSafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelableCreatorAndWriter; + +@SafeParcelable.Class +public class Landmark extends AbstractSafeParcelable { + + @Field(1) + public int id; + @Field(2) + public Float x; + @Field(3) + public Float y; + @Field(4) + public int type; + + @Constructor + public Landmark(@Param(1) int id, @Param(2) Float x, @Param(3) Float y, @Param(4) int type) { + this.id = id; + this.x = x; + this.y = y; + this.type = type; + } + + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + CREATOR.writeToParcel(this, dest, flags); + } + + public static final SafeParcelableCreatorAndWriter CREATOR = findCreator(Landmark.class); + +} diff --git a/play-services-vision/src/main/java/com/google/android/gms/vision/face/internal/client/DetectionOptions.java b/play-services-vision/src/main/java/com/google/android/gms/vision/face/internal/client/DetectionOptions.java new file mode 100644 index 0000000000..67c2c2f7d0 --- /dev/null +++ b/play-services-vision/src/main/java/com/google/android/gms/vision/face/internal/client/DetectionOptions.java @@ -0,0 +1,38 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.face.internal.client; + +import android.os.Parcel; + +import androidx.annotation.NonNull; + +import com.google.android.gms.common.internal.safeparcel.AbstractSafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelableCreatorAndWriter; + +@SafeParcelable.Class +public class DetectionOptions extends AbstractSafeParcelable { + + @Field(2) + public int mode; + @Field(3) + public int landmarkType; + @Field(4) + public int classificationType; + @Field(5) + public boolean prominentFaceOnly; + @Field(6) + public boolean trackingEnabled; + @Field(7) + public float minFaceSize; + + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + CREATOR.writeToParcel(this, dest, flags); + } + + public static final SafeParcelableCreatorAndWriter CREATOR = findCreator(DetectionOptions.class); +} diff --git a/play-services-vision/src/main/java/com/google/android/gms/vision/face/internal/client/FaceParcel.java b/play-services-vision/src/main/java/com/google/android/gms/vision/face/internal/client/FaceParcel.java new file mode 100644 index 0000000000..32f079ed46 --- /dev/null +++ b/play-services-vision/src/main/java/com/google/android/gms/vision/face/internal/client/FaceParcel.java @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: 2025 microG Project Team + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.google.android.gms.vision.face.internal.client; + +import android.os.Parcel; + +import androidx.annotation.NonNull; + +import com.google.android.gms.common.internal.safeparcel.AbstractSafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelable; +import com.google.android.gms.common.internal.safeparcel.SafeParcelableCreatorAndWriter; +import com.google.android.gms.vision.face.Contour; +import com.google.android.gms.vision.face.Landmark; + +@SafeParcelable.Class +public class FaceParcel extends AbstractSafeParcelable { + + @Field(value = 1, defaultValue = "1") + public final int versionCode; + @Field(2) + public final int id; + @Field(3) + public final float centerX; + @Field(4) + public final float centerY; + @Field(5) + public final float width; + @Field(6) + public final float height; + @Field(7) + public final float eulerY; + @Field(8) + public final float eulerZ; + @Field(14) + public final float eulerX; + @Field(9) + public final Landmark[] landmarks; + @Field(10) + public final float leftEyeOpenProbability; + @Field(11) + public final float rightEyeOpenProbability; + @Field(12) + public final float smileProbability; + @Field(13) + public final Contour[] contours; + @Field(value = 15, defaultValue = "-1.0f") + public final float confidenceScore; + + @Constructor + public FaceParcel(@Param(1) int versionCode, @Param(2) int id, @Param(3) float centerX, @Param(4) float centerY, @Param(5) float width, @Param(6) float height, @Param(7) float eulerY, @Param(8) float eulerZ, @Param(14) float eulerX, @Param(9) Landmark[] landmarks, @Param(10) float leftEyeOpenProbability, @Param(11) float rightEyeOpenProbability, @Param(12) float smileProbability, @Param(13) Contour[] contours, @Param(15) float confidenceScore) { + this.versionCode = versionCode; + this.id = id; + this.centerX = centerX; + this.centerY = centerY; + this.width = width; + this.height = height; + this.eulerY = eulerY; + this.eulerZ = eulerZ; + this.eulerX = eulerX; + this.landmarks = landmarks; + this.leftEyeOpenProbability = leftEyeOpenProbability; + this.rightEyeOpenProbability = rightEyeOpenProbability; + this.smileProbability = smileProbability; + this.contours = contours; + this.confidenceScore = confidenceScore; + } + + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + CREATOR.writeToParcel(this, dest, flags); + } + + public static final SafeParcelableCreatorAndWriter CREATOR = findCreator(FaceParcel.class); +} diff --git a/settings.gradle b/settings.gradle index a4c85dafd1..2bfd78f358 100644 --- a/settings.gradle +++ b/settings.gradle @@ -46,6 +46,7 @@ include ':play-services-location' include ':play-services-maps' include ':play-services-measurement-base' sublude ':play-services-mlkit:barcode-scanning' +sublude ':play-services-mlkit:face-detection' include ':play-services-nearby' include ':play-services-oss-licenses' include ':play-services-panorama'