Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[wip] implement android face contour detection #53

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[wip] implement android face contour detection #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions android/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,5 +39,6 @@ android {
api 'com.google.firebase:firebase-ml-vision-image-label-model:17.+'
implementation 'com.google.firebase:firebase-ml-model-interpreter:16.+'
api "com.android.support:exifinterface:28.0.0"
implementation 'com.google.firebase:firebase-ml-vision-face-model:17.+'
}
}
43 changes: 43 additions & 0 deletions android/src/main/java/com/azihsoyn/flutter/mlkit/MlkitPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@
import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcode;
import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetector;
import com.google.firebase.ml.vision.common.FirebaseVisionImage;
import com.google.firebase.ml.vision.common.FirebaseVisionPoint;
import com.google.firebase.ml.vision.face.FirebaseVisionFace;
import com.google.firebase.ml.vision.face.FirebaseVisionFaceContour;
import com.google.firebase.ml.vision.face.FirebaseVisionFaceDetector;
import com.google.firebase.ml.vision.face.FirebaseVisionFaceDetectorOptions;
import com.google.firebase.ml.vision.face.FirebaseVisionFaceLandmark;
Expand Down Expand Up @@ -76,6 +78,23 @@ public class MlkitPlugin implements MethodCallHandler {
add(FirebaseVisionFaceLandmark.LEFT_CHEEK);
add(FirebaseVisionFaceLandmark.NOSE_BASE);
}});

private static final List<Integer> ContourTypes = Collections.unmodifiableList(new ArrayList<Integer>() {{
add(FirebaseVisionFaceContour.ALL_POINTS);
add(FirebaseVisionFaceContour.FACE);
add(FirebaseVisionFaceContour.LEFT_EYEBROW_TOP);
add(FirebaseVisionFaceContour.LEFT_EYEBROW_BOTTOM);
add(FirebaseVisionFaceContour.RIGHT_EYEBROW_TOP);
add(FirebaseVisionFaceContour.RIGHT_EYEBROW_BOTTOM);
add(FirebaseVisionFaceContour.LEFT_EYE);
add(FirebaseVisionFaceContour.RIGHT_EYE);
add(FirebaseVisionFaceContour.UPPER_LIP_TOP);
add(FirebaseVisionFaceContour.UPPER_LIP_BOTTOM);
add(FirebaseVisionFaceContour.LOWER_LIP_TOP);
add(FirebaseVisionFaceContour.LOWER_LIP_BOTTOM);
add(FirebaseVisionFaceContour.NOSE_BRIDGE);
add(FirebaseVisionFaceContour.NOSE_BOTTOM);
}});
private static Context context;
private static Activity activity;

Expand Down Expand Up @@ -217,6 +236,7 @@ public void onFailure(@NonNull Exception e) {
.setPerformanceMode((int) optionsMap.get("modeType"))
.setLandmarkMode((int) optionsMap.get("landmarkType"))
.setClassificationMode((int) optionsMap.get("classificationType"))
.setContourMode((int)optionsMap.get("contourMode"))
.setMinFaceSize((float) (double) optionsMap.get("minFaceSize"));
if((boolean) optionsMap.get("isTrackingEnabled")){
builder.enableTracking();
Expand Down Expand Up @@ -608,6 +628,7 @@ private ImmutableList<ImmutableMap<String, Object>> processFaceDetectionResult(L
ImmutableList.<ImmutableMap<String, Object>>builder();

for (FirebaseVisionFace face : faces) {
Log.d("face : ", face.toString());
ImmutableMap.Builder<String, Object> faceBuilder = ImmutableMap.<String, Object>builder();
faceBuilder.put("rect_bottom", (double) face.getBoundingBox().bottom);
faceBuilder.put("rect_top", (double) face.getBoundingBox().top);
Expand Down Expand Up @@ -637,6 +658,28 @@ private ImmutableList<ImmutableMap<String, Object>> processFaceDetectionResult(L
}
faceBuilder.put("landmarks", landmarksBuilder.build());

// contour
ImmutableMap.Builder<Integer, Object> contoursBuilder = ImmutableMap.<Integer, Object>builder();
for (Integer contourType : ContourTypes) {
ImmutableMap.Builder<String, Object> contourBuilder = ImmutableMap.<String, Object>builder();
ImmutableList.Builder<Object> pointsBuilder = ImmutableList.<Object>builder();
FirebaseVisionFaceContour contour = face.getContour(contourType);
if (contour != null) {
for (FirebaseVisionPoint point: contour.getPoints()){
ImmutableMap.Builder<String, Object> pointBuilder = ImmutableMap.<String, Object>builder();
pointBuilder.put("x", point.getX());
pointBuilder.put("y", point.getY());
if (point.getZ() != null) {
pointBuilder.put("z", point.getZ());
}
pointsBuilder.add(pointBuilder.build());
}
contourBuilder.put("points", pointsBuilder.build());
contourBuilder.put("type", contourType);
contoursBuilder.put(contourType, contourBuilder.build());
}
}
faceBuilder.put("contours", contoursBuilder.build());

dataBuilder.add(faceBuilder.build());
}
Expand Down
48 changes: 48 additions & 0 deletions lib/mlkit.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -422,11 +422,13 @@ class VisionFaceDetectorOptions {
final VisionFaceDetectorLandmark landmarkType;
final double minFaceSize;
final bool isTrackingEnabled;
VisionFaceDetectorContourMode contourMode;

VisionFaceDetectorOptions(
{this.classificationType: VisionFaceDetectorClassification.None,
this.modeType: VisionFaceDetectorMode.Fast,
this.landmarkType: VisionFaceDetectorLandmark.None,
this.contourMode: VisionFaceDetectorContourMode.NO_CONTOURS,
this.minFaceSize: 0.1,
this.isTrackingEnabled: false});

Expand All @@ -437,6 +439,7 @@ class VisionFaceDetectorOptions {
"landmarkType": landmarkType.value,
"minFaceSize": minFaceSize,
"isTrackingEnabled": isTrackingEnabled,
"contourMode": contourMode.value,
};
}
}
Expand Down Expand Up @@ -465,6 +468,13 @@ class VisionFaceDetectorLandmark {
static const All = const VisionFaceDetectorLandmark._(2);
}

class VisionFaceDetectorContourMode {
final int value;
const VisionFaceDetectorContourMode._(int value) : value = value;
static const NO_CONTOURS = const VisionFaceDetectorContourMode._(1);
static const ALL_CONTOURS = const VisionFaceDetectorContourMode._(2);
}

class VisionFaceLandmark {
final FaceLandmarkType type;
final VisionPoint position;
Expand All @@ -474,6 +484,17 @@ class VisionFaceLandmark {
position = VisionPoint._(data['position']);
}

class VisionFaceContour {
final FaceContourType type;
final List<VisionPoint> points;

VisionFaceContour._(Map<dynamic, dynamic> data)
: type = FaceContourType._(data['type']),
points = data['points']
.map<VisionPoint>((dynamic item) => VisionPoint._(item))
.toList();
}

// ios
// https://firebase.google.com/docs/reference/ios/firebasemlvision/api/reference/Classes/FIRVisionPoint
class VisionPoint {
Expand Down Expand Up @@ -511,6 +532,28 @@ class FaceLandmarkType {
static const MouthRight = const FaceLandmarkType._(11);
}

// android
// https://firebase.google.com/docs/reference/android/com/google/firebase/ml/vision/face/FirebaseVisionFaceContour
class FaceContourType {
final int value;

const FaceContourType._(int value) : value = value;
static const AllPoints = const FaceContourType._(1);
static const Face = const FaceContourType._(2);
static const LeftEeybrowTop = const FaceContourType._(3);
static const LeftEeybrowBottom = const FaceContourType._(4);
static const RightEyebrowTop = const FaceContourType._(5);
static const RightEyebrowBottom = const FaceContourType._(6);
static const LeftEye = const FaceContourType._(7);
static const RightEye = const FaceContourType._(8);
static const UpperLipTop = const FaceContourType._(9);
static const UpperLipBottom = const FaceContourType._(10);
static const LowerLipTop = const FaceContourType._(11);
static const LowerLipBottom = const FaceContourType._(12);
static const NoseBridge = const FaceContourType._(13);
static const NoseBottom = const FaceContourType._(14);
}

class VisionFace {
final Map<dynamic, dynamic> _data;

Expand Down Expand Up @@ -540,6 +583,11 @@ class VisionFace {
_data['landmarks'][type.value] == null
? null
: VisionFaceLandmark._(_data['landmarks'][type.value]);

VisionFaceContour getContour(FaceContourType type) =>
_data['contours'][type.value] == null
? null
: VisionFaceContour._(_data['contours'][type.value]);
}

// ios
Expand Down