AI: Adjust Pigo parameters for improved Face Detection photoprism#5167

Signed-off-by: Michael Mayer <michael@photoprism.app>

Author: lastzero

internal/ai/face/detector.go

@@ -208,35 +208,45 @@ func (d *Detector) Faces(det []pigo.Detection, params pigo.CascadeParams, findLa
 	results = make(Faces, 0, len(det))
 
 	for _, face := range det {
-		// Skip result if quality is too low.
-		if face.Q < QualityThreshold(face.Scale) {
-			continue
+		score := face.Q
+		scale := face.Scale
+		requiredScore := QualityThreshold(scale)
+		scaleMin := LandmarkQualityScaleMin
+		scaleMax := LandmarkQualityScaleMax
+		fallbackCandidate := false
+		if !findLandmarks && score < requiredScore && score >= LandmarkQualityFloor && scale >= scaleMin && scale <= scaleMax && requiredScore-score <= LandmarkQualitySlack {
+			fallbackCandidate = true
 		}
 
 		faceCoord := NewArea(
 			"face",
 			face.Row,
 			face.Col,
-			face.Scale,
+			scale,
 		)
 
 		var eyesCoords []Area
 		var landmarkCoords []Area
+		var eyesFound bool
+
+		needLandmarks := (findLandmarks || fallbackCandidate) && scale > 50
 
-		if findLandmarks && face.Scale > 50 {
-			eyesCoords = make([]Area, 0, 2)
+		if needLandmarks {
+			if findLandmarks {
+				eyesCoords = make([]Area, 0, 2)
+			}
 
-			scale := float32(face.Scale)
+			scaleF := float32(scale)
 			leftCandidate := pigo.Puploc{
-				Row:      face.Row - int(0.075*scale),
-				Col:      face.Col - int(0.175*scale),
-				Scale:    scale * 0.25,
+				Row:      face.Row - int(0.075*scaleF),
+				Col:      face.Col - int(0.175*scaleF),
+				Scale:    scaleF * 0.25,
 				Perturbs: d.perturb,
 			}
 
 			leftEye := plc.RunDetector(leftCandidate, params.ImageParams, d.landmarkAngle, false)
 			leftEyeFound := leftEye.Row > 0 && leftEye.Col > 0
-			if leftEyeFound {
+			if leftEyeFound && findLandmarks {
 				eyesCoords = append(eyesCoords, NewArea(
 					"eye_l",
 					leftEye.Row,
@@ -246,15 +256,15 @@ func (d *Detector) Faces(det []pigo.Detection, params pigo.CascadeParams, findLa
 			}
 
 			rightCandidate := pigo.Puploc{
-				Row:      face.Row - int(0.075*scale),
-				Col:      face.Col + int(0.185*scale),
-				Scale:    scale * 0.25,
+				Row:      face.Row - int(0.075*scaleF),
+				Col:      face.Col + int(0.185*scaleF),
+				Scale:    scaleF * 0.25,
 				Perturbs: d.perturb,
 			}
 
 			rightEye := plc.RunDetector(rightCandidate, params.ImageParams, d.landmarkAngle, false)
 			rightEyeFound := rightEye.Row > 0 && rightEye.Col > 0
-			if rightEyeFound {
+			if rightEyeFound && findLandmarks {
 				eyesCoords = append(eyesCoords, NewArea(
 					"eye_r",
 					rightEye.Row,
@@ -264,71 +274,83 @@ func (d *Detector) Faces(det []pigo.Detection, params pigo.CascadeParams, findLa
 			}
 
 			if leftEyeFound && rightEyeFound {
-				landmarkCapacity := len(eyeCascades)*2 + len(mouthCascades) + 1
-				landmarkCoords = make([]Area, 0, landmarkCapacity)
-
-				for _, eye := range eyeCascades {
-					for _, flpc := range flpcs[eye] {
-						if flpc == nil {
-							continue
-						}
-
-						flp := flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, d.perturb, false)
-						if flp.Row > 0 && flp.Col > 0 {
-							landmarkCoords = append(landmarkCoords, NewArea(
-								eye,
-								flp.Row,
-								flp.Col,
-								int(flp.Scale),
-							))
-						}
-
-						flp = flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, d.perturb, true)
-						if flp.Row > 0 && flp.Col > 0 {
-							landmarkCoords = append(landmarkCoords, NewArea(
-								eye+"_v",
-								flp.Row,
-								flp.Col,
-								int(flp.Scale),
-							))
+				eyesFound = true
+
+				if findLandmarks {
+					landmarkCapacity := len(eyeCascades)*2 + len(mouthCascades) + 1
+					landmarkCoords = make([]Area, 0, landmarkCapacity)
+
+					for _, eye := range eyeCascades {
+						for _, flpc := range flpcs[eye] {
+							if flpc == nil {
+								continue
+							}
+
+							flp := flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, d.perturb, false)
+							if flp.Row > 0 && flp.Col > 0 {
+								landmarkCoords = append(landmarkCoords, NewArea(
+									eye,
+									flp.Row,
+									flp.Col,
+									int(flp.Scale),
+								))
+							}
+
+							flp = flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, d.perturb, true)
+							if flp.Row > 0 && flp.Col > 0 {
+								landmarkCoords = append(landmarkCoords, NewArea(
+									eye+"_v",
+									flp.Row,
+									flp.Col,
+									int(flp.Scale),
+								))
+							}
 						}
 					}
-				}
 
-				for _, mouth := range mouthCascades {
-					for _, flpc := range flpcs[mouth] {
-						if flpc == nil {
-							continue
-						}
-
-						flp := flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, d.perturb, false)
-						if flp.Row > 0 && flp.Col > 0 {
-							landmarkCoords = append(landmarkCoords, NewArea(
-								"mouth_"+mouth,
-								flp.Row,
-								flp.Col,
-								int(flp.Scale),
-							))
+					for _, mouth := range mouthCascades {
+						for _, flpc := range flpcs[mouth] {
+							if flpc == nil {
+								continue
+							}
+
+							flp := flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, d.perturb, false)
+							if flp.Row > 0 && flp.Col > 0 {
+								landmarkCoords = append(landmarkCoords, NewArea(
+									"mouth_"+mouth,
+									flp.Row,
+									flp.Col,
+									int(flp.Scale),
+								))
+							}
 						}
 					}
-				}
 
-				if cascades := flpcs["lp84"]; len(cascades) > 0 {
-					if flpc := cascades[0]; flpc != nil {
-						flp := flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, d.perturb, true)
-						if flp.Row > 0 && flp.Col > 0 {
-							landmarkCoords = append(landmarkCoords, NewArea(
-								"lp84",
-								flp.Row,
-								flp.Col,
-								int(flp.Scale),
-							))
+					if cascades := flpcs["lp84"]; len(cascades) > 0 {
+						if flpc := cascades[0]; flpc != nil {
+							flp := flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, d.perturb, true)
+							if flp.Row > 0 && flp.Col > 0 {
+								landmarkCoords = append(landmarkCoords, NewArea(
+									"lp84",
+									flp.Row,
+									flp.Col,
+									int(flp.Scale),
+								))
+							}
 						}
 					}
 				}
 			}
 		}
 
+		if eyesFound && fallbackCandidate && requiredScore > LandmarkQualityFloor {
+			requiredScore = LandmarkQualityFloor
+		}
+
+		if score < requiredScore {
+			continue
+		}
+
 		// Create face.
 		f := Face{
 			Rows:      params.ImageParams.Rows,

internal/ai/face/detector_test.go

@@ -14,6 +14,8 @@ import (
 	"github.com/photoprism/photoprism/pkg/fs/fastwalk"
 )
 
+var benchmarkFacesCount int
+
 func TestDetect(t *testing.T) {
 	expected := map[string]int{
 		"1.jpg":  1,
@@ -152,7 +154,23 @@ func TestDetectLandmarkCounts(t *testing.T) {
 	}
 }
 
-var benchmarkFacesCount int
+func TestDetectQualityFallback(t *testing.T) {
+	t.SkipNow()
+	faces, err := Detect("testdata/<no public test image available>.jpg", false, 20)
+	require.NoError(t, err)
+	require.NotEmpty(t, faces)
+
+	found := false
+
+	for _, face := range faces {
+		if face.Score < int(QualityThreshold(face.Area.Scale)) {
+			found = true
+			break
+		}
+	}
+
+	require.Truef(t, found, "expected at least one face below the quality threshold, got %+v", faces)
+}
 
 func BenchmarkDetectorFacesLandmarks(b *testing.B) {
 	const sample = "testdata/18.jpg"

internal/ai/face/faces.go

@@ -26,6 +26,17 @@ func (faces Faces) Count() int {
 	return len(faces)
 }
 
+// MaxScale returns the largest face scale in pixels.
+func (faces Faces) MaxScale() (max int) {
+	for _, f := range faces {
+		if f.Area.Scale > max {
+			max = f.Area.Scale
+		}
+	}
+
+	return max
+}
+
 // Uncertainty returns the max face detection uncertainty in percent.
 func (faces Faces) Uncertainty() int {
 	if len(faces) < 1 {

internal/ai/face/thresholds.go

@@ -8,6 +8,10 @@ var CropSize = crop.Sizes[crop.Tile160]          // Face image crop size for Fac
 var OverlapThreshold = 42                        // Face area overlap threshold in percent.
 var OverlapThresholdFloor = OverlapThreshold - 1 // Reduced overlap area to avoid rounding inconsistencies.
 var ScoreThreshold = 9.0                         // Min face score.
+var LandmarkQualityFloor = float32(5.0)          // Min score when both eyes are located.
+var LandmarkQualityScaleMin = 60                 // Min face size eligible for landmark-based quality fallback.
+var LandmarkQualityScaleMax = 90                 // Max face size eligible for landmark-based quality fallback.
+var LandmarkQualitySlack = float32(4.0)          // Max allowed gap between quality threshold and score.
 var ClusterScoreThreshold = 15                   // Min score for faces forming a cluster.
 var SizeThreshold = 50                           // Min face size in pixels.
 var ClusterSizeThreshold = 80                    // Min size for faces forming a cluster in pixels.

internal/entity/src.go

@@ -33,6 +33,7 @@ const (
 	SrcLocation Src = classify.SrcLocation // Prio 8
 	SrcMarker   Src = "marker"             // Prio 8
 	SrcImage    Src = classify.SrcImage    // Prio 8
+	SrcONNX     Src = "onnx"
 	SrcOllama   Src = "ollama"
 	SrcOpenAI   Src = "openai"
 	SrcTitle    Src = classify.SrcTitle   // Prio 16
@@ -69,6 +70,7 @@ var SrcPriority = Priorities{
 	SrcLocation: 8,
 	SrcMarker:   8,
 	SrcImage:    8,
+	SrcONNX:     16,
 	SrcOllama:   16,
 	SrcOpenAI:   16,
 	SrcTitle:    16,
@@ -117,6 +119,7 @@ var SrcDesc = map[Src]string{
 	SrcLocation: "GPS Position",
 	SrcMarker:   "Object Detection",
 	SrcImage:    "Computer Vision (default)",
+	SrcONNX:     "Computer Vision (ONNX)",
 	SrcOllama:   "Computer Vision (Ollama)",
 	SrcOpenAI:   "Computer Vision (OpenAI)",
 	SrcTitle:    "Picture Title",

internal/photoprism/index_faces.go

@@ -46,6 +46,31 @@ func (ind *Index) Faces(jpeg *MediaFile, expected int) face.Faces {
 		log.Debugf("vision: %s in %s (detect faces)", err, clean.Log(jpeg.BaseName()))
 	}
 
+	if thumbSize != thumb.Fit1280 {
+		needRetry := len(faces) == 0
+
+		if !needRetry && expected > 0 && len(faces) < expected {
+			needRetry = true
+		}
+
+		if !needRetry && len(faces) > 0 && faces.MaxScale() < 96 {
+			needRetry = true
+		}
+
+		if needRetry {
+			if altThumb, altErr := jpeg.Thumbnail(Config().ThumbCachePath(), thumb.Fit1280); altErr != nil {
+				log.Debugf("vision: %s in %s (detect faces @1280)", altErr, clean.Log(jpeg.BaseName()))
+			} else if altThumb == "" {
+				log.Debugf("vision: thumb %s not found in %s (detect faces @1280)", thumb.Fit1280, clean.Log(jpeg.BaseName()))
+			} else if retryFaces, retryErr := vision.Faces(altThumb, Config().FaceSize(), true, expected); retryErr != nil {
+				log.Debugf("vision: %s in %s (detect faces @1280)", retryErr, clean.Log(jpeg.BaseName()))
+			} else if len(retryFaces) > 0 {
+				log.Debugf("vision: retry face detection for %s using %s", clean.Log(jpeg.BaseName()), thumb.Fit1280)
+				faces = retryFaces
+			}
+		}
+	}
+
 	if l := len(faces); l > 0 {
 		log.Infof("vision: found %s in %s [%s]", english.Plural(l, "face", "faces"), clean.Log(jpeg.BaseName()), time.Since(start))
 	}