From b03570758feea9f9daf587a48d2f256b33cbc6a9 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Wed, 11 Apr 2018 15:58:47 +0300 Subject: [PATCH 01/29] Change the imports in Go code, reflecting the fork --- nfnt/resizer.go | 2 +- smartcrop.go | 2 +- smartcrop_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nfnt/resizer.go b/nfnt/resizer.go index 0f1741a..7709bd5 100644 --- a/nfnt/resizer.go +++ b/nfnt/resizer.go @@ -30,8 +30,8 @@ package nfnt import ( "image" - "github.com/muesli/smartcrop/options" "github.com/nfnt/resize" + "github.com/svkoskin/smartcrop/options" ) type nfntResizer struct { diff --git a/smartcrop.go b/smartcrop.go index 5796ac4..222c9c7 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -40,7 +40,7 @@ import ( "math" "time" - "github.com/muesli/smartcrop/options" + "github.com/svkoskin/smartcrop/options" "golang.org/x/image/draw" ) diff --git a/smartcrop_test.go b/smartcrop_test.go index fbb2ae7..c3d39bc 100644 --- a/smartcrop_test.go +++ b/smartcrop_test.go @@ -38,7 +38,7 @@ import ( "strings" "testing" - "github.com/muesli/smartcrop/nfnt" + "github.com/svkoskin/smartcrop/nfnt" ) var ( From 9fa7dac00c3b8cc5428404ca6d44eca5cb554b9f Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Thu, 12 Apr 2018 15:20:38 +0300 Subject: [PATCH 02/29] Declare and use an interface for edge, skin and saturation detectors --- smartcrop.go | 71 +++++++++++++++++++++++++++++++++++------------ smartcrop_test.go | 3 +- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/smartcrop.go b/smartcrop.go index 222c9c7..0db48e5 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -102,6 +102,15 @@ type Logger struct { Log *log.Logger } +/* + Detector contains a method that detects either skin, features or saturation. Its Detect method writes + the detected skin, features or saturation to red, green and blue channels, respectively. +*/ +type Detector interface { + Name() string + Detect(original *image.RGBA, sharedResult *image.RGBA) error +} + type smartcropAnalyzer struct { logger Logger options.Resizer @@ -249,25 +258,30 @@ func score(output *image.RGBA, crop Crop) Score { return score } -func analyse(logger Logger, img *image.RGBA, cropWidth, cropHeight, realMinScale float64) (image.Rectangle, error) { +func analyse(logger Logger, detectors []Detector, img *image.RGBA, cropWidth, cropHeight, realMinScale float64) (image.Rectangle, error) { o := image.NewRGBA(img.Bounds()) - now := time.Now() - edgeDetect(img, o) - logger.Log.Println("Time elapsed edge:", time.Since(now)) - debugOutput(logger.DebugMode, o, "edge") - - now = time.Now() - skinDetect(img, o) - logger.Log.Println("Time elapsed skin:", time.Since(now)) - debugOutput(logger.DebugMode, o, "skin") + detectors := []Detector{ + &EdgeDetector{}, + &SkinDetector{}, + &SaturationDetector{}, + } - now = time.Now() - saturationDetect(img, o) - logger.Log.Println("Time elapsed sat:", time.Since(now)) - debugOutput(logger.DebugMode, o, "saturation") + /* + Run each detector. They write to R (skin), G (features) and B (saturation) channels on image 'o'. + The score function will use that information. + */ + for _, d := range detectors { + start := time.Now() + err := d.Detect(img, o) + if err != nil { + return image.Rectangle{}, err + } + logger.Log.Printf("Time elapsed detecting %s: %s\n", d.Name(), time.Since(start)) + debugOutput(logger.DebugMode, o, d.Name()) + } - now = time.Now() + now := time.Now() var topCrop Crop topScore := -1.0 cs := crops(o, cropWidth, cropHeight, realMinScale) @@ -361,7 +375,13 @@ func makeCies(img *image.RGBA) []float64 { return cies } -func edgeDetect(i *image.RGBA, o *image.RGBA) { +type EdgeDetector struct{} + +func (d *EdgeDetector) Name() string { + return "edge" +} + +func (d *EdgeDetector) Detect(i *image.RGBA, o *image.RGBA) error { width := i.Bounds().Dx() height := i.Bounds().Dy() cies := makeCies(i) @@ -384,9 +404,16 @@ func edgeDetect(i *image.RGBA, o *image.RGBA) { o.SetRGBA(x, y, nc) } } + return nil } -func skinDetect(i *image.RGBA, o *image.RGBA) { +type SkinDetector struct{} + +func (d *SkinDetector) Name() string { + return "skin" +} + +func (d *SkinDetector) Detect(i *image.RGBA, o *image.RGBA) error { width := i.Bounds().Dx() height := i.Bounds().Dy() @@ -406,9 +433,16 @@ func skinDetect(i *image.RGBA, o *image.RGBA) { } } } + return nil +} + +type SaturationDetector struct{} + +func (d *SaturationDetector) Name() string { + return "saturation" } -func saturationDetect(i *image.RGBA, o *image.RGBA) { +func (d *SaturationDetector) Detect(i *image.RGBA, o *image.RGBA) error { width := i.Bounds().Dx() height := i.Bounds().Dy() @@ -428,6 +462,7 @@ func saturationDetect(i *image.RGBA, o *image.RGBA) { } } } + return nil } func crops(i image.Image, cropWidth, cropHeight, realMinScale float64) []Crop { diff --git a/smartcrop_test.go b/smartcrop_test.go index c3d39bc..cd77a6e 100644 --- a/smartcrop_test.go +++ b/smartcrop_test.go @@ -118,8 +118,9 @@ func BenchmarkEdge(b *testing.B) { rgbaImg := toRGBA(img) b.ResetTimer() for i := 0; i < b.N; i++ { + d := EdgeDetector{} o := image.NewRGBA(img.Bounds()) - edgeDetect(rgbaImg, o) + d.Detect(rgbaImg, o) } } From fa369abf7594d7d3f2bf87f863c331bb2ae67167 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Thu, 12 Apr 2018 15:28:08 +0300 Subject: [PATCH 03/29] Allow Analyzer users to override Detectors --- smartcrop.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/smartcrop.go b/smartcrop.go index 0db48e5..0ab52ac 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -81,6 +81,7 @@ const ( // width and height returns an error if invalid type Analyzer interface { FindBestCrop(img image.Image, width, height int) (image.Rectangle, error) + SetDetectors(ds []Detector) } // Score contains values that classify matches @@ -112,7 +113,8 @@ type Detector interface { } type smartcropAnalyzer struct { - logger Logger + detectors []Detector + logger Logger options.Resizer } @@ -130,7 +132,19 @@ func NewAnalyzerWithLogger(resizer options.Resizer, logger Logger) Analyzer { if logger.Log == nil { logger.Log = log.New(ioutil.Discard, "", 0) } - return &smartcropAnalyzer{Resizer: resizer, logger: logger} + + // Set default detectors here + detectors := []Detector{ + &EdgeDetector{}, + &SkinDetector{}, + &SaturationDetector{}, + } + + return &smartcropAnalyzer{detectors: detectors, Resizer: resizer, logger: logger} +} + +func (o *smartcropAnalyzer) SetDetectors(ds []Detector) { + o.detectors = ds } func (o smartcropAnalyzer) FindBestCrop(img image.Image, width, height int) (image.Rectangle, error) { @@ -172,7 +186,7 @@ func (o smartcropAnalyzer) FindBestCrop(img image.Image, width, height int) (ima o.logger.Log.Printf("original resolution: %dx%d\n", img.Bounds().Dx(), img.Bounds().Dy()) o.logger.Log.Printf("scale: %f, cropw: %f, croph: %f, minscale: %f\n", scale, cropWidth, cropHeight, realMinScale) - topCrop, err := analyse(o.logger, lowimg, cropWidth, cropHeight, realMinScale) + topCrop, err := analyse(o.logger, o.detectors, lowimg, cropWidth, cropHeight, realMinScale) if err != nil { return topCrop, err } @@ -261,12 +275,6 @@ func score(output *image.RGBA, crop Crop) Score { func analyse(logger Logger, detectors []Detector, img *image.RGBA, cropWidth, cropHeight, realMinScale float64) (image.Rectangle, error) { o := image.NewRGBA(img.Bounds()) - detectors := []Detector{ - &EdgeDetector{}, - &SkinDetector{}, - &SaturationDetector{}, - } - /* Run each detector. They write to R (skin), G (features) and B (saturation) channels on image 'o'. The score function will use that information. From 654bf38b6dc542ffe5de854d7a540d4366d4d51f Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Wed, 11 Apr 2018 16:06:28 +0300 Subject: [PATCH 04/29] Include a tool for debug runs (based on README) --- smartcrop-rundebug/main.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 smartcrop-rundebug/main.go diff --git a/smartcrop-rundebug/main.go b/smartcrop-rundebug/main.go new file mode 100644 index 0000000..3871319 --- /dev/null +++ b/smartcrop-rundebug/main.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "log" + "os" + + "github.com/svkoskin/smartcrop" + "github.com/svkoskin/smartcrop/nfnt" +) + +func main() { + if len(os.Args) < 2 { + fmt.Println("Please give me an argument") + os.Exit(1) + } + + f, _ := os.Open(os.Args[1]) + img, _, _ := image.Decode(f) + + l := smartcrop.Logger{ + DebugMode: true, + Log: log.New(os.Stderr, "", 0), + } + + analyzer := smartcrop.NewAnalyzerWithLogger(nfnt.NewDefaultResizer(), l) + topCrop, _ := analyzer.FindBestCrop(img, 300, 200) + + // The crop will have the requested aspect ratio, but you need to copy/scale it yourself + fmt.Printf("Top crop: %+v\n", topCrop) +} From d6b4b4cf0d243338840f15239c018ecd51804aa3 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Thu, 12 Apr 2018 15:53:31 +0300 Subject: [PATCH 05/29] Restore face detection deleted in 'f81c194e7d11e4d031b45c072aea29043c698500' Restore pieces of deleted face detection code, to its new location. --- gocv/face.go | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 gocv/face.go diff --git a/gocv/face.go b/gocv/face.go new file mode 100644 index 0000000..be4c239 --- /dev/null +++ b/gocv/face.go @@ -0,0 +1,62 @@ +package gocv + +import ( + "fmt" + "image" + "image/color" + "log" + "os" + + "github.com/llgcode/draw2d/draw2dimg" + "github.com/llgcode/draw2d/draw2dkit" +) + +type FaceDetector struct { + FaceDetectionHaarCascadeFilepath string + DebugMode bool +} + +func (d *FaceDetector) Name() string { + return "face" +} + +func (d *FaceDetector) Detect(i *image.RGBA, o *image.RGBA) error { + // TODO: Fix to use gocv + + if d.FaceDetectionHaarCascadeFilepath == "" { + return fmt.Errorf("FaceDetector's FaceDetectionHaarCascadeFilepath not specified") + } + + _, err := os.Stat(d.FaceDetectionHaarCascadeFilepath) + if err != nil { + return err + } + cascade := opencv.LoadHaarClassifierCascade(d.FaceDetectionHaarCascadeFilepath) + defer cascade.Release() + + cvImage := opencv.FromImage(i) + defer cvImage.Release() + + faces := cascade.DetectObjects(cvImage) + + gc := draw2dimg.NewGraphicContext(o) + + if d.DebugMode == true { + log.Println("Faces detected:", len(faces)) + } + + for _, face := range faces { + if d.DebugMode == true { + log.Printf("Face: x: %d y: %d w: %d h: %d\n", face.X(), face.Y(), face.Width(), face.Height()) + } + draw2dkit.Ellipse( + gc, + float64(face.X()+(face.Width()/2)), + float64(face.Y()+(face.Height()/2)), + float64(face.Width()/2), + float64(face.Height())/2) + gc.SetFillColor(color.RGBA{255, 0, 0, 255}) + gc.Fill() + } + return nil +} From 41573b11851772ef4eec11a4b9350c2e96bb488d Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Thu, 12 Apr 2018 16:21:31 +0300 Subject: [PATCH 06/29] gocv api --- gocv/face.go | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/gocv/face.go b/gocv/face.go index be4c239..0ef49a5 100644 --- a/gocv/face.go +++ b/gocv/face.go @@ -9,6 +9,7 @@ import ( "github.com/llgcode/draw2d/draw2dimg" "github.com/llgcode/draw2d/draw2dkit" + "gocv.io/x/gocv" ) type FaceDetector struct { @@ -21,8 +22,12 @@ func (d *FaceDetector) Name() string { } func (d *FaceDetector) Detect(i *image.RGBA, o *image.RGBA) error { - // TODO: Fix to use gocv - + if i == nil { + return fmt.Errorf("i can't be nil") + } + if o == nil { + return fmt.Errorf("o can't be nil") + } if d.FaceDetectionHaarCascadeFilepath == "" { return fmt.Errorf("FaceDetector's FaceDetectionHaarCascadeFilepath not specified") } @@ -31,13 +36,18 @@ func (d *FaceDetector) Detect(i *image.RGBA, o *image.RGBA) error { if err != nil { return err } - cascade := opencv.LoadHaarClassifierCascade(d.FaceDetectionHaarCascadeFilepath) - defer cascade.Release() - cvImage := opencv.FromImage(i) - defer cvImage.Release() + classifier := gocv.NewCascadeClassifier() + defer classifier.Close() + if !classifier.Load(d.FaceDetectionHaarCascadeFilepath) { + return fmt.Errorf("FaceDetector failed loading cascade file") + } + + // image.NRGBA-compatible params + cvMat := gocv.NewMatFromBytes(i.Rect.Dy(), i.Rect.Dx(), gocv.MatTypeCV8UC4, i.Pix) + defer cvMat.Close() - faces := cascade.DetectObjects(cvImage) + faces := classifier.DetectMultiScale(cvMat) gc := draw2dimg.NewGraphicContext(o) @@ -46,15 +56,24 @@ func (d *FaceDetector) Detect(i *image.RGBA, o *image.RGBA) error { } for _, face := range faces { + // Upper left corner of detected face-rectangle + x := face.Min.X + y := face.Min.Y + + width := face.Dx() + height := face.Dy() + if d.DebugMode == true { - log.Printf("Face: x: %d y: %d w: %d h: %d\n", face.X(), face.Y(), face.Width(), face.Height()) + log.Printf("Face: x: %d y: %d w: %d h: %d\n", x, y, width, height) } + + // Draw a filled circle where the face is draw2dkit.Ellipse( gc, - float64(face.X()+(face.Width()/2)), - float64(face.Y()+(face.Height()/2)), - float64(face.Width()/2), - float64(face.Height())/2) + float64(x+(width/2)), + float64(y+(height/2)), + float64(width/2), + float64(height)/2) gc.SetFillColor(color.RGBA{255, 0, 0, 255}) gc.Fill() } From e15f8103c03e1072caf2d50e0eb38ea0b5078360 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Thu, 12 Apr 2018 16:34:14 +0300 Subject: [PATCH 07/29] Reset skin bias --- smartcrop.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartcrop.go b/smartcrop.go index 0ab52ac..ce23fae 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -54,7 +54,7 @@ var ( const ( detailWeight = 0.2 - skinBias = 0.01 + skinBias = 0.9 skinBrightnessMin = 0.2 skinBrightnessMax = 1.0 skinThreshold = 0.8 From 96dc0c110cc4e1904154531dd5bf21435fcc94f3 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Thu, 12 Apr 2018 16:40:36 +0300 Subject: [PATCH 08/29] Describe example usage of gocv features --- smartcrop-rundebug/main.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/smartcrop-rundebug/main.go b/smartcrop-rundebug/main.go index 3871319..e0f49e7 100644 --- a/smartcrop-rundebug/main.go +++ b/smartcrop-rundebug/main.go @@ -10,6 +10,7 @@ import ( "github.com/svkoskin/smartcrop" "github.com/svkoskin/smartcrop/nfnt" + // "github.com/svkoskin/smartcrop/gocv" ) func main() { @@ -27,6 +28,17 @@ func main() { } analyzer := smartcrop.NewAnalyzerWithLogger(nfnt.NewDefaultResizer(), l) + + /* + To replace skin detection with gocv-based face detection: + + analyzer.SetDetectors([]smartcrop.Detector{ + &smartcrop.EdgeDetector{}, + &gocv.FaceDetector{"./cascade.xml", true}, + &smartcrop.SaturationDetector{}, + }) + */ + topCrop, _ := analyzer.FindBestCrop(img, 300, 200) // The crop will have the requested aspect ratio, but you need to copy/scale it yourself From a3a40ff9611c38019a903bba056a8366552e6747 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Mon, 16 Apr 2018 15:10:35 +0300 Subject: [PATCH 09/29] Restore imports --- nfnt/resizer.go | 2 +- smartcrop-rundebug/main.go | 6 +++--- smartcrop.go | 2 +- smartcrop_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nfnt/resizer.go b/nfnt/resizer.go index 7709bd5..0f1741a 100644 --- a/nfnt/resizer.go +++ b/nfnt/resizer.go @@ -30,8 +30,8 @@ package nfnt import ( "image" + "github.com/muesli/smartcrop/options" "github.com/nfnt/resize" - "github.com/svkoskin/smartcrop/options" ) type nfntResizer struct { diff --git a/smartcrop-rundebug/main.go b/smartcrop-rundebug/main.go index e0f49e7..e53d8f9 100644 --- a/smartcrop-rundebug/main.go +++ b/smartcrop-rundebug/main.go @@ -8,9 +8,9 @@ import ( "log" "os" - "github.com/svkoskin/smartcrop" - "github.com/svkoskin/smartcrop/nfnt" - // "github.com/svkoskin/smartcrop/gocv" + "github.com/muesli/smartcrop" + // "github.com/muesli/smartcrop/gocv" + "github.com/muesli/smartcrop/nfnt" ) func main() { diff --git a/smartcrop.go b/smartcrop.go index ce23fae..637c7fc 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -40,7 +40,7 @@ import ( "math" "time" - "github.com/svkoskin/smartcrop/options" + "github.com/muesli/smartcrop/options" "golang.org/x/image/draw" ) diff --git a/smartcrop_test.go b/smartcrop_test.go index cd77a6e..5f1b764 100644 --- a/smartcrop_test.go +++ b/smartcrop_test.go @@ -38,7 +38,7 @@ import ( "strings" "testing" - "github.com/svkoskin/smartcrop/nfnt" + "github.com/muesli/smartcrop/nfnt" ) var ( From 6da9bd309e17a16e4712efc1f19b4827ab3cac40 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Tue, 5 Jun 2018 13:32:48 +0300 Subject: [PATCH 10/29] Revert "Restore imports" This reverts commit a3a40ff9611c38019a903bba056a8366552e6747. --- nfnt/resizer.go | 2 +- smartcrop-rundebug/main.go | 6 +++--- smartcrop.go | 2 +- smartcrop_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nfnt/resizer.go b/nfnt/resizer.go index 0f1741a..7709bd5 100644 --- a/nfnt/resizer.go +++ b/nfnt/resizer.go @@ -30,8 +30,8 @@ package nfnt import ( "image" - "github.com/muesli/smartcrop/options" "github.com/nfnt/resize" + "github.com/svkoskin/smartcrop/options" ) type nfntResizer struct { diff --git a/smartcrop-rundebug/main.go b/smartcrop-rundebug/main.go index e53d8f9..e0f49e7 100644 --- a/smartcrop-rundebug/main.go +++ b/smartcrop-rundebug/main.go @@ -8,9 +8,9 @@ import ( "log" "os" - "github.com/muesli/smartcrop" - // "github.com/muesli/smartcrop/gocv" - "github.com/muesli/smartcrop/nfnt" + "github.com/svkoskin/smartcrop" + "github.com/svkoskin/smartcrop/nfnt" + // "github.com/svkoskin/smartcrop/gocv" ) func main() { diff --git a/smartcrop.go b/smartcrop.go index 637c7fc..ce23fae 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -40,7 +40,7 @@ import ( "math" "time" - "github.com/muesli/smartcrop/options" + "github.com/svkoskin/smartcrop/options" "golang.org/x/image/draw" ) diff --git a/smartcrop_test.go b/smartcrop_test.go index 5f1b764..cd77a6e 100644 --- a/smartcrop_test.go +++ b/smartcrop_test.go @@ -38,7 +38,7 @@ import ( "strings" "testing" - "github.com/muesli/smartcrop/nfnt" + "github.com/svkoskin/smartcrop/nfnt" ) var ( From 0b7246fba030e4c41110619fb6973c8c601f60e9 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Wed, 30 May 2018 16:56:30 +0300 Subject: [PATCH 11/29] Tests: Remove a noisy debug print --- smartcrop_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/smartcrop_test.go b/smartcrop_test.go index cd77a6e..6af1dee 100644 --- a/smartcrop_test.go +++ b/smartcrop_test.go @@ -29,7 +29,6 @@ package smartcrop import ( "errors" - "fmt" "image" _ "image/jpeg" _ "image/png" @@ -147,7 +146,6 @@ func BenchmarkImageDir(b *testing.B) { b.Error(err) continue } - fmt.Printf("Top crop: %+v\n", topCrop) sub, ok := img.(SubImager) if ok { @@ -159,5 +157,4 @@ func BenchmarkImageDir(b *testing.B) { } } } - // fmt.Println("average time/image:", b.t) } From 15bd6aceaa10229c07724b560332b637e5151467 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Mon, 4 Jun 2018 13:56:23 +0300 Subject: [PATCH 12/29] Make analyse a method of smartcropAnalyzer smartcropAnalyzer will soon have the configuraed detectors as its members, and a method can use them without having to pass them as parameters --- smartcrop.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/smartcrop.go b/smartcrop.go index ce23fae..4ad773d 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -186,7 +186,7 @@ func (o smartcropAnalyzer) FindBestCrop(img image.Image, width, height int) (ima o.logger.Log.Printf("original resolution: %dx%d\n", img.Bounds().Dx(), img.Bounds().Dy()) o.logger.Log.Printf("scale: %f, cropw: %f, croph: %f, minscale: %f\n", scale, cropWidth, cropHeight, realMinScale) - topCrop, err := analyse(o.logger, o.detectors, lowimg, cropWidth, cropHeight, realMinScale) + topCrop, err := o.analyse(lowimg, cropWidth, cropHeight, realMinScale) if err != nil { return topCrop, err } @@ -272,42 +272,42 @@ func score(output *image.RGBA, crop Crop) Score { return score } -func analyse(logger Logger, detectors []Detector, img *image.RGBA, cropWidth, cropHeight, realMinScale float64) (image.Rectangle, error) { +func (a *smartcropAnalyzer) analyse(img *image.RGBA, cropWidth, cropHeight, realMinScale float64) (image.Rectangle, error) { o := image.NewRGBA(img.Bounds()) /* Run each detector. They write to R (skin), G (features) and B (saturation) channels on image 'o'. The score function will use that information. */ - for _, d := range detectors { + for _, d := range a.detectors { start := time.Now() err := d.Detect(img, o) if err != nil { return image.Rectangle{}, err } - logger.Log.Printf("Time elapsed detecting %s: %s\n", d.Name(), time.Since(start)) - debugOutput(logger.DebugMode, o, d.Name()) + a.logger.Log.Printf("Time elapsed detecting %s: %s\n", d.Name(), time.Since(start)) + debugOutput(a.logger.DebugMode, o, d.Name()) } now := time.Now() var topCrop Crop topScore := -1.0 cs := crops(o, cropWidth, cropHeight, realMinScale) - logger.Log.Println("Time elapsed crops:", time.Since(now), len(cs)) + a.logger.Log.Println("Time elapsed crops:", time.Since(now), len(cs)) now = time.Now() for _, crop := range cs { nowIn := time.Now() crop.Score = score(o, crop) - logger.Log.Println("Time elapsed single-score:", time.Since(nowIn)) + a.logger.Log.Println("Time elapsed single-score:", time.Since(nowIn)) if crop.totalScore() > topScore { topCrop = crop topScore = crop.totalScore() } } - logger.Log.Println("Time elapsed score:", time.Since(now)) + a.logger.Log.Println("Time elapsed score:", time.Since(now)) - if logger.DebugMode { + if a.logger.DebugMode { drawDebugCrop(topCrop, o) debugOutput(true, o, "final") } From 964bc81c59e59b3db71041799289fa88178d8316 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Mon, 4 Jun 2018 14:01:14 +0300 Subject: [PATCH 13/29] Introduce a new interface and a field to describe EdgeDetector EdgeDetector's results are handled a bit differently in the scoring phase, so introduce a new interface and a field for it --- smartcrop.go | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/smartcrop.go b/smartcrop.go index 4ad773d..764cb5f 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -81,6 +81,7 @@ const ( // width and height returns an error if invalid type Analyzer interface { FindBestCrop(img image.Image, width, height int) (image.Rectangle, error) + SetDetailDetector(d DetailDetector) SetDetectors(ds []Detector) } @@ -103,6 +104,14 @@ type Logger struct { Log *log.Logger } +/* + DetailDetector detects detail that other Detectors can use. +*/ +type DetailDetector interface { + Name() string + Detect(original *image.RGBA, sharedResult *image.RGBA) error +} + /* Detector contains a method that detects either skin, features or saturation. Its Detect method writes the detected skin, features or saturation to red, green and blue channels, respectively. @@ -113,8 +122,9 @@ type Detector interface { } type smartcropAnalyzer struct { - detectors []Detector - logger Logger + detailDetector DetailDetector + detectors []Detector + logger Logger options.Resizer } @@ -134,19 +144,23 @@ func NewAnalyzerWithLogger(resizer options.Resizer, logger Logger) Analyzer { } // Set default detectors here + detailDetector := &EdgeDetector{} detectors := []Detector{ - &EdgeDetector{}, &SkinDetector{}, &SaturationDetector{}, } - return &smartcropAnalyzer{detectors: detectors, Resizer: resizer, logger: logger} + return &smartcropAnalyzer{detailDetector: detailDetector, detectors: detectors, Resizer: resizer, logger: logger} } func (o *smartcropAnalyzer) SetDetectors(ds []Detector) { o.detectors = ds } +func (o *smartcropAnalyzer) SetDetailDetector(d DetailDetector) { + o.detailDetector = d +} + func (o smartcropAnalyzer) FindBestCrop(img image.Image, width, height int) (image.Rectangle, error) { if width == 0 && height == 0 { return image.Rectangle{}, ErrInvalidDimensions @@ -279,6 +293,16 @@ func (a *smartcropAnalyzer) analyse(img *image.RGBA, cropWidth, cropHeight, real Run each detector. They write to R (skin), G (features) and B (saturation) channels on image 'o'. The score function will use that information. */ + + d := a.detailDetector + start := time.Now() + err := d.Detect(img, o) + if err != nil { + return image.Rectangle{}, err + } + a.logger.Log.Printf("Time elapsed detecting %s: %s\n", d.Name(), time.Since(start)) + debugOutput(a.logger.DebugMode, o, d.Name()) + for _, d := range a.detectors { start := time.Now() err := d.Detect(img, o) From d607a1b1c6dcb105ec0531b7e2a493ee62aa0dfa Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Mon, 4 Jun 2018 14:13:04 +0300 Subject: [PATCH 14/29] Alter face detector initialization --- smartcrop-rundebug/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/smartcrop-rundebug/main.go b/smartcrop-rundebug/main.go index e0f49e7..c60dd61 100644 --- a/smartcrop-rundebug/main.go +++ b/smartcrop-rundebug/main.go @@ -33,7 +33,6 @@ func main() { To replace skin detection with gocv-based face detection: analyzer.SetDetectors([]smartcrop.Detector{ - &smartcrop.EdgeDetector{}, &gocv.FaceDetector{"./cascade.xml", true}, &smartcrop.SaturationDetector{}, }) From ff0aec371fbd29b105191140475b0add74ce080c Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Mon, 4 Jun 2018 14:22:27 +0300 Subject: [PATCH 15/29] Detectors shall provide their Bias and Weight --- smartcrop.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/smartcrop.go b/smartcrop.go index 764cb5f..ffe988e 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -110,15 +110,19 @@ type Logger struct { type DetailDetector interface { Name() string Detect(original *image.RGBA, sharedResult *image.RGBA) error + Weight() float64 } /* Detector contains a method that detects either skin, features or saturation. Its Detect method writes the detected skin, features or saturation to red, green and blue channels, respectively. + Detector contains a method that detects features like skin or saturation. */ type Detector interface { Name() string Detect(original *image.RGBA, sharedResult *image.RGBA) error + Bias() float64 + Weight() float64 } type smartcropAnalyzer struct { @@ -413,6 +417,10 @@ func (d *EdgeDetector) Name() string { return "edge" } +func (d *EdgeDetector) Weight() float64 { + return detailWeight +} + func (d *EdgeDetector) Detect(i *image.RGBA, o *image.RGBA) error { width := i.Bounds().Dx() height := i.Bounds().Dy() @@ -445,6 +453,14 @@ func (d *SkinDetector) Name() string { return "skin" } +func (d *SkinDetector) Bias() float64 { + return skinBias +} + +func (d *SkinDetector) Weight() float64 { + return skinWeight +} + func (d *SkinDetector) Detect(i *image.RGBA, o *image.RGBA) error { width := i.Bounds().Dx() height := i.Bounds().Dy() @@ -474,6 +490,14 @@ func (d *SaturationDetector) Name() string { return "saturation" } +func (d *SaturationDetector) Bias() float64 { + return saturationBias +} + +func (d *SaturationDetector) Weight() float64 { + return saturationWeight +} + func (d *SaturationDetector) Detect(i *image.RGBA, o *image.RGBA) error { width := i.Bounds().Dx() height := i.Bounds().Dy() From 9acfd050f4492c1ca0eebfcde3f7d7f64b0aa202 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Mon, 4 Jun 2018 14:23:50 +0300 Subject: [PATCH 16/29] FaceDetector: Return bias and weight --- gocv/face.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gocv/face.go b/gocv/face.go index 0ef49a5..918dcf5 100644 --- a/gocv/face.go +++ b/gocv/face.go @@ -21,6 +21,14 @@ func (d *FaceDetector) Name() string { return "face" } +func (d *FaceDetector) Bias() float64 { + return 0.9 +} + +func (d *FaceDetector) Weight() float64 { + return 1.8 +} + func (d *FaceDetector) Detect(i *image.RGBA, o *image.RGBA) error { if i == nil { return fmt.Errorf("i can't be nil") From f8c5cd0127854521b4fe77d94c5517ae16f8791b Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Mon, 4 Jun 2018 14:24:23 +0300 Subject: [PATCH 17/29] Revert "Reset skin bias" Revert skin bias to the value in master branch that users are depending on. This reverts commit e15f8103c03e1072caf2d50e0eb38ea0b5078360. --- smartcrop.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartcrop.go b/smartcrop.go index ffe988e..4531a00 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -54,7 +54,7 @@ var ( const ( detailWeight = 0.2 - skinBias = 0.9 + skinBias = 0.01 skinBrightnessMin = 0.2 skinBrightnessMax = 1.0 skinThreshold = 0.8 From cef7d23b0c29153a9109f5428c8ce0b083875679 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Tue, 5 Jun 2018 16:30:46 +0300 Subject: [PATCH 18/29] Satisfy the new Detect interface in detectors --- smartcrop.go | 58 ++++++++++++++++++++++------------------------- smartcrop_test.go | 3 +-- 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/smartcrop.go b/smartcrop.go index 4531a00..6ebee2b 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -105,22 +105,20 @@ type Logger struct { } /* - DetailDetector detects detail that other Detectors can use. + DetailDetector detects detail that can be handled in the scoring phase in a different way. */ type DetailDetector interface { Name() string - Detect(original *image.RGBA, sharedResult *image.RGBA) error + Detect(original *image.RGBA) ([][]uint8, error) Weight() float64 } /* - Detector contains a method that detects either skin, features or saturation. Its Detect method writes - the detected skin, features or saturation to red, green and blue channels, respectively. Detector contains a method that detects features like skin or saturation. */ type Detector interface { Name() string - Detect(original *image.RGBA, sharedResult *image.RGBA) error + Detect(original *image.RGBA) ([][]uint8, error) Bias() float64 Weight() float64 } @@ -421,14 +419,17 @@ func (d *EdgeDetector) Weight() float64 { return detailWeight } -func (d *EdgeDetector) Detect(i *image.RGBA, o *image.RGBA) error { +func (d *EdgeDetector) Detect(i *image.RGBA) ([][]uint8, error) { width := i.Bounds().Dx() height := i.Bounds().Dy() cies := makeCies(i) + res := make([][]uint8, width) + var lightness float64 - for y := 0; y < height; y++ { - for x := 0; x < width; x++ { + for x := 0; x < width; x++ { + res[x] = make([]uint8, height) + for y := 0; y < height; y++ { if x == 0 || x >= width-1 || y == 0 || y >= height-1 { //lightness = cie((*i).At(x, y)) lightness = 0 @@ -440,11 +441,10 @@ func (d *EdgeDetector) Detect(i *image.RGBA, o *image.RGBA) error { cies[x+(y+1)*width] } - nc := color.RGBA{0, uint8(bounds(lightness)), 0, 255} - o.SetRGBA(x, y, nc) + res[x][y] = uint8(bounds(lightness)) } } - return nil + return res, nil } type SkinDetector struct{} @@ -461,27 +461,25 @@ func (d *SkinDetector) Weight() float64 { return skinWeight } -func (d *SkinDetector) Detect(i *image.RGBA, o *image.RGBA) error { +func (d *SkinDetector) Detect(i *image.RGBA) ([][]uint8, error) { width := i.Bounds().Dx() height := i.Bounds().Dy() - for y := 0; y < height; y++ { - for x := 0; x < width; x++ { + res := make([][]uint8, width) + + for x := 0; x < width; x++ { + res[x] = make([]uint8, height) + for y := 0; y < height; y++ { lightness := cie(i.RGBAAt(x, y)) / 255.0 skin := skinCol(i.RGBAAt(x, y)) - c := o.RGBAAt(x, y) if skin > skinThreshold && lightness >= skinBrightnessMin && lightness <= skinBrightnessMax { r := (skin - skinThreshold) * (255.0 / (1.0 - skinThreshold)) - nc := color.RGBA{uint8(bounds(r)), c.G, c.B, 255} - o.SetRGBA(x, y, nc) - } else { - nc := color.RGBA{0, c.G, c.B, 255} - o.SetRGBA(x, y, nc) + res[x][y] = uint8(bounds(r)) } } } - return nil + return res, nil } type SaturationDetector struct{} @@ -498,27 +496,25 @@ func (d *SaturationDetector) Weight() float64 { return saturationWeight } -func (d *SaturationDetector) Detect(i *image.RGBA, o *image.RGBA) error { +func (d *SaturationDetector) Detect(i *image.RGBA) ([][]uint8, error) { width := i.Bounds().Dx() height := i.Bounds().Dy() - for y := 0; y < height; y++ { - for x := 0; x < width; x++ { + res := make([][]uint8, width) + + for x := 0; x < width; x++ { + res[x] = make([]uint8, height) + for y := 0; y < height; y++ { lightness := cie(i.RGBAAt(x, y)) / 255.0 saturation := saturation(i.RGBAAt(x, y)) - c := o.RGBAAt(x, y) if saturation > saturationThreshold && lightness >= saturationBrightnessMin && lightness <= saturationBrightnessMax { b := (saturation - saturationThreshold) * (255.0 / (1.0 - saturationThreshold)) - nc := color.RGBA{c.R, c.G, uint8(bounds(b)), 255} - o.SetRGBA(x, y, nc) - } else { - nc := color.RGBA{c.R, c.G, 0, 255} - o.SetRGBA(x, y, nc) + res[x][y] = uint8(bounds(b)) } } } - return nil + return res, nil } func crops(i image.Image, cropWidth, cropHeight, realMinScale float64) []Crop { diff --git a/smartcrop_test.go b/smartcrop_test.go index 6af1dee..de65cc5 100644 --- a/smartcrop_test.go +++ b/smartcrop_test.go @@ -118,8 +118,7 @@ func BenchmarkEdge(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { d := EdgeDetector{} - o := image.NewRGBA(img.Bounds()) - d.Detect(rgbaImg, o) + d.Detect(rgbaImg) } } From 751d4f93badd0db54b8df7d9531474267d62d57d Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Tue, 5 Jun 2018 16:31:07 +0300 Subject: [PATCH 19/29] Make scoring work with the new detector structure --- smartcrop.go | 98 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 58 insertions(+), 40 deletions(-) diff --git a/smartcrop.go b/smartcrop.go index 6ebee2b..aa590a3 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -87,9 +87,8 @@ type Analyzer interface { // Score contains values that classify matches type Score struct { - Detail float64 - Saturation float64 - Skin float64 + Detail float64 + PerDetector []float64 } // Crop contains results @@ -105,7 +104,7 @@ type Logger struct { } /* - DetailDetector detects detail that can be handled in the scoring phase in a different way. + DetailDetector detects detail that Detectors can use. */ type DetailDetector interface { Name() string @@ -217,8 +216,18 @@ func (o smartcropAnalyzer) FindBestCrop(img image.Image, width, height int) (ima return topCrop.Canon(), nil } -func (c Crop) totalScore() float64 { - return (c.Score.Detail*detailWeight + c.Score.Skin*skinWeight + c.Score.Saturation*saturationWeight) / float64(c.Dx()) / float64(c.Dy()) +func (o *smartcropAnalyzer) totalScoreForCrop(c Crop) float64 { + t := 0.0 + + t += c.Score.Detail * o.detailDetector.Weight() + + for i := range c.Score.PerDetector { + t += c.Score.PerDetector[i] * o.detectors[i].Weight() + } + + t = t / float64(c.Dx()) / float64(c.Dy()) + + return t } func chop(x float64) float64 { @@ -260,83 +269,92 @@ func importance(crop Crop, x, y int) float64 { return s + d } -func score(output *image.RGBA, crop Crop) Score { - width := output.Bounds().Dx() - height := output.Bounds().Dy() +func score(detailDetection detection, detections []detection, crop Crop) Score { + width := len(detailDetection.Pix) + height := len(detailDetection.Pix[0]) score := Score{} + score.PerDetector = make([]float64, len(detections)) + // same loops but with downsampling //for y := 0; y < height; y++ { //for x := 0; x < width; x++ { for y := 0; y <= height-scoreDownSample; y += scoreDownSample { for x := 0; x <= width-scoreDownSample; x += scoreDownSample { - c := output.RGBAAt(x, y) - r8 := float64(c.R) - g8 := float64(c.G) - b8 := float64(c.B) - imp := importance(crop, int(x), int(y)) - det := g8 / 255.0 + det := float64(detailDetection.Pix[x][y]) / 255.0 - score.Skin += r8 / 255.0 * (det + skinBias) * imp score.Detail += det * imp - score.Saturation += b8 / 255.0 * (det + saturationBias) * imp + + for i, d := range detections { + score.PerDetector[i] += float64(d.Pix[x][y]) / 255.0 * (det + d.Bias) * imp + } } } return score } -func (a *smartcropAnalyzer) analyse(img *image.RGBA, cropWidth, cropHeight, realMinScale float64) (image.Rectangle, error) { - o := image.NewRGBA(img.Bounds()) - - /* - Run each detector. They write to R (skin), G (features) and B (saturation) channels on image 'o'. - The score function will use that information. - */ +type detection struct { + Pix [][]uint8 + Weight float64 + Bias float64 +} +func (a *smartcropAnalyzer) analyse(img *image.RGBA, cropWidth, cropHeight, realMinScale float64) (image.Rectangle, error) { d := a.detailDetector start := time.Now() - err := d.Detect(img, o) + detailPix, err := d.Detect(img) if err != nil { return image.Rectangle{}, err } a.logger.Log.Printf("Time elapsed detecting %s: %s\n", d.Name(), time.Since(start)) - debugOutput(a.logger.DebugMode, o, d.Name()) + //debugOutput(a.logger.DebugMode, o, d.Name()) + + detailDetection := detection{Pix: detailPix, Weight: a.detailDetector.Weight()} + + detections := make([]detection, len(a.detectors)) - for _, d := range a.detectors { + for i, d := range a.detectors { start := time.Now() - err := d.Detect(img, o) + pix, err := d.Detect(img) if err != nil { return image.Rectangle{}, err } + + detections[i] = detection{Pix: pix, Weight: d.Weight(), Bias: d.Bias()} + a.logger.Log.Printf("Time elapsed detecting %s: %s\n", d.Name(), time.Since(start)) - debugOutput(a.logger.DebugMode, o, d.Name()) + //debugOutput(a.logger.DebugMode, o, d.Name()) } now := time.Now() var topCrop Crop topScore := -1.0 - cs := crops(o, cropWidth, cropHeight, realMinScale) + cs := crops(img.Bounds(), cropWidth, cropHeight, realMinScale) a.logger.Log.Println("Time elapsed crops:", time.Since(now), len(cs)) now = time.Now() for _, crop := range cs { nowIn := time.Now() - crop.Score = score(o, crop) + crop.Score = score(detailDetection, detections, crop) a.logger.Log.Println("Time elapsed single-score:", time.Since(nowIn)) - if crop.totalScore() > topScore { + + totalScore := a.totalScoreForCrop(crop) + if totalScore > topScore { topCrop = crop - topScore = crop.totalScore() + topScore = totalScore } } a.logger.Log.Println("Time elapsed score:", time.Since(now)) - if a.logger.DebugMode { - drawDebugCrop(topCrop, o) - debugOutput(true, o, "final") - } + /* + if a.logger.DebugMode { + drawDebugCrop(topCrop, o) + debugOutput(true, o, "final") + } + */ return topCrop.Rectangle, nil } @@ -517,10 +535,10 @@ func (d *SaturationDetector) Detect(i *image.RGBA) ([][]uint8, error) { return res, nil } -func crops(i image.Image, cropWidth, cropHeight, realMinScale float64) []Crop { +func crops(bounds image.Rectangle, cropWidth, cropHeight, realMinScale float64) []Crop { res := []Crop{} - width := i.Bounds().Dx() - height := i.Bounds().Dy() + width := bounds.Dx() + height := bounds.Dy() minDimension := math.Min(float64(width), float64(height)) var cropW, cropH float64 From fe4a524687a89a5e492c109113587e1612f4ad51 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Mon, 4 Jun 2018 15:32:19 +0300 Subject: [PATCH 20/29] FaceDetector: Satisfy the new interface --- gocv/face.go | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/gocv/face.go b/gocv/face.go index 918dcf5..d0c41d4 100644 --- a/gocv/face.go +++ b/gocv/face.go @@ -3,12 +3,9 @@ package gocv import ( "fmt" "image" - "image/color" "log" "os" - "github.com/llgcode/draw2d/draw2dimg" - "github.com/llgcode/draw2d/draw2dkit" "gocv.io/x/gocv" ) @@ -29,36 +26,36 @@ func (d *FaceDetector) Weight() float64 { return 1.8 } -func (d *FaceDetector) Detect(i *image.RGBA, o *image.RGBA) error { - if i == nil { - return fmt.Errorf("i can't be nil") +func (d *FaceDetector) Detect(img *image.RGBA) ([][]uint8, error) { + res := make([][]uint8, img.Bounds().Dx()) + for x := range res { + res[x] = make([]uint8, img.Bounds().Dy()) } - if o == nil { - return fmt.Errorf("o can't be nil") + + if img == nil { + return res, fmt.Errorf("img can't be nil") } if d.FaceDetectionHaarCascadeFilepath == "" { - return fmt.Errorf("FaceDetector's FaceDetectionHaarCascadeFilepath not specified") + return res, fmt.Errorf("FaceDetector's FaceDetectionHaarCascadeFilepath not specified") } _, err := os.Stat(d.FaceDetectionHaarCascadeFilepath) if err != nil { - return err + return res, err } classifier := gocv.NewCascadeClassifier() defer classifier.Close() if !classifier.Load(d.FaceDetectionHaarCascadeFilepath) { - return fmt.Errorf("FaceDetector failed loading cascade file") + return res, fmt.Errorf("FaceDetector failed loading cascade file") } // image.NRGBA-compatible params - cvMat := gocv.NewMatFromBytes(i.Rect.Dy(), i.Rect.Dx(), gocv.MatTypeCV8UC4, i.Pix) + cvMat := gocv.NewMatFromBytes(img.Rect.Dy(), img.Rect.Dx(), gocv.MatTypeCV8UC4, img.Pix) defer cvMat.Close() faces := classifier.DetectMultiScale(cvMat) - gc := draw2dimg.NewGraphicContext(o) - if d.DebugMode == true { log.Println("Faces detected:", len(faces)) } @@ -75,15 +72,12 @@ func (d *FaceDetector) Detect(i *image.RGBA, o *image.RGBA) error { log.Printf("Face: x: %d y: %d w: %d h: %d\n", x, y, width, height) } - // Draw a filled circle where the face is - draw2dkit.Ellipse( - gc, - float64(x+(width/2)), - float64(y+(height/2)), - float64(width/2), - float64(height)/2) - gc.SetFillColor(color.RGBA{255, 0, 0, 255}) - gc.Fill() + // Mark the rectangle in our [][]uint8 result + for i := 0; i < width; i++ { + for j := 0; j < height; j++ { + res[x+i][y+j] = 255 + } + } } - return nil + return res, nil } From fc9447038eebefa422a062f4badae411a09da1ca Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Tue, 5 Jun 2018 13:12:46 +0300 Subject: [PATCH 21/29] Restore debug output functionality --- debug.go | 92 +++++++++++++++++++++++++++++++++++++++++++++++++--- smartcrop.go | 22 ++++++++----- 2 files changed, 102 insertions(+), 12 deletions(-) diff --git a/debug.go b/debug.go index d3395fd..3576b70 100644 --- a/debug.go +++ b/debug.go @@ -41,12 +41,94 @@ import ( "path/filepath" ) -func debugOutput(debug bool, img *image.RGBA, debugType string) { - if debug { - writeImage("png", img, "./smartcrop_"+debugType+".png") +// debugImage carries debug output image and has methods for updating and writing it +type DebugImage struct { + img *image.RGBA + colors []color.RGBA + nextColorIdx int +} + +func NewDebugImage(bounds image.Rectangle) *DebugImage { + di := DebugImage{} + + // Set up the actual image + di.img = image.NewRGBA(bounds) + for x := bounds.Min.X; x < bounds.Max.X; x++ { + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + di.img.Set(x, y, color.Black) + } + } + + // Set up an array of colors used for debug outputs + di.colors = []color.RGBA{ + {0, 255, 0, 255}, // default edges + {255, 0, 0, 255}, // default skin + {0, 0, 255, 255}, // default saturation + {255, 128, 0, 255}, // a few extra... + {128, 0, 128, 255}, + {64, 255, 255, 255}, + {255, 64, 255, 255}, + {255, 255, 64, 255}, + {255, 255, 255, 255}, + } + di.nextColorIdx = 0 + return &di +} + +func (di *DebugImage) popNextColor() color.RGBA { + c := di.colors[di.nextColorIdx] + di.nextColorIdx++ + + // Wrap around if necessary (if someone ever implements and sets a tenth detector) + if di.nextColorIdx >= len(di.colors) { + di.nextColorIdx = 0 + } + return c +} + +func scaledColorComponent(factor uint8, oldComponent uint8, newComponent uint8) uint8 { + if factor < 1 { + return oldComponent + } + + return uint8(bounds(float64(factor) / 255.0 * float64(newComponent))) +} + +func (di *DebugImage) AddDetected(d [][]uint8) { + baseColor := di.popNextColor() + + minX := di.img.Bounds().Min.X + minY := di.img.Bounds().Min.Y + + maxX := di.img.Bounds().Max.X + maxY := di.img.Bounds().Max.Y + if maxX > len(d) { + maxX = len(d) + } + if maxY > len(d[0]) { + maxY = len(d[0]) + } + + for x := minX; x < maxX; x++ { + for y := minY; y < maxY; y++ { + if d[x][y] > 0 { + c := di.img.RGBAAt(x, y) + nc := color.RGBA{} + nc.R = scaledColorComponent(d[x][y], c.R, baseColor.R) + nc.G = scaledColorComponent(d[x][y], c.G, baseColor.G) + nc.B = scaledColorComponent(d[x][y], c.B, baseColor.B) + nc.A = 255 + + di.img.SetRGBA(x, y, nc) + } + } } } +func (di *DebugImage) DebugOutput(debugType string) { + writeImage("png", di.img, "./smartcrop_"+debugType+".png") +} + func writeImage(imgtype string, img image.Image, name string) error { if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil { panic(err) @@ -82,7 +164,9 @@ func writeImageToPng(img image.Image, name string) error { return png.Encode(fso, img) } -func drawDebugCrop(topCrop Crop, o *image.RGBA) { +func (di *DebugImage) DrawDebugCrop(topCrop Crop) { + o := di.img + width := o.Bounds().Dx() height := o.Bounds().Dy() diff --git a/smartcrop.go b/smartcrop.go index aa590a3..9a893f4 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -303,6 +303,8 @@ type detection struct { } func (a *smartcropAnalyzer) analyse(img *image.RGBA, cropWidth, cropHeight, realMinScale float64) (image.Rectangle, error) { + debugImg := NewDebugImage(img.Bounds()) + d := a.detailDetector start := time.Now() detailPix, err := d.Detect(img) @@ -310,7 +312,10 @@ func (a *smartcropAnalyzer) analyse(img *image.RGBA, cropWidth, cropHeight, real return image.Rectangle{}, err } a.logger.Log.Printf("Time elapsed detecting %s: %s\n", d.Name(), time.Since(start)) - //debugOutput(a.logger.DebugMode, o, d.Name()) + if a.logger.DebugMode { + debugImg.AddDetected(detailPix) + debugImg.DebugOutput(d.Name()) + } detailDetection := detection{Pix: detailPix, Weight: a.detailDetector.Weight()} @@ -326,7 +331,10 @@ func (a *smartcropAnalyzer) analyse(img *image.RGBA, cropWidth, cropHeight, real detections[i] = detection{Pix: pix, Weight: d.Weight(), Bias: d.Bias()} a.logger.Log.Printf("Time elapsed detecting %s: %s\n", d.Name(), time.Since(start)) - //debugOutput(a.logger.DebugMode, o, d.Name()) + if a.logger.DebugMode { + debugImg.AddDetected(detections[i].Pix) + debugImg.DebugOutput(d.Name()) + } } now := time.Now() @@ -349,12 +357,10 @@ func (a *smartcropAnalyzer) analyse(img *image.RGBA, cropWidth, cropHeight, real } a.logger.Log.Println("Time elapsed score:", time.Since(now)) - /* - if a.logger.DebugMode { - drawDebugCrop(topCrop, o) - debugOutput(true, o, "final") - } - */ + if a.logger.DebugMode { + debugImg.DrawDebugCrop(topCrop) + debugImg.DebugOutput("final") + } return topCrop.Rectangle, nil } From 9b8c1bb48a862f4c897b7813c583870e66bc6c14 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Tue, 24 Apr 2018 15:49:08 +0300 Subject: [PATCH 22/29] Extract logger --- gocv/face.go | 13 +++++++------ logger/logger.go | 11 +++++++++++ smartcrop-rundebug/main.go | 3 ++- smartcrop.go | 13 ++++--------- 4 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 logger/logger.go diff --git a/gocv/face.go b/gocv/face.go index d0c41d4..ffc7f6a 100644 --- a/gocv/face.go +++ b/gocv/face.go @@ -3,15 +3,16 @@ package gocv import ( "fmt" "image" - "log" "os" "gocv.io/x/gocv" + + sclogger "github.com/svkoskin/smartcrop/logger" ) type FaceDetector struct { FaceDetectionHaarCascadeFilepath string - DebugMode bool + Logger *sclogger.Logger } func (d *FaceDetector) Name() string { @@ -56,8 +57,8 @@ func (d *FaceDetector) Detect(img *image.RGBA) ([][]uint8, error) { faces := classifier.DetectMultiScale(cvMat) - if d.DebugMode == true { - log.Println("Faces detected:", len(faces)) + if d.Logger.DebugMode == true { + d.Logger.Log.Printf("Number of faces detected: %d\n", len(faces)) } for _, face := range faces { @@ -68,8 +69,8 @@ func (d *FaceDetector) Detect(img *image.RGBA) ([][]uint8, error) { width := face.Dx() height := face.Dy() - if d.DebugMode == true { - log.Printf("Face: x: %d y: %d w: %d h: %d\n", x, y, width, height) + if d.Logger.DebugMode == true { + d.Logger.Log.Printf("Face: x: %d y: %d w: %d h: %d\n", x, y, width, height) } // Mark the rectangle in our [][]uint8 result diff --git a/logger/logger.go b/logger/logger.go new file mode 100644 index 0000000..3290966 --- /dev/null +++ b/logger/logger.go @@ -0,0 +1,11 @@ +package logger + +import ( + "log" +) + +// Logger contains a logger. +type Logger struct { + DebugMode bool + Log *log.Logger +} diff --git a/smartcrop-rundebug/main.go b/smartcrop-rundebug/main.go index c60dd61..96a4412 100644 --- a/smartcrop-rundebug/main.go +++ b/smartcrop-rundebug/main.go @@ -9,6 +9,7 @@ import ( "os" "github.com/svkoskin/smartcrop" + sclogger "github.com/svkoskin/smartcrop/logger" "github.com/svkoskin/smartcrop/nfnt" // "github.com/svkoskin/smartcrop/gocv" ) @@ -22,7 +23,7 @@ func main() { f, _ := os.Open(os.Args[1]) img, _, _ := image.Decode(f) - l := smartcrop.Logger{ + l := sclogger.Logger{ DebugMode: true, Log: log.New(os.Stderr, "", 0), } diff --git a/smartcrop.go b/smartcrop.go index 9a893f4..702d972 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -40,6 +40,7 @@ import ( "math" "time" + sclogger "github.com/svkoskin/smartcrop/logger" "github.com/svkoskin/smartcrop/options" "golang.org/x/image/draw" @@ -97,12 +98,6 @@ type Crop struct { Score Score } -// Logger contains a logger. -type Logger struct { - DebugMode bool - Log *log.Logger -} - /* DetailDetector detects detail that Detectors can use. */ @@ -125,13 +120,13 @@ type Detector interface { type smartcropAnalyzer struct { detailDetector DetailDetector detectors []Detector - logger Logger + logger sclogger.Logger options.Resizer } // NewAnalyzer returns a new Analyzer using the given Resizer. func NewAnalyzer(resizer options.Resizer) Analyzer { - logger := Logger{ + logger := sclogger.Logger{ DebugMode: false, } @@ -139,7 +134,7 @@ func NewAnalyzer(resizer options.Resizer) Analyzer { } // NewAnalyzerWithLogger returns a new analyzer with the given Resizer and Logger. -func NewAnalyzerWithLogger(resizer options.Resizer, logger Logger) Analyzer { +func NewAnalyzerWithLogger(resizer options.Resizer, logger sclogger.Logger) Analyzer { if logger.Log == nil { logger.Log = log.New(ioutil.Discard, "", 0) } From d2c290a5e123961f7607fbba7f605cb9596d831c Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Mon, 16 Apr 2018 15:10:35 +0300 Subject: [PATCH 23/29] Restore imports --- gocv/face.go | 2 +- nfnt/resizer.go | 2 +- smartcrop-rundebug/main.go | 8 ++++---- smartcrop.go | 6 +++--- smartcrop_test.go | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gocv/face.go b/gocv/face.go index ffc7f6a..dd76ec2 100644 --- a/gocv/face.go +++ b/gocv/face.go @@ -7,7 +7,7 @@ import ( "gocv.io/x/gocv" - sclogger "github.com/svkoskin/smartcrop/logger" + sclogger "github.com/muesli/smartcrop/logger" ) type FaceDetector struct { diff --git a/nfnt/resizer.go b/nfnt/resizer.go index 7709bd5..0f1741a 100644 --- a/nfnt/resizer.go +++ b/nfnt/resizer.go @@ -30,8 +30,8 @@ package nfnt import ( "image" + "github.com/muesli/smartcrop/options" "github.com/nfnt/resize" - "github.com/svkoskin/smartcrop/options" ) type nfntResizer struct { diff --git a/smartcrop-rundebug/main.go b/smartcrop-rundebug/main.go index 96a4412..9fd949e 100644 --- a/smartcrop-rundebug/main.go +++ b/smartcrop-rundebug/main.go @@ -8,10 +8,10 @@ import ( "log" "os" - "github.com/svkoskin/smartcrop" - sclogger "github.com/svkoskin/smartcrop/logger" - "github.com/svkoskin/smartcrop/nfnt" - // "github.com/svkoskin/smartcrop/gocv" + "github.com/muesli/smartcrop" + sclogger "github.com/muesli/smartcrop/logger" + "github.com/muesli/smartcrop/nfnt" + // "github.com/muesli/smartcrop/gocv" ) func main() { diff --git a/smartcrop.go b/smartcrop.go index 702d972..96cf14d 100644 --- a/smartcrop.go +++ b/smartcrop.go @@ -40,10 +40,10 @@ import ( "math" "time" - sclogger "github.com/svkoskin/smartcrop/logger" - "github.com/svkoskin/smartcrop/options" - "golang.org/x/image/draw" + + sclogger "github.com/muesli/smartcrop/logger" + "github.com/muesli/smartcrop/options" ) var ( diff --git a/smartcrop_test.go b/smartcrop_test.go index de65cc5..c3f4d70 100644 --- a/smartcrop_test.go +++ b/smartcrop_test.go @@ -37,7 +37,7 @@ import ( "strings" "testing" - "github.com/svkoskin/smartcrop/nfnt" + "github.com/muesli/smartcrop/nfnt" ) var ( From bbd2e7105e9fc8944cb19716f1714f2ade334d7d Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Tue, 5 Jun 2018 16:42:20 +0300 Subject: [PATCH 24/29] Skip building gocv-pieces in Travis CI for now --- gocv/face.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gocv/face.go b/gocv/face.go index dd76ec2..d78cc2a 100644 --- a/gocv/face.go +++ b/gocv/face.go @@ -1,3 +1,5 @@ +// +build !ci + package gocv import ( From 1961e95ae68557795d94fc59467ccffa9f65cd1c Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Wed, 6 Jun 2018 12:54:11 +0300 Subject: [PATCH 25/29] Face detector: Mark faces with a filled circle instead of a filled rectangle, like it used to before Detector interface changes --- gocv/face.go | 48 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/gocv/face.go b/gocv/face.go index d78cc2a..b24868a 100644 --- a/gocv/face.go +++ b/gocv/face.go @@ -75,12 +75,48 @@ func (d *FaceDetector) Detect(img *image.RGBA) ([][]uint8, error) { d.Logger.Log.Printf("Face: x: %d y: %d w: %d h: %d\n", x, y, width, height) } - // Mark the rectangle in our [][]uint8 result - for i := 0; i < width; i++ { - for j := 0; j < height; j++ { - res[x+i][y+j] = 255 - } - } + drawAFilledCircle(res, x+(width/2), y+(height/2), width/2) } return res, nil } + +func drawAFilledCircle(pix [][]uint8, x0, y0, r int) { + x := r - 1 + y := 0 + dx := 1 + dy := 1 + err := dx - (r << 1) + + for { + if x < y { + return + } + + for i := -x; i <= x; i++ { + putPixel(pix, x0+i, y0+y) + putPixel(pix, x0+i, y0-y) + putPixel(pix, x0+y, y0+i) + putPixel(pix, x0-y, y0+i) + } + + if err <= 0 { + y++ + err += dy + dy += 2 + } else { + x-- + dx += 2 + err += dx - (r << 1) + } + } +} + +func putPixel(pix [][]uint8, x, y int) { + if x >= len(pix) { + return + } + if y >= len(pix[x]) { + return + } + pix[x][y] = uint8(255) +} From f09e3d5364c016cbfdd85f98fb33d3aafd522e84 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Thu, 7 Jun 2018 13:17:27 +0300 Subject: [PATCH 26/29] Debug: Try to blend colors a bit --- debug.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug.go b/debug.go index 3576b70..dc5968b 100644 --- a/debug.go +++ b/debug.go @@ -91,7 +91,7 @@ func scaledColorComponent(factor uint8, oldComponent uint8, newComponent uint8) return oldComponent } - return uint8(bounds(float64(factor) / 255.0 * float64(newComponent))) + return uint8(bounds(((float64(factor)/255.0*float64(newComponent))+float64(oldComponent))/2.0) * 2.0) } func (di *DebugImage) AddDetected(d [][]uint8) { From 2f3124cb58cf75242d67bbcfa303e02b86de0143 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Thu, 7 Jun 2018 13:18:49 +0300 Subject: [PATCH 27/29] Face detector: Fix initialization of gocv.Mat, the constructor has different signature in current gocv --- gocv/face.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gocv/face.go b/gocv/face.go index b24868a..5a715d1 100644 --- a/gocv/face.go +++ b/gocv/face.go @@ -54,8 +54,11 @@ func (d *FaceDetector) Detect(img *image.RGBA) ([][]uint8, error) { } // image.NRGBA-compatible params - cvMat := gocv.NewMatFromBytes(img.Rect.Dy(), img.Rect.Dx(), gocv.MatTypeCV8UC4, img.Pix) + cvMat, err := gocv.NewMatFromBytes(img.Rect.Dy(), img.Rect.Dx(), gocv.MatTypeCV8UC4, img.Pix) defer cvMat.Close() + if err != nil { + return res, err + } faces := classifier.DetectMultiScale(cvMat) From 490810117a336613a0c7300242eb23810f63c327 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Thu, 7 Jun 2018 13:20:39 +0300 Subject: [PATCH 28/29] Fix FaceDetector init example --- smartcrop-rundebug/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartcrop-rundebug/main.go b/smartcrop-rundebug/main.go index 9fd949e..44a6ad8 100644 --- a/smartcrop-rundebug/main.go +++ b/smartcrop-rundebug/main.go @@ -34,7 +34,7 @@ func main() { To replace skin detection with gocv-based face detection: analyzer.SetDetectors([]smartcrop.Detector{ - &gocv.FaceDetector{"./cascade.xml", true}, + &gocv.FaceDetector{"./cascade.xml", &l}, &smartcrop.SaturationDetector{}, }) */ From 58f6608b9f2d3b7285b8024a26388cc7a4ae2fb1 Mon Sep 17 00:00:00 2001 From: Sami Koskinen Date: Thu, 7 Jun 2018 13:34:37 +0300 Subject: [PATCH 29/29] Travis CI: Try to skip building gocv and OpenCV --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index b82ac19..8d53597 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,10 @@ before_install: # - sudo apt-get update -qq # - sudo apt-get install libcv-dev libopencv-dev libopencv-contrib-dev libhighgui-dev libopencv-photo-dev libopencv-imgproc-dev libopencv-stitching-dev libopencv-superres-dev libopencv-ts-dev libopencv-videostab-dev +install: +# It's complicated to get OpenCV built on each platform, so exclude the OpenCV-based detector and its dependencies this way. + - go get -t $(go list ./... | grep -v gocv | xargs) + script: - go test -v -tags ci ./... - if [[ $TRAVIS_GO_VERSION == 1.9* ]]; then $GOPATH/bin/goveralls -service=travis-ci; fi