diff --git a/app/background.cc b/app/background.cc index 51266bd..d3cc8b0 100644 --- a/app/background.cc +++ b/app/background.cc @@ -9,6 +9,16 @@ #include #include +#include + +#if CV_VERSION_MAJOR < 4 ||\ + CV_VERSION_MAJOR == 4 && CV_VERSION_MINOR < 5 ||\ + CV_VERSION_MAJOR == 4 && CV_VERSION_MINOR == 5 && CV_VERSION_REVISION < 3 +# define HAVE_IMCOUNT 0 +#else +# define HAVE_IMCOUNT 1 +#endif + // Internal state of background processing struct background_t { int debug; @@ -16,8 +26,11 @@ struct background_t { volatile bool run; cv::VideoCapture cap; int frame; + int cnt; double fps; + cv::Rect cropRegion; cv::Mat raw; + bool bg_stored; std::mutex rawmux; cv::Mat thumb; std::mutex thumbmux; @@ -58,6 +71,11 @@ static void read_thread(std::weak_ptr weak) { grab.copyTo(pbkd->raw); pbkd->frame += 1; } + // if last frame reached (gstreamer and GIF file) set to frame 0 + if (pbkd->frame == pbkd->cnt) { + pbkd->cap.set(cv::CAP_PROP_POS_FRAMES, 0); + pbkd->frame = 0; + } // grab timing point auto now = std::chrono::steady_clock::now(); // generate thumbnail frame with overlay info if double debug enabled @@ -130,6 +148,14 @@ std::shared_ptr load_background(const std::string& path, int debug pbkd->debug = debug; pbkd->video = false; pbkd->run = false; + pbkd->bg_stored = false; + pbkd->cropRegion = {-1, -1, -1, -1}; + +#if HAVE_IMCOUNT + size_t images = cv::imcount(path.c_str()); +#else + size_t images = 0; +#endif pbkd->cap.open(path, cv::CAP_ANY); // explicitly ask for auto-detection of backend if (!pbkd->cap.isOpened()) { if (pbkd->debug) fprintf(stderr, "background: cap cannot open: %s\n", path.c_str()); @@ -139,23 +165,39 @@ std::shared_ptr load_background(const std::string& path, int debug int fcc = (int)pbkd->cap.get(cv::CAP_PROP_FOURCC); pbkd->fps = pbkd->cap.get(cv::CAP_PROP_FPS); int cnt = (int)pbkd->cap.get(cv::CAP_PROP_FRAME_COUNT); + pbkd->cnt = cnt; +#if !HAVE_IMCOUNT + if ( cnt < 0 ) + images = 1; +#endif // Here be the logic... - // if: can read 2 video frames => it's a video - // else: is loaded as an image => it's an image + // if: count > 1 it's a video + // else: if cnt < 1 it´s is a still image + // else: if cnt = 1 it´s is a static gif file // else: it's not usable. - if (pbkd->cap.read(pbkd->raw) && pbkd->cap.read(pbkd->raw)) { + if (cnt > 1) { + if (!pbkd->cap.read(pbkd->raw)) { + if (pbkd->debug) fprintf(stderr, "background: read failed %s\n", path.c_str()); + return nullptr; + } // it's a video, try a reset and start reader thread.. if (pbkd->cap.set(cv::CAP_PROP_POS_FRAMES, 0)) pbkd->frame = 0; else - pbkd->frame = 2; // unable to reset, so we're 2 frames in + pbkd->frame = 1; // unable to reset, so we're 1 frames in pbkd->video = true; pbkd->run = true; pbkd->thread = std::thread(read_thread, std::weak_ptr(pbkd)); } else { // static image file, try loading.. - pbkd->cap.release(); - pbkd->raw = cv::imread(path); + if (cnt < 0) { + pbkd->cap.release(); + pbkd->raw = cv::imread(path); + } else if (cnt == 1) { + // this is a static gif file with 1 frame + pbkd->cap.read(pbkd->raw); + pbkd->cap.release(); + } if (pbkd->raw.empty()) { if (pbkd->debug) fprintf(stderr, "background: imread cannot open: %s\n", path.c_str()); return nullptr; @@ -179,15 +221,33 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c if (!pbkd) return -1; // static image or video? - int frm ; + int frm; if (pbkd->video) { // grab frame & frame no. under mutex std::unique_lock hold(pbkd->rawmux); - cv::resize(pbkd->raw, out, cv::Size(width, height)); + if ( pbkd->cropRegion.x < 0 ) { // calculate cropping if not already done + pbkd->cropRegion = bs_calc_cropping(pbkd->raw.cols, pbkd->raw.rows, width, height); + } + // Perform cropping if necessary + if ( pbkd->cropRegion.x || pbkd->cropRegion.y ) { + cv::Mat tmp; + pbkd->raw(pbkd->cropRegion).copyTo(tmp); + cv::resize(tmp, out, cv::Size(width, height)); + } else { + cv::resize(pbkd->raw, out, cv::Size(width, height)); + } frm = pbkd->frame; } else { - // resize still image as requested into out - cv::resize(pbkd->raw, out, cv::Size(width, height)); + if (!pbkd->bg_stored) { + // resize still image as requested into out + cv::Rect crop = bs_calc_cropping(pbkd->raw.cols, pbkd->raw.rows, width, height); + // Under some circumstances we must do the job in two steps! + // Otherwise this resize(pbkd->raw(crop), pbkd->raw, ...) may fail. + pbkd->raw(crop).copyTo(pbkd->raw); + cv::resize(pbkd->raw, pbkd->raw, cv::Size(width, height)); + pbkd->bg_stored = true; + } + out = pbkd->raw; frm = 1; } return frm; diff --git a/app/deepseg.cc b/app/deepseg.cc index 55733ea..1a2d195 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -84,7 +84,7 @@ std::optional> geometryFromString(const std::string& i } // OpenCV helper functions -cv::Mat convert_rgb_to_yuyv( cv::Mat input ) { +cv::Mat convert_rgb_to_yuyv(cv::Mat input) { cv::Mat tmp; cv::cvtColor(input, tmp, cv::COLOR_RGB2YUV); std::vector yuv; @@ -372,6 +372,7 @@ int main(int argc, char* argv[]) try { bool flipVertical = false; int fourcc = 0; size_t blur_strength = 0; + cv::Rect crop_region(0, 0, 0, 0); const char* modelname = "selfiesegmentation_mlkit-256x256-2021_01_19-v1215.f16.tflite"; @@ -568,6 +569,12 @@ int main(int argc, char* argv[]) try { if (expWidth != vidGeo.value().first) { fprintf(stderr, "Warning: virtual camera aspect ratio does not match capture device.\n"); } + // calculate crop region, only if result always smaller + if (capGeo != vidGeo) { + crop_region = bs_calc_cropping( + capGeo->first, capGeo->second, + vidGeo->first, vidGeo->second); + } // dump settings.. printf("debug: %d\n", debug); @@ -600,7 +607,11 @@ int main(int argc, char* argv[]) try { } } // default green screen background (at capture true geometry) - cv::Mat bg = cv::Mat(capGeo.value().second, capGeo.value().first, CV_8UC3, cv::Scalar(0, 255, 0)); + std::pair bg_dim = *capGeo; + if (crop_region.x || crop_region.y) { + bg_dim = {crop_region.width, crop_region.height}; + } + cv::Mat bg(bg_dim.second, bg_dim.first, CV_8UC3, cv::Scalar(0, 255, 0)); // Virtual camera (at specified geometry) int lbfd = loopback_init(s_vcam, vidGeo.value().first, vidGeo.value().second, debug); @@ -613,11 +624,24 @@ int main(int argc, char* argv[]) try { loopback_free(lbfd); }); - // Processing components, all at capture true geometry - cv::Mat mask(capGeo.value().second, capGeo.value().first, CV_8U); + std::pair mask_dim = *capGeo; + if (crop_region.x || crop_region.y) { + mask_dim = {crop_region.width, crop_region.height}; + } + cv::Mat mask(mask_dim.second, mask_dim.first, CV_8U); + cv::Mat raw; - CalcMask ai(s_model.value(), threads, capGeo.value().first, capGeo.value().second); + int aiw,aih; + if (!(crop_region.x || crop_region.y)) { + aiw=capGeo->first; + aih=capGeo->second; + } else { + aiw=crop_region.width; + aih=crop_region.height; + } + CalcMask ai(*s_model, threads, aiw, aih); + ti.lastns = timestamp(); printf("Startup: %ldns\n", diffnanosecs(ti.lastns,ti.bootns)); @@ -631,22 +655,35 @@ int main(int argc, char* argv[]) try { // copy new frame to buffer cap.retrieve(raw); ti.retrns = timestamp(); + + if (raw.rows == 0 || raw.cols == 0) continue; // sanity check + + if (crop_region.x || crop_region.y) { + raw(crop_region).copyTo(raw); + } ai.set_input_frame(raw); ti.copyns = timestamp(); - if (raw.rows == 0 || raw.cols == 0) continue; // sanity check + // do background detection magic + ai.get_output_mask(mask); + ti.copyns = timestamp(); if (filterActive) { - // do background detection magic - ai.get_output_mask(mask); - // get background frame: // - specified source if set // - copy of input video if blur_strength != 0 // - default green (initial value) bool canBlur = false; if (pbk) { - if (grab_background(pbk, capGeo.value().first, capGeo.value().second, bg)<0) + int tw,th; + if (crop_region.x || crop_region.y) { + tw = crop_region.width; + th = crop_region.height; + } else { + tw = capGeo->first; + th = capGeo->second; + } + if (grab_background(pbk, tw, th, bg) < 0) throw "Failed to read background frame"; canBlur = true; } else if (blur_strength) { diff --git a/lib/libbackscrub.cc b/lib/libbackscrub.cc index fb69be5..7acf82d 100644 --- a/lib/libbackscrub.cc +++ b/lib/libbackscrub.cc @@ -365,7 +365,11 @@ bool bs_maskgen_process(void *context, cv::Mat &frame, cv::Mat &mask) { // scale up into full-sized mask cv::Mat tmpbuf; - cv::resize(ctx.ofinal(ctx.in_roidim),tmpbuf,ctx.mroi.size()); + // with body-pix-float-050-8.tflite the size of ctx.ofinal is 33x33 + // and the wanted roi may be greater as 33x33 so we can crash with + // cv::resize(ctx.ofinal(ctx.in_roidim),tmpbuf,ctx.mroi.size()); + ctx.ofinal.copyTo(tmpbuf); + cv::resize(tmpbuf, tmpbuf, ctx.mroi.size()); // blur at full size for maximum smoothness cv::blur(tmpbuf,ctx.mroi,ctx.blur); @@ -375,3 +379,24 @@ bool bs_maskgen_process(void *context, cv::Mat &frame, cv::Mat &mask) { return true; } +cv::Rect bs_calc_cropping(int inWidth, int inHeight, int targetWidth, int targetHigh) { + // if the input and output aspect ratio are not the same + // we can crop the source image. For example if the + // input image has a 16:9 (1280x720) ratio and the output is 4:3 (960x720) + // we will return the cropRegion set as x=160, width=960, y=0, height=720 + // which is the centered part of the original image + cv::Rect cropRegion = {0, 0, 0, 0}; + float sc = (float)targetWidth / inWidth; + float st = (float)targetHigh / inHeight; + sc = st > sc ? st : sc; + + int sx = (int)(targetWidth / sc) - inWidth; + cropRegion.x = (sx < 0 ? -sx : sx) / 2; + + int sy = (int)(targetHigh / sc) - inHeight; + cropRegion.y = (sy < 0 ? -sy : sy) / 2; + + cropRegion.width = inWidth - cropRegion.x * 2; + cropRegion.height = inHeight - cropRegion.y * 2; + return cropRegion; +} diff --git a/lib/libbackscrub.h b/lib/libbackscrub.h index 8eed1f0..08a4e13 100644 --- a/lib/libbackscrub.h +++ b/lib/libbackscrub.h @@ -38,4 +38,6 @@ extern void bs_maskgen_delete(void *context); // Process a video frame into a mask extern bool bs_maskgen_process(void *context, cv::Mat& frame, cv::Mat &mask); +extern cv::Rect bs_calc_cropping(int inWidth, int inHeight, int targetWidth, int targetHight); + #endif