Version 1.2 (#8)

* small tweak to unique frame detection

* extracting into functions

* frame cropping

* Frame detection speed greatly improved

* Fixes #7 with how skill names were made uppercase

* Added resiliency in the form of error handling

.vscode/launch.json

@@ -4,17 +4,18 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-
-    {
-        "name": "Node encoder",
-        "program": "${workspaceFolder}/js_encoder.js",
-        "request": "launch",
-        "skipFiles": [
-            "<node_internals>/**"
-        ],
-        "type": "pwa-node",
-        "args": ["charms.single.json"]
-    },
+        {
+            "name": "Node encoder",
+            "program": "${workspaceFolder}/js_encoder.js",
+            "request": "launch",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "type": "pwa-node",
+            "args": [
+                "charms.single.json"
+            ]
+        },
         {
             "name": "Python: Current File",
             "type": "python",
@@ -23,25 +24,37 @@
             "console": "integratedTerminal"
         },
         {
-            "name": "decompress",
+            "name": "Frame Extraction",
             "type": "python",
             "request": "launch",
-            "program":"mp4_decompress.py",
-            "console": "integratedTerminal"
+            "program": "main.py",
+            "console": "integratedTerminal",
+            "args": [
+                "--skip-charms"
+            ]
         },
         {
-            "name": "get_charms",
+            "name": "Charm Extraction",
             "type": "python",
             "request": "launch",
-            "program":"charm_extraction.py",
-            "console": "integratedTerminal"
+            "program": "main.py",
+            "console": "integratedTerminal",
+            "args": [
+                "--skip-frames"
+            ]
         },
         {
-            "name": "masking",
+            "name": "Debug Charm Extraction",
             "type": "python",
             "request": "launch",
-            "program":"mp4_masking.py",
-            "console": "integratedTerminal"
+            "program": "main.py",
+            "console": "integratedTerminal",
+            "args": [
+                "--skip-frames",
+                "--frames",
+                "debug-frames"
+
+            ]
         }
     ]
 }

skill_corrections.csv

@@ -899,3 +899,4 @@ slugs“,slugger
 sIug99',slugger
 slugs",slugger
 SDSIS,spare
+Free:,Free

src/charm_extraction.py

@@ -155,7 +155,7 @@ def extract_charm(frame_loc, slots, skills, skill_text):
             continue
 
         logger.debug(f"Added {skill}, {level}")
-        charm.add_skill(fix_skill_name(skill), level)
+        charm.add_skill(fix_skill_name(all_skills, skill), level)
 
     logger.debug(f"Finished charm for {frame_loc}")
     logger.debug(f"{frame_loc}: {charm.to_dict()}")
@@ -169,22 +169,29 @@ def extract_charms(frame_dir):
         with tqdm(list(os.scandir(frame_dir)), desc="Parsing skills")as tqdm_iter:
             for frame_loc in tqdm_iter:
                 frame_loc = frame_loc.path
-                tqdm_iter.set_description(f"Parsing {frame_loc}")
-                frame = cv2.imread(frame_loc)
+                try:
+                    tqdm_iter.set_description(f"Parsing {frame_loc}")
+                    frame = cv2.imread(frame_loc)
 
-                skill_only_im = remove_non_skill_info(frame)
-                slots = get_slots(skill_only_im)
+                    skill_only_im = remove_non_skill_info(frame)
+                    slots = get_slots(skill_only_im)
 
-                inverted = cv2.bitwise_not(skill_only_im)
+                    inverted = cv2.bitwise_not(skill_only_im)
 
-                trunc_tr = apply_trunc_threshold(inverted)  # appears to work best
+                    trunc_tr = apply_trunc_threshold(inverted)  # appears to work best
 
-                skills = get_skills(trunc_tr, True)
+                    skills = get_skills(trunc_tr, True)
 
-                skill_text = read_text_from_skill_tuple(skills)
+                    skill_text = read_text_from_skill_tuple(skills)
 
-                charm = extract_charm(frame_loc, slots, skills, skill_text)
-                charms.append(charm)
+                except Exception as e:
+                    logger.error(f"An error occured when analysing frame {frame_loc}. Error: {e}")
+
+                try:
+                    charm = extract_charm(frame_loc, slots, skills, skill_text)
+                    charms.append(charm)
+                except Exception as e:
+                    logger.error(f"An error occured when extracting charm on {frame_loc}. Error: {e}")
 
     except Exception as e:
         logger.error(f"Crashed with {e}")

src/frame_extraction.py

@@ -1,61 +1,81 @@
 import os
 import cv2
-from .utils import pre_crop_mask, get_charm_borders, only_keep_shiny_border, get_resource_path
+from .utils import apply_pre_crop_mask, get_frame_change_observation_section, get_resource_path
 from tqdm import tqdm
 from math import floor
 from skimage.metrics import structural_similarity
+import numpy as np
+
+
+def crop_frame(frame):
+    x = 620
+    y = 175
+    x2 = x+630
+    y2 = y+440
+    pre_crop = apply_pre_crop_mask(frame)
+    cropped = pre_crop[y:y2, x:x2]
+    charm_only = get_frame_change_observation_section(cropped)
+    return cropped, charm_only
+
+
+def crop_frames(capture_device):
+    results = []
+    for i, f in read_frames(capture_device):
+        yield i, crop_frame(f)
+
+
+def read_frames(capture_device):
+    i = 0
+    while(True):
+        ret, frame = capture_device.read()
+        if not ret:
+            break
+        yield i, frame
+        i += 1
+
+
+def is_new_frame(previous_charm_marker, charm_only):
+    diff = cv2.absdiff(previous_charm_marker, charm_only)
+    ret, threshold = cv2.threshold(diff, 30,255, cv2.THRESH_BINARY_INV)
+
+    return 0 in threshold[:, ]
 
 
 def extract_unique_frames(input_dir, frame_dir):
-    overlay_file_name = get_resource_path("mask")
     charm_count = 0
     currentFrame = 0
 
     input_files = list(
         filter(lambda x: x.name.endswith(".mp4"), os.scandir(input_dir)))
     print(f"Total input files to scan: {len(input_files)}")
+
     for f_loc in input_files:
         f_name = f_loc.name
         f_loc = f_loc.path
 
         cap = cv2.VideoCapture(f_loc)
-        # 620, 175, 630, 440
-        x = 620
-        y = 175
-        x2 = x+630
-        y2 = y+440
-        previous_frame = None
-        with tqdm(total=floor(cap.get(cv2.CAP_PROP_FRAME_COUNT)), desc=f"{f_name},  Total Estimated charms found: {charm_count}") as tqdm_iter:
-            while(True):
-                ret, frame = cap.read()
-                if not ret:
-                    break
-                currentFrame += 1
-                name = os.path.join(frame_dir, f"frame{currentFrame}.png")
+        frame_count = floor(cap.get(cv2.CAP_PROP_FRAME_COUNT))
 
-                pre_crop = pre_crop_mask(frame, overlay_file_name)
-                cropped = pre_crop[y:y2, x:x2]
-
-                charm_border = get_charm_borders(cropped)
-                shiny = only_keep_shiny_border(charm_border)
+        previous_charm_marker = None
+        with tqdm(crop_frames(cap), total=frame_count, desc=f"{f_name},  Total Estimated charms/frames found: {charm_count}") as frame_pbar:
+            for i, cropped_tuple in frame_pbar:
+                cropped, charm_only = cropped_tuple
+                name = os.path.join(frame_dir, f"frame{currentFrame}.png")
+                if previous_charm_marker is not None:
 
-                if previous_frame is not None:
-                    shiny_grayscale = cv2.cvtColor(shiny, cv2.COLOR_BGR2GRAY)
-                    score = structural_similarity(
-                        previous_frame, shiny_grayscale)
-                    if score < 0.996:
+                    if is_new_frame(previous_charm_marker, charm_only):
                         charm_count += 1
 
                         cv2.imwrite(name, cropped)
                 else:
-                    charm_count = 1
+                    charm_count += 1
                     cv2.imwrite(name, cropped)
 
-                previous_frame = cv2.cvtColor(shiny, cv2.COLOR_BGR2GRAY)
+                previous_charm_marker = charm_only
 
-                tqdm_iter.set_description(
-                    f"{f_name},  Total Estimated charms found: {charm_count}")
-                tqdm_iter.update(1)
+                frame_pbar.set_description(
+                    f"{f_name},  Total Estimated charms/frames found: {charm_count}")
+                currentFrame += 1
 
         cap.release()
         cv2.destroyAllWindows()

src/utils.py

@@ -21,36 +21,17 @@ def apply_black_white_mask(img, mask_img):
     return cv2.bitwise_and(img, img, mask=mask)
 
 
-def pre_crop_mask(img, mask_location):
-    pre_crop_filter = _load_potentially_transparent(mask_location)
+def apply_pre_crop_mask(img):
+    pre_crop_filter = _load_potentially_transparent(get_resource_path("mask"))
     return apply_black_white_mask(img, pre_crop_filter)
 
 
-def get_charm_borders(img):
-    hsv = [0, 179, 0, 255, 1, 255]
+def get_frame_change_observation_section(img):
     charm_only_filter_path = get_resource_path("charm_only")
     charm_only_filter = cv2.imread(charm_only_filter_path)
 
-    lower = np.array([hsv[0], hsv[2], hsv[4]])
-    upper = np.array([hsv[1], hsv[3], hsv[5]])
-
-    mask = cv2.inRange(charm_only_filter, lower, upper)
-
-    return cv2.bitwise_or(img, img, mask=mask)
-
-
-def only_keep_shiny_border(img):
-    imgHSV = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
-    hsv = [0, 39, 0, 255, 109, 255]
-
-    lower = np.array([hsv[0], hsv[2], hsv[4]])
-    upper = np.array([hsv[1], hsv[3], hsv[5]])
-
-    mask = cv2.inRange(imgHSV, lower, upper)
-    imgResult = cv2.bitwise_and(img, img, mask=mask)
-
-    ret, imgResult = cv2.threshold(imgResult, 50, 255, cv2.THRESH_BINARY)
-    return imgResult
+    charm_only = apply_black_white_mask(img, charm_only_filter)
+    return cv2.cvtColor(charm_only, cv2.COLOR_BGR2GRAY)
 
 
 def remove_non_skill_info(img):
@@ -237,6 +218,23 @@ def print_licenses():
         print("\n\n")
 
 
+def batchify(lst, batch_size):
+    return list(batchify_lazy(lst, batch_size))
+
+
+def batchify_lazy(lst, batch_size):
+    batch = []
+    i = 0
+    for item in lst:
+        if i > batch_size:
+            i = 0
+            yield batch
+            batch = []
+        batch.append(item)
+        i += 1
+    yield batch
+
+
 _resources = {
     'skill_dict': _alter_resource_path(os.path.join("data", "skill_dict.freq")),
     'skill_list': _alter_resource_path(os.path.join("data", "skill_list.txt")),