Merge pull request kaegi#237 from ianki/analyzer_fixes

analyzer fixes

morph/config.py

@@ -83,13 +83,18 @@
     # A card with an unknown morph that matches some of the last words in frequency.txt will get almost no bonus.
     'frequency.txt bonus': 100000,
 
+    # Cards with unknown morphemes outside of frequency.txt or priority.db will get this penalty.  This will make
+    # i+2 or i+n cards with morphemes that are in frequency.txt or prioritydb get a higher priority.
+    'no priority penalty': 1000000,
+
     # lite update
     # this reduces how many notes are changed and thus sync burden by not updating notes that aren't as important
     'only update k+2 and below': False,
 
     # only these can have deck overrides
     # skip cards with focusMorph that was already seen or aren't k+1
     'next new card feature': True,
+
     # fill new card queue with cards from all child decks instead of sequentially. also enforce a minimum due value
     'new card merged fill': False,
     # k+1 by default. this mostly is to boost performance of 'next new card feature'

morph/main.py

@@ -385,6 +385,9 @@ def updateNotes(allDb):
 
         # add bonus for morphs in priority.db and frequency.txt
         frequencyBonus = C('frequency.txt bonus')
+        noPriorityPenalty = C('no priority penalty')
+        reinforceNewVocabWeight = C('reinforce new vocab weight')
+        priorityDbWeight = C('priority.db weight')
         isPriority = False
         isFrequency = False
 
@@ -397,7 +400,7 @@ def updateNotes(allDb):
 
             if priorityDb.frequency(focusMorph) > 0:
                 isPriority = True
-                usefulness += C('priority.db weight')
+                usefulness += priorityDbWeight
 
             if frequency_has_morphemes:
                 focusMorphIndex = frequency_map.get(focusMorph, -1)
@@ -420,7 +423,7 @@ def updateNotes(allDb):
             if locs:
                 ivl = min(1, max(loc.maturity for loc in locs))
                 # TODO: maybe average this so it doesnt favor long sentences
-                usefulness += C('reinforce new vocab weight') // ivl
+                usefulness += reinforceNewVocabWeight // ivl
 
         if any(morpheme.pos == '動詞' for morpheme in unknowns):  # FIXME: this isn't working???
             usefulness += C('verb bonus')
@@ -439,6 +442,10 @@ def updateNotes(allDb):
         ts = [t for t in ts if t not in (
             notReadyTag, compTag, vocabTag, freshTag)]
 
+        # apply penalty for cards that aren't prioritized for learning
+        if not (isPriority or isFrequency):
+            usefulness += noPriorityPenalty
+
         # determine card type
         if N_m == 0:  # sentence comprehension card, m+0
             ts.append(compTag)

morph/readability.py

@@ -619,7 +619,9 @@ def get_line_readability(self, show, known_db):
         return line_readability
 
     def get_master_freq(self):
-        return self.master_score * 100.0 / self.master_total_instances
+        if self.master_total_instances > 0:
+            return self.master_score * 100.0 / self.master_total_instances
+        return 0
 
     def sourceStudyPlan(self, f, source, known_db, unknown_db):
         missing_morphs = []
@@ -1203,12 +1205,12 @@ def output_study_result(source, study_result, old_line_readability, new_line_rea
 
                 matched_sources = set()
 
-                optmize_master_freq = self.optimal_master_target > 0
-
+                optmize_master_freq = (self.optimal_master_target > 0 and self.master_total_instances > 0)
                 cumulative_master_freq = self.get_master_freq()
 
                 while len(matched_sources) < len(sources):
                     find_optimal_source = (optmize_master_freq and cumulative_master_freq < self.optimal_master_target)
+                    self.writeOutput("{} {} {} {}\n".format(find_optimal_source, optmize_master_freq, cumulative_master_freq, self.optimal_master_target))
                     source_results = []
 
                     if optmize_master_freq and not find_optimal_source: