SeqTM

dialectid/__init__.py

@@ -22,5 +22,5 @@
 
 __version__ = '0.0.5'
 
-from dialectid.text_repr import BoW
+from dialectid.text_repr import BoW, SeqTM
 from dialectid.model import DialectId

dialectid/tests/test_text_repr.py

@@ -49,7 +49,10 @@ def test_subwords():
 def test_SeqTM():
     """Test SeqTM class"""
 
-    seq = SeqTM(language='es', subwords=True, voc_size_exponent=13)
+    seq = SeqTM(language='es', subwords=True,
+                sequence=False,
+                voc_selection='most_common_by_type',
+                voc_size_exponent=13)
     assert seq.language == 'es'
     assert seq.voc_size_exponent == 13
     _ = [['dias', 'q:~dur', 'q:os~']]
@@ -62,7 +65,21 @@ def test_SeqTM():
 def test_SeqTM_bug():
     """Test SeqTM class"""
 
-    seq = SeqTM(language='es', subwords=True, voc_size_exponent=13)
+    seq = SeqTM(language='es', subwords=True,
+                sequence=False,
+                voc_selection='most_common_by_type',
+                voc_size_exponent=13)
+    res1 = seq.tokenize('mira pinche a')
+    res2 = seq.tokenize('a pinche a')
+    assert res1[1:] == res2[1:]
+
+
+def test_SeqTM_seq():
+    """Test SeqTM seq option"""
+
+    seq = SeqTM(language='es', sequence=True,
+                voc_selection='most_common',
+                voc_size_exponent=13)
     res1 = seq.tokenize('mira pinche a')
     res2 = seq.tokenize('a pinche a')
     assert res1[1:] == res2[1:]

dialectid/text_repr.py

@@ -106,10 +106,13 @@ def __init__(self, language='es',
                  voc_selection: str='most_common_by_type',
                  loc: str=None,
                  subwords: bool=True,
+                 sequence: bool=True,
                  lang=None,
                  **kwargs):
         assert lang is None
-        if subwords:
+        if sequence and subwords:
+            loc = 'seq'
+        elif subwords:
             assert loc is None
             loc = 'qgrams'
         self._map = {}