Krzysztofwrobel/sc 26238/improve nameai tokenizations for important (#…

…586) * more domain specific tokens * update version * regenerate data
namehash · Jan 30, 2025 · 9a3d509 · 9a3d509
1 parent 9209c29
commit 9a3d509
Show file tree

Hide file tree

Showing 5 changed files with 2 additions and 5 deletions.
diff --git a/apps/api.nameai.dev/nameai/data/static/nameai.all_tokenizer.AllTokenizer.automaton.pickle b/apps/api.nameai.dev/nameai/data/static/nameai.all_tokenizer.AllTokenizer.automaton.pickle
diff --git a/apps/api.nameai.dev/nameai/data/static/nameai.ngrams.Ngrams._unigrams_and_count.pickle b/apps/api.nameai.dev/nameai/data/static/nameai.ngrams.Ngrams._unigrams_and_count.pickle
diff --git a/apps/api.nameai.dev/pyproject.toml b/apps/api.nameai.dev/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "nameai"
-version = "0.1.5"
+version = "0.1.6"
 description = "NameHash NameAI API"
 authors = ["NameHash Team <devops@namehash.io>"]
 maintainers = ["NameHash Team <devops@namehash.io>"]

diff --git a/apps/api.nameai.dev/tests/test_api.py b/apps/api.nameai.dev/tests/test_api.py
@@ -1,6 +1,5 @@
 import pytest
 from fastapi.testclient import TestClient
-from pprint import pprint
 
 from mocked_static_property import mock_static_property
 from nameguard.utils import MAX_INSPECTED_NAME_CHARACTERS
@@ -48,7 +47,6 @@ def test_inspect_name_post_long(test_client):
     assert res_json['nameguard']['highest_risk']['check'] == 'normalized'
     assert res_json['nameguard']['inspected']
     assert res_json['nameai']['analysis']['status'] == 'unnormalized'
-    pprint(res_json)
 
 
 def test_inspect_name_post_too_long(test_client):
@@ -62,7 +60,6 @@ def test_inspect_name_post_too_long(test_client):
     assert res_json['nameguard']['normalization'] == 'unnormalized'
     assert not res_json['nameguard']['inspected']
     assert res_json['nameai']['analysis'] is None
-    pprint(res_json)
 
 
 def test_inspect_name_post_too_long_normalized(test_client):
@@ -75,4 +72,3 @@ def test_inspect_name_post_too_long_normalized(test_client):
     assert res_json['nameguard']['highest_risk']['check'] == 'uninspected'
     assert res_json['nameguard']['normalization'] == 'normalized'
     assert res_json['nameai']['analysis'] is None
-    pprint(res_json)
diff --git a/apps/api.nameai.dev/tests/test_nameai.py b/apps/api.nameai.dev/tests/test_nameai.py
@@ -68,3 +68,4 @@ def top_tok(label):
     assert top_tok('expertsexchange') == ['experts', 'exchange']
     # higher prob than [i, teach]
     assert top_tok('iteach') == ['it', 'each']
+    assert top_tok('vitalik') == ['vitalik']