Merge thaidd/develop to master

ngocbh · ngocbh · commit cc94bfd57d7e · 2020-09-07T14:34:40.000+07:00
diff --git a/sctokenizer/assets/c_keywords.py b/sctokenizer/assets/c_keywords.py
@@ -1,4 +1,4 @@
-keyword_set = {
+c_keyword_set = {
     'auto',
     'break',
     'case',
diff --git a/sctokenizer/assets/c_operators.py b/sctokenizer/assets/c_operators.py
@@ -0,0 +1,8 @@
+c_operator_set = {
+    '++', '--', '.', '->', '~', '!', '+', '-', \
+    '&', '_Alignof', 'sizeof', '?:', ',', '*', \
+    '/', '%', '<<', '>>', '<', '>', '<=', '>=', \
+    '==', '!=', '^', '|', '&&', '||', '=', '*=', \
+    '/=', '%=', '+=', '-=', '>>=', '<<=', '&=', \
+    '^=', '|=', '?', ':'
+} # Becareful with ?: operator
diff --git a/sctokenizer/assets/cpp_keywords.py b/sctokenizer/assets/cpp_keywords.py
@@ -1,4 +1,4 @@
-keyword_set = {
+cpp_keyword_set = {
     'alignas',
     'alignof',
     'and',
diff --git a/sctokenizer/assets/cpp_operators.py b/sctokenizer/assets/cpp_operators.py
@@ -4,5 +4,5 @@
     '*', '/', '%', '<<', '>>', '<', '>', '<=', '>=', \
     '==', '!=', '^', '|', '&&', '||', '=', '*=', '/=', \
     '%=', '+=', '-=', '>>=', '<<=', '&=', '^=', '|=', ',', \
-    '?:', 'and', 'or','xor', 'not'
+    '?:', 'and', 'or','xor', 'not', '?', ':'
 } # Becareful with ?: operator
diff --git a/sctokenizer/assets/java_keywords.py b/sctokenizer/assets/java_keywords.py
@@ -1,4 +1,4 @@
-keyword_set = {
+java_keyword_set = {
     'abstract',
     'assert',
     'boolean',
diff --git a/sctokenizer/assets/java_operators.py b/sctokenizer/assets/java_operators.py
@@ -1,8 +1,7 @@
-cpp_operator_set = {
-    '::', '++', '--', '.', '->', '~', '!', '+', '-', \
-    '&', '*', 'new', 'delete', 'sizeof', '.*', '->*', \
-    '*', '/', '%', '<<', '>>', '<', '>', '<=', '>=', \
+java_operator_set = {
+    '::', '++', '--', '~', '!', '+', '-', 'instanceof', \
+    '&', '*', '/', '%', '<<', '>>', '<', '>', '<=', '>=', \
     '==', '!=', '^', '|', '&&', '||', '=', '*=', '/=', \
-    '%=', '+=', '-=', '>>=', '<<=', '&=', '^=', '|=', ',', \
-    '?:'
+    '%=', '+=', '-=', '>>=', '<<=', '&=', '^=', '|=', \
+    '?:', '>>>', '>>>=', '?', ':'
 } # Becareful with ?: operator
diff --git a/sctokenizer/assets/python_keywords.py b/sctokenizer/assets/python_keywords.py
@@ -1,4 +1,4 @@
-keyword_set = {
+python_keyword_set = {
     'and',
     'as',
     'assert',
diff --git a/sctokenizer/assets/python_operators.py b/sctokenizer/assets/python_operators.py
@@ -0,0 +1,7 @@
+python_operator_set = {
+    '+', '-', '*', '/', '%', '**', '//', '=', \
+    '+=', '-=', '*=', '/=', '%=', '//=', '**=', \
+    '&=', '|=', '^=', '>>=', '<<=', '==', '!=', \
+    '>', '<', '>=', '<=', 'and', 'or', 'not', 'is', \
+    'in', '&', '|', '^', '~', '<<', '>>'
+} 
diff --git a/sctokenizer/c_tokenizer.py b/sctokenizer/c_tokenizer.py
@@ -1,11 +1,13 @@
 from __future__ import absolute_import
 
 from sctokenizer.cpp_tokenizer import CppTokenizer
-from sctokenizer.assets.c_keywords import keyword_set
-# from sctokenizer.aset.c_operator_set import c_operator_set
+from sctokenizer.assets.c_keywords import c_keyword_set
+from sctokenizer.assets.c_operators import c_operator_set
 from sctokenizer.token import TokenType, Token
 
 class CTokenizer(CppTokenizer):
     def __init__(self):
         super().__init__()
-        self.keyword_set = keyword_set
+        self.keyword_set = c_keyword_set
+        self.operator_set = c_operator_set
+
diff --git a/sctokenizer/cpp_tokenizer.py b/sctokenizer/cpp_tokenizer.py
@@ -1,14 +1,14 @@
 from __future__ import absolute_import
 
 from sctokenizer.tokenizer import Tokenizer, TokenizerState
-from sctokenizer.assets.cpp_keywords import keyword_set
+from sctokenizer.assets.cpp_keywords import cpp_keyword_set
 from sctokenizer.assets.cpp_operators import cpp_operator_set
 from sctokenizer.token import TokenType, Token
 
 class CppTokenizer(Tokenizer):
     def __init__(self):
         super().__init__()
-        self.keyword_set = keyword_set
+        self.keyword_set = cpp_keyword_set
         self.operator_set = cpp_operator_set
 
     def tokenize(self, source_str):
@@ -50,6 +50,7 @@ def tokenize(self, source_str):
                 if first_no_space_in_word == '':
                     first_no_space_in_word = cur
                     self.colnumber = i
+
             if state == TokenizerState.IN_COMMENT:
                 # Check end of block comment
                 if cur == '*':
@@ -59,10 +60,12 @@ def tokenize(self, source_str):
                         i += 1
                         state = TokenizerState.REGULAR
                         continue
+
             elif state == TokenizerState.IN_LINECOMMENT:
                 # Check end of line comment
                 if cur == '\n':
                     state = TokenizerState.REGULAR
+
             elif state == TokenizerState.IN_MACRO:
                 # Get first char after # in marco
                 if cur == ' ' or cur == '\t':
@@ -88,6 +91,7 @@ def tokenize(self, source_str):
                     first_no_space_in_macro = ''
                 if self.is_alpha(cur):
                     pending += cur
+
             elif state == TokenizerState.IN_INCLUDE:
                 if cur == '<' or cur == '"':
                     state = TokenizerState.IN_INCLUDE_HEADER
@@ -98,6 +102,7 @@ def tokenize(self, source_str):
                     self.add_pending(tokens, cur, TokenType.SPECIAL_SYMBOL, len_lines, t)
                 elif cur != ' ' and cur != '\t':
                     pending += cur
+
             elif state == TokenizerState.IN_INCLUDE_HEADER:
                 if cur == '>' or cur == '"':
                     state = TokenizerState.REGULAR
@@ -108,6 +113,7 @@ def tokenize(self, source_str):
                     self.add_pending(tokens, cur, TokenType.SPECIAL_SYMBOL, len_lines, t)
                 elif cur != ' ' and cur != '\t':
                     pending += cur
+
             elif state == TokenizerState.IN_STRING:
                 # Check end of string
                 if cur == '"' and prev != '\\':
@@ -119,6 +125,7 @@ def tokenize(self, source_str):
                     self.add_pending(tokens, cur, TokenType.SPECIAL_SYMBOL, len_lines, t)
                 else:
                     pending += cur
+
             elif state == TokenizerState.IN_CHAR:
                 # Check end of char
                 if cur == "'" and prev != '\\':
@@ -130,26 +137,36 @@ def tokenize(self, source_str):
                     self.add_pending(tokens, cur, TokenType.SPECIAL_SYMBOL, len_lines, t)
                 else:
                     pending += cur
+
             elif state == TokenizerState.IN_NUMBER:
                 if (cur >= '0' and cur <= '9') or \
                     cur == '.' or cur == 'E' or cur == 'e':
                     pending += cur
                     i += 1
-                    # self.colnumber += 1
                     continue
                 if (cur == '-' or cur == '+') and \
                     (prev == 'E' or prev == 'e'):
                     pending += cur
                     i += 1
-                    # self.colnumber += 1
                     continue
                 self.add_pending(tokens, pending, TokenType.CONSTANT, len_lines, t)
-                pending = ''
-                first_no_space_in_word = ''
+                first_no_space_in_word = cur
+                pending = cur
                 self.colnumber = i
-                self.add_pending(tokens, cur, TokenType.SPECIAL_SYMBOL, len_lines, t)
                 state = TokenizerState.REGULAR 
+
             elif state == TokenizerState.REGULAR:
+                if pending in self.operator_set:
+                    if (pending + cur) in self.operator_set:
+                        pending += cur
+                        i += 1
+                        continue
+                    else:
+                        self.add_pending(tokens, pending, TokenType.OPERATOR, len_lines, t)
+                        pending = ''
+                        first_no_space_in_word = cur
+                        self.colnumber = i
+                
                 if cur == '/':
                     if next == '*': # Begin block comments
                         state = TokenizerState.IN_COMMENT
@@ -194,27 +211,25 @@ def tokenize(self, source_str):
                     if first_no_space_in_word == cur:
                         state = TokenizerState.IN_NUMBER
                         self.add_pending(tokens, pending, TokenType.IDENTIFIER, len_lines, t)
+                        self.colnumber = i
                         # first_no_space_in_word = ''
                         pending = cur
                     else:
                         pending += cur
                 elif self.is_alpha(cur): 
                     pending += cur
                 elif cur in self.operator_set: # cur = + - * / , ...
-                    if pending in self.operator_set and (pending+cur) in self.operator_set:
-                        pending += cur
-                    else:
-                        self.add_pending(tokens, pending, TokenType.IDENTIFIER, len_lines, t)
-                        pending = cur
-                        first_no_space_in_word = ''
+                    self.add_pending(tokens, pending, TokenType.IDENTIFIER, len_lines, t)
+                    pending = cur
+                    first_no_space_in_word = cur
+                    self.colnumber = i
                 else: # cur = ;, ', space
                     self.add_pending(tokens, pending, TokenType.IDENTIFIER, len_lines, t)
                     pending = ''
                     first_no_space_in_word = ''
                     if cur > ' ': 
                         self.colnumber = i
                         self.add_pending(tokens, cur, TokenType.SPECIAL_SYMBOL, len_lines, t)
-                        pending = ''
             i += 1
         # is Cpp always ends with } ?
         if len(cur) > 1 or self.is_alpha(cur):
diff --git a/sctokenizer/java_tokenizer.py b/sctokenizer/java_tokenizer.py
diff --git a/sctokenizer/python_tokenizer.py b/sctokenizer/python_tokenizer.py
diff --git a/tests/data/a.cpp b/tests/data/a.cpp
diff --git a/tests/data/java_a.java b/tests/data/java_a.java
diff --git a/tests/data/python_a.py b/tests/data/python_a.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-keyword_set = {`
	`1`	`+c_keyword_set = {`
`2`	`2`	`'auto',`
`3`	`3`	`'break',`
`4`	`4`	`'case',`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-keyword_set = {`
	`1`	`+cpp_keyword_set = {`
`2`	`2`	`'alignas',`
`3`	`3`	`'alignof',`
`4`	`4`	`'and',`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-keyword_set = {`
	`1`	`+java_keyword_set = {`
`2`	`2`	`'abstract',`
`3`	`3`	`'assert',`
`4`	`4`	`'boolean',`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-keyword_set = {`
	`1`	`+python_keyword_set = {`
`2`	`2`	`'and',`
`3`	`3`	`'as',`
`4`	`4`	`'assert',`