-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathpreprocess.py
159 lines (128 loc) · 5.2 KB
/
preprocess.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
import pandas as pd
import re
import string
try:
rawData = pd.read_json('arabicDialects.json', encoding='utf-8',
typ='series', convert_axes=False)
except Exception as e:
%run -i fetch.py
rawData = pd.read_json('arabicDialects.json', encoding='utf-8',
typ='series', convert_axes=False)
rawData.index = rawData.index.astype('int64')
datasetIds = pd.read_csv('dialect_dataset.csv', index_col='id')
datasetIds = datasetIds.loc[rawData.index]
# https://github.com/motazsaad/process-arabic-text/blob/f950f9b3aeba13ac3c16cd10502a8eafbdfa1262/clean_arabic_text.py
# I found this useful and quick to use, I hope it doesn't violate the task guideline.
# I'm using it and I know how it works and I'm referring to the links for fidelity.
# I modified methods that remove unwanted characters by replacing with empty string
# to replace with a space, and then a final step to remove redudant white spaces.
arabic_punctuations = '''`÷×؛<>_()*&^%][ـ،/:"؟.,'{}~¦+|!”…“–ـ'''
english_punctuations = string.punctuation
punctuations_list = arabic_punctuations + english_punctuations
punctuations_map = {p:' ' for p in punctuations_list}
def remove_diacritics(text):
arabic_diacritics = re.compile('''
ّ | # Tashdid
َ | # Fatha
ً | # Tanwin Fath
ُ | # Damma
ٌ | # Tanwin Damm
ِ | # Kasra
ٍ | # Tanwin Kasr
ْ | # Sukun
ـ # Tatwil/Kashida
''', re.VERBOSE)
return re.sub(arabic_diacritics, '', text)
def normalize_arabic(text):
text = re.sub('[إأآا]', 'ا', text)
text = re.sub('ى', 'ي', text)
text = re.sub('ؤ', 'ء', text)
text = re.sub('ئ', 'ء', text)
text = re.sub('ة', 'ه', text)
text = re.sub('گ', 'ك', text)
return text
def remove_punctuations(text):
# translator = str.maketrans('', '', punctuations_list)
translator = str.maketrans(punctuations_map)
return text.translate(translator)
def remove_repeating_char(text):
# # I think skipping هههه might make sense
# # this actually have some meaning/sentiment
# text = re.sub(r'([^ه])\1+', r'\1', text)
# # but then remove redundant هه more than, say 5
# # this is a common limit.
# # the pattern might be distorted, but it counts
# # correctly {4,}\1
# return re.sub(r'(ه)\1{4,}', r'\1\1\1\1', text)
return re.sub(r'(.)\1+', r'\1', text)
def remove_mentions_links(text):
# https://stackoverflow.com/a/56659272/12896502
mention_links_pattern = re.compile(r'(@|https?)\S+|#')
return mention_links_pattern.sub(r' ', text)
def remove_emojis(text):
# https://stackoverflow.com/a/33417311/12896502
# emojis_pattern = re.compile('['
# u'\U0001F600-\U0001F64F' # emoticons
# u'\U0001F300-\U0001F5FF' # symbols & pictographs
# u'\U0001F680-\U0001F6FF' # transport & map symbols
# u'\U0001F1E0-\U0001F1FF' # flags (iOS)
# ']+', flags=re.UNICODE)
# more emoji, previous wasn't enough, hopefully not overkill
# https://stackoverflow.com/a/58356570/12896502
emoj = re.compile('['
u'\U0001F600-\U0001F64F' # emoticons
u'\U0001F300-\U0001F5FF' # symbols & pictographs
u'\U0001F680-\U0001F6FF' # transport & map symbols
u'\U0001F1E0-\U0001F1FF' # flags (iOS)
u'\U00002500-\U00002BEF' # chinese char
u'\U00002702-\U000027B0'
u'\U00002702-\U000027B0'
u'\U000024C2-\U0001F251'
u'\U0001f926-\U0001f937'
u'\U00010000-\U0010ffff'
u'\u2640-\u2642'
u'\u2600-\u2B55'
u'\u200d'
u'\u23cf'
u'\u23e9'
u'\u231a'
u'\ufe0f' # dingbats
u'\u3030'
']+', re.UNICODE)
return emoj.sub(' ', text)
def remove_numbers(text):
text = re.sub(r'[0-9]+', ' ', text)
# https://stackoverflow.com/a/4134156/12896502
return re.sub(r'[\u0660-\u0669]+', ' ', text)
def remove_newlines(text):
return re.sub(r'\n', ' ', text)
def remove_whitespaces(text):
return re.sub(r'\s+', ' ', text)
def cleaning_text(text):
# Regarding removing URLs and mentions, according
# to the paper I'm following for developing the model,
# they normalized, rather than removed them, by
# replacing them by URL and USER tokens, respectively.
# So, this may be another trial to go for.
# Paper; arxiv:2005.06557
pipeline = [
remove_emojis,
remove_mentions_links,
remove_diacritics,
normalize_arabic,
remove_punctuations,
remove_numbers,
remove_repeating_char,
remove_newlines,
remove_whitespaces,
]
clean_text = text
for step in pipeline:
clean_text = step(clean_text)
return clean_text.strip()
datasetIds['text'] = rawData.apply(cleaning_text)
dialects = sorted(datasetIds.dialect.unique())
label_map = dict(zip(dialects, range(len(dialects))))
inv_label_map = dict(enumerate(dialects))
datasetIds['dialect'] = datasetIds.dialect.apply(lambda x:label_map[x])
data = datasetIds[datasetIds['text'] != '']