forked from minimaxir/facebook-page-post-scraper
-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathmakefile
208 lines (183 loc) · 6.13 KB
/
makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
JsonDir = ./json/
CsvDir = ./csv/
UNAME := $(shell uname)
face_file1 = venv/src/facebook-sdk/facebook/__init__.py
face_file2 = venv/src/facebook-sdk/facebook/version.py
default: test
travis:
@make style
@make clean
green3 tests/test_page_scraper_unit.py -vv -f
test:
ifeq ($(OS), Windows_NT)
make clean
green3 -vv
else
@make clean
green3 -vv
endif
data:
python3 -c 'from scraper.collector import collect_new_data; collect_new_data()'
run:
ifeq ($(OS), Windows_NT)
python -m scraper.collector
else
python3 -m scraper.collector
endif
heroku:
ifeq ($(OS), Windows_NT)
python3 -m heroku_clock
else
python3 -m heroku_clock
endif
tm:
ifeq ($(OS), Windows_NT)
green3 tests\test_token_manager.py -vv
else
green3 tests/test_token_manager.py -vvv -f
endif
ps:
ifeq ($(OS), Windows_NT)
make clean
green3 tests\test_page_scraper_inte.py -vv
else
make clean
green3 tests/test_page_scraper.py -vvv -f
endif
style:
ifeq ($(OS), Windows_NT)
pycodestyle tests\. scraper\. server\. --ignore=E402,W504,E127
else
pycodestyle tests/. scraper/. server/. --ignore=E402,W504,E127
endif
doc:
ifeq ($(OS), Windows_NT)
pydocstyle tests\. scraper\. server\. --ignore=E402,W504
else
pydocstyle tests/. scraper/. server/.
endif
cov:
ifeq ($(OS), Windows_NT)
make clean
coverage run -m py.test tests/test_page_scraper.py tests/test_token_manager.py
coverage report -m scraper/page_scraper.py scraper/token_manager.py
coverage html scraper/page_scraper.py scraper/token_manager.py
else
make clean
coverage run -m py.test tests/test_page_scraper.py tests/test_token_manager.py
coverage report -m scraper/page_scraper.py scraper/token_manager.py
coverage html scraper/page_scraper.py scraper/token_manager.py
endif
full:
ifeq ($(OS), Windows_NT)
make clean
green3 -vvv --run-coverage -o $(face_file1),$(face_file2)
coverage html scraper\page_scraper.py scraper\token_manager.py
make style
else
@make clean
green3 -vvv --run-coverage -f -o $(face_file1),$(face_file2)
coverage html scraper/page_scraper.py scraper/token_manager.py
make style
endif
documentation:style doc
cc:
radon cc scraper -s
mi:
radon mi scraper -s
install:
pip3 install -r requirements.txt
.PHONY: autotoken
autotoken:
python3 -m scraper.token_manager
.PHONY: chromedriver
chromedriver:
ifeq ($(UNAME), Linux)
cd $(HOME)/Downloads
wget https://chromedriver.storage.googleapis.com/2.37/chromedriver_linux64.zip
unzip chromedriver_linux64.zip
mkdir -p $(HOME)/bin
mv chromedriver $(HOME)/bin
echo "export PATH=$(PATH):$(HOME)/bin" >> $(HOME)/.bash_profile
rm -f chromedriver_linux64.zip
endif
# Call for creating a Json dir and moves all json files there
.PHONY: json
json:
mkdir -p $(JsonDir)
mv *.json $(JsonDir); true
# Call for *.json clean up
.PHONY: clean
clean:
ifeq ($(OS), Windows_NT)
rmdir .\json /S /Q
rmdir .\csv /S /Q
rmdir .\htmlcov /S /Q
rmdir .\.coverage /S /Q
else
rm -f $(CsvDir)*.csv
rm -rf ./htmlcov
rm -f ./.coverage
rm -rf ./json/
endif
pylint:
pylint -j 2 scraper/page_scraper.py scraper/token_manager.py --reports=y
.PHONY: createconfig
createconfig:
echo '[DEFAULT]\ntoken = YOURTOKENHERE' >./scraper/config.ini
# Call for help with this makefile's commands
.PHONY: help
help:
ifeq ($(OS), Windows_NT)
@echo.
@echo Makefile of Facebook scrapper from UnB
@echo.
@echo make.............= Runs the default function
@echo make test........= Runs the tests using green3
@echo make tm..........= Runs only the tests on token management
@echo make ps..........= Runs only the tests on page scraping
@echo make run.........= Collects all pages in entidades.csv
@echo make install.....= Installs the requirements necessary for this project
@echo make style.......= Checks if your code is using our pattern of coding for this
@echo project
@echo make json........= Creates a json dir and moves all .json files there
@echo make cov.........= Checks tests coverage
@echo make full........= Runs make test, cov and style
@echo make autotoken...= Open Facebook Developers web page so that the user can
@echo take a new token and update config.ini file
@echo make json........= Creates a json dir and moves all .json files there
@echo make clean.......= Removes all .json files
@echo make createconfig= Creates config.ini in scraper dir with the expected way
@echo to use it
@echo make chromedriver= Install chromedriver for get the token automatically,
@echo works in Linux
@echo make heroku......= Realiza o agendamento do serviço de coleta de dados
@echo diariamente. A configuração é feita no arquivo heroku_clock.
@echo.
@echo End of Makefile Help
@echo.
else
@echo "\n\t Makefile of Facebook scrapper from UnB\n"
@echo " make.............= Runs the default function"
@echo " make test........= Runs the tests using green3"
@echo " make tm..........= Runs only the tests on token management"
@echo " make ps..........= Runs only the tests on page scraping"
@echo " make run.........= Collects all pages in entidades.csv"
@echo " make install.....= Installs the requirements necessary for this project"
@echo " make style.......= Checks if your code is our pattern of coding for this "
@echo " project"
@echo " make json........= Creates a json dir and moves all .json files there"
@echo " make cov.........= Checks test's coverage"
@echo " make full........= Runs make test, cov and style"
@echo " make autotoken...= Open Facebook Developers web page so that the user can "
@echo " take a new token and update config.ini file"
@echo " make json........= Creates a json dir and moves all .json files there"
@echo " make clean.......= Removes all .json files"
@echo " make createconfig= Creates config.ini in scraper dir with the expected way"
@echo " to use it"
@echo " make chromedriver= Install chromedriver for get the token automatically, "
@echo " works in Linux"
@echo " make heroku......= Realiza o agendamento do serviço de coleta de dados"
@echo " diariamente. A configuração é feita no arquivo heroku_clock."
@echo "\n\t End of Makefile Help\n"
endif