Skip to content
This repository was archived by the owner on Jun 17, 2024. It is now read-only.

Commit a3a3894

Browse files
tdoshea90tdoshea90
committed
Hourly data mining working
1 parent ee8803d commit a3a3894

7 files changed

+328
-24
lines changed

‎.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/.project
22
/.pydevproject
33
*.pyc
4-
*.sh
4+
/setup_env.sh
55

66
# Elastic Beanstalk Files
77
.elasticbeanstalk/*

‎README.md

+2-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,3 @@
1-
Notes:
2-
- virtualenv -p python3 envname
3-
- Need to set ST_API_KEY and APP_SECRET_KEY env variables
4-
51
TODO:
62
- Ads (obviously)
73
- Spell checking
@@ -10,4 +6,5 @@ TODO:
106
- Show 'trending' on ST
117
- Reverse lookup, search for a word and show word cloud of stocks.
128
- Automation that posts on a stock at the end of day stating what the most mentioned word(s) were
13-
for that specific stock. Links back to site.
9+
for that specific stock. Links back to site.
10+
- Buy domain, do a security pass.

‎stocktwits_wrapper.py

+17-18
Original file line numberDiff line numberDiff line change
@@ -58,17 +58,17 @@ def get_ticker(self, ticker):
5858
get_ticker_url = 'https://api.stocktwits.com/api/2/streams/symbol/'
5959
url_postfix = '.json?'
6060
pagination_since = self.__get_last_message(ticker)
61+
6162
# TODO: use visitors token for watchlist, etc.
6263
# only time we don't have access_token is on localhost so kind of an unnecessary check
6364
# if 'access_token' in session:
6465
# url_postfix = '%s&access_token=%s' % (url_postfix, session['access_token'])
6566

67+
# TODO: make a system account on ST
6668
# use my token for all automated calls
6769
url_postfix = '%s&access_token=%s' % (url_postfix, self.oauth_token)
6870

69-
# TODO: only messages 'since' show up.
70-
# TODO: store last 30 messages in DB or always fetch last 30 and do fake pagination here?
71-
# TODO: or use one of the embedded widget things
71+
# TODO: only messages 'since' show up, use visitors token and get last 30 messages.
7272
if pagination_since != -1:
7373
url_postfix = '%s&since=%s' % (url_postfix, str(pagination_since))
7474

@@ -91,15 +91,13 @@ def get_ticker(self, ticker):
9191

9292
st_compliant_posts = []
9393
simple_messages = []
94-
# index = 1
95-
for message in response_json['messages']:
9694

97-
# TODO: first message is new 'last_message' to use for pagination
98-
# TODO: turn this on once automation is set up
99-
# self.__update_last_message(ticker, message['id'])
95+
# first message is the most recent
96+
if(len(response_json['messages']) > 0):
97+
last_message = response_json['messages'][0]['id']
98+
self.__update_last_message(ticker, last_message)
10099

101-
# print('%d:%s:%s' % (index, message['id'], message['body']))
102-
# index = index + 1
100+
for message in response_json['messages']:
103101

104102
# 1. https://stocktwits.com/developers/docs/display_requirement
105103
st_compliant_posts.append(self.__build_st_compliant_post(message))
@@ -114,10 +112,9 @@ def get_ticker(self, ticker):
114112

115113
word_map = self.__build_word_map(simple_messages)
116114

117-
# TODO: find the right place for this
118-
self.__update_word_frequencies(ticker, 'DERP', 2)
115+
self.__update_word_frequencies(ticker, word_map)
119116

120-
# TODO: send to DB and update, return results from DB and st_compliant_posts.
117+
# TODO: retrieve ticker words from db
121118

122119
return GetTickerResponse(rate_remaining, rate_reset, symbol, co_name, st_compliant_posts, word_map)
123120

@@ -141,8 +138,9 @@ def __build_word_map(self, simple_messages):
141138
# http://www.artima.com/weblogs/viewpost.jsp?thread=98196
142139
word_map = {}
143140
for word in all_words:
144-
frequency = word_map.get(word, 0)
145-
word_map[word] = frequency + 1
141+
if(len(word) > 1):
142+
frequency = word_map.get(word, 0)
143+
word_map[word] = frequency + 1
146144

147145
return word_map
148146

@@ -213,10 +211,11 @@ def __check_response_code(self, response_json, rate_remaining):
213211
abort(response_code)
214212

215213
@classmethod
216-
def __update_word_frequencies(self, ticker, word, count):
214+
def __update_word_frequencies(self, ticker, word_map):
217215
with closing(self.mysql_conn.cursor()) as cursor:
218-
# ticker, word, count
219-
cursor.callproc('update_word_frequencies', (ticker, word, count))
216+
for word in word_map:
217+
# ticker, word, count
218+
cursor.callproc('update_word_frequencies', (ticker, word, word_map[word]))
220219
self.mysql_conn.commit()
221220

222221
@classmethod

‎create_wordtwits_db.sql ‎supplemental_materials/create_wordtwits_db.sql

File renamed without changes.

‎supplemental_materials/curl_tickers.sh

+204
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
#!/bin/bash
2+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/AAPL/ | echo ""
3+
sleep 10
4+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/ABBV/ | echo ""
5+
sleep 10
6+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/ABT/ | echo ""
7+
sleep 10
8+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/ACN/ | echo ""
9+
sleep 10
10+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/AGN/ | echo ""
11+
sleep 10
12+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/AIG/ | echo ""
13+
sleep 10
14+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/ALL/ | echo ""
15+
sleep 10
16+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/AMGN/ | echo ""
17+
sleep 10
18+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/AMZN/ | echo ""
19+
sleep 10
20+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/AXP/ | echo ""
21+
sleep 10
22+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/BA/ | echo ""
23+
sleep 10
24+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/BAC/ | echo ""
25+
sleep 10
26+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/BIIB/ | echo ""
27+
sleep 10
28+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/BK/ | echo ""
29+
sleep 10
30+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/BLK/ | echo ""
31+
sleep 10
32+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/BMY/ | echo ""
33+
sleep 10
34+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/BRK.B/ | echo ""
35+
sleep 10
36+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/C/ | echo ""
37+
sleep 10
38+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/CAT/ | echo ""
39+
sleep 10
40+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/CELG/ | echo ""
41+
sleep 10
42+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/CL/ | echo ""
43+
sleep 10
44+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/CMCSA/ | echo ""
45+
sleep 10
46+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/COF/ | echo ""
47+
sleep 10
48+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/COP/ | echo ""
49+
sleep 10
50+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/COST/ | echo ""
51+
sleep 10
52+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/CSCO/ | echo ""
53+
sleep 10
54+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/CVS/ | echo ""
55+
sleep 10
56+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/CVX/ | echo ""
57+
sleep 10
58+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/DD/ | echo ""
59+
sleep 10
60+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/DHR/ | echo ""
61+
sleep 10
62+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/DIS/ | echo ""
63+
sleep 10
64+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/DOW/ | echo ""
65+
sleep 10
66+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/DUK/ | echo ""
67+
sleep 10
68+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/EMC/ | echo ""
69+
sleep 10
70+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/EMR/ | echo ""
71+
sleep 10
72+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/EXC/ | echo ""
73+
sleep 10
74+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/F/ | echo ""
75+
sleep 10
76+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/FB/ | echo ""
77+
sleep 10
78+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/FDX/ | echo ""
79+
sleep 10
80+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/FOX/ | echo ""
81+
sleep 10
82+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/FOXA/ | echo ""
83+
sleep 10
84+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/GD/ | echo ""
85+
sleep 10
86+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/GE/ | echo ""
87+
sleep 10
88+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/GILD/ | echo ""
89+
sleep 10
90+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/GM/ | echo ""
91+
sleep 10
92+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/GOOG/ | echo ""
93+
sleep 10
94+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/GOOGL/ | echo ""
95+
sleep 10
96+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/GS/ | echo ""
97+
sleep 10
98+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/HAL/ | echo ""
99+
sleep 10
100+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/HD/ | echo ""
101+
sleep 10
102+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/HON/ | echo ""
103+
sleep 10
104+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/IBM/ | echo ""
105+
sleep 10
106+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/INTC/ | echo ""
107+
sleep 10
108+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/JNJ/ | echo ""
109+
sleep 10
110+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/JPM/ | echo ""
111+
sleep 10
112+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/KMI/ | echo ""
113+
sleep 10
114+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/KO/ | echo ""
115+
sleep 10
116+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/LLY/ | echo ""
117+
sleep 10
118+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/LMT/ | echo ""
119+
sleep 10
120+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/LOW/ | echo ""
121+
sleep 10
122+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/MA/ | echo ""
123+
sleep 10
124+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/MCD/ | echo ""
125+
sleep 10
126+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/MDLZ/ | echo ""
127+
sleep 10
128+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/MDT/ | echo ""
129+
sleep 10
130+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/MET/ | echo ""
131+
sleep 10
132+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/MMM/ | echo ""
133+
sleep 10
134+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/MO/ | echo ""
135+
sleep 10
136+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/MON/ | echo ""
137+
sleep 10
138+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/MRK/ | echo ""
139+
sleep 10
140+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/MS/ | echo ""
141+
sleep 10
142+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/MSFT/ | echo ""
143+
sleep 10
144+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/NEE/ | echo ""
145+
sleep 10
146+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/NKE/ | echo ""
147+
sleep 10
148+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/ORCL/ | echo ""
149+
sleep 10
150+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/OXY/ | echo ""
151+
sleep 10
152+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/PCLN/ | echo ""
153+
sleep 10
154+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/PEP/ | echo ""
155+
sleep 10
156+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/PFE/ | echo ""
157+
sleep 10
158+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/PG/ | echo ""
159+
sleep 10
160+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/PM/ | echo ""
161+
sleep 10
162+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/PYPL/ | echo ""
163+
sleep 10
164+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/QCOM/ | echo ""
165+
sleep 10
166+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/RTN/ | echo ""
167+
sleep 10
168+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/SBUX/ | echo ""
169+
sleep 10
170+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/SLB/ | echo ""
171+
sleep 10
172+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/SO/ | echo ""
173+
sleep 10
174+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/SPG/ | echo ""
175+
sleep 10
176+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/T/ | echo ""
177+
sleep 10
178+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/TGT/ | echo ""
179+
sleep 10
180+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/TWX/ | echo ""
181+
sleep 10
182+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/TXN/ | echo ""
183+
sleep 10
184+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/UNH/ | echo ""
185+
sleep 10
186+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/UNP/ | echo ""
187+
sleep 10
188+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/UPS/ | echo ""
189+
sleep 10
190+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/USB/ | echo ""
191+
sleep 10
192+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/UTX/ | echo ""
193+
sleep 10
194+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/V/ | echo ""
195+
sleep 10
196+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/VZ/ | echo ""
197+
sleep 10
198+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/WBA/ | echo ""
199+
sleep 10
200+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/WFC/ | echo ""
201+
sleep 10
202+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/WMT/ | echo ""
203+
sleep 10
204+
curl -s http://stocktwitsanalyzer-env.us-west-2.elasticbeanstalk.com/XOM/ | echo ""

‎supplemental_materials/truncate_daily.sql

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
TRUNCATE WordTwitsDatabase.word_frequencies;
2+
UPDATE WordTwitsDatabase.tickers SET last_message = -1;

0 commit comments

Comments
 (0)