Pagination working

Author: tdoshea90

create_wordtwits_db.sql

@@ -30,11 +30,12 @@ CREATE TABLE `tickers` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `sector` varchar(64) DEFAULT NULL,
   `industry` varchar(64) DEFAULT NULL,
+  `last_message` int(11) NOT NULL DEFAULT '-1',
   PRIMARY KEY (`id`),
   UNIQUE KEY `ticker_UNIQUE` (`ticker`),
   UNIQUE KEY `id_UNIQUE` (`id`),
   UNIQUE KEY `company_name_UNIQUE` (`company_name`)
-) ENGINE=InnoDB AUTO_INCREMENT=13258 DEFAULT CHARSET=latin1 COMMENT='Dimension table for tickers';
+) ENGINE=InnoDB AUTO_INCREMENT=13275 DEFAULT CHARSET=latin1 COMMENT='Table for all tickers';
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 --
@@ -54,12 +55,55 @@ CREATE TABLE `word_frequencies` (
   UNIQUE KEY `uq_ticker_idx` (`ticker_id`,`word`),
   KEY `fk_ticker_id_idx` (`ticker_id`),
   CONSTRAINT `fk_ticker_id` FOREIGN KEY (`ticker_id`) REFERENCES `tickers` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
-) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=latin1 COMMENT='Word Frequency fact table';
+) ENGINE=InnoDB AUTO_INCREMENT=27 DEFAULT CHARSET=latin1 COMMENT='Word Frequency fact table';
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 --
 -- Dumping routines for database 'WordTwitsDatabase'
 --
+/*!50003 DROP PROCEDURE IF EXISTS `get_last_message` */;
+/*!50003 SET @saved_cs_client      = @@character_set_client */ ;
+/*!50003 SET @saved_cs_results     = @@character_set_results */ ;
+/*!50003 SET @saved_col_connection = @@collation_connection */ ;
+/*!50003 SET character_set_client  = utf8 */ ;
+/*!50003 SET character_set_results = utf8 */ ;
+/*!50003 SET collation_connection  = utf8_general_ci */ ;
+/*!50003 SET @saved_sql_mode       = @@sql_mode */ ;
+/*!50003 SET sql_mode              = 'NO_ENGINE_SUBSTITUTION' */ ;
+DELIMITER ;;
+CREATE DEFINER=`tdoshea90`@`%` PROCEDURE `get_last_message`(IN ticker_arg CHAR(5), OUT last_message_out INT)
+BEGIN
+
+	SET last_message_out = (SELECT IFNULL((SELECT last_message FROM tickers WHERE ticker=ticker_arg), -1));
+    
+END ;;
+DELIMITER ;
+/*!50003 SET sql_mode              = @saved_sql_mode */ ;
+/*!50003 SET character_set_client  = @saved_cs_client */ ;
+/*!50003 SET character_set_results = @saved_cs_results */ ;
+/*!50003 SET collation_connection  = @saved_col_connection */ ;
+/*!50003 DROP PROCEDURE IF EXISTS `update_last_message` */;
+/*!50003 SET @saved_cs_client      = @@character_set_client */ ;
+/*!50003 SET @saved_cs_results     = @@character_set_results */ ;
+/*!50003 SET @saved_col_connection = @@collation_connection */ ;
+/*!50003 SET character_set_client  = utf8 */ ;
+/*!50003 SET character_set_results = utf8 */ ;
+/*!50003 SET collation_connection  = utf8_general_ci */ ;
+/*!50003 SET @saved_sql_mode       = @@sql_mode */ ;
+/*!50003 SET sql_mode              = 'NO_ENGINE_SUBSTITUTION' */ ;
+DELIMITER ;;
+CREATE DEFINER=`tdoshea90`@`%` PROCEDURE `update_last_message`(IN ticker_arg CHAR(5), IN last_message_arg INT)
+BEGIN
+
+	UPDATE tickers SET last_message=last_message_arg
+	WHERE ticker=ticker_arg;
+
+END ;;
+DELIMITER ;
+/*!50003 SET sql_mode              = @saved_sql_mode */ ;
+/*!50003 SET character_set_client  = @saved_cs_client */ ;
+/*!50003 SET character_set_results = @saved_cs_results */ ;
+/*!50003 SET collation_connection  = @saved_col_connection */ ;
 /*!50003 DROP PROCEDURE IF EXISTS `update_word_frequencies` */;
 /*!50003 SET @saved_cs_client      = @@character_set_client */ ;
 /*!50003 SET @saved_cs_results     = @@character_set_results */ ;
@@ -70,7 +114,7 @@ CREATE TABLE `word_frequencies` (
 /*!50003 SET @saved_sql_mode       = @@sql_mode */ ;
 /*!50003 SET sql_mode              = 'NO_ENGINE_SUBSTITUTION' */ ;
 DELIMITER ;;
-CREATE PROCEDURE `update_word_frequencies`(ticker_arg CHAR(5), word_arg VARCHAR(32), count_arg INT)
+CREATE DEFINER=`tdoshea90`@`%` PROCEDURE `update_word_frequencies`(IN ticker_arg CHAR(5), IN word_arg VARCHAR(32), IN count_arg INT)
 BEGIN
 
 	# 1. Get ticker or insert
@@ -97,4 +141,4 @@ DELIMITER ;
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
--- Dump completed on 2016-11-06 12:57:07
+-- Dump completed

stocktwits_wrapper.py

@@ -53,14 +53,20 @@ def get_ticker(self, ticker):
         self.__check_rate_reset()
 
         get_ticker_url = 'https://api.stocktwits.com/api/2/streams/symbol/'
-        url_postfix = '.json'
+        url_postfix = '.json?'
+        pagination_since = self.__get_last_message(ticker)
+        # only time we don't have access_token is on localhost so kind of an unnecessary check
         if 'access_token' in session:
-            url_postfix = '%s?access_token=%s' % (url_postfix, session['access_token'])
+            url_postfix = '%s&access_token=%s' % (url_postfix, session['access_token'])
 
-        # call ST api
-        request_url = get_ticker_url + ticker + url_postfix
+        # TODO: only messages 'since' show up.
+        # TODO: store last 30 messages in DB or always fetch last 30 and do fake pagination here?
+        # TODO: or use one of the embedded widget things
+        if pagination_since != -1:
+            url_postfix = '%s&since=%s' % (url_postfix, str(pagination_since))
 
-        # TODO: use 'cursor' parameter to only get newest messages
+        # call ST api
+        request_url = '%s%s%s' % (get_ticker_url, ticker, url_postfix)
 
         response = requests.get(request_url)
         response_json = response.json()
@@ -78,8 +84,16 @@ def get_ticker(self, ticker):
 
         st_compliant_posts = []
         simple_messages = []
+        # index = 1
         for message in response_json['messages']:
 
+            # TODO: first message is new 'last_message' to use for pagination
+            # TODO: turn this on once automation is set up
+            # self.__update_last_message(ticker, message['id'])
+
+            #             print('%d:%s:%s' % (index, message['id'], message['body']))
+            #             index = index + 1
+
             # 1. https://stocktwits.com/developers/docs/display_requirement
             st_compliant_posts.append(self.__build_st_compliant_post(message))
 
@@ -94,7 +108,7 @@ def get_ticker(self, ticker):
         word_map = self.__build_word_map(simple_messages)
 
         # TODO: find the right place for this
-        # self.__update_db('AMZN', 'DERP', 10)
+        # self.__update_word_frequencies('AMZN', 'DERP', 10)
 
         # TODO: send to DB and update, return results from DB and st_compliant_posts.
 
@@ -192,22 +206,25 @@ def __check_response_code(self, response_json, rate_remaining):
             abort(response_code)
 
     @classmethod
-    def __update_db(self, ticker, word, count):
+    def __update_word_frequencies(self, ticker, word, count):
         with closing(self.mysql_conn.cursor()) as cursor:
             # ticker, word, count
             cursor.callproc('update_word_frequencies', (ticker, word, count))
             self.mysql_conn.commit()
 
     @classmethod
-    def __query_db(self):
-        query = 'SELECT tickers.ticker, word_frequencies.word, word_frequencies.frequency \
-                 FROM word_frequencies \
-                 INNER JOIN tickers ON word_frequencies.ticker_id = tickers.id;'
+    def __get_last_message(self, ticker):
+        with closing(self.mysql_conn.cursor()) as cursor:
+            # ticker, last_message return value
+            result_args = cursor.callproc('get_last_message', (ticker, 0))
+            return result_args[1]
 
+    @classmethod
+    def __update_last_message(self, ticker, last_message):
         with closing(self.mysql_conn.cursor()) as cursor:
-            cursor.execute(query)
-            for (ticker, word, frequency) in cursor:
-                print('%s\t%s\t%s' % (ticker, word, frequency))
+            # ticker, last_message
+            cursor.callproc('update_last_message', (ticker, last_message))
+            self.mysql_conn.commit()
 
 
 class GetTickerResponse: