Merge pull request #50 from NicolasConstant/develop

0.7.0 PR
NicolasConstant · Jan 14, 2021 · 72f6337 · 72f6337
2 parents ce5c843 + f0a0bb9
commit 72f6337
Show file tree

Hide file tree

Showing 11 changed files with 186 additions and 27 deletions.
diff --git a/src/BirdsiteLive.Domain/Statistics/ExtractionStatisticsHandler.cs b/src/BirdsiteLive.Domain/Statistics/ExtractionStatisticsHandler.cs
@@ -0,0 +1,80 @@
+using System.Threading;
+using System.Timers;
+
+namespace BirdsiteLive.Domain.Statistics
+{
+    public interface IExtractionStatisticsHandler
+    {
+        void ExtractedDescription(int mentionsCount);
+        void ExtractedStatus(int mentionsCount);
+        ExtractionStatistics GetStatistics();
+    }
+
+    public class ExtractionStatisticsHandler : IExtractionStatisticsHandler
+    {
+        private static int _lastDescriptionMentionsExtracted;
+        private static int _lastStatusMentionsExtracted;
+
+        private static int _descriptionMentionsExtracted;
+        private static int _statusMentionsExtracted;
+
+        private static System.Timers.Timer _resetTimer;
+
+        #region Ctor
+        public ExtractionStatisticsHandler()
+        {
+            if (_resetTimer == null)
+            {
+                _resetTimer = new System.Timers.Timer();
+                _resetTimer.Elapsed += OnTimeResetEvent;
+                _resetTimer.Interval = 24 * 60 * 60 * 1000; // 24h
+                _resetTimer.Enabled = true;
+            }
+        }
+        #endregion
+
+        private void OnTimeResetEvent(object sender, ElapsedEventArgs e)
+        {
+            _lastDescriptionMentionsExtracted = _descriptionMentionsExtracted;
+            _lastStatusMentionsExtracted = _statusMentionsExtracted;
+
+            // Reset
+            Interlocked.Exchange(ref _descriptionMentionsExtracted, 0);
+            Interlocked.Exchange(ref _statusMentionsExtracted, 0);
+        }
+
+        public void ExtractedDescription(int mentionsCount)
+        {
+            Interlocked.Increment(ref _descriptionMentionsExtracted);
+        }
+
+        public void ExtractedStatus(int mentionsCount)
+        {
+            Interlocked.Increment(ref _statusMentionsExtracted);
+        }
+
+        public ExtractionStatistics GetStatistics()
+        {
+            return new ExtractionStatistics(_descriptionMentionsExtracted, _statusMentionsExtracted, _lastDescriptionMentionsExtracted, _lastStatusMentionsExtracted);
+        }
+    }
+
+    public class ExtractionStatistics
+    {
+        #region Ctor
+        public ExtractionStatistics(int mentionsInDescriptionsExtraction, int mentionsInStatusesExtraction, int lastMentionsInDescriptionsExtraction, int lastMentionsInStatusesExtraction)
+        {
+            MentionsInDescriptionsExtraction = mentionsInDescriptionsExtraction;
+            MentionsInStatusesExtraction = mentionsInStatusesExtraction;
+            LastMentionsInDescriptionsExtraction = lastMentionsInDescriptionsExtraction;
+            LastMentionsInStatusesExtraction = lastMentionsInStatusesExtraction;
+        }
+        #endregion
+
+        public int MentionsInDescriptionsExtraction { get; }
+        public int MentionsInStatusesExtraction { get; }
+
+        public int LastMentionsInDescriptionsExtraction { get; }
+        public int LastMentionsInStatusesExtraction { get; }
+    }
+}
diff --git a/src/BirdsiteLive.Domain/StatusService.cs b/src/BirdsiteLive.Domain/StatusService.cs
@@ -7,6 +7,7 @@
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
+using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter.Models;
 using Tweetinvi.Models;
@@ -23,12 +24,14 @@ public class StatusService : IStatusService
     {
         private readonly InstanceSettings _instanceSettings;
         private readonly IStatusExtractor _statusExtractor;
-
+        private readonly IExtractionStatisticsHandler _statisticsHandler;
+
         #region Ctor
-        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor)
+        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler)
         {
             _instanceSettings = instanceSettings;
             _statusExtractor = statusExtractor;
+            _statisticsHandler = statisticsHandler;
         }
         #endregion
 
@@ -41,6 +44,7 @@ public Note GetStatus(string username, ExtractedTweet tweet)
             var apPublic = "https://www.w3.org/ns/activitystreams#Public";
 
             var extractedTags = _statusExtractor.ExtractTags(tweet.MessageContent);
+            _statisticsHandler.ExtractedStatus(extractedTags.tags.Count(x => x.type == "Mention"));
 
             string inReplyTo = null;
             if (tweet.InReplyToStatusId != default)

diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs
@@ -10,6 +10,7 @@
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Cryptography;
 using BirdsiteLive.Domain.BusinessUseCases;
+using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter.Models;
 using Tweetinvi.Core.Exceptions;
@@ -33,16 +34,18 @@ public class UserService : IUserService
         private readonly ICryptoService _cryptoService;
         private readonly IActivityPubService _activityPubService;
         private readonly IStatusExtractor _statusExtractor;
+        private readonly IExtractionStatisticsHandler _statisticsHandler;
 
         #region Ctor
-        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor)
+        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler)
         {
             _instanceSettings = instanceSettings;
             _cryptoService = cryptoService;
             _activityPubService = activityPubService;
             _processFollowUser = processFollowUser;
             _processUndoFollowUser = processUndoFollowUser;
             _statusExtractor = statusExtractor;
+            _statisticsHandler = statisticsHandler;
             //_host = $"https://{instanceSettings.Domain.Replace("https://",string.Empty).Replace("http://", string.Empty).TrimEnd('/')}";
         }
         #endregion
@@ -58,6 +61,8 @@ public Actor GetUser(TwitterUser twitterUser)
             {
                 var extracted = _statusExtractor.ExtractTags(description);
                 description = extracted.content;
+
+                _statisticsHandler.ExtractedDescription(extracted.tags.Count(x => x.type == "Mention"));
             }
 
             var user = new Actor

diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs
@@ -45,6 +45,10 @@ public string ExtractMessage(ITweet tweet)
                     message = message.Replace("RT", "[RT]");
             }
 
+            // Expand URLs
+            foreach (var url in tweet.Urls.OrderByDescending(x => x.URL.Length))
+                message = message.Replace(url.URL, url.ExpandedURL);
+
             return message;
         }
 

diff --git a/src/BirdsiteLive.Twitter/Models/ApiStatistics.cs b/src/BirdsiteLive.Twitter/Models/ApiStatistics.cs
@@ -2,11 +2,17 @@
 {
     public class ApiStatistics
     {
-        public int UserCallsCount { get; set; }
+        public int UserCallsCountMin { get; set; }
+        public int UserCallsCountAvg { get; set; }
+        public int UserCallsCountMax { get; set; }
         public int UserCallsMax { get; set; }
-        public int TweetCallsCount { get; set; }
+        public int TweetCallsCountMin { get; set; }
+        public int TweetCallsCountAvg { get; set; }
+        public int TweetCallsCountMax { get; set; }
         public int TweetCallsMax { get; set; }
-        public int TimelineCallsCount { get; set; }
+        public int TimelineCallsCountMin { get; set; }
+        public int TimelineCallsCountAvg { get; set; }
+        public int TimelineCallsCountMax { get; set; }
         public int TimelineCallsMax { get; set; }
     }
 }
diff --git a/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs b/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs
@@ -1,4 +1,7 @@
-using System.Threading;
+using System;
+using System.Collections.Concurrent;
+using System.Linq;
+using System.Threading;
 using System.Timers;
 using BirdsiteLive.Twitter.Models;
 
@@ -15,14 +18,12 @@ public interface ITwitterStatisticsHandler
     //Rate limits: https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits
     public class TwitterStatisticsHandler : ITwitterStatisticsHandler
     {
-        private static int _previousUserCalls;
-        private static int _previousTweetCalls;
-        private static int _previousTimelineCalls;
-
         private static int _userCalls;
         private static int _tweetCalls;
         private static int _timelineCalls;
 
+        private static ConcurrentDictionary<DateTime, ApiStatisticsSnapshot> _snapshots = new ConcurrentDictionary<DateTime, ApiStatisticsSnapshot>();
+
         private static System.Timers.Timer _resetTimer;
 
         #region Ctor
@@ -40,13 +41,23 @@ public TwitterStatisticsHandler()
 
         private void OnTimeResetEvent(object sender, ElapsedEventArgs e)
         {
-            _previousUserCalls = _userCalls;
-            _previousTweetCalls = _tweetCalls;
-            _previousTimelineCalls = _timelineCalls;
-
+            // Add snapshot
+            var snapshot = new ApiStatisticsSnapshot(_userCalls, _tweetCalls, _timelineCalls);
+            bool success;
+            do
+            {
+                success = _snapshots.TryAdd(snapshot.SnapshotDate, snapshot);
+            } while (!success);
+
+            // Reset
             Interlocked.Exchange(ref _userCalls, 0);
             Interlocked.Exchange(ref _tweetCalls, 0);
             Interlocked.Exchange(ref _timelineCalls, 0);
+
+            // Clean up 
+            var now = DateTime.UtcNow;
+            var oldSnapshots = _snapshots.Keys.Where(x => (now - x).TotalHours > 24).ToList();
+            foreach (var old in oldSnapshots) _snapshots.TryRemove(old, out var data);
         }
 
         public void CalledUserApi()  //GET users/show - 900/15mins
@@ -66,15 +77,44 @@ public void CalledTimelineApi()  // GET statuses/user_timeline - 1500/15 mins
 
         public ApiStatistics GetStatistics()
         {
+            var snapshots = _snapshots.Values.ToList();
+            var userCalls = snapshots.Select(x => x.UserCalls).ToList();
+            var tweetCalls = snapshots.Select(x => x.TweetCalls).ToList();
+            var timelineCalls = snapshots.Select(x => x.TimelineCalls).ToList();
+
             return new ApiStatistics
             {
-                UserCallsCount = _previousUserCalls,
+                UserCallsCountMin = userCalls.Any() ? userCalls.Min() : 0,
+                UserCallsCountAvg = userCalls.Any() ? (int)userCalls.Average() : 0,
+                UserCallsCountMax = userCalls.Any() ? userCalls.Max() : 0,
                 UserCallsMax = 900,
-                TweetCallsCount = _previousTweetCalls,
+                TweetCallsCountMin = tweetCalls.Any() ? tweetCalls.Min() : 0,
+                TweetCallsCountAvg = tweetCalls.Any() ? (int)tweetCalls.Average() : 0,
+                TweetCallsCountMax = tweetCalls.Any() ? tweetCalls.Max() : 0,
                 TweetCallsMax = 300,
-                TimelineCallsCount = _previousTimelineCalls,
+                TimelineCallsCountMin = timelineCalls.Any() ? timelineCalls.Min() : 0,
+                TimelineCallsCountAvg = timelineCalls.Any() ? (int)timelineCalls.Average() : 0,
+                TimelineCallsCountMax = timelineCalls.Any() ? timelineCalls.Max() : 0,
                 TimelineCallsMax = 1500
             };
         }
     }
+
+    internal class ApiStatisticsSnapshot 
+    {
+        #region Ctor
+        public ApiStatisticsSnapshot(int userCalls, int tweetCalls, int timelineCalls)
+        {
+            UserCalls = userCalls;
+            TweetCalls = tweetCalls;
+            TimelineCalls = timelineCalls;
+            SnapshotDate = DateTime.UtcNow;
+        }
+        #endregion
+
+        public DateTime SnapshotDate { get;  }
+        public int UserCalls { get; set; }
+        public int TweetCalls { get; set; }
+        public int TimelineCalls { get; set; }
+    }
 }
diff --git a/src/BirdsiteLive.Twitter/TwitterService.cs b/src/BirdsiteLive.Twitter/TwitterService.cs
@@ -43,11 +43,16 @@ public TwitterUser GetUser(string username)
             _statisticsHandler.CalledUserApi();
             if (user == null) return null;
 
+            // Expand URLs
+            var description = user.Description;
+            foreach (var descriptionUrl in user.Entities?.Description?.Urls?.OrderByDescending(x => x.URL.Length))
+                description = description.Replace(descriptionUrl.URL, descriptionUrl.ExpandedURL);
+
             return new TwitterUser
             {
                 Acct = username,
                 Name = user.Name,
-                Description = user.Description,
+                Description = description,
                 Url = $"https://twitter.com/{username}",
                 ProfileImageUrl = user.ProfileImageUrlFullSize,
                 ProfileBackgroundImageUrl = user.ProfileBackgroundImageUrlHttps,

diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj
@@ -4,7 +4,7 @@
     <TargetFramework>netcoreapp3.1</TargetFramework>
     <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-    <Version>0.6.1</Version>
+    <Version>0.7.0</Version>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/src/BirdsiteLive/Controllers/StatisticsController.cs b/src/BirdsiteLive/Controllers/StatisticsController.cs
@@ -3,6 +3,7 @@
 using System.Linq;
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Statistics.Domain;
 using Microsoft.AspNetCore.Mvc;
 
@@ -13,13 +14,16 @@ public class StatisticsController : Controller
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly IFollowersDal _followersDal;
         private readonly ITwitterStatisticsHandler _twitterStatistics;
+        private readonly IExtractionStatisticsHandler _extractionStatistics;
+
 
         #region Ctor
-        public StatisticsController(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ITwitterStatisticsHandler twitterStatistics)
+        public StatisticsController(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ITwitterStatisticsHandler twitterStatistics, IExtractionStatisticsHandler extractionStatistics)
         {
             _twitterUserDal = twitterUserDal;
             _followersDal = followersDal;
             _twitterStatistics = twitterStatistics;
+            _extractionStatistics = extractionStatistics;
         }
         #endregion
 
@@ -29,7 +33,8 @@ public async Task<IActionResult> Index()
             {
                 FollowersCount = await _followersDal.GetFollowersCountAsync(),
                 TwitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync(),
-                TwitterStatistics = _twitterStatistics.GetStatistics()
+                TwitterStatistics = _twitterStatistics.GetStatistics(),
+                ExtractionStatistics = _extractionStatistics.GetStatistics(),
             };
             return View(stats);
         }

diff --git a/src/BirdsiteLive/Models/StatisticsModels/Statistics.cs b/src/BirdsiteLive/Models/StatisticsModels/Statistics.cs
@@ -1,4 +1,5 @@
-using BirdsiteLive.Twitter.Models;
+using BirdsiteLive.Domain.Statistics;
+using BirdsiteLive.Twitter.Models;
 
 namespace BirdsiteLive.Models.StatisticsModels
 {
@@ -7,5 +8,6 @@ public class Statistics
         public int FollowersCount { get; set; }
         public int TwitterUserCount { get; set; }
         public ApiStatistics TwitterStatistics { get; set; }
+        public ExtractionStatistics ExtractionStatistics { get; set; }
     }
 }
diff --git a/src/BirdsiteLive/Views/Statistics/Index.cshtml b/src/BirdsiteLive/Views/Statistics/Index.cshtml
@@ -12,9 +12,17 @@
     <li>Followers: @Model.FollowersCount</li>
 </ul>
 
-<h4>Twitter API</h4>
+<h4>Twitter API (Min, Avg, Max for the last 24h)</h4>
 <ul>
-    <li>Users Calls: @Model.TwitterStatistics.UserCallsCount / @Model.TwitterStatistics.UserCallsMax</li>
-    <li>Tweets Calls: @Model.TwitterStatistics.TweetCallsCount / @Model.TwitterStatistics.TweetCallsMax</li>
-    <li>Timeline Calls: @Model.TwitterStatistics.TimelineCallsCount / @Model.TwitterStatistics.TimelineCallsMax</li>
+    <li>Users Calls: @Model.TwitterStatistics.UserCallsCountMin ;  @Model.TwitterStatistics.UserCallsCountAvg ;  @Model.TwitterStatistics.UserCallsCountMax / @Model.TwitterStatistics.UserCallsMax</li>
+    <li>Tweets Calls: @Model.TwitterStatistics.TweetCallsCountMin ; @Model.TwitterStatistics.TweetCallsCountAvg ; @Model.TwitterStatistics.TweetCallsCountMax / @Model.TwitterStatistics.TweetCallsMax</li>
+    <li>Timeline Calls: @Model.TwitterStatistics.TimelineCallsCountMin ; @Model.TwitterStatistics.TimelineCallsCountAvg ; @Model.TwitterStatistics.TimelineCallsCountMax / @Model.TwitterStatistics.TimelineCallsMax</li>
+</ul>
+
+<h4>Mentions Extraction</h4>
+<ul>
+    <li>Current day in descriptions:  @Model.ExtractionStatistics.MentionsInDescriptionsExtraction</li>
+    <li>Current day in statuses: @Model.ExtractionStatistics.MentionsInStatusesExtraction</li>
+    <li>Last day in descriptions: @Model.ExtractionStatistics.LastMentionsInDescriptionsExtraction</li>
+    <li>Last day in statuses: @Model.ExtractionStatistics.LastMentionsInStatusesExtraction</li>
 </ul>