diff --git a/.github/workflows/dotnet-core.yml b/.github/workflows/dotnet-core.yml index d28cb08..ae430cc 100644 --- a/.github/workflows/dotnet-core.yml +++ b/.github/workflows/dotnet-core.yml @@ -16,7 +16,7 @@ jobs: - name: Setup .NET Core uses: actions/setup-dotnet@v1 with: - dotnet-version: 3.1.101 + dotnet-version: 6.0.x - name: Install dependencies run: dotnet restore working-directory: ${{env.working-directory}} diff --git a/Dockerfile b/Dockerfile index 11a4422..a59f7f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. -FROM mcr.microsoft.com/dotnet/aspnet:3.1-buster-slim AS base +FROM mcr.microsoft.com/dotnet/aspnet:6.0-bullseye-slim AS base WORKDIR /app EXPOSE 80 EXPOSE 443 -FROM mcr.microsoft.com/dotnet/sdk:3.1-buster AS publish +FROM mcr.microsoft.com/dotnet/sdk:6.0-bullseye-slim AS publish COPY ./src/ ./src/ RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish diff --git a/INSTALLATION.md b/INSTALLATION.md index 74d9131..569c2a1 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -186,6 +186,70 @@ services: + command: --interval 300 ``` +## IP Whitelisting + +If you want to use the IP Whitelisting functionality (see related [variable](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md)) and you are using the nginx reverse proxy set as before, please add the following: + +``` +sudo nano /etc/nginx/sites-enabled/{your-domain-name.com} +``` + +``` diff +server { + listen 80; + server_name {your-domain-name.com}; + location / { + proxy_pass http://localhost:5000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection keep-alive; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; ++ proxy_set_header X-Real-IP $remote_addr; + } +} +``` + +And edit the docker-compose file as follow: + +```diff +version: "3" + +networks: + birdsitelivenetwork: + external: false + +services: + server: + image: nicolasconstant/birdsitelive:latest + restart: always + container_name: birdsitelive + environment: + - Instance:Domain=domain.name + - Instance:AdminEmail=name@domain.ext ++ - Instance:IpWhiteListing=127.0.0.1;127.0.0.2 ++ - Instance:EnableXRealIpHeader=true + - Db:Type=postgres + - Db:Host=db + - Db:Name=birdsitelive + - Db:User=birdsitelive + - Db:Password=birdsitelive + - Twitter:ConsumerKey=twitter.api.key + - Twitter:ConsumerSecret=twitter.api.key + networks: + - birdsitelivenetwork + ports: + - "5000:80" + depends_on: + - db + + db: + image: postgres:9.6 + [...] +``` + ## More options You can find more options available [here](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md) diff --git a/VARIABLES.md b/VARIABLES.md index 2ef0cdf..bb8bf61 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -51,6 +51,9 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act * `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins) * `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc) * `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance. +* `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided). +* `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy). +* `Instance:MaxTweetRetention` (default: 20, min: 1, max: 90) Number of days before synchronized tweets get deleted # Docker Compose full example diff --git a/src/BSLManager/BSLManager.csproj b/src/BSLManager/BSLManager.csproj index 52e5cde..1a725e9 100644 --- a/src/BSLManager/BSLManager.csproj +++ b/src/BSLManager/BSLManager.csproj @@ -2,15 +2,15 @@ Exe - netcoreapp3.1 + net6.0 - - - - - + + + + + diff --git a/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj b/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj index a690b63..f914c67 100644 --- a/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj +++ b/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj @@ -6,8 +6,8 @@ - - + + diff --git a/src/BirdsiteLive.ActivityPub/Models/Tombstone.cs b/src/BirdsiteLive.ActivityPub/Models/Tombstone.cs new file mode 100644 index 0000000..49dcabe --- /dev/null +++ b/src/BirdsiteLive.ActivityPub/Models/Tombstone.cs @@ -0,0 +1,9 @@ +namespace BirdsiteLive.ActivityPub.Models +{ + public class Tombstone + { + public string id { get; set; } + public readonly string type = "Tombstone"; + public string atomUrl { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index 0f701ff..0ece251 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -16,5 +16,9 @@ public class InstanceSettings public int FailingFollowerCleanUpThreshold { get; set; } = -1; public int UserCacheCapacity { get; set; } + public string IpWhiteListing { get; set; } + public bool EnableXRealIpHeader { get; set; } + + public int MaxTweetRetention { get; set; } } } diff --git a/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj b/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj index f0d9f4f..4fc9f61 100644 --- a/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj +++ b/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj @@ -6,8 +6,8 @@ - - + + diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index cab0c58..9a893c7 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -9,6 +9,7 @@ using BirdsiteLive.ActivityPub.Converters; using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Models; using Microsoft.Extensions.Logging; using Newtonsoft.Json; @@ -22,6 +23,7 @@ public interface IActivityPubService Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox); Task DeleteUserAsync(string username, string targetHost, string targetInbox); + Task DeleteNoteAsync(SyncTweet tweet); } public class WebFinger @@ -108,6 +110,30 @@ public async Task DeleteUserAsync(string username, string targetHost, string tar } } + public async Task DeleteNoteAsync(SyncTweet tweet) + { + var acct = tweet.Acct.ToLowerInvariant().Trim(); + + var actor = $"https://{_instanceSettings.Domain}/users/{acct}"; + var noteId = $"https://{_instanceSettings.Domain}/users/{acct}/statuses/{tweet.TweetId}"; + + var delete = new ActivityDelete + { + context = "https://www.w3.org/ns/activitystreams", + id = $"{noteId}#delete", + type = "Delete", + actor = actor, + to = new[] { "https://www.w3.org/ns/activitystreams#Public" }, + apObject = new Tombstone + { + id = noteId, + atomUrl = noteId + } + }; + + await PostDataAsync(delete, tweet.Host, actor, tweet.Inbox); + } + public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox) { try diff --git a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj index 7bc9873..57219d8 100644 --- a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj +++ b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs index a35b6c8..a36c963 100644 --- a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs +++ b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs @@ -9,6 +9,7 @@ public interface IProcessDeleteUser { Task ExecuteAsync(Follower follower); Task ExecuteAsync(string followerUsername, string followerDomain); + Task ExecuteAsync(string actorId); } public class ProcessDeleteUser : IProcessDeleteUser @@ -33,6 +34,15 @@ public async Task ExecuteAsync(string followerUsername, string followerDomain) await ExecuteAsync(follower); } + public async Task ExecuteAsync(string actorId) + { + // Get Follower and Twitter Users + var follower = await _followersDal.GetFollowerAsync(actorId); + if (follower == null) return; + + await ExecuteAsync(follower); + } + public async Task ExecuteAsync(Follower follower) { // Remove twitter users if no more followers diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 6f88543..ed31c62 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -280,15 +280,21 @@ public async Task UndoFollowRequestedAsync(string signature, string method public async Task DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityDelete activity, string body) { - // Validate - var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body); - if (!sigValidation.SignatureIsValidated) return false; + if (activity.apObject is string apObject) + { + if (!string.Equals(activity.actor.Trim(), apObject.Trim(), StringComparison.InvariantCultureIgnoreCase)) return true; - // Remove user and followings - var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation); - var followerHost = SigValidationResultExtractor.GetHost(sigValidation); + try + { + // Validate + var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body); + if (!sigValidation.SignatureIsValidated) return false; + } + catch (FollowerIsGoneException){} - await _processDeleteUser.ExecuteAsync(followerUserName, followerHost); + // Remove user and followings + await _processDeleteUser.ExecuteAsync(activity.actor.Trim()); + } return true; } diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index 8601b19..bf20e96 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -1,14 +1,14 @@  - netstandard2.0 + net6.0 latest - - - + + + @@ -19,6 +19,7 @@ + diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRefreshTwitterUserStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRefreshTwitterUserStatusProcessor.cs similarity index 85% rename from src/BirdsiteLive.Pipeline/Contracts/IRefreshTwitterUserStatusProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/Federation/IRefreshTwitterUserStatusProcessor.cs index 9f20e59..bf74efd 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/IRefreshTwitterUserStatusProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRefreshTwitterUserStatusProcessor.cs @@ -3,7 +3,7 @@ using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Models; -namespace BirdsiteLive.Pipeline.Contracts +namespace BirdsiteLive.Pipeline.Contracts.Federation { public interface IRefreshTwitterUserStatusProcessor { diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveFollowersProcessor.cs similarity index 89% rename from src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveFollowersProcessor.cs index a9ef35c..567d41b 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveFollowersProcessor.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using BirdsiteLive.Pipeline.Models; -namespace BirdsiteLive.Pipeline.Contracts +namespace BirdsiteLive.Pipeline.Contracts.Federation { public interface IRetrieveFollowersProcessor { diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTweetsProcessor.cs similarity index 84% rename from src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTweetsProcessor.cs index 49712c2..0cbf2cb 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTweetsProcessor.cs @@ -3,7 +3,7 @@ using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Models; -namespace BirdsiteLive.Pipeline.Contracts +namespace BirdsiteLive.Pipeline.Contracts.Federation { public interface IRetrieveTweetsProcessor { diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTwitterUsersProcessor.cs similarity index 85% rename from src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTwitterUsersProcessor.cs index b71ae93..c6d08d0 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/Federation/IRetrieveTwitterUsersProcessor.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks.Dataflow; using BirdsiteLive.DAL.Models; -namespace BirdsiteLive.Pipeline.Contracts +namespace BirdsiteLive.Pipeline.Contracts.Federation { public interface IRetrieveTwitterUsersProcessor { diff --git a/src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/Federation/ISaveProgressionProcessor.cs similarity index 82% rename from src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/Federation/ISaveProgressionProcessor.cs index 6b1c9ba..f0320cb 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/Federation/ISaveProgressionProcessor.cs @@ -2,7 +2,7 @@ using System.Threading.Tasks; using BirdsiteLive.Pipeline.Models; -namespace BirdsiteLive.Pipeline.Contracts +namespace BirdsiteLive.Pipeline.Contracts.Federation { public interface ISaveProgressionProcessor { diff --git a/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/Federation/ISendTweetsToFollowersProcessor.cs similarity index 83% rename from src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/Federation/ISendTweetsToFollowersProcessor.cs index 33db423..49d47f0 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/Federation/ISendTweetsToFollowersProcessor.cs @@ -2,7 +2,7 @@ using System.Threading.Tasks; using BirdsiteLive.Pipeline.Models; -namespace BirdsiteLive.Pipeline.Contracts +namespace BirdsiteLive.Pipeline.Contracts.Federation { public interface ISendTweetsToFollowersProcessor { diff --git a/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IDeleteTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IDeleteTweetsProcessor.cs new file mode 100644 index 0000000..c74659d --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IDeleteTweetsProcessor.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp +{ + public interface IDeleteTweetsProcessor + { + Task ProcessAsync(TweetToDelete tweet, CancellationToken ct); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IRetrieveTweetsToDeleteProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IRetrieveTweetsToDeleteProcessor.cs new file mode 100644 index 0000000..45f9442 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IRetrieveTweetsToDeleteProcessor.cs @@ -0,0 +1,12 @@ +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp +{ + public interface IRetrieveTweetsToDeleteProcessor + { + Task GetTweetsAsync(BufferBlock tweetsBufferBlock, CancellationToken ct); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/ISaveDeletedTweetStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/ISaveDeletedTweetStatusProcessor.cs new file mode 100644 index 0000000..0486fb9 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/ISaveDeletedTweetStatusProcessor.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp +{ + public interface ISaveDeletedTweetStatusProcessor + { + Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Models/TweetToDelete.cs b/src/BirdsiteLive.Pipeline/Models/TweetToDelete.cs new file mode 100644 index 0000000..dd411f9 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Models/TweetToDelete.cs @@ -0,0 +1,10 @@ +using BirdsiteLive.DAL.Models; + +namespace BirdsiteLive.Pipeline.Models +{ + public class TweetToDelete + { + public SyncTweet Tweet { get; set; } + public bool DeleteSuccessful { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/RefreshTwitterUserStatusProcessor.cs similarity index 97% rename from src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/RefreshTwitterUserStatusProcessor.cs index 739d50b..a3e7ab2 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/RefreshTwitterUserStatusProcessor.cs @@ -6,12 +6,12 @@ using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Moderation.Actions; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; -namespace BirdsiteLive.Pipeline.Processors +namespace BirdsiteLive.Pipeline.Processors.Federation { public class RefreshTwitterUserStatusProcessor : IRefreshTwitterUserStatusProcessor { diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveFollowersProcessor.cs similarity index 89% rename from src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveFollowersProcessor.cs index 57e3e49..7f696f7 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveFollowersProcessor.cs @@ -2,10 +2,10 @@ using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Models; -namespace BirdsiteLive.Pipeline.Processors +namespace BirdsiteLive.Pipeline.Processors.Federation { public class RetrieveFollowersProcessor : IRetrieveFollowersProcessor { diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTweetsProcessor.cs similarity index 96% rename from src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTweetsProcessor.cs index 321fbf0..d38feb2 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTweetsProcessor.cs @@ -5,14 +5,14 @@ using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; using Microsoft.Extensions.Logging; using Tweetinvi.Models; -namespace BirdsiteLive.Pipeline.Processors +namespace BirdsiteLive.Pipeline.Processors.Federation { public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor { @@ -64,7 +64,7 @@ public async Task ProcessAsync(UserWithDataToSync[] syncTw private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user) { var tweets = new ExtractedTweet[0]; - + try { if (user.LastTweetPostedId == -1) diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTwitterUsersProcessor.cs similarity index 93% rename from src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTwitterUsersProcessor.cs index d9d0ffb..7d16ac5 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/RetrieveTwitterUsersProcessor.cs @@ -7,18 +7,18 @@ using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Tools; using Microsoft.Extensions.Logging; -namespace BirdsiteLive.Pipeline.Processors +namespace BirdsiteLive.Pipeline.Processors.Federation { public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor { private readonly ITwitterUserDal _twitterUserDal; private readonly IMaxUsersNumberProvider _maxUsersNumberProvider; private readonly ILogger _logger; - + public int WaitFactor = 1000 * 60; //1 min #region Ctor @@ -42,7 +42,7 @@ public async Task GetTwitterUsersAsync(BufferBlock twitterUse var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false); var userCount = users.Any() ? users.Length : 1; - var splitNumber = (int) Math.Ceiling(userCount / 15d); + var splitNumber = (int)Math.Ceiling(userCount / 15d); var splitUsers = users.Split(splitNumber).ToList(); foreach (var u in splitUsers) diff --git a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/SaveProgressionProcessor.cs similarity index 94% rename from src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/SaveProgressionProcessor.cs index 1f94871..0b29769 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/SaveProgressionProcessor.cs @@ -5,11 +5,11 @@ using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Moderation.Actions; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Models; using Microsoft.Extensions.Logging; -namespace BirdsiteLive.Pipeline.Processors +namespace BirdsiteLive.Pipeline.Processors.Federation { public class SaveProgressionProcessor : ISaveProgressionProcessor { @@ -36,13 +36,13 @@ public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, Cancella await UpdateUserSyncDateAsync(userWithTweetsToSync.User); return; } - if(userWithTweetsToSync.Followers.Length == 0) + if (userWithTweetsToSync.Followers.Length == 0) { _logger.LogInformation("No Followers found for {User}", userWithTweetsToSync.User.Acct); await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User); return; } - + var userId = userWithTweetsToSync.User.Id; var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList(); var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/SendTweetsToFollowersProcessor.cs similarity index 97% rename from src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/SendTweetsToFollowersProcessor.cs index e210f39..3a52544 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/SendTweetsToFollowersProcessor.cs @@ -10,7 +10,7 @@ using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; using BirdsiteLive.Moderation.Actions; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors.SubTasks; using BirdsiteLive.Twitter; @@ -18,7 +18,7 @@ using Microsoft.Extensions.Logging; using Tweetinvi.Models; -namespace BirdsiteLive.Pipeline.Processors +namespace BirdsiteLive.Pipeline.Processors.Federation { public class SendTweetsToFollowersProcessor : ISendTweetsToFollowersProcessor { @@ -83,7 +83,7 @@ private async Task ProcessFollowersWithSharedInboxAsync(ExtractedTweet[] tweets, } } } - + private async Task ProcessFollowersWithInboxAsync(ExtractedTweet[] tweets, List followerWtInbox, SyncTwitterUser user) { foreach (var follower in followerWtInbox) @@ -114,7 +114,7 @@ private async Task ProcessFailingUserAsync(Follower follower) { follower.PostingErrorCount++; - if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold + if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold && _instanceSettings.FailingFollowerCleanUpThreshold > 0 || follower.PostingErrorCount > 2147483600) { diff --git a/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsTaskBase.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsTaskBase.cs new file mode 100644 index 0000000..1cb9be9 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsTaskBase.cs @@ -0,0 +1,32 @@ +using System; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; + +namespace BirdsiteLive.Pipeline.Processors.SubTasks +{ + public class SendTweetsTaskBase + { + private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; + + #region Ctor + protected SendTweetsTaskBase(ISyncTweetsPostgresDal syncTweetsPostgresDal) + { + _syncTweetsPostgresDal = syncTweetsPostgresDal; + } + #endregion + + protected async Task SaveSyncTweetAsync(string acct, long tweetId, string host, string inbox) + { + var tweet = new SyncTweet + { + Acct = acct, + TweetId = tweetId, + PublishedAt = DateTime.UtcNow, + Inbox = inbox, + Host = host + }; + await _syncTweetsPostgresDal.SaveTweetAsync(tweet); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsToInboxTask.cs similarity index 92% rename from src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsToInboxTask.cs index a6f6982..0c91750 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsToInboxTask.cs @@ -17,17 +17,16 @@ public interface ISendTweetsToInboxTask Task ExecuteAsync(IEnumerable tweets, Follower follower, SyncTwitterUser user); } - public class SendTweetsToInboxTask : ISendTweetsToInboxTask + public class SendTweetsToInboxTask : SendTweetsTaskBase, ISendTweetsToInboxTask { private readonly IActivityPubService _activityPubService; private readonly IStatusService _statusService; private readonly IFollowersDal _followersDal; private readonly InstanceSettings _settings; private readonly ILogger _logger; - - + #region Ctor - public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger) + public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger, ISyncTweetsPostgresDal syncTweetsPostgresDal): base(syncTweetsPostgresDal) { _activityPubService = activityPubService; _statusService = statusService; @@ -61,6 +60,7 @@ public async Task ExecuteAsync(IEnumerable tweets, Follower foll { var note = _statusService.GetStatus(user.Acct, tweet); await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); + await SaveSyncTweetAsync(user.Acct, tweet.Id, follower.Host, inbox); } } catch (ArgumentException e) diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsToSharedInboxTask.cs similarity index 91% rename from src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs rename to src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsToSharedInboxTask.cs index 1abe183..95f32fd 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/Federation/SubTasks/SendTweetsToSharedInboxTask.cs @@ -2,6 +2,7 @@ using System.Linq; using System.Net; using System.Threading.Tasks; +using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; @@ -16,7 +17,7 @@ public interface ISendTweetsToSharedInboxTask Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance); } - public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask + public class SendTweetsToSharedInboxTask : SendTweetsTaskBase, ISendTweetsToSharedInboxTask { private readonly IStatusService _statusService; private readonly IActivityPubService _activityPubService; @@ -25,7 +26,7 @@ public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask private readonly ILogger _logger; #region Ctor - public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger) + public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger, ISyncTweetsPostgresDal syncTweetsPostgresDal): base(syncTweetsPostgresDal) { _activityPubService = activityPubService; _statusService = statusService; @@ -61,6 +62,7 @@ public async Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, st { var note = _statusService.GetStatus(user.Acct, tweet); await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); + await SaveSyncTweetAsync(user.Acct, tweet.Id, host, inbox); } } catch (ArgumentException e) diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/Base/RetentionBase.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/Base/RetentionBase.cs new file mode 100644 index 0000000..58942ac --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/Base/RetentionBase.cs @@ -0,0 +1,16 @@ +using BirdsiteLive.Common.Settings; +using System; + +namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base +{ + public class RetentionBase + { + protected int GetRetentionTime(InstanceSettings settings) + { + var retentionTime = Math.Abs(settings.MaxTweetRetention); + if (retentionTime < 1) retentionTime = 1; + if (retentionTime > 90) retentionTime = 90; + return retentionTime; + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs new file mode 100644 index 0000000..a9841a9 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/DeleteTweetsProcessor.cs @@ -0,0 +1,54 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Domain; +using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; +using BirdsiteLive.Pipeline.Models; +using Microsoft.Extensions.Logging; + +namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp +{ + public class DeleteTweetsProcessor : IDeleteTweetsProcessor + { + private readonly IActivityPubService _activityPubService; + private readonly ILogger _logger; + + #region Ctor + public DeleteTweetsProcessor(IActivityPubService activityPubService, ILogger logger) + { + _activityPubService = activityPubService; + _logger = logger; + } + #endregion + + public async Task ProcessAsync(TweetToDelete tweet, CancellationToken ct) + { + try + { + await _activityPubService.DeleteNoteAsync(tweet.Tweet); + tweet.DeleteSuccessful = true; + } + catch (HttpRequestException e) + { + var code = e.StatusCode; + if (code is HttpStatusCode.Gone or HttpStatusCode.NotFound) + { + _logger.LogInformation("Tweet already deleted"); + tweet.DeleteSuccessful = true; + } + else + { + _logger.LogError(e.Message, e); + } + } + catch (Exception e) + { + _logger.LogError(e.Message, e); + } + + return tweet; + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs new file mode 100644 index 0000000..daf4729 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs @@ -0,0 +1,64 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; +using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base; + +namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp +{ + public class RetrieveTweetsToDeleteProcessor : RetentionBase, IRetrieveTweetsToDeleteProcessor + { + private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; + private readonly InstanceSettings _instanceSettings; + + #region Ctor + public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings) + { + _syncTweetsPostgresDal = syncTweetsPostgresDal; + _instanceSettings = instanceSettings; + } + #endregion + + public async Task GetTweetsAsync(BufferBlock tweetsBufferBlock, CancellationToken ct) + { + var batchSize = 100; + + for (;;) + { + ct.ThrowIfCancellationRequested(); + + var now = DateTime.UtcNow; + + + var from = now.AddDays(-GetRetentionTime(_instanceSettings)); + var dbBrowsingEnded = false; + var lastId = -1L; + + do + { + var tweets = await _syncTweetsPostgresDal.GetTweetsOlderThanAsync(from, lastId, batchSize); + + foreach (var syncTweet in tweets) + { + var tweet = new TweetToDelete + { + Tweet = syncTweet + }; + await tweetsBufferBlock.SendAsync(tweet, ct); + } + + if (tweets.Any()) lastId = tweets.Last().Id; + if (tweets.Count < batchSize) dbBrowsingEnded = true; + + } while (!dbBrowsingEnded); + + await Task.Delay(TimeSpan.FromHours(3), ct); + } + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs new file mode 100644 index 0000000..817d0be --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs @@ -0,0 +1,36 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; +using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base; + +namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp +{ + public class SaveDeletedTweetStatusProcessor : RetentionBase, ISaveDeletedTweetStatusProcessor + { + private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal; + private readonly InstanceSettings _instanceSettings; + + #region Ctor + public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings) + { + _syncTweetsPostgresDal = syncTweetsPostgresDal; + _instanceSettings = instanceSettings; + } + #endregion + + public async Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct) + { + var retentionTime = GetRetentionTime(_instanceSettings); + retentionTime += 20; // Delay until last retry + var highLimitDate = DateTime.UtcNow.AddDays(-retentionTime); + if (tweetToDelete.DeleteSuccessful || tweetToDelete.Tweet.PublishedAt < highLimitDate) + { + await _syncTweetsPostgresDal.DeleteTweetAsync(tweetToDelete.Tweet.Id); + } + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs index c6917e7..f0a81e6 100644 --- a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs +++ b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs @@ -4,7 +4,7 @@ using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using BirdsiteLive.DAL.Models; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using BirdsiteLive.Pipeline.Models; using Microsoft.Extensions.Logging; diff --git a/src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs b/src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs new file mode 100644 index 0000000..10f69c0 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/TweetCleanUpPipeline.cs @@ -0,0 +1,76 @@ +using System.Threading.Tasks; +using System.Threading; +using System.Threading.Tasks.Dataflow; +using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; +using BirdsiteLive.Pipeline.Models; +using Microsoft.Extensions.Logging; + +namespace BirdsiteLive.Pipeline +{ + public interface ITweetCleanUpPipeline + { + Task ExecuteAsync(CancellationToken ct); + } + + public class TweetCleanUpPipeline : ITweetCleanUpPipeline + { + private readonly IRetrieveTweetsToDeleteProcessor _retrieveTweetsToDeleteProcessor; + private readonly IDeleteTweetsProcessor _deleteTweetsProcessor; + private readonly ISaveDeletedTweetStatusProcessor _saveDeletedTweetStatusProcessor; + private readonly ILogger _logger; + + #region Ctor + public TweetCleanUpPipeline(IRetrieveTweetsToDeleteProcessor retrieveTweetsToDeleteProcessor, IDeleteTweetsProcessor deleteTweetsProcessor, ISaveDeletedTweetStatusProcessor saveDeletedTweetStatusProcessor, ILogger logger) + { + _retrieveTweetsToDeleteProcessor = retrieveTweetsToDeleteProcessor; + _deleteTweetsProcessor = deleteTweetsProcessor; + _saveDeletedTweetStatusProcessor = saveDeletedTweetStatusProcessor; + _logger = logger; + } + #endregion + + public async Task ExecuteAsync(CancellationToken ct) + { + // Create blocks + var tweetsToDeleteBufferBlock = new BufferBlock(new DataflowBlockOptions + { + BoundedCapacity = 200, + CancellationToken = ct + }); + var deleteTweetsBlock = new TransformBlock( + async x => await _deleteTweetsProcessor.ProcessAsync(x, ct), + new ExecutionDataflowBlockOptions() + { + MaxDegreeOfParallelism = 5, + CancellationToken = ct + }); + var deletedTweetsBufferBlock = new BufferBlock(new DataflowBlockOptions + { + BoundedCapacity = 200, + CancellationToken = ct + }); + var saveProgressionBlock = new ActionBlock( + async x => await _saveDeletedTweetStatusProcessor.ProcessAsync(x, ct), + new ExecutionDataflowBlockOptions + { + MaxDegreeOfParallelism = 5, + CancellationToken = ct + }); + + // Link pipeline + var dataflowLinkOptions = new DataflowLinkOptions { PropagateCompletion = true }; + tweetsToDeleteBufferBlock.LinkTo(deleteTweetsBlock, dataflowLinkOptions); + deleteTweetsBlock.LinkTo(deletedTweetsBufferBlock, dataflowLinkOptions); + deletedTweetsBufferBlock.LinkTo(saveProgressionBlock, dataflowLinkOptions); + + // Launch tweet retriever + var retrieveTweetsToDeleteTask = _retrieveTweetsToDeleteProcessor.GetTweetsAsync(tweetsToDeleteBufferBlock, ct); + + // Wait + await Task.WhenAny(new[] { retrieveTweetsToDeleteTask, saveProgressionBlock.Completion }); + + var ex = retrieveTweetsToDeleteTask.IsFaulted ? retrieveTweetsToDeleteTask.Exception : saveProgressionBlock.Completion.Exception; + _logger.LogCritical(ex, "An error occurred, pipeline stopped"); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj index 438b4e1..49ddda9 100644 --- a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj +++ b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 122ff88..acdd0be 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -1,17 +1,17 @@  - netcoreapp3.1 + net6.0 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.22.0 + 0.23.0 - - - - + + + + diff --git a/src/BirdsiteLive/Controllers/DebugingController.cs b/src/BirdsiteLive/Controllers/DebugingController.cs index 8f37f22..f3e7d4e 100644 --- a/src/BirdsiteLive/Controllers/DebugingController.cs +++ b/src/BirdsiteLive/Controllers/DebugingController.cs @@ -56,6 +56,35 @@ public async Task Follow() return View("Index"); } + private static string _noteId; + + [HttpPost] + public async Task DeleteNote() + { + var username = "twitter"; + var actor = $"https://{_instanceSettings.Domain}/users/{username}"; + var targetHost = "ioc.exchange"; + var target = $"https://{targetHost}/users/test"; + var inbox = $"/inbox"; + + var delete = new ActivityDelete + { + context = "https://www.w3.org/ns/activitystreams", + id = $"{_noteId}#delete", + type = "Delete", + actor = actor, + to = new[] { "https://www.w3.org/ns/activitystreams#Public" }, + apObject = new Tombstone + { + id = _noteId, + atomUrl = _noteId + } + }; + + await _activityPubService.PostDataAsync(delete, targetHost, actor, inbox); + return View("Index"); + } + [HttpPost] public async Task PostNote() { @@ -70,6 +99,8 @@ public async Task PostNote() var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteGuid}"; var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteGuid}"; + _noteId = noteId; + var to = $"{actor}/followers"; to = target; diff --git a/src/BirdsiteLive/Controllers/InboxController.cs b/src/BirdsiteLive/Controllers/InboxController.cs index f92a0a6..57825af 100644 --- a/src/BirdsiteLive/Controllers/InboxController.cs +++ b/src/BirdsiteLive/Controllers/InboxController.cs @@ -49,14 +49,15 @@ public async Task Inbox() case "Delete": { var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityDelete, body); + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), + activity as ActivityDelete, body); if (succeeded) return Accepted(); else return Unauthorized(); } } } } - catch (FollowerIsGoneException) { } //TODO: check if user in DB + catch (FollowerIsGoneException) { } return Accepted(); } diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs index f2cde09..8d4b816 100644 --- a/src/BirdsiteLive/Controllers/MigrationController.cs +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -3,7 +3,6 @@ using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; -using Npgsql.TypeHandlers; using BirdsiteLive.Domain; using BirdsiteLive.Domain.Enum; using BirdsiteLive.DAL.Contracts; diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index e66dac5..c636c0e 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -3,7 +3,6 @@ using System.IO; using System.Linq; using System.Net.Mime; -using System.Runtime.InteropServices.WindowsRuntime; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; diff --git a/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs b/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs new file mode 100644 index 0000000..da4c275 --- /dev/null +++ b/src/BirdsiteLive/Middlewares/IpWhitelistingMiddleware.cs @@ -0,0 +1,82 @@ +using BirdsiteLive.Common.Settings; +using BirdsiteLive.Domain.Tools; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; + +namespace BirdsiteLive.Middlewares +{ + public class IpWhitelistingMiddleware + { + private readonly RequestDelegate _next; + private readonly ILogger _logger; + private readonly InstanceSettings _instanceSettings; + private readonly byte[][] _safelist; + private readonly bool _ipWhitelistingSet; + + public IpWhitelistingMiddleware( + RequestDelegate next, + ILogger logger, + InstanceSettings instanceSettings) + { + if (!string.IsNullOrWhiteSpace(instanceSettings.IpWhiteListing)) + { + var ips = PatternsParser.Parse(instanceSettings.IpWhiteListing); + _safelist = new byte[ips.Length][]; + for (var i = 0; i < ips.Length; i++) + { + _safelist[i] = IPAddress.Parse(ips[i]).GetAddressBytes(); + } + _ipWhitelistingSet = true; + } + + _next = next; + _logger = logger; + _instanceSettings = instanceSettings; + } + + public async Task Invoke(HttpContext context) + { + if (_ipWhitelistingSet) + { + var remoteIp = context.Connection.RemoteIpAddress; + + if (_instanceSettings.EnableXRealIpHeader) + { + var forwardedIp = context.Request.Headers.FirstOrDefault(x => x.Key == "X-Real-IP").Value + .ToString(); + if (!string.IsNullOrWhiteSpace(forwardedIp)) + { + _logger.LogDebug("Redirected IP address detected"); + remoteIp = IPAddress.Parse(forwardedIp); + } + } + + _logger.LogDebug("Request from Remote IP address: {RemoteIp}", remoteIp); + + var bytes = remoteIp.GetAddressBytes(); + var badIp = true; + foreach (var address in _safelist) + { + if (address.SequenceEqual(bytes)) + { + badIp = false; + break; + } + } + + if (badIp) + { + _logger.LogWarning("Forbidden Request from Remote IP address: {RemoteIp}", remoteIp); + context.Response.StatusCode = (int)HttpStatusCode.NotFound; + return; + } + } + + await _next.Invoke(context); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Program.cs b/src/BirdsiteLive/Program.cs index d238b02..7f29e76 100644 --- a/src/BirdsiteLive/Program.cs +++ b/src/BirdsiteLive/Program.cs @@ -29,6 +29,7 @@ public static IHostBuilder CreateHostBuilder(string[] args) => .ConfigureServices(services => { services.AddHostedService(); + services.AddHostedService(); }); } } diff --git a/src/BirdsiteLive/Services/FederationService.cs b/src/BirdsiteLive/Services/FederationService.cs index 0b0faed..32e8b5e 100644 --- a/src/BirdsiteLive/Services/FederationService.cs +++ b/src/BirdsiteLive/Services/FederationService.cs @@ -6,6 +6,7 @@ using BirdsiteLive.DAL.Contracts; using BirdsiteLive.Moderation; using BirdsiteLive.Pipeline; +using BirdsiteLive.Tools; using Microsoft.Extensions.Hosting; namespace BirdsiteLive.Services @@ -32,6 +33,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) try { await _databaseInitializer.InitAndMigrateDbAsync(); + InitStateSynchronization.IsDbInitialized = true; await _moderationPipeline.ApplyModerationSettingsAsync(); await _statusPublicationPipeline.ExecuteAsync(stoppingToken); } diff --git a/src/BirdsiteLive/Services/TweetCleanUpService.cs b/src/BirdsiteLive/Services/TweetCleanUpService.cs new file mode 100644 index 0000000..c2d81bd --- /dev/null +++ b/src/BirdsiteLive/Services/TweetCleanUpService.cs @@ -0,0 +1,43 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Pipeline; +using BirdsiteLive.Tools; +using Microsoft.Extensions.Hosting; + +namespace BirdsiteLive.Services +{ + public class TweetCleanUpService : BackgroundService + { + private readonly ITweetCleanUpPipeline _cleanUpPipeline; + private readonly IHostApplicationLifetime _applicationLifetime; + + #region Ctor + public TweetCleanUpService(IHostApplicationLifetime applicationLifetime, ITweetCleanUpPipeline cleanUpPipeline) + { + _applicationLifetime = applicationLifetime; + _cleanUpPipeline = cleanUpPipeline; + } + #endregion + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + // Wait for initialization + while (!InitStateSynchronization.IsDbInitialized) + { + if (stoppingToken.IsCancellationRequested) return; + await Task.Delay(250, stoppingToken); + } + + await _cleanUpPipeline.ExecuteAsync(stoppingToken); + } + catch (Exception) + { + await Task.Delay(1000 * 30); + _applicationLifetime.StopApplication(); + } + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Startup.cs b/src/BirdsiteLive/Startup.cs index 16c7ee4..e088a56 100644 --- a/src/BirdsiteLive/Startup.cs +++ b/src/BirdsiteLive/Startup.cs @@ -8,6 +8,7 @@ using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Postgres.DataAccessLayers; using BirdsiteLive.DAL.Postgres.Settings; +using BirdsiteLive.Middlewares; using BirdsiteLive.Models; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Tools; @@ -18,6 +19,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; namespace BirdsiteLive { @@ -132,6 +134,9 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) app.UseAuthorization(); + var instanceSettings = Configuration.GetSection("Instance").Get(); + app.UseMiddleware(instanceSettings); + app.UseEndpoints(endpoints => { endpoints.MapControllerRoute( diff --git a/src/BirdsiteLive/Tools/InitStateSynchronization.cs b/src/BirdsiteLive/Tools/InitStateSynchronization.cs new file mode 100644 index 0000000..c2a2e42 --- /dev/null +++ b/src/BirdsiteLive/Tools/InitStateSynchronization.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Tools +{ + public static class InitStateSynchronization + { + public static bool IsDbInitialized { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Views/Debuging/Index.cshtml b/src/BirdsiteLive/Views/Debuging/Index.cshtml index e343cf2..886b88d 100644 --- a/src/BirdsiteLive/Views/Debuging/Index.cshtml +++ b/src/BirdsiteLive/Views/Debuging/Index.cshtml @@ -18,6 +18,11 @@ +
+ + + +
diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 4cee8ba..9628532 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -25,7 +25,8 @@ "SensitiveTwitterAccounts": null, "FailingTwitterUserCleanUpThreshold": 700, "FailingFollowerCleanUpThreshold": 30000, - "UserCacheCapacity": 10000 + "UserCacheCapacity": 10000, + "MaxTweetRetention": 20 }, "Db": { "Type": "postgres", diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj index 690c878..b72a179 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj @@ -5,8 +5,8 @@ - - + + diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index 55b38f6..24ccf6e 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -23,7 +23,7 @@ internal class DbVersion public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal { private readonly PostgresTools _tools; - private readonly Version _currentVersion = new Version(2, 5); + private readonly Version _currentVersion = new Version(2, 6); private const string DbVersionType = "db-version"; #region Ctor @@ -136,7 +136,8 @@ public Tuple[] GetMigrationPatterns() new Tuple(new Version(2,1), new Version(2,2)), new Tuple(new Version(2,2), new Version(2,3)), new Tuple(new Version(2,3), new Version(2,4)), - new Tuple(new Version(2,4), new Version(2,5)) + new Tuple(new Version(2,4), new Version(2,5)), + new Tuple(new Version(2,5), new Version(2,6)) }; } @@ -184,6 +185,23 @@ public async Task MigrateDbAsync(Version from, Version to) var addDeletedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD deleted BOOLEAN"; await _tools.ExecuteRequestAsync(addDeletedToAcct); } + else if (from == new Version(2, 5) && to == new Version(2, 6)) + { + // Create Synchronized Tweets table + var createFollowers = $@"CREATE TABLE {_settings.SynchronizedTweetsTableName} + ( + id SERIAL PRIMARY KEY, + + acct VARCHAR(50) NOT NULL, + tweetId BIGINT NOT NULL, + inbox VARCHAR(2048) NOT NULL, + host VARCHAR(253) NOT NULL, + publishedAt TIMESTAMP (2) WITHOUT TIME ZONE NOT NULL, + + UNIQUE (acct, tweetId, inbox) + );"; + await _tools.ExecuteRequestAsync(createFollowers); + } else { throw new NotImplementedException(); @@ -212,7 +230,8 @@ public async Task DeleteAllAsync() $@"DROP TABLE {_settings.DbVersionTableName};", $@"DROP TABLE {_settings.TwitterUserTableName};", $@"DROP TABLE {_settings.FollowersTableName};", - $@"DROP TABLE {_settings.CachedTweetsTableName};" + $@"DROP TABLE {_settings.CachedTweetsTableName};", + $@"DROP TABLE {_settings.SynchronizedTweetsTableName};" }; foreach (var r in dropsRequests) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs index db2f9f7..c93ad5e 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs @@ -82,6 +82,21 @@ public async Task GetFollowerAsync(string acct, string host) } } + public async Task GetFollowerAsync(string actorId) + { + var query = $"SELECT * FROM {_settings.FollowersTableName} WHERE actorid = @actorid"; + + actorId = actorId.ToLowerInvariant().Trim(); + + using (var dbConnection = Connection) + { + dbConnection.Open(); + + var result = (await dbConnection.QueryAsync(query, new { actorId })).FirstOrDefault(); + return Convert(result); + } + } + public async Task GetFollowersAsync(int followedUserId) { if (followedUserId == default) throw new ArgumentException("followedUserId"); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/SyncTweetsPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/SyncTweetsPostgresDal.cs new file mode 100644 index 0000000..ff05db4 --- /dev/null +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/SyncTweetsPostgresDal.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.DAL.Postgres.DataAccessLayers.Base; +using BirdsiteLive.DAL.Postgres.Settings; +using Dapper; + +namespace BirdsiteLive.DAL.Postgres.DataAccessLayers +{ + public class SyncTweetsPostgresDal : PostgresBase, ISyncTweetsPostgresDal + { + #region Ctor + public SyncTweetsPostgresDal(PostgresSettings settings) : base(settings) + { + } + #endregion + + public async Task SaveTweetAsync(SyncTweet tweet) + { + if (tweet.PublishedAt == default) throw new ArgumentException("publishedAt"); + if (tweet.TweetId == default) throw new ArgumentException("tweetId"); + if (string.IsNullOrWhiteSpace(tweet.Acct)) throw new ArgumentException("acct"); + if (string.IsNullOrWhiteSpace(tweet.Inbox)) throw new ArgumentException("inbox"); + if (string.IsNullOrWhiteSpace(tweet.Host)) throw new ArgumentException("host"); + + var acct = tweet.Acct.ToLowerInvariant().Trim(); + var inbox = tweet.Inbox.ToLowerInvariant().Trim(); + var host = tweet.Host.ToLowerInvariant().Trim(); + + using (var dbConnection = Connection) + { + dbConnection.Open(); + + return (await dbConnection.QueryAsync( + $"INSERT INTO {_settings.SynchronizedTweetsTableName} (acct,tweetId,inbox,host,publishedAt) VALUES(@acct,@tweetId,@inbox,@host,@publishedAt) RETURNING id", + new + { + acct, + tweetId = tweet.TweetId, + inbox, + host, + publishedAt = tweet.PublishedAt.ToUniversalTime() + })).First(); + } + } + + public async Task GetTweetAsync(long id) + { + if (id == default) throw new ArgumentException("id"); + + var query = $"SELECT * FROM {_settings.SynchronizedTweetsTableName} WHERE id = @id"; + + using (var dbConnection = Connection) + { + dbConnection.Open(); + + var result = (await dbConnection.QueryAsync(query, new { id })).FirstOrDefault(); + return result; + } + } + + public async Task DeleteTweetAsync(long id) + { + if (id == default) throw new ArgumentException("id"); + + var query = $"DELETE FROM {_settings.SynchronizedTweetsTableName} WHERE id = @id"; + + using (var dbConnection = Connection) + { + dbConnection.Open(); + + await dbConnection.QueryAsync(query, new { id }); + } + } + + public async Task> GetTweetsOlderThanAsync(DateTime date, long from = -1, int size = 100) + { + if (date == default) throw new ArgumentException("date"); + + var query = $"SELECT * FROM {_settings.SynchronizedTweetsTableName} WHERE id > @from AND publishedAt < @date ORDER BY id ASC LIMIT @size"; + + using (var dbConnection = Connection) + { + dbConnection.Open(); + + var result = await dbConnection.QueryAsync(query, new + { + from, + date, + size + }); + return result.ToList(); + } + } + } +} \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs index c7504ef..6be43da 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs @@ -8,5 +8,6 @@ public class PostgresSettings public string TwitterUserTableName { get; set; } = "twitter_users"; public string FollowersTableName { get; set; } = "followers"; public string CachedTweetsTableName { get; set; } = "cached_tweets"; + public string SynchronizedTweetsTableName { get; set; } = "sync_tweets"; } } \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs index fe87b28..cb97b8e 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs @@ -7,6 +7,7 @@ namespace BirdsiteLive.DAL.Contracts public interface IFollowersDal { Task GetFollowerAsync(string acct, string host); + Task GetFollowerAsync(string actorId); Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, string actorId, int[] followings = null, Dictionary followingSyncStatus = null); Task GetFollowersAsync(int followedUserId); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ISyncTweetsPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ISyncTweetsPostgresDal.cs new file mode 100644 index 0000000..42f843f --- /dev/null +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ISyncTweetsPostgresDal.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Models; + +namespace BirdsiteLive.DAL.Contracts +{ + public interface ISyncTweetsPostgresDal + { + Task SaveTweetAsync(SyncTweet tweet); + Task GetTweetAsync(long id); + Task DeleteTweetAsync(long id); + Task> GetTweetsOlderThanAsync(DateTime date, long from = -1, int size = 100); + } +} \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTweet.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTweet.cs new file mode 100644 index 0000000..b727a1c --- /dev/null +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTweet.cs @@ -0,0 +1,15 @@ +using System; + +namespace BirdsiteLive.DAL.Models +{ + public class SyncTweet + { + public long Id { get; set; } + + public string Acct { get; set; } + public string Host { get; set; } //TODO + public string Inbox { get; set; } + public long TweetId { get; set; } + public DateTime PublishedAt { get; set; } + } +} \ No newline at end of file diff --git a/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj b/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj index 033cfe1..7f9a210 100644 --- a/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj +++ b/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj @@ -1,16 +1,19 @@ - + - netcoreapp3.1 + net6.0 false - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj b/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj index 611d29e..05bd857 100644 --- a/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj +++ b/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj @@ -1,16 +1,19 @@ - netcoreapp3.1 + net6.0 false - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj b/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj index 0a52603..59ee199 100644 --- a/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj +++ b/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj @@ -1,16 +1,19 @@ - + - netcoreapp3.1 + net6.0 false - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj b/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj index 0c9c0a6..ed1ed50 100644 --- a/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj +++ b/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj @@ -1,16 +1,19 @@ - + - netcoreapp3.1 + net6.0 false - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj index da05ef2..ae70505 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj @@ -1,16 +1,19 @@ - + - netcoreapp3.1 + net6.0 false - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs index 72bf352..00f0f9c 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs @@ -19,6 +19,7 @@ public PostgresTestingBase() CachedTweetsTableName = "CachedTweetsTableName" + RandomGenerator.GetString(4), FollowersTableName = "FollowersTableName" + RandomGenerator.GetString(4), TwitterUserTableName = "TwitterUserTableName" + RandomGenerator.GetString(4), + SynchronizedTweetsTableName = "SynchronizedTweetsTableName" + RandomGenerator.GetString(4), }; _tools = new PostgresTools(_settings); } diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/SyncTweetsPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/SyncTweetsPostgresDalTests.cs new file mode 100644 index 0000000..c65cf24 --- /dev/null +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/SyncTweetsPostgresDalTests.cs @@ -0,0 +1,183 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using BirdsiteLive.DAL.Postgres.DataAccessLayers; +using BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers.Base; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Models; + +namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers +{ + [TestClass] + public class SyncTweetsPostgresDalTests : PostgresTestingBase + { + [TestInitialize] + public async Task TestInit() + { + var dal = new DbInitializerPostgresDal(_settings, _tools); + var init = new DatabaseInitializer(dal); + await init.InitAndMigrateDbAsync(); + } + + [TestCleanup] + public async Task CleanUp() + { + var dal = new DbInitializerPostgresDal(_settings, _tools); + await dal.DeleteAllAsync(); + } + + [TestMethod] + public async Task CreateAndGetTweets() + { + var tweet = new SyncTweet + { + Acct = "test", + PublishedAt = DateTime.UtcNow, + Inbox = "/inbox", + Host = "instance.ext", + TweetId = 4567889 + }; + + var dal = new SyncTweetsPostgresDal(_settings); + + var id = await dal.SaveTweetAsync(tweet); + var result = await dal.GetTweetAsync(id); + + Assert.IsNotNull(result); + + Assert.IsTrue(result.Id > 0); + Assert.AreEqual(tweet.Acct, result.Acct); + Assert.AreEqual(tweet.Inbox, result.Inbox); + Assert.AreEqual(tweet.Host, result.Host); + Assert.AreEqual(tweet.TweetId, result.TweetId); + Assert.IsTrue(Math.Abs((tweet.PublishedAt - result.PublishedAt).Seconds) < 5); + } + + [TestMethod] + public async Task CreateDeleteAndGetTweets() + { + var tweet = new SyncTweet + { + Acct = "test", + PublishedAt = DateTime.UtcNow, + Inbox = "/inbox", + Host = "instance.ext", + TweetId = 4567889 + }; + + var dal = new SyncTweetsPostgresDal(_settings); + + var id = await dal.SaveTweetAsync(tweet); + await dal.DeleteTweetAsync(id); + var result = await dal.GetTweetAsync(id); + + Assert.IsNull(result); + } + + [TestMethod] + public async Task CreateAndGetTweetsByDate() + { + var now = DateTime.UtcNow; + var dal = new SyncTweetsPostgresDal(_settings); + + var allData = new List(); + + for (var i = 0; i < 100; i++) + { + var tweet = new SyncTweet + { + Acct = "test", + PublishedAt = now.AddDays(-10 - i), + Inbox = "/inbox", + Host = "instance.ext", + TweetId = 4567889 + i + }; + allData.Add(tweet); + await dal.SaveTweetAsync(tweet); + } + + var date = now.AddDays(-20); + var result = await dal.GetTweetsOlderThanAsync(date, -1, 10); + + Assert.IsNotNull(result); + Assert.IsTrue(result.Count == 10); + + foreach (var res in result) + { + Assert.IsTrue(res.PublishedAt < date); + Assert.IsTrue(res.Id > 10); + Assert.IsTrue(res.Id < 25); + + var original = allData.First(x => x.Acct == res.Acct && x.TweetId == res.TweetId); + Assert.AreEqual(original.Acct, res.Acct); + Assert.AreEqual(original.Inbox, res.Inbox); + Assert.AreEqual(original.Host, res.Host); + Assert.AreEqual(original.TweetId, res.TweetId); + Assert.IsTrue(Math.Abs((original.PublishedAt - res.PublishedAt).Seconds) < 5); + } + } + + [TestMethod] + public async Task CreateAndGetTweetsByDate_Offset() + { + var now = DateTime.UtcNow; + var dal = new SyncTweetsPostgresDal(_settings); + + for (var i = 0; i < 100; i++) + { + var tweet = new SyncTweet + { + Acct = "test", + PublishedAt = now.AddDays(-10 - i), + Inbox = "/inbox", + Host = "instance.ext", + TweetId = 4567889 + i + }; + await dal.SaveTweetAsync(tweet); + } + + var date = now.AddDays(-20); + var result = await dal.GetTweetsOlderThanAsync(date, 1000, 10); + + Assert.IsNotNull(result); + Assert.IsTrue(result.Count == 0); + } + + [TestMethod] + public async Task CreateAndGetTweetsByDate_Iteration() + { + var now = DateTime.UtcNow; + var dal = new SyncTweetsPostgresDal(_settings); + + for (var i = 0; i < 100; i++) + { + var tweet = new SyncTweet + { + Acct = "test", + PublishedAt = now.AddDays(-10 - i), + Inbox = "/inbox", + Host = "instance.ext", + TweetId = 4567889 + i + }; + await dal.SaveTweetAsync(tweet); + } + + var date = now.AddDays(-20); + var result = await dal.GetTweetsOlderThanAsync(date, -1, 10); + var result2 = await dal.GetTweetsOlderThanAsync(date, result.Last().Id, 10); + + var global = result.ToList(); + global.AddRange(result2); + var d = global.GroupBy(x => x.Id).Count(); + Assert.AreEqual(20, d); + + foreach (var res in global) + { + Assert.IsTrue(res.PublishedAt < date); + Assert.IsTrue(res.Id > 10); + Assert.IsTrue(res.Id < 35); + } + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj index 0992b02..8bc759d 100644 --- a/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj +++ b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj @@ -1,17 +1,20 @@ - + - netcoreapp3.1 + net6.0 false - - - - - + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj b/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj index 626c941..852e4b5 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj +++ b/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj @@ -1,17 +1,20 @@ - + - netcoreapp3.1 + net6.0 false - - - - - + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj b/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj index e85b592..9b86065 100644 --- a/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj +++ b/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj @@ -1,17 +1,20 @@ - + - netcoreapp3.1 + net6.0 false - - - - - + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj index d1cfd06..7d8e345 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj @@ -1,20 +1,23 @@ - + - netcoreapp3.1 - + net6.0 false - - - - - + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RefreshTwitterUserStatusProcessorTests.cs similarity index 98% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RefreshTwitterUserStatusProcessorTests.cs index cd2d116..a1c9757 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RefreshTwitterUserStatusProcessorTests.cs @@ -7,14 +7,13 @@ using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Moderation.Actions; -using BirdsiteLive.Pipeline.Models; -using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.Federation; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; -namespace BirdsiteLive.Pipeline.Tests.Processors +namespace BirdsiteLive.Pipeline.Tests.Processors.Federation { [TestClass] public class RefreshTwitterUserStatusProcessorTests @@ -61,7 +60,7 @@ public async Task ProcessAsync_Test() var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); #region Validations - Assert.AreEqual(2 , result.Length); + Assert.AreEqual(2, result.Length); Assert.IsTrue(result.Any(x => x.User.Id == userId1)); Assert.IsTrue(result.Any(x => x.User.Id == userId2)); @@ -289,7 +288,7 @@ public async Task ProcessAsync_Exception_Test() twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Throws(new Exception()); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -361,7 +360,7 @@ public async Task ProcessAsync_Error_Test() twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Returns((TwitterUser)null); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -433,7 +432,7 @@ public async Task ProcessAsync_Error_OverThreshold_Test() twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Returns((TwitterUser)null); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -507,7 +506,7 @@ public async Task ProcessAsync_Protected_Test() { Protected = true }); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -582,7 +581,7 @@ public async Task ProcessAsync_Protected_OverThreshold_Test() { Protected = true }); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -610,14 +609,14 @@ public async Task ProcessAsync_Protected_OverThreshold_Test() removeTwitterAccountActionMock.VerifyAll(); #endregion } - + [TestMethod] public async Task ProcessAsync_Error_NotInit_Test() { #region Stubs var userId1 = 1; var acct1 = "user1"; - + var users = new List { new SyncTwitterUser @@ -641,7 +640,7 @@ public async Task ProcessAsync_Error_NotInit_Test() twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct1))) .Returns((TwitterUser)null); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct1))) @@ -709,7 +708,7 @@ public async Task ProcessAsync_RateLimited_Test() twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Throws(new RateLimitExceededException()); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveFollowersProcessorTests.cs similarity index 95% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveFollowersProcessorTests.cs index 4679259..d687541 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveFollowersProcessorTests.cs @@ -5,11 +5,11 @@ using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Models; -using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.Federation; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; -namespace BirdsiteLive.Pipeline.Tests.Processors +namespace BirdsiteLive.Pipeline.Tests.Processors.Federation { [TestClass] public class RetrieveFollowersProcessorTests diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveTweetsProcessorTests.cs similarity index 99% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveTweetsProcessorTests.cs index f95ad82..1f69775 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveTweetsProcessorTests.cs @@ -5,7 +5,7 @@ using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Models; -using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.Federation; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; using Microsoft.Extensions.Logging; diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveTwitterUsersProcessorTests.cs similarity index 97% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveTwitterUsersProcessorTests.cs index daf0bfa..f8956d3 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/RetrieveTwitterUsersProcessorTests.cs @@ -6,13 +6,13 @@ using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; -using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.Federation; using BirdsiteLive.Pipeline.Tools; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; -namespace BirdsiteLive.Pipeline.Tests.Processors +namespace BirdsiteLive.Pipeline.Tests.Processors.Federation { [TestClass] public class RetrieveTwitterUsersProcessorTests @@ -43,7 +43,7 @@ public async Task GetTwitterUsersAsync_Test() It.Is(y => y == maxUsers), It.Is(y => y == false))) .ReturnsAsync(users); - + var loggerMock = new Mock>(); #endregion @@ -100,7 +100,7 @@ public async Task GetTwitterUsersAsync_Multi_Test() var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.WhenAny(t, Task.Delay(300)); - + #region Validations maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); @@ -139,7 +139,7 @@ public async Task GetTwitterUsersAsync_Multi2_Test() .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]); - + var loggerMock = new Mock>(); #endregion @@ -190,7 +190,7 @@ public async Task GetTwitterUsersAsync_NoUsers_Test() var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 1; - var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None); + var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.WhenAny(t, Task.Delay(50)); @@ -200,7 +200,7 @@ public async Task GetTwitterUsersAsync_NoUsers_Test() Assert.AreEqual(0, buffer.Count); #endregion } - + [TestMethod] public async Task GetTwitterUsersAsync_Exception_Test() { @@ -258,7 +258,7 @@ public async Task GetTwitterUsersAsync_Cancellation_Test() .ReturnsAsync(maxUsers); var twitterUserDalMock = new Mock(MockBehavior.Strict); - + var loggerMock = new Mock>(); #endregion diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SaveProgressionProcessorTests.cs similarity index 99% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SaveProgressionProcessorTests.cs index d245713..f38d806 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SaveProgressionProcessorTests.cs @@ -2,7 +2,7 @@ using BirdsiteLive.DAL.Models; using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Models; -using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.Federation; using BirdsiteLive.Twitter.Models; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SendTweetsToFollowersProcessorTests.cs similarity index 99% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SendTweetsToFollowersProcessorTests.cs index 8a78038..bd03747 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SendTweetsToFollowersProcessorTests.cs @@ -7,7 +7,7 @@ using BirdsiteLive.DAL.Models; using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Models; -using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.Federation; using BirdsiteLive.Pipeline.Processors.SubTasks; using BirdsiteLive.Twitter.Models; using Microsoft.Extensions.Logging; diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SubTasks/SendTweetsToInboxTaskTests.cs similarity index 85% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SubTasks/SendTweetsToInboxTaskTests.cs index 367a642..76b7a7f 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SubTasks/SendTweetsToInboxTaskTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Net; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Threading.Tasks; using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; @@ -87,15 +88,26 @@ public async Task ExecuteAsync_SingleTweet_Test() .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == inbox + && y.Host == host + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -155,15 +167,18 @@ public async Task ExecuteAsync_SingleTweet_Reply_Test() .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -237,15 +252,26 @@ public async Task ExecuteAsync_SingleTweet_ReplyThread_Test() .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == inbox + && y.Host == host + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -319,15 +345,26 @@ public async Task ExecuteAsync_SingleTweet_PublishReply_Test() .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == inbox + && y.Host == host + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -402,15 +439,29 @@ public async Task ExecuteAsync_MultipleTweets_Test() .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + foreach (var tweetId in new[] { tweetId2, tweetId3 }) + { + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == inbox + && y.Host == host + && y.PublishedAt != default))) + .ReturnsAsync(-1); + } #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -493,9 +544,20 @@ public async Task ExecuteAsync_MultipleTweets_Error_Test() .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId2 + && y.Inbox == inbox + && y.Host == host + && y.PublishedAt != default))) + .ReturnsAsync(-1); + #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); try { @@ -507,6 +569,7 @@ public async Task ExecuteAsync_MultipleTweets_Error_Test() activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } } @@ -571,15 +634,18 @@ public async Task ExecuteAsync_SingleTweet_ParsingError_Test() .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -634,9 +700,11 @@ public async Task ExecuteAsync_SingleTweet_ArgumentException_Test() var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); try { @@ -649,6 +717,7 @@ public async Task ExecuteAsync_SingleTweet_ArgumentException_Test() activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } } diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SubTasks/SendTweetsToSharedInboxTests.cs similarity index 88% rename from src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs rename to src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SubTasks/SendTweetsToSharedInboxTests.cs index 7ab06a2..26db9f6 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/Federation/SubTasks/SendTweetsToSharedInboxTests.cs @@ -109,15 +109,26 @@ public async Task ExecuteAsync_SingleTweet_Test() } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == inbox + && y.Host == host + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -199,15 +210,18 @@ public async Task ExecuteAsync_SingleTweet_Reply_Test() } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -302,15 +316,26 @@ public async Task ExecuteAsync_SingleTweet_ReplyThread_Test() } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == inbox + && y.Host == host + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -405,15 +430,26 @@ public async Task ExecuteAsync_SingleTweet_PublishReply_Test() } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == inbox + && y.Host == host + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -509,15 +545,29 @@ public async Task ExecuteAsync_MultipleTweets_Test() } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + foreach (var tweetId in new[] { tweetId2, tweetId3 }) + { + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId + && y.Inbox == inbox + && y.Host == host + && y.PublishedAt != default))) + .ReturnsAsync(-1); + } #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -621,9 +671,19 @@ public async Task ExecuteAsync_MultipleTweets_Error_Test() } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); + syncTweetDalMock + .Setup(x => x.SaveTweetAsync( + It.Is(y => y.Acct == twitterHandle + && y.TweetId == tweetId2 + && y.Inbox == inbox + && y.Host == host + && y.PublishedAt != default))) + .ReturnsAsync(-1); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); try { @@ -635,6 +695,7 @@ public async Task ExecuteAsync_MultipleTweets_Error_Test() activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } } @@ -720,15 +781,18 @@ public async Task ExecuteAsync_SingleTweet_ParsingError_Test() } var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } @@ -800,9 +864,11 @@ public async Task ExecuteAsync_SingleTweet_ArgumentException_Test() var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var syncTweetDalMock = new Mock(MockBehavior.Strict); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object, syncTweetDalMock.Object); try { @@ -815,6 +881,7 @@ public async Task ExecuteAsync_SingleTweet_ArgumentException_Test() activityPubService.VerifyAll(); statusServiceMock.VerifyAll(); followersDalMock.VerifyAll(); + syncTweetDalMock.VerifyAll(); #endregion } } diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/DeleteTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/DeleteTweetsProcessorTests.cs new file mode 100644 index 0000000..d81c581 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/DeleteTweetsProcessorTests.cs @@ -0,0 +1,160 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain; +using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.Federation; +using BirdsiteLive.Pipeline.Processors.TweetsCleanUp; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Processors.TweetsCleanUp +{ + [TestClass] + public class DeleteTweetsProcessorTests + { + [TestMethod] + public async Task Process_DeleteTweet_Test() + { + #region Stubs + var tweetId = 42; + var tweet = new TweetToDelete + { + Tweet = new SyncTweet + { + Id = tweetId + } + }; + #endregion + + #region Mocks + var serviceMock = new Mock(MockBehavior.Strict); + serviceMock + .Setup(x => x.DeleteNoteAsync(It.Is(y => y.Id == tweetId))) + .Returns(Task.CompletedTask); + + var loggerMock = new Mock>(); + #endregion + + var processor = new DeleteTweetsProcessor(serviceMock.Object, loggerMock.Object); + var result = await processor.ProcessAsync(tweet, CancellationToken.None); + + #region Validations + serviceMock.VerifyAll(); + loggerMock.VerifyAll(); + + Assert.IsNotNull(result); + Assert.IsTrue(result.DeleteSuccessful); + #endregion + } + + [TestMethod] + public async Task Process_DeleteTweet_NotFound_Test() + { + #region Stubs + var tweetId = 42; + var tweet = new TweetToDelete + { + Tweet = new SyncTweet + { + Id = tweetId + } + }; + #endregion + + #region Mocks + var serviceMock = new Mock(MockBehavior.Strict); + serviceMock + .Setup(x => x.DeleteNoteAsync(It.Is(y => y.Id == tweetId))) + .Throws(new HttpRequestException("not found", null, HttpStatusCode.NotFound)); + + var loggerMock = new Mock>(); + #endregion + + var processor = new DeleteTweetsProcessor(serviceMock.Object, loggerMock.Object); + var result = await processor.ProcessAsync(tweet, CancellationToken.None); + + #region Validations + serviceMock.VerifyAll(); + loggerMock.VerifyAll(); + + Assert.IsNotNull(result); + Assert.IsTrue(result.DeleteSuccessful); + #endregion + } + + [TestMethod] + public async Task Process_DeleteTweet_HttpError_Test() + { + #region Stubs + var tweetId = 42; + var tweet = new TweetToDelete + { + Tweet = new SyncTweet + { + Id = tweetId + } + }; + #endregion + + #region Mocks + var serviceMock = new Mock(MockBehavior.Strict); + serviceMock + .Setup(x => x.DeleteNoteAsync(It.Is(y => y.Id == tweetId))) + .Throws(new HttpRequestException("bad gateway", null, HttpStatusCode.BadGateway)); + + var loggerMock = new Mock>(); + #endregion + + var processor = new DeleteTweetsProcessor(serviceMock.Object, loggerMock.Object); + var result = await processor.ProcessAsync(tweet, CancellationToken.None); + + #region Validations + serviceMock.VerifyAll(); + loggerMock.VerifyAll(); + + Assert.IsNotNull(result); + Assert.IsFalse(result.DeleteSuccessful); + #endregion + } + + [TestMethod] + public async Task Process_DeleteTweet_GenericException_Test() + { + #region Stubs + var tweetId = 42; + var tweet = new TweetToDelete + { + Tweet = new SyncTweet + { + Id = tweetId + } + }; + #endregion + + #region Mocks + var serviceMock = new Mock(MockBehavior.Strict); + serviceMock + .Setup(x => x.DeleteNoteAsync(It.Is(y => y.Id == tweetId))) + .Throws(new Exception()); + + var loggerMock = new Mock>(); + #endregion + + var processor = new DeleteTweetsProcessor(serviceMock.Object, loggerMock.Object); + var result = await processor.ProcessAsync(tweet, CancellationToken.None); + + #region Validations + serviceMock.VerifyAll(); + loggerMock.VerifyAll(); + + Assert.IsNotNull(result); + Assert.IsFalse(result.DeleteSuccessful); + #endregion + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessorTests.cs new file mode 100644 index 0000000..7d7f7d1 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessorTests.cs @@ -0,0 +1,131 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using System.Xml.Linq; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.TweetsCleanUp; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Org.BouncyCastle.Crypto; + +namespace BirdsiteLive.Pipeline.Tests.Processors.TweetsCleanUp +{ + [TestClass] + public class RetrieveTweetsToDeleteProcessorTests + { + [TestMethod] + public async Task Process_Test() + { + #region Stubs + var settings = new InstanceSettings + { + + }; + var bufferBlock = new BufferBlock(); + + var tweetId1 = 42; + var tweetId2 = 43; + + var tweet1 = new SyncTweet + { + Id = tweetId1 + }; + var tweet2 = new SyncTweet + { + Id = tweetId2 + }; + var batch = new List + { + tweet1, + tweet2 + }; + #endregion + + #region Mocks + var dalMock = new Mock(MockBehavior.Strict); + dalMock + .Setup(x => x.GetTweetsOlderThanAsync(It.IsAny(), It.Is(y => y == -1), It.Is(y => y == 100))) + .ReturnsAsync(batch); + #endregion + + var processor = new RetrieveTweetsToDeleteProcessor(dalMock.Object, settings); + processor.GetTweetsAsync(bufferBlock, CancellationToken.None); + + await Task.Delay(TimeSpan.FromSeconds(1)); + + #region Validations + dalMock.VerifyAll(); + + Assert.AreEqual(batch.Count, bufferBlock.Count); + + bufferBlock.TryReceiveAll(out var all); + foreach (var tweet in all) + { + Assert.IsTrue(batch.Any(x => x.Id == tweet.Tweet.Id)); + } + #endregion + } + + [TestMethod] + public async Task Process_Pagination_Test() + { + #region Stubs + var settings = new InstanceSettings { }; + var bufferBlock = new BufferBlock(); + + + var batch = new List(); + var batch2 = new List(); + + for (var i = 0; i < 100; i++) + { + var tweet = new SyncTweet + { + Id = i + }; + batch.Add(tweet); + } + + for (var i = 100; i < 120; i++) + { + var tweet = new SyncTweet + { + Id = i + }; + batch2.Add(tweet); + } + + #endregion + + #region Mocks + var dalMock = new Mock(MockBehavior.Strict); + dalMock + .Setup(x => x.GetTweetsOlderThanAsync(It.IsAny(), It.Is(y => y == -1), It.Is(y => y == 100))) + .ReturnsAsync(batch); + + dalMock + .Setup(x => x.GetTweetsOlderThanAsync(It.IsAny(), It.Is(y => y == 99), It.Is(y => y == 100))) + .ReturnsAsync(batch2); + #endregion + + var processor = new RetrieveTweetsToDeleteProcessor(dalMock.Object, settings); + processor.GetTweetsAsync(bufferBlock, CancellationToken.None); + + await Task.Delay(TimeSpan.FromSeconds(1)); + + #region Validations + dalMock.VerifyAll(); + + Assert.AreEqual(batch.Count + batch2.Count, bufferBlock.Count); + #endregion + } + + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessorTests.cs new file mode 100644 index 0000000..ea002cb --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessorTests.cs @@ -0,0 +1,80 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.TweetsCleanUp; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Processors.TweetsCleanUp +{ + [TestClass] + public class SaveDeletedTweetStatusProcessorTests + { + [TestMethod] + public async Task Process_DeleteSuccessfull_Test() + { + #region Stubs + var settings = new InstanceSettings { }; + var tweetId = 42; + var tweetToDelete = new TweetToDelete + { + DeleteSuccessful = true, + Tweet = new SyncTweet + { + Id = tweetId + } + }; + #endregion + + #region Mocks + var dalMock = new Mock(MockBehavior.Strict); + dalMock + .Setup(x => x.DeleteTweetAsync(It.Is(y => y == tweetId))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SaveDeletedTweetStatusProcessor(dalMock.Object, settings); + await processor.ProcessAsync(tweetToDelete, CancellationToken.None); + + #region Validations + dalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task Process_Expired_Test() + { + #region Stubs + var settings = new InstanceSettings { }; + var tweetId = 42; + var tweetToDelete = new TweetToDelete + { + DeleteSuccessful = false, + Tweet = new SyncTweet + { + Id = tweetId, + PublishedAt = DateTime.UtcNow.AddDays(-30) + } + }; + #endregion + + #region Mocks + var dalMock = new Mock(MockBehavior.Strict); + dalMock + .Setup(x => x.DeleteTweetAsync(It.Is(y => y == tweetId))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SaveDeletedTweetStatusProcessor(dalMock.Object, settings); + await processor.ProcessAsync(tweetToDelete, CancellationToken.None); + + #region Validations + dalMock.VerifyAll(); + #endregion + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs index 81eeb59..d21d296 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs @@ -2,7 +2,7 @@ using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using BirdsiteLive.DAL.Models; -using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Contracts.Federation; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/TweetCleanUpPipelineTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/TweetCleanUpPipelineTests.cs new file mode 100644 index 0000000..a7f2751 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/TweetCleanUpPipelineTests.cs @@ -0,0 +1,48 @@ +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Contracts.Federation; +using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp; +using BirdsiteLive.Pipeline.Models; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests +{ + [TestClass] + public class TweetCleanUpPipelineTests + { + [TestMethod] + public async Task Process() + { + #region Stubs + var ct = new CancellationTokenSource(10); + #endregion + + #region Mocks + var retrieveTweetsToDeleteProcessor = new Mock(MockBehavior.Strict); + retrieveTweetsToDeleteProcessor + .Setup(x => x.GetTweetsAsync( + It.IsAny>(), + It.IsAny())) + .Returns(Task.Delay(0)); + + var deleteTweetsProcessor = new Mock(MockBehavior.Strict); + var saveDeletedTweetStatusProcessor = new Mock(MockBehavior.Strict); + var logger = new Mock>(); + + #endregion + + var pipeline = new TweetCleanUpPipeline(retrieveTweetsToDeleteProcessor.Object, deleteTweetsProcessor.Object, saveDeletedTweetStatusProcessor.Object, logger.Object); + await pipeline.ExecuteAsync(ct.Token); + + #region Validations + retrieveTweetsToDeleteProcessor.VerifyAll(); + deleteTweetsProcessor.VerifyAll(); + saveDeletedTweetStatusProcessor.VerifyAll(); + #endregion + } + } +} \ No newline at end of file