Merge pull request #190 from NicolasConstant/develop

0.23.0 PR
NicolasConstant · Mar 17, 2023 · cf0797a · cf0797a
2 parents 8b1a61c + 899a595
commit cf0797a
Show file tree

Hide file tree

Showing 85 changed files with 1,728 additions and 175 deletions.
diff --git a/.github/workflows/dotnet-core.yml b/.github/workflows/dotnet-core.yml
@@ -16,7 +16,7 @@ jobs:
     - name: Setup .NET Core
       uses: actions/setup-dotnet@v1
       with:
-        dotnet-version: 3.1.101
+        dotnet-version: 6.0.x
     - name: Install dependencies
       run: dotnet restore
       working-directory: ${{env.working-directory}}

diff --git a/Dockerfile b/Dockerfile
@@ -1,11 +1,11 @@
 #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging.
 
-FROM mcr.microsoft.com/dotnet/aspnet:3.1-buster-slim AS base
+FROM mcr.microsoft.com/dotnet/aspnet:6.0-bullseye-slim AS base
 WORKDIR /app
 EXPOSE 80
 EXPOSE 443
 
-FROM mcr.microsoft.com/dotnet/sdk:3.1-buster AS publish
+FROM mcr.microsoft.com/dotnet/sdk:6.0-bullseye-slim AS publish
 COPY ./src/ ./src/
 RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish
 RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish

diff --git a/INSTALLATION.md b/INSTALLATION.md
@@ -186,6 +186,70 @@ services:
 +       command: --interval 300
 ```
 
+## IP Whitelisting
+
+If you want to use the IP Whitelisting functionality (see related [variable](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md)) and you are using the nginx reverse proxy set as before, please add the following: 
+
+```
+sudo nano /etc/nginx/sites-enabled/{your-domain-name.com}
+```
+
+``` diff
+server {
+    listen        80;
+    server_name   {your-domain-name.com};
+    location / {
+        proxy_pass         http://localhost:5000;
+        proxy_http_version 1.1;
+        proxy_set_header   Upgrade $http_upgrade;
+        proxy_set_header   Connection keep-alive;
+        proxy_set_header   Host $host;
+        proxy_cache_bypass $http_upgrade;
+        proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header   X-Forwarded-Proto $scheme;
++       proxy_set_header   X-Real-IP $remote_addr;
+    }
+}
+```
+
+And edit the docker-compose file as follow: 
+
+```diff
+version: "3"
+
+networks:
+    birdsitelivenetwork:
+        external: false
+
+services:
+    server:
+        image: nicolasconstant/birdsitelive:latest
+        restart: always
+        container_name: birdsitelive
+        environment:
+            - Instance:Domain=domain.name
+            - Instance:AdminEmail=name@domain.ext
++           - Instance:IpWhiteListing=127.0.0.1;127.0.0.2
++           - Instance:EnableXRealIpHeader=true
+            - Db:Type=postgres
+            - Db:Host=db
+            - Db:Name=birdsitelive
+            - Db:User=birdsitelive
+            - Db:Password=birdsitelive
+            - Twitter:ConsumerKey=twitter.api.key
+            - Twitter:ConsumerSecret=twitter.api.key
+        networks:
+            - birdsitelivenetwork
+        ports:
+            - "5000:80"
+        depends_on:
+            - db
+
+    db:
+        image: postgres:9.6
+        [...]
+```
+
 ## More options 
 
 You can find more options available [here](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md)
diff --git a/VARIABLES.md b/VARIABLES.md
@@ -51,6 +51,9 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
 * `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins)
 * `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc)
 * `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
+* `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided).
+* `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy).
+* `Instance:MaxTweetRetention` (default: 20, min: 1, max: 90) Number of days before synchronized tweets get deleted
 
 # Docker Compose full example
 

diff --git a/src/BSLManager/BSLManager.csproj b/src/BSLManager/BSLManager.csproj
@@ -2,15 +2,15 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Lamar" Version="5.0.3" />
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
-    <PackageReference Include="Terminal.Gui" Version="1.0.0-beta.11" />
+    <PackageReference Include="Lamar" Version="10.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="7.0.0" />
+    <PackageReference Include="Terminal.Gui" Version="1.9.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj b/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj
@@ -6,8 +6,8 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
-    <PackageReference Include="System.Text.Json" Version="4.7.2" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+    <PackageReference Include="System.Text.Json" Version="7.0.1" />
   </ItemGroup>
 
 </Project>
diff --git a/src/BirdsiteLive.ActivityPub/Models/Tombstone.cs b/src/BirdsiteLive.ActivityPub/Models/Tombstone.cs
@@ -0,0 +1,9 @@
+namespace BirdsiteLive.ActivityPub.Models
+{
+    public class Tombstone
+    {
+        public string id { get; set; }
+        public readonly string type = "Tombstone";
+        public string atomUrl { get; set; }
+    }
+}
diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs
@@ -16,5 +16,9 @@ public class InstanceSettings
         public int FailingFollowerCleanUpThreshold { get; set; } = -1;
 
         public int UserCacheCapacity { get; set; }
+        public string IpWhiteListing { get; set; }
+        public bool EnableXRealIpHeader { get; set; }
+
+        public int MaxTweetRetention { get; set; }
     }
 }
diff --git a/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj b/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj
@@ -6,8 +6,8 @@
 
   <ItemGroup>
     <PackageReference Include="Asn1" Version="1.0.9" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
-    <PackageReference Include="Portable.BouncyCastle" Version="1.8.6.7" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+    <PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
   </ItemGroup>
 
 </Project>
diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs
@@ -9,6 +9,7 @@
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Models;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
 
@@ -22,6 +23,7 @@ public interface IActivityPubService
         Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
             string targetInbox);
         Task DeleteUserAsync(string username, string targetHost, string targetInbox);
+        Task DeleteNoteAsync(SyncTweet tweet);
     }
 
     public class WebFinger
@@ -108,6 +110,30 @@ public async Task DeleteUserAsync(string username, string targetHost, string tar
             }
         }
 
+        public async Task DeleteNoteAsync(SyncTweet tweet)
+        {
+            var acct = tweet.Acct.ToLowerInvariant().Trim();
+
+            var actor = $"https://{_instanceSettings.Domain}/users/{acct}";
+            var noteId = $"https://{_instanceSettings.Domain}/users/{acct}/statuses/{tweet.TweetId}";
+
+            var delete = new ActivityDelete
+            {
+                context = "https://www.w3.org/ns/activitystreams",
+                id = $"{noteId}#delete",
+                type = "Delete",
+                actor = actor,
+                to = new[] { "https://www.w3.org/ns/activitystreams#Public" },
+                apObject = new Tombstone
+                {
+                    id = noteId,
+                    atomUrl = noteId
+                }
+            };
+
+            await PostDataAsync(delete, tweet.Host, actor, tweet.Inbox);
+        }
+
         public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
         {
             try

diff --git a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj
@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs
@@ -9,6 +9,7 @@ public interface IProcessDeleteUser
     {
         Task ExecuteAsync(Follower follower);
         Task ExecuteAsync(string followerUsername, string followerDomain);
+        Task ExecuteAsync(string actorId);
     }
 
     public class ProcessDeleteUser : IProcessDeleteUser
@@ -33,6 +34,15 @@ public async Task ExecuteAsync(string followerUsername, string followerDomain)
             await ExecuteAsync(follower);
         }
 
+        public async Task ExecuteAsync(string actorId)
+        {
+            // Get Follower and Twitter Users
+            var follower = await _followersDal.GetFollowerAsync(actorId);
+            if (follower == null) return;
+
+            await ExecuteAsync(follower);
+        }
+
         public async Task ExecuteAsync(Follower follower)
         {
             // Remove twitter users if no more followers

diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs
@@ -280,15 +280,21 @@ public async Task<bool> UndoFollowRequestedAsync(string signature, string method
         public async Task<bool> DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders,
             ActivityDelete activity, string body)
         {
-            // Validate
-            var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
-            if (!sigValidation.SignatureIsValidated) return false;
+            if (activity.apObject is string apObject)
+            {
+                if (!string.Equals(activity.actor.Trim(), apObject.Trim(), StringComparison.InvariantCultureIgnoreCase)) return true;
 
-            // Remove user and followings
-            var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation);
-            var followerHost = SigValidationResultExtractor.GetHost(sigValidation);
+                try
+                {
+                    // Validate
+                    var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
+                    if (!sigValidation.SignatureIsValidated) return false;
+                }
+                catch (FollowerIsGoneException){}
 
-            await _processDeleteUser.ExecuteAsync(followerUserName, followerHost);
+                // Remove user and followings
+                await _processDeleteUser.ExecuteAsync(activity.actor.Trim());
+            }
 
             return true;
         }

diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj
@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
-    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.1" />
+    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
+    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>
@@ -19,6 +19,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <Folder Include="Processors\TweetsCleanUp\Base\" />
     <Folder Include="Tools\" />
   </ItemGroup>
 

diff --git a/...cts/IRefreshTwitterUserStatusProcessor.cs → ...ion/IRefreshTwitterUserStatusProcessor.cs b/...cts/IRefreshTwitterUserStatusProcessor.cs → ...ion/IRefreshTwitterUserStatusProcessor.cs
@@ -3,7 +3,7 @@
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRefreshTwitterUserStatusProcessor
     {

diff --git a/.../Contracts/IRetrieveFollowersProcessor.cs → ...Federation/IRetrieveFollowersProcessor.cs b/.../Contracts/IRetrieveFollowersProcessor.cs → ...Federation/IRetrieveFollowersProcessor.cs
@@ -3,7 +3,7 @@
 using System.Threading.Tasks;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRetrieveFollowersProcessor
     {

diff --git a/...ine/Contracts/IRetrieveTweetsProcessor.cs → ...ts/Federation/IRetrieveTweetsProcessor.cs b/...ine/Contracts/IRetrieveTweetsProcessor.cs → ...ts/Federation/IRetrieveTweetsProcessor.cs
@@ -3,7 +3,7 @@
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRetrieveTweetsProcessor
     {

diff --git a/...ntracts/IRetrieveTwitterUsersProcessor.cs → ...eration/IRetrieveTwitterUsersProcessor.cs b/...ntracts/IRetrieveTwitterUsersProcessor.cs → ...eration/IRetrieveTwitterUsersProcessor.cs
@@ -3,7 +3,7 @@
 using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.DAL.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRetrieveTwitterUsersProcessor
     {

diff --git a/...ne/Contracts/ISaveProgressionProcessor.cs → ...s/Federation/ISaveProgressionProcessor.cs b/...ne/Contracts/ISaveProgressionProcessor.cs → ...s/Federation/ISaveProgressionProcessor.cs
@@ -2,7 +2,7 @@
 using System.Threading.Tasks;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface ISaveProgressionProcessor
     {

diff --git a/...tracts/ISendTweetsToFollowersProcessor.cs → ...ration/ISendTweetsToFollowersProcessor.cs b/...tracts/ISendTweetsToFollowersProcessor.cs → ...ration/ISendTweetsToFollowersProcessor.cs
@@ -2,7 +2,7 @@
 using System.Threading.Tasks;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface ISendTweetsToFollowersProcessor
     {

diff --git a/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IDeleteTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IDeleteTweetsProcessor.cs
@@ -0,0 +1,11 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
+{
+    public interface IDeleteTweetsProcessor
+    {
+        Task<TweetToDelete> ProcessAsync(TweetToDelete tweet, CancellationToken ct);
+    }
+}
diff --git a/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IRetrieveTweetsToDeleteProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IRetrieveTweetsToDeleteProcessor.cs
@@ -0,0 +1,12 @@
+using System.Threading;
+using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
+{
+    public interface IRetrieveTweetsToDeleteProcessor
+    {
+        Task GetTweetsAsync(BufferBlock<TweetToDelete> tweetsBufferBlock, CancellationToken ct);
+    }
+}
diff --git a/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/ISaveDeletedTweetStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/ISaveDeletedTweetStatusProcessor.cs
@@ -0,0 +1,11 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
+{
+    public interface ISaveDeletedTweetStatusProcessor
+    {
+        Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct);
+    }
+}