From 4240bccd0c56fa65ecef9fd86ec02fb3beb3611e Mon Sep 17 00:00:00 2001 From: Arlo Godfrey Date: Sat, 14 Sep 2024 15:58:07 -0500 Subject: [PATCH] Release 0.19.0. See release notes or expand full commit message. [New] Added IAddFileToGetCid interface. This is functionally identical to IGetCid, but instead of simply returning a CID already in ipfs, it computes the CID by providing data to ipfs using preferences in the AddFileOptions parameter. [Breaking] StorableKuboExtensions.GetCidAsync now takes an AddFileOptions parameter. ContentAddressedSystemFile and ContentAddressedSystemFolder now implement IAddFileToGetCid instead of IGetCid. [Fixes] Inherited fixes from OwlCore.ComponentModel 0.9.1. Fixed issues with MfsStream where it would return before the task was complete. MfsStream.ReadAsync and MfsStream.WriteAsync now respect the requested offset when operating on the provided buffer. [Improvement] Updated to IpfsShipyard.Ipfs.Http.Client 0.5.1. MfsStream.WriteAsync now supplies Flush = false when writing to mfs, instead of the default of flushing after every write. This improves performance when writing large files, but requires a manual call to FlushAsync to persist the changes. --- src/Extensions/IAddFileToGetCid.cs | 19 ++++++ .../ContentAddressedSystemFile.cs | 10 +--- .../ContentAddressedSystemFolder.cs | 10 +--- src/Extensions/StorableKuboExtensions.cs | 33 +++++++---- src/IpnsFolder.cs | 2 +- src/MfsFolder.Modifiable.cs | 58 ++++++++++++++----- src/MfsStream.cs | 12 ++-- src/OwlCore.Kubo.csproj | 31 +++++++--- .../OwlCore.Kubo.Tests.csproj | 2 +- 9 files changed, 120 insertions(+), 57 deletions(-) create mode 100644 src/Extensions/IAddFileToGetCid.cs diff --git a/src/Extensions/IAddFileToGetCid.cs b/src/Extensions/IAddFileToGetCid.cs new file mode 100644 index 0000000..2948042 --- /dev/null +++ b/src/Extensions/IAddFileToGetCid.cs @@ -0,0 +1,19 @@ +using Ipfs; +using Ipfs.CoreApi; +using OwlCore.Storage; + +namespace OwlCore.Kubo; + +/// +/// Implementations are capable of providing a CID for the current content by adding it ipfs. +/// +public partial interface IAddFileToGetCid : IStorable +{ + /// + /// Gets the CID of the storable item. + /// + /// The add file options to use when computing the cid for this storable. + /// A token that can be used to cancel the ongoing operation. + /// + public Task GetCidAsync(AddFileOptions addFileOptions, CancellationToken cancellationToken); +} diff --git a/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFile.cs b/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFile.cs index 12d20e4..bbb4295 100644 --- a/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFile.cs +++ b/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFile.cs @@ -8,7 +8,7 @@ namespace OwlCore.Storage.System.IO; /// /// An implementation of with added support for . /// -public class ContentAddressedSystemFile : SystemFile, IGetCid +public class ContentAddressedSystemFile : SystemFile, IAddFileToGetCid { /// /// Creates a new instance of . @@ -27,13 +27,9 @@ public ContentAddressedSystemFile(string path, ICoreApi client) public ICoreApi Client { get; } /// - public async Task GetCidAsync(CancellationToken cancellationToken) + public async Task GetCidAsync(AddFileOptions addFileOptions, CancellationToken cancellationToken) { - var res = await Client.FileSystem.AddFileAsync(Id, new() - { - OnlyHash = true, - Pin = false - }, cancellationToken); + var res = await Client.FileSystem.AddFileAsync(Id, addFileOptions, cancellationToken); Guard.IsFalse(res.IsDirectory); return res.ToLink().Id; diff --git a/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFolder.cs b/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFolder.cs index 21d1d0c..c8f151b 100644 --- a/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFolder.cs +++ b/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFolder.cs @@ -8,7 +8,7 @@ namespace OwlCore.Storage.System.IO; /// /// An implementation of with added support for . /// -public class ContentAddressedSystemFolder : SystemFolder, IGetCid +public class ContentAddressedSystemFolder : SystemFolder, IAddFileToGetCid { /// /// Creates a new instance of . @@ -27,13 +27,9 @@ public ContentAddressedSystemFolder(string path, ICoreApi client) public ICoreApi Client { get; } /// - public async Task GetCidAsync(CancellationToken cancellationToken) + public async Task GetCidAsync(AddFileOptions addFileOptions, CancellationToken cancellationToken) { - var res = await Client.FileSystem.AddDirectoryAsync(Id, recursive: true, new() - { - OnlyHash = true, - Pin = false, - }, cancellationToken); + var res = await Client.FileSystem.AddDirectoryAsync(Id, recursive: true, addFileOptions, cancellationToken); Guard.IsTrue(res.IsDirectory); return res.ToLink().Id; diff --git a/src/Extensions/StorableKuboExtensions.cs b/src/Extensions/StorableKuboExtensions.cs index f9fa842..c1b3662 100644 --- a/src/Extensions/StorableKuboExtensions.cs +++ b/src/Extensions/StorableKuboExtensions.cs @@ -12,8 +12,16 @@ namespace OwlCore.Kubo; /// public static partial class StorableKuboExtensions { - /// - public static async Task GetCidAsync(this IStorable item, ICoreApi client, CancellationToken cancellationToken) + /// + /// Gets a CID for the provided . If possible, a CID will be provided without adding the item to ipfs, otherwise the will be used to add content to ipfs and compute the cid. + /// + /// The storable to get the cid for. + /// The client to use for communicating with ipfs. + /// The options to use when adding content from the to ipfs. + /// A token that can be used to cancel the ongoing operation. + /// A task containing the cid of the . + /// An unsupported implementation of was provided for . + public static async Task GetCidAsync(this IStorable item, ICoreApi client, AddFileOptions addFileOptions, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -24,15 +32,21 @@ public static async Task GetCidAsync(this IStorable item, ICoreApi client, // You'd typically only do this if you can interact your Kubo node faster than a manually piped stream, // or if your implementation of IGetCid has enhanced capabilities (e.g. folder support). // --- - if (item is not IGetCid && item is SystemFile systemFile) + + // Get cid without adding to ipfs, if possible + if (item is IGetCid getCid) + return await getCid.GetCidAsync(cancellationToken); + + // Get cid by adding content to ipfs. + if (item is not IAddFileToGetCid && item is SystemFile systemFile) item = new ContentAddressedSystemFile(systemFile.Path, client); - if (item is not IGetCid && item is SystemFolder systemFolder) + if (item is not IAddFileToGetCid && item is SystemFolder systemFolder) item = new ContentAddressedSystemFolder(systemFolder.Path, client); // If the implementation can handle content addressing directly, use that. - if (item is IGetCid contentAddressedStorable) - return await contentAddressedStorable.GetCidAsync(cancellationToken); + if (item is IAddFileToGetCid contentAddressedStorable) + return await contentAddressedStorable.GetCidAsync(addFileOptions, cancellationToken); // Otherwise, a fallback approach that manually connects the streams together. // The Kubo API doesn't support this scenario for folders, without assuming that the Id is a local path, @@ -40,12 +54,7 @@ public static async Task GetCidAsync(this IStorable item, ICoreApi client, if (item is IFile file) { using var stream = await file.OpenStreamAsync(FileAccess.Read, cancellationToken); - - var res = await client.FileSystem.AddAsync(stream, file.Name, new() - { - OnlyHash = true, - Pin = false, - }, cancellationToken); + var res = await client.FileSystem.AddAsync(stream, file.Name, addFileOptions, cancellationToken); Guard.IsFalse(res.IsDirectory); return res.ToLink().Id; diff --git a/src/IpnsFolder.cs b/src/IpnsFolder.cs index 31c0f88..482c550 100644 --- a/src/IpnsFolder.cs +++ b/src/IpnsFolder.cs @@ -71,7 +71,7 @@ public IpnsFolder(string ipnsAddress, ICoreApi client) /// public virtual async IAsyncEnumerable GetItemsAsync(StorableType type = StorableType.All, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var cid = await GetCidAsync(cancellationToken); + var cid = await GetCidAsync(Id, cancellationToken); var itemInfo = await Client.FileSystem.ListAsync(cid, cancellationToken); Guard.IsTrue(itemInfo.IsDirectory); diff --git a/src/MfsFolder.Modifiable.cs b/src/MfsFolder.Modifiable.cs index acd0bd7..e9c1e74 100644 --- a/src/MfsFolder.Modifiable.cs +++ b/src/MfsFolder.Modifiable.cs @@ -1,4 +1,5 @@ using CommunityToolkit.Diagnostics; +using Ipfs.CoreApi; using OwlCore.Kubo.FolderWatchers; using OwlCore.Storage; @@ -11,6 +12,11 @@ public partial class MfsFolder : IModifiableFolder, IMoveFrom, ICreateCopyOf /// public TimeSpan UpdateCheckInterval { get; } = TimeSpan.FromSeconds(10); + /// + /// The options to use when adding content to this folder on ipfs. + /// + public AddFileOptions AddFileOptions { get; set; } = new(); + /// public virtual async Task DeleteAsync(IStorableChild item, CancellationToken cancellationToken = default) { @@ -21,24 +27,40 @@ public virtual async Task DeleteAsync(IStorableChild item, CancellationToken can } /// - public Task CreateCopyOfAsync(IFile fileToCopy, bool overwrite, CancellationToken cancellationToken, - CreateCopyOfDelegate fallback) + public async Task CreateCopyOfAsync(IFile fileToCopy, bool overwrite, CancellationToken cancellationToken, CreateCopyOfDelegate fallback) { if (fileToCopy is MfsFile mfsFile) - return CreateCopyOfAsync(mfsFile, overwrite, cancellationToken); + return await CreateCopyOfAsync(mfsFile, overwrite, cancellationToken); if (fileToCopy is IpfsFile ipfsFile) - return CreateCopyOfAsync(ipfsFile, overwrite, cancellationToken); + return await CreateCopyOfAsync(ipfsFile, overwrite, cancellationToken); if (fileToCopy is IpnsFile ipnsFile) - return CreateCopyOfAsync(ipnsFile, overwrite, cancellationToken); + return await CreateCopyOfAsync(ipnsFile, overwrite, cancellationToken); + + if (fileToCopy is IGetCid getCid) + { + var cid = await getCid.GetCidAsync(cancellationToken); + + var newPath = $"{Path}{fileToCopy.Name}"; + await Client.Mfs.CopyAsync($"/ipfs/{cid}", newPath, cancel: cancellationToken); + return new MfsFile(newPath, Client); + } - return fallback(this, fileToCopy, overwrite, cancellationToken); + if (fileToCopy is IAddFileToGetCid addFileToGetCid) + { + var cid = await addFileToGetCid.GetCidAsync(AddFileOptions, cancellationToken); + + var newPath = $"{Path}{fileToCopy.Name}"; + await Client.Mfs.CopyAsync($"/ipfs/{cid}", newPath, cancel: cancellationToken); + return new MfsFile(newPath, Client); + } + + return await fallback(this, fileToCopy, overwrite, cancellationToken); } /// - public Task MoveFromAsync(IChildFile fileToMove, IModifiableFolder source, bool overwrite, CancellationToken cancellationToken, - MoveFromDelegate fallback) + public Task MoveFromAsync(IChildFile fileToMove, IModifiableFolder source, bool overwrite, CancellationToken cancellationToken, MoveFromDelegate fallback) { if (fileToMove is MfsFile mfsFile) return MoveFromAsync(mfsFile, source, overwrite, cancellationToken); @@ -51,8 +73,9 @@ public virtual async Task CreateCopyOfAsync(MfsFile fileToCopy, bool { cancellationToken.ThrowIfCancellationRequested(); - await Client.Mfs.CopyAsync(fileToCopy.Path, Path, cancel: cancellationToken); - return new MfsFile($"{Path}{fileToCopy.Name}", Client); + var newPath = $"{Path}{fileToCopy.Name}"; + await Client.Mfs.CopyAsync(fileToCopy.Path, newPath, cancel: cancellationToken); + return new MfsFile(newPath, Client); } /// @@ -60,8 +83,9 @@ public virtual async Task CreateCopyOfAsync(IpfsFile fileToCopy, boo { cancellationToken.ThrowIfCancellationRequested(); - await Client.Mfs.CopyAsync($"/ipfs/{fileToCopy.Id}", Path, cancel: cancellationToken); - return new MfsFile($"{Path}{fileToCopy.Name}", Client); + var newPath = $"{Path}{fileToCopy.Name}"; + await Client.Mfs.CopyAsync($"/ipfs/{fileToCopy.Id}", newPath, cancel: cancellationToken); + return new MfsFile(newPath, Client); } /// @@ -69,10 +93,11 @@ public virtual async Task CreateCopyOfAsync(IpnsFile fileToCopy, boo { cancellationToken.ThrowIfCancellationRequested(); + var newPath = $"{Path}{fileToCopy.Name}"; var cid = await fileToCopy.GetCidAsync(cancellationToken); - await Client.Mfs.CopyAsync($"/ipfs/{cid}", Path, cancel: cancellationToken); + await Client.Mfs.CopyAsync($"/ipfs/{cid}", newPath, cancel: cancellationToken); - return new MfsFile($"{Path}{fileToCopy.Name}", Client); + return new MfsFile(newPath, Client); } /// @@ -80,8 +105,9 @@ public virtual async Task MoveFromAsync(MfsFile fileToMove, IModifia { cancellationToken.ThrowIfCancellationRequested(); - await Client.Mfs.MoveAsync(fileToMove.Path, $"{Path}{fileToMove.Name}", cancellationToken); - return new MfsFile($"{Path}{fileToMove.Name}", Client); + var newPath = $"{Path}{fileToMove.Name}"; + await Client.Mfs.MoveAsync(fileToMove.Path, newPath, cancellationToken); + return new MfsFile(newPath, Client); } /// diff --git a/src/MfsStream.cs b/src/MfsStream.cs index a6fab89..e6a94ae 100644 --- a/src/MfsStream.cs +++ b/src/MfsStream.cs @@ -57,7 +57,7 @@ public MfsStream(string path, long length, ICoreApi client) /// public override void Flush() { - _ = Client.Mfs.FlushAsync(Path).Result; + Client.Mfs.FlushAsync(Path).Wait(); } /// @@ -80,8 +80,7 @@ public override async Task ReadAsync(byte[] buffer, int offset, int count, var result = await Client.Mfs.ReadFileStreamAsync(Path, offset: Position, count: count, cancellationToken); var bytes = await result.ToBytesAsync(cancellationToken); - for (var i = 0; i < bytes.Length; i++) - buffer[i] = bytes[i]; + bytes.CopyTo(buffer, offset); Position += bytes.Length; @@ -108,7 +107,7 @@ public override long Seek(long offset, SeekOrigin origin) if (origin == SeekOrigin.Current) { Guard.IsLessThanOrEqualTo(Position + offset, Length); - Position = Position + offset; + Position += offset; } return Position; @@ -124,7 +123,7 @@ public override void SetLength(long value) /// public override void Write(byte[] buffer, int offset, int count) { - WriteAsync(buffer, offset, count, CancellationToken.None).GetResultOrDefault(); + WriteAsync(buffer, offset, count, CancellationToken.None).Wait(); } /// @@ -153,7 +152,8 @@ public override async Task WriteAsync(byte[] buffer, int offset, int count, Canc SetLength(Position + count); } - await Client.Mfs.WriteAsync(Path, buffer, new() { Offset = Position, Count = count, Create = true }, cancellationToken); + await Client.Mfs.WriteAsync(Path, buffer.Skip(offset).ToArray(), new() { Offset = Position, Count = count, Create = true, Flush = false }, cancellationToken); + Position += count; } diff --git a/src/OwlCore.Kubo.csproj b/src/OwlCore.Kubo.csproj index 7402fa7..cfeb379 100644 --- a/src/OwlCore.Kubo.csproj +++ b/src/OwlCore.Kubo.csproj @@ -14,13 +14,30 @@ $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb Arlo Godfrey - 0.18.0 + 0.19.0 OwlCore An essential toolkit for Kubo, IPFS and the distributed web. LICENSE.txt +--- 0.19.0 --- +[New] +Added IAddFileToGetCid interface. This is functionally identical to IGetCid, but instead of simply returning a CID already in ipfs, it computes the CID by providing data to ipfs using preferences in the AddFileOptions parameter. + +[Breaking] +StorableKuboExtensions.GetCidAsync now takes an AddFileOptions parameter. +ContentAddressedSystemFile and ContentAddressedSystemFolder now implement IAddFileToGetCid instead of IGetCid. + +[Fixes] +Inherited fixes from OwlCore.ComponentModel 0.9.1. +Fixed issues with MfsStream where it would return before the task was complete. +MfsStream.ReadAsync and MfsStream.WriteAsync now respect the requested offset when operating on the provided buffer. + +[Improvement] +Updated to IpfsShipyard.Ipfs.Http.Client 0.5.1. +MfsStream.WriteAsync now supplies Flush = false when writing to mfs, instead of the default of flushing after every write. This improves performance when writing large files, but requires a manual call to FlushAsync to persist the changes. + --- 0.18.0 --- [Breaking] Inherited breaking changes from OwlCore.Storage 0.12.0 and OwlCore.ComponentModel 0.9.0. @@ -443,15 +460,15 @@ Added unit tests. - - - + + + - + - - + + diff --git a/tests/OwlCore.Kubo.Tests/OwlCore.Kubo.Tests.csproj b/tests/OwlCore.Kubo.Tests/OwlCore.Kubo.Tests.csproj index a312a73..247d00a 100644 --- a/tests/OwlCore.Kubo.Tests/OwlCore.Kubo.Tests.csproj +++ b/tests/OwlCore.Kubo.Tests/OwlCore.Kubo.Tests.csproj @@ -17,7 +17,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + runtime; build; native; contentfiles; analyzers; buildtransitive all