From e1c3fa80861d57ec270142cf8642de4b70475ad9 Mon Sep 17 00:00:00 2001 From: Matt Sherman Date: Fri, 21 Jun 2024 13:45:11 -0400 Subject: [PATCH] Deprecate the extension methods --- README.md | 9 ++++---- uax29/Examples.Test.cs | 4 ++-- uax29/README.md | 9 ++++---- uax29/StreamTokenizer.Test.cs | 4 ++-- uax29/Tokenizer.Graphemes.cs | 26 +++++++++++----------- uax29/Tokenizer.Sentences.cs | 26 +++++++++++----------- uax29/Tokenizer.Test.cs | 42 +---------------------------------- uax29/Tokenizer.Words.cs | 26 +++++++++++----------- 8 files changed, 52 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index b9db7b6..852e3c7 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ var example = "Hello, 🌏 world. δ½ ε₯½οΌŒδΈ–η•Œ."; // The tokenizer can split words, graphemes or sentences. // It operates on strings, UTF-8 bytes, and streams. -var words = example.GetWords(); +var words = Tokenizer.GetWords(example); // Iterate over the tokens foreach (var word in words) @@ -46,9 +46,8 @@ world . */ - var utf8bytes = Encoding.UTF8.GetBytes(example); -var graphemes = utf8bytes.GetGraphemes(); +var graphemes = Tokenizer.GetGraphemes(utf8bytes); // Iterate over the tokens foreach (var grapheme in graphemes) @@ -109,9 +108,9 @@ If you are using v1.x of this package, v2 has been renamed: We now use extension methods: -`Tokenizer.Create(input)` β†’ `input.GetWords()` +`Tokenizer.Create(input)` β†’ `Tokenizer.GetWords(input)` -`Tokenizer.Create(input, TokenType.Graphemes)` β†’ `input.GetGraphemes()` +`Tokenizer.Create(input, TokenType.Graphemes)` β†’ `Tokenizer.GetGraphemes(input)` ### Performance diff --git a/uax29/Examples.Test.cs b/uax29/Examples.Test.cs index 7634cfa..608d3d8 100644 --- a/uax29/Examples.Test.cs +++ b/uax29/Examples.Test.cs @@ -21,7 +21,7 @@ public void Readme() // The tokenizer can split words, graphemes or sentences. // It operates on strings, UTF-8 bytes, and streams. - var words = example.GetWords(); + var words = Tokenizer.GetWords(example); // Iterate over the tokens foreach (var word in words) @@ -49,7 +49,7 @@ public void Readme() */ var utf8bytes = Encoding.UTF8.GetBytes(example); - var graphemes = utf8bytes.GetGraphemes(); + var graphemes = Tokenizer.GetGraphemes(utf8bytes); // Iterate over the tokens foreach (var grapheme in graphemes) diff --git a/uax29/README.md b/uax29/README.md index b9db7b6..852e3c7 100644 --- a/uax29/README.md +++ b/uax29/README.md @@ -19,7 +19,7 @@ var example = "Hello, 🌏 world. δ½ ε₯½οΌŒδΈ–η•Œ."; // The tokenizer can split words, graphemes or sentences. // It operates on strings, UTF-8 bytes, and streams. -var words = example.GetWords(); +var words = Tokenizer.GetWords(example); // Iterate over the tokens foreach (var word in words) @@ -46,9 +46,8 @@ world . */ - var utf8bytes = Encoding.UTF8.GetBytes(example); -var graphemes = utf8bytes.GetGraphemes(); +var graphemes = Tokenizer.GetGraphemes(utf8bytes); // Iterate over the tokens foreach (var grapheme in graphemes) @@ -109,9 +108,9 @@ If you are using v1.x of this package, v2 has been renamed: We now use extension methods: -`Tokenizer.Create(input)` β†’ `input.GetWords()` +`Tokenizer.Create(input)` β†’ `Tokenizer.GetWords(input)` -`Tokenizer.Create(input, TokenType.Graphemes)` β†’ `input.GetGraphemes()` +`Tokenizer.Create(input, TokenType.Graphemes)` β†’ `Tokenizer.GetGraphemes(input)` ### Performance diff --git a/uax29/StreamTokenizer.Test.cs b/uax29/StreamTokenizer.Test.cs index 98676c3..a4a89c2 100644 --- a/uax29/StreamTokenizer.Test.cs +++ b/uax29/StreamTokenizer.Test.cs @@ -27,10 +27,10 @@ public void Stream() foreach (var input in examples) { var bytes = Encoding.UTF8.GetBytes(input); - var staticTokens = bytes.GetWords(); + var staticTokens = Tokenizer.GetWords(bytes); using var stream = new MemoryStream(bytes); - var streamTokens = stream.GetWords(); + var streamTokens = Tokenizer.GetWords(stream); foreach (var streamToken in streamTokens) { diff --git a/uax29/Tokenizer.Graphemes.cs b/uax29/Tokenizer.Graphemes.cs index 6f27f15..28e9116 100644 --- a/uax29/Tokenizer.Graphemes.cs +++ b/uax29/Tokenizer.Graphemes.cs @@ -11,7 +11,7 @@ public static partial class Tokenizer /// /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// - public static Tokenizer GetGraphemes(this Span input) => new(input, Graphemes.SplitUtf8Bytes); + public static Tokenizer GetGraphemes(Span input) => new(input, Graphemes.SplitUtf8Bytes); /// /// Split the graphemes in the given of UTF-8 encoded bytes, according to the Unicode UAX #29 spec. https://unicode.org/reports/tr29/ @@ -20,7 +20,7 @@ public static partial class Tokenizer /// /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// - public static Tokenizer GetGraphemes(this ReadOnlySpan input) => new(input, Graphemes.SplitUtf8Bytes); + public static Tokenizer GetGraphemes(ReadOnlySpan input) => new(input, Graphemes.SplitUtf8Bytes); /// /// Split the graphemes in the given of UTF-8 encoded bytes. @@ -29,7 +29,7 @@ public static partial class Tokenizer /// /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// - public static Tokenizer GetGraphemes(this Memory input) => new(input.Span, Graphemes.SplitUtf8Bytes); + public static Tokenizer GetGraphemes(Memory input) => new(input.Span, Graphemes.SplitUtf8Bytes); /// /// Split the graphemes in the given of UTF-8 encoded bytes. @@ -38,7 +38,7 @@ public static partial class Tokenizer /// /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// - public static Tokenizer GetGraphemes(this ReadOnlyMemory input) => new(input.Span, Graphemes.SplitUtf8Bytes); + public static Tokenizer GetGraphemes(ReadOnlyMemory input) => new(input.Span, Graphemes.SplitUtf8Bytes); /// /// Split the graphemes in the given array of UTF-8 encoded bytes. @@ -47,7 +47,7 @@ public static partial class Tokenizer /// /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// - public static Tokenizer GetGraphemes(this byte[] input) => new(input.AsSpan(), Graphemes.SplitUtf8Bytes); + public static Tokenizer GetGraphemes(byte[] input) => new(input.AsSpan(), Graphemes.SplitUtf8Bytes); /// /// Split the graphemes in the given string. @@ -56,7 +56,7 @@ public static partial class Tokenizer /// /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// - public static Tokenizer GetGraphemes(this string input) => new(input.AsSpan(), Graphemes.SplitChars); + public static Tokenizer GetGraphemes(string input) => new(input.AsSpan(), Graphemes.SplitChars); /// /// Split the graphemes in the given string. @@ -65,7 +65,7 @@ public static partial class Tokenizer /// /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// - public static Tokenizer GetGraphemes(this char[] input) => new(input.AsSpan(), Graphemes.SplitChars); + public static Tokenizer GetGraphemes(char[] input) => new(input.AsSpan(), Graphemes.SplitChars); /// /// Split the graphemes in the given of . @@ -75,7 +75,7 @@ public static partial class Tokenizer /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// /// - public static Tokenizer GetGraphemes(this Span input) => new(input, Graphemes.SplitChars); + public static Tokenizer GetGraphemes(Span input) => new(input, Graphemes.SplitChars); /// /// Split the graphemes in the given of . @@ -84,7 +84,7 @@ public static partial class Tokenizer /// /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// - public static Tokenizer GetGraphemes(this ReadOnlySpan input) => new(input, Graphemes.SplitChars); + public static Tokenizer GetGraphemes(ReadOnlySpan input) => new(input, Graphemes.SplitChars); /// /// Split the graphemes in the given of . @@ -93,7 +93,7 @@ public static partial class Tokenizer /// /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// - public static Tokenizer GetGraphemes(this Memory input) => new(input.Span, Graphemes.SplitChars); + public static Tokenizer GetGraphemes(Memory input) => new(input.Span, Graphemes.SplitChars); /// /// Split the graphemes in the given of . @@ -102,7 +102,7 @@ public static partial class Tokenizer /// /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// - public static Tokenizer GetGraphemes(this ReadOnlyMemory input) => new(input.Span, Graphemes.SplitChars); + public static Tokenizer GetGraphemes(ReadOnlyMemory input) => new(input.Span, Graphemes.SplitChars); /// /// Split the graphemes in the given of UTF-8 encoded bytes. @@ -127,7 +127,7 @@ public static partial class Tokenizer /// /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// - public static StreamTokenizer GetGraphemes(this Stream stream, int minBufferBytes = 1024, byte[]? bufferStorage = null) + public static StreamTokenizer GetGraphemes(Stream stream, int minBufferBytes = 1024, byte[]? bufferStorage = null) { bufferStorage ??= new byte[minBufferBytes * 2]; var buffer = new Buffer(stream.Read, minBufferBytes, bufferStorage); @@ -157,7 +157,7 @@ public static StreamTokenizer GetGraphemes(this Stream stream, int minBuff /// /// An enumerator of graphemes. Use foreach (var grapheme in graphemes). /// - public static StreamTokenizer GetGraphemes(this TextReader stream, int minBufferChars = 1024, char[]? bufferStorage = null) + public static StreamTokenizer GetGraphemes(TextReader stream, int minBufferChars = 1024, char[]? bufferStorage = null) { bufferStorage ??= new char[minBufferChars * 2]; var buffer = new Buffer(stream.Read, minBufferChars, bufferStorage); diff --git a/uax29/Tokenizer.Sentences.cs b/uax29/Tokenizer.Sentences.cs index 1bd5669..6442bca 100644 --- a/uax29/Tokenizer.Sentences.cs +++ b/uax29/Tokenizer.Sentences.cs @@ -11,7 +11,7 @@ public static partial class Tokenizer /// /// An enumerator of sentences. Use foreach (var sentence in sentences). /// - public static Tokenizer GetSentences(this Span input) => new(input, Sentences.SplitUtf8Bytes); + public static Tokenizer GetSentences(Span input) => new(input, Sentences.SplitUtf8Bytes); /// /// Split the sentences in the given of UTF-8 encoded bytes, according to the Unicode UAX #29 spec. https://unicode.org/reports/tr29/ @@ -20,7 +20,7 @@ public static partial class Tokenizer /// /// An enumerator of sentences. Use foreach (var sentence in sentences). /// - public static Tokenizer GetSentences(this ReadOnlySpan input) => new(input, Sentences.SplitUtf8Bytes); + public static Tokenizer GetSentences(ReadOnlySpan input) => new(input, Sentences.SplitUtf8Bytes); /// /// Split the sentences in the given of UTF-8 encoded bytes. @@ -29,7 +29,7 @@ public static partial class Tokenizer /// /// An enumerator of sentences. Use foreach (var sentence in sentences). /// - public static Tokenizer GetSentences(this Memory input) => new(input.Span, Sentences.SplitUtf8Bytes); + public static Tokenizer GetSentences(Memory input) => new(input.Span, Sentences.SplitUtf8Bytes); /// /// Split the sentences in the given of UTF-8 encoded bytes. @@ -38,7 +38,7 @@ public static partial class Tokenizer /// /// An enumerator of sentences. Use foreach (var sentence in sentences). /// - public static Tokenizer GetSentences(this ReadOnlyMemory input) => new(input.Span, Sentences.SplitUtf8Bytes); + public static Tokenizer GetSentences(ReadOnlyMemory input) => new(input.Span, Sentences.SplitUtf8Bytes); /// /// Split the sentences in the given array of UTF-8 encoded bytes. @@ -47,7 +47,7 @@ public static partial class Tokenizer /// /// An enumerator of sentences. Use foreach (var sentence in sentences). /// - public static Tokenizer GetSentences(this byte[] input) => new(input.AsSpan(), Sentences.SplitUtf8Bytes); + public static Tokenizer GetSentences(byte[] input) => new(input.AsSpan(), Sentences.SplitUtf8Bytes); /// /// Split the sentences in the given string. @@ -56,7 +56,7 @@ public static partial class Tokenizer /// /// An enumerator of sentences. Use foreach (var sentence in sentences). /// - public static Tokenizer GetSentences(this string input) => new(input.AsSpan(), Sentences.SplitChars); + public static Tokenizer GetSentences(string input) => new(input.AsSpan(), Sentences.SplitChars); /// /// Split the sentences in the given string. @@ -65,7 +65,7 @@ public static partial class Tokenizer /// /// An enumerator of sentences. Use foreach (var sentence in sentences). /// - public static Tokenizer GetSentences(this char[] input) => new(input.AsSpan(), Sentences.SplitChars); + public static Tokenizer GetSentences(char[] input) => new(input.AsSpan(), Sentences.SplitChars); /// /// Split the sentences in the given of . @@ -75,7 +75,7 @@ public static partial class Tokenizer /// An enumerator of sentences. Use foreach (var sentence in sentences). /// /// - public static Tokenizer GetSentences(this Span input) => new(input, Sentences.SplitChars); + public static Tokenizer GetSentences(Span input) => new(input, Sentences.SplitChars); /// /// Split the sentences in the given of . @@ -84,7 +84,7 @@ public static partial class Tokenizer /// /// An enumerator of sentences. Use foreach (var sentence in sentences). /// - public static Tokenizer GetSentences(this ReadOnlySpan input) => new(input, Sentences.SplitChars); + public static Tokenizer GetSentences(ReadOnlySpan input) => new(input, Sentences.SplitChars); /// /// Split the sentences in the given of . @@ -93,7 +93,7 @@ public static partial class Tokenizer /// /// An enumerator of sentences. Use foreach (var sentence in sentences). /// - public static Tokenizer GetSentences(this Memory input) => new(input.Span, Sentences.SplitChars); + public static Tokenizer GetSentences(Memory input) => new(input.Span, Sentences.SplitChars); /// /// Split the sentences in the given of . @@ -102,7 +102,7 @@ public static partial class Tokenizer /// /// An enumerator of sentences. Use foreach (var sentence in sentences). /// - public static Tokenizer GetSentences(this ReadOnlyMemory input) => new(input.Span, Sentences.SplitChars); + public static Tokenizer GetSentences(ReadOnlyMemory input) => new(input.Span, Sentences.SplitChars); @@ -129,7 +129,7 @@ public static partial class Tokenizer /// /// An enumerator of sentences. Use foreach (var sentence in sentences). /// - public static StreamTokenizer GetSentences(this Stream stream, int minBufferBytes = 1024, byte[]? bufferStorage = null) + public static StreamTokenizer GetSentences(Stream stream, int minBufferBytes = 1024, byte[]? bufferStorage = null) { bufferStorage ??= new byte[minBufferBytes * 2]; var buffer = new Buffer(stream.Read, minBufferBytes, bufferStorage); @@ -159,7 +159,7 @@ public static StreamTokenizer GetSentences(this Stream stream, int minBuff /// /// An enumerator of sentences. Use foreach (var sentence in sentences). /// - public static StreamTokenizer GetSentences(this TextReader stream, int minBufferChars = 1024, char[]? bufferStorage = null) + public static StreamTokenizer GetSentences(TextReader stream, int minBufferChars = 1024, char[]? bufferStorage = null) { bufferStorage ??= new char[minBufferChars * 2]; var buffer = new Buffer(stream.Read, minBufferChars, bufferStorage); diff --git a/uax29/Tokenizer.Test.cs b/uax29/Tokenizer.Test.cs index 2ffedf3..650517a 100644 --- a/uax29/Tokenizer.Test.cs +++ b/uax29/Tokenizer.Test.cs @@ -85,7 +85,6 @@ static int ExpectedOverloads() expected++; // Stream expected++; // TextReader - expected *= 2; // One regular call, one extension call expected *= 3; // Words, Graphemes, Sentences return expected; @@ -108,30 +107,23 @@ public void Overloads() { // chars - input.GetWords(); got++; Tokenizer.GetWords(input); got++; var array = input.ToCharArray(); - array.GetWords(); got++; Tokenizer.GetWords(array); got++; var span = new Span(array); - span.GetWords(); got++; Tokenizer.GetWords(span); got++; ReadOnlySpan rspan = input.AsSpan(); - rspan.GetWords(); got++; Tokenizer.GetWords(rspan); got++; var mem = new Memory(array); - mem.GetWords(); got++; Tokenizer.GetWords(mem); got++; ReadOnlyMemory rmem = input.AsMemory(); - rmem.GetWords(); got++; Tokenizer.GetWords(rmem); got++; - reader.GetWords(); got++; Tokenizer.GetWords(reader); got++; } @@ -139,30 +131,23 @@ public void Overloads() { // chars - input.GetGraphemes(); got++; Tokenizer.GetGraphemes(input); got++; var array = input.ToCharArray(); - array.GetGraphemes(); got++; Tokenizer.GetGraphemes(array); got++; var span = new Span(array); - span.GetGraphemes(); got++; Tokenizer.GetGraphemes(span); got++; ReadOnlySpan rspan = input.AsSpan(); - rspan.GetGraphemes(); got++; Tokenizer.GetGraphemes(rspan); got++; var mem = new Memory(array); - mem.GetGraphemes(); got++; Tokenizer.GetGraphemes(mem); got++; ReadOnlyMemory rmem = input.AsMemory(); - rmem.GetGraphemes(); got++; Tokenizer.GetGraphemes(rmem); got++; - reader.GetGraphemes(); got++; Tokenizer.GetGraphemes(reader); got++; } @@ -170,56 +155,43 @@ public void Overloads() { // chars - input.GetSentences(); got++; Tokenizer.GetSentences(input); got++; var array = input.ToCharArray(); - array.GetSentences(); got++; Tokenizer.GetSentences(array); got++; var span = new Span(array); - span.GetSentences(); got++; Tokenizer.GetSentences(span); got++; ReadOnlySpan rspan = input.AsSpan(); - rspan.GetSentences(); got++; Tokenizer.GetSentences(rspan); got++; var mem = new Memory(array); - mem.GetSentences(); got++; Tokenizer.GetSentences(mem); got++; ReadOnlyMemory rmem = input.AsMemory(); - rmem.GetSentences(); got++; Tokenizer.GetSentences(rmem); got++; - reader.GetSentences(); got++; Tokenizer.GetSentences(reader); got++; } { // bytes - bytes.GetWords(); got++; Tokenizer.GetWords(bytes); got++; Span span = bytes.AsSpan(); - span.GetWords(); got++; Tokenizer.GetWords(span); got++; ReadOnlySpan rspan = bytes.AsSpan(); - rspan.GetWords(); got++; Tokenizer.GetWords(rspan); got++; Memory mem = bytes.AsMemory(); - mem.GetWords(); got++; Tokenizer.GetWords(mem); got++; ReadOnlyMemory rmem = bytes.AsMemory(); - rmem.GetWords(); got++; Tokenizer.GetWords(rmem); got++; - stream.GetWords(); got++; Tokenizer.GetWords(stream); got++; } @@ -227,26 +199,20 @@ public void Overloads() { // bytes - bytes.GetGraphemes(); got++; Tokenizer.GetGraphemes(bytes); got++; Span span = bytes.AsSpan(); - span.GetGraphemes(); got++; Tokenizer.GetGraphemes(span); got++; ReadOnlySpan rspan = bytes.AsSpan(); - rspan.GetGraphemes(); got++; Tokenizer.GetGraphemes(rspan); got++; Memory mem = bytes.AsMemory(); - mem.GetGraphemes(); got++; Tokenizer.GetGraphemes(mem); got++; ReadOnlyMemory rmem = bytes.AsMemory(); - rmem.GetGraphemes(); got++; Tokenizer.GetGraphemes(rmem); got++; - stream.GetGraphemes(); got++; Tokenizer.GetGraphemes(stream); got++; } @@ -254,26 +220,20 @@ public void Overloads() { // bytes - bytes.GetSentences(); got++; Tokenizer.GetSentences(bytes); got++; Span span = bytes.AsSpan(); - span.GetSentences(); got++; Tokenizer.GetSentences(span); got++; ReadOnlySpan rspan = bytes.AsSpan(); - rspan.GetSentences(); got++; Tokenizer.GetSentences(rspan); got++; Memory mem = bytes.AsMemory(); - mem.GetSentences(); got++; Tokenizer.GetSentences(mem); got++; ReadOnlyMemory rmem = bytes.AsMemory(); - rmem.GetSentences(); got++; Tokenizer.GetSentences(rmem); got++; - stream.GetSentences(); got++; Tokenizer.GetSentences(stream); got++; } @@ -286,7 +246,7 @@ public void Enumerator() var input = "Hello, how are you?"; var mem = input.AsMemory(); var bytes = Encoding.UTF8.GetBytes(input); - mem.GetWords(); + Tokenizer.GetWords(mem); var tokens = Tokenizer.GetWords(input); var first = new List(); diff --git a/uax29/Tokenizer.Words.cs b/uax29/Tokenizer.Words.cs index 4ae14df..ff4ee9c 100644 --- a/uax29/Tokenizer.Words.cs +++ b/uax29/Tokenizer.Words.cs @@ -11,7 +11,7 @@ public static partial class Tokenizer /// /// An enumerator of words. Use foreach (var word in words). /// - public static Tokenizer GetWords(this Span input) => new(input, Words.SplitUtf8Bytes); + public static Tokenizer GetWords(Span input) => new(input, Words.SplitUtf8Bytes); /// /// Split the words in the given of UTF-8 encoded bytes, according to the Unicode UAX #29 spec. https://unicode.org/reports/tr29/ @@ -20,7 +20,7 @@ public static partial class Tokenizer /// /// An enumerator of words. Use foreach (var word in words). /// - public static Tokenizer GetWords(this ReadOnlySpan input) => new(input, Words.SplitUtf8Bytes); + public static Tokenizer GetWords(ReadOnlySpan input) => new(input, Words.SplitUtf8Bytes); /// /// Split the words in the given of UTF-8 encoded bytes. @@ -29,7 +29,7 @@ public static partial class Tokenizer /// /// An enumerator of words. Use foreach (var word in words). /// - public static Tokenizer GetWords(this Memory input) => new(input.Span, Words.SplitUtf8Bytes); + public static Tokenizer GetWords(Memory input) => new(input.Span, Words.SplitUtf8Bytes); /// /// Split the words in the given of UTF-8 encoded bytes. @@ -38,7 +38,7 @@ public static partial class Tokenizer /// /// An enumerator of words. Use foreach (var word in words). /// - public static Tokenizer GetWords(this ReadOnlyMemory input) => new(input.Span, Words.SplitUtf8Bytes); + public static Tokenizer GetWords(ReadOnlyMemory input) => new(input.Span, Words.SplitUtf8Bytes); /// /// Split the words in the given array of UTF-8 encoded bytes. @@ -47,7 +47,7 @@ public static partial class Tokenizer /// /// An enumerator of words. Use foreach (var word in words). /// - public static Tokenizer GetWords(this byte[] input) => new(input.AsSpan(), Words.SplitUtf8Bytes); + public static Tokenizer GetWords(byte[] input) => new(input.AsSpan(), Words.SplitUtf8Bytes); /// /// Split the words in the given string. @@ -56,7 +56,7 @@ public static partial class Tokenizer /// /// An enumerator of words. Use foreach (var word in words). /// - public static Tokenizer GetWords(this string input) => new(input.AsSpan(), Words.SplitChars); + public static Tokenizer GetWords(string input) => new(input.AsSpan(), Words.SplitChars); /// /// Split the words in the given string. @@ -65,7 +65,7 @@ public static partial class Tokenizer /// /// An enumerator of words. Use foreach (var word in words). /// - public static Tokenizer GetWords(this char[] input) => new(input.AsSpan(), Words.SplitChars); + public static Tokenizer GetWords(char[] input) => new(input.AsSpan(), Words.SplitChars); /// /// Split the words in the given of . @@ -75,7 +75,7 @@ public static partial class Tokenizer /// An enumerator of words. Use foreach (var word in words). /// /// - public static Tokenizer GetWords(this Span input) => new(input, Words.SplitChars); + public static Tokenizer GetWords(Span input) => new(input, Words.SplitChars); /// /// Split the words in the given of . @@ -84,7 +84,7 @@ public static partial class Tokenizer /// /// An enumerator of words. Use foreach (var word in words). /// - public static Tokenizer GetWords(this ReadOnlySpan input) => new(input, Words.SplitChars); + public static Tokenizer GetWords(ReadOnlySpan input) => new(input, Words.SplitChars); /// /// Split the words in the given of . @@ -93,7 +93,7 @@ public static partial class Tokenizer /// /// An enumerator of words. Use foreach (var word in words). /// - public static Tokenizer GetWords(this Memory input) => new(input.Span, Words.SplitChars); + public static Tokenizer GetWords(Memory input) => new(input.Span, Words.SplitChars); /// /// Split the words in the given of . @@ -102,7 +102,7 @@ public static partial class Tokenizer /// /// An enumerator of words. Use foreach (var word in words). /// - public static Tokenizer GetWords(this ReadOnlyMemory input) => new(input.Span, Words.SplitChars); + public static Tokenizer GetWords(ReadOnlyMemory input) => new(input.Span, Words.SplitChars); /// /// Split the words in the given of UTF-8 encoded bytes. @@ -127,7 +127,7 @@ public static partial class Tokenizer /// /// An enumerator of words. Use foreach (var word in words). /// - public static StreamTokenizer GetWords(this Stream stream, int minBufferBytes = 1024, byte[]? bufferStorage = null) + public static StreamTokenizer GetWords(Stream stream, int minBufferBytes = 1024, byte[]? bufferStorage = null) { bufferStorage ??= new byte[minBufferBytes * 2]; var buffer = new Buffer(stream.Read, minBufferBytes, bufferStorage); @@ -157,7 +157,7 @@ public static StreamTokenizer GetWords(this Stream stream, int minBufferBy /// /// An enumerator of words. Use foreach (var word in words). /// - public static StreamTokenizer GetWords(this TextReader stream, int minBufferChars = 1024, char[]? bufferStorage = null) + public static StreamTokenizer GetWords(TextReader stream, int minBufferChars = 1024, char[]? bufferStorage = null) { bufferStorage ??= new char[minBufferChars * 2]; var buffer = new Buffer(stream.Read, minBufferChars, bufferStorage);