diff --git a/README.md b/README.md
index b9db7b6..852e3c7 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ var example = "Hello, π world. δ½ ε₯½οΌδΈη.";
// The tokenizer can split words, graphemes or sentences.
// It operates on strings, UTF-8 bytes, and streams.
-var words = example.GetWords();
+var words = Tokenizer.GetWords(example);
// Iterate over the tokens
foreach (var word in words)
@@ -46,9 +46,8 @@ world
.
*/
-
var utf8bytes = Encoding.UTF8.GetBytes(example);
-var graphemes = utf8bytes.GetGraphemes();
+var graphemes = Tokenizer.GetGraphemes(utf8bytes);
// Iterate over the tokens
foreach (var grapheme in graphemes)
@@ -109,9 +108,9 @@ If you are using v1.x of this package, v2 has been renamed:
We now use extension methods:
-`Tokenizer.Create(input)` β `input.GetWords()`
+`Tokenizer.Create(input)` β `Tokenizer.GetWords(input)`
-`Tokenizer.Create(input, TokenType.Graphemes)` β `input.GetGraphemes()`
+`Tokenizer.Create(input, TokenType.Graphemes)` β `Tokenizer.GetGraphemes(input)`
### Performance
diff --git a/uax29/Examples.Test.cs b/uax29/Examples.Test.cs
index 7634cfa..608d3d8 100644
--- a/uax29/Examples.Test.cs
+++ b/uax29/Examples.Test.cs
@@ -21,7 +21,7 @@ public void Readme()
// The tokenizer can split words, graphemes or sentences.
// It operates on strings, UTF-8 bytes, and streams.
- var words = example.GetWords();
+ var words = Tokenizer.GetWords(example);
// Iterate over the tokens
foreach (var word in words)
@@ -49,7 +49,7 @@ public void Readme()
*/
var utf8bytes = Encoding.UTF8.GetBytes(example);
- var graphemes = utf8bytes.GetGraphemes();
+ var graphemes = Tokenizer.GetGraphemes(utf8bytes);
// Iterate over the tokens
foreach (var grapheme in graphemes)
diff --git a/uax29/README.md b/uax29/README.md
index b9db7b6..852e3c7 100644
--- a/uax29/README.md
+++ b/uax29/README.md
@@ -19,7 +19,7 @@ var example = "Hello, π world. δ½ ε₯½οΌδΈη.";
// The tokenizer can split words, graphemes or sentences.
// It operates on strings, UTF-8 bytes, and streams.
-var words = example.GetWords();
+var words = Tokenizer.GetWords(example);
// Iterate over the tokens
foreach (var word in words)
@@ -46,9 +46,8 @@ world
.
*/
-
var utf8bytes = Encoding.UTF8.GetBytes(example);
-var graphemes = utf8bytes.GetGraphemes();
+var graphemes = Tokenizer.GetGraphemes(utf8bytes);
// Iterate over the tokens
foreach (var grapheme in graphemes)
@@ -109,9 +108,9 @@ If you are using v1.x of this package, v2 has been renamed:
We now use extension methods:
-`Tokenizer.Create(input)` β `input.GetWords()`
+`Tokenizer.Create(input)` β `Tokenizer.GetWords(input)`
-`Tokenizer.Create(input, TokenType.Graphemes)` β `input.GetGraphemes()`
+`Tokenizer.Create(input, TokenType.Graphemes)` β `Tokenizer.GetGraphemes(input)`
### Performance
diff --git a/uax29/StreamTokenizer.Test.cs b/uax29/StreamTokenizer.Test.cs
index 98676c3..a4a89c2 100644
--- a/uax29/StreamTokenizer.Test.cs
+++ b/uax29/StreamTokenizer.Test.cs
@@ -27,10 +27,10 @@ public void Stream()
foreach (var input in examples)
{
var bytes = Encoding.UTF8.GetBytes(input);
- var staticTokens = bytes.GetWords();
+ var staticTokens = Tokenizer.GetWords(bytes);
using var stream = new MemoryStream(bytes);
- var streamTokens = stream.GetWords();
+ var streamTokens = Tokenizer.GetWords(stream);
foreach (var streamToken in streamTokens)
{
diff --git a/uax29/Tokenizer.Graphemes.cs b/uax29/Tokenizer.Graphemes.cs
index 6f27f15..28e9116 100644
--- a/uax29/Tokenizer.Graphemes.cs
+++ b/uax29/Tokenizer.Graphemes.cs
@@ -11,7 +11,7 @@ public static partial class Tokenizer
///
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
- public static Tokenizer GetGraphemes(this Span input) => new(input, Graphemes.SplitUtf8Bytes);
+ public static Tokenizer GetGraphemes(Span input) => new(input, Graphemes.SplitUtf8Bytes);
///
/// Split the graphemes in the given of UTF-8 encoded bytes, according to the Unicode UAX #29 spec. https://unicode.org/reports/tr29/
@@ -20,7 +20,7 @@ public static partial class Tokenizer
///
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
- public static Tokenizer GetGraphemes(this ReadOnlySpan input) => new(input, Graphemes.SplitUtf8Bytes);
+ public static Tokenizer GetGraphemes(ReadOnlySpan input) => new(input, Graphemes.SplitUtf8Bytes);
///
/// Split the graphemes in the given of UTF-8 encoded bytes.
@@ -29,7 +29,7 @@ public static partial class Tokenizer
///
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
- public static Tokenizer GetGraphemes(this Memory input) => new(input.Span, Graphemes.SplitUtf8Bytes);
+ public static Tokenizer GetGraphemes(Memory input) => new(input.Span, Graphemes.SplitUtf8Bytes);
///
/// Split the graphemes in the given of UTF-8 encoded bytes.
@@ -38,7 +38,7 @@ public static partial class Tokenizer
///
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
- public static Tokenizer GetGraphemes(this ReadOnlyMemory input) => new(input.Span, Graphemes.SplitUtf8Bytes);
+ public static Tokenizer GetGraphemes(ReadOnlyMemory input) => new(input.Span, Graphemes.SplitUtf8Bytes);
///
/// Split the graphemes in the given array of UTF-8 encoded bytes.
@@ -47,7 +47,7 @@ public static partial class Tokenizer
///
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
- public static Tokenizer GetGraphemes(this byte[] input) => new(input.AsSpan(), Graphemes.SplitUtf8Bytes);
+ public static Tokenizer GetGraphemes(byte[] input) => new(input.AsSpan(), Graphemes.SplitUtf8Bytes);
///
/// Split the graphemes in the given string.
@@ -56,7 +56,7 @@ public static partial class Tokenizer
///
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
- public static Tokenizer GetGraphemes(this string input) => new(input.AsSpan(), Graphemes.SplitChars);
+ public static Tokenizer GetGraphemes(string input) => new(input.AsSpan(), Graphemes.SplitChars);
///
/// Split the graphemes in the given string.
@@ -65,7 +65,7 @@ public static partial class Tokenizer
///
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
- public static Tokenizer GetGraphemes(this char[] input) => new(input.AsSpan(), Graphemes.SplitChars);
+ public static Tokenizer GetGraphemes(char[] input) => new(input.AsSpan(), Graphemes.SplitChars);
///
/// Split the graphemes in the given of .
@@ -75,7 +75,7 @@ public static partial class Tokenizer
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
///
- public static Tokenizer GetGraphemes(this Span input) => new(input, Graphemes.SplitChars);
+ public static Tokenizer GetGraphemes(Span input) => new(input, Graphemes.SplitChars);
///
/// Split the graphemes in the given of .
@@ -84,7 +84,7 @@ public static partial class Tokenizer
///
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
- public static Tokenizer GetGraphemes(this ReadOnlySpan input) => new(input, Graphemes.SplitChars);
+ public static Tokenizer GetGraphemes(ReadOnlySpan input) => new(input, Graphemes.SplitChars);
///
/// Split the graphemes in the given of .
@@ -93,7 +93,7 @@ public static partial class Tokenizer
///
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
- public static Tokenizer GetGraphemes(this Memory input) => new(input.Span, Graphemes.SplitChars);
+ public static Tokenizer GetGraphemes(Memory input) => new(input.Span, Graphemes.SplitChars);
///
/// Split the graphemes in the given of .
@@ -102,7 +102,7 @@ public static partial class Tokenizer
///
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
- public static Tokenizer GetGraphemes(this ReadOnlyMemory input) => new(input.Span, Graphemes.SplitChars);
+ public static Tokenizer GetGraphemes(ReadOnlyMemory input) => new(input.Span, Graphemes.SplitChars);
///
/// Split the graphemes in the given of UTF-8 encoded bytes.
@@ -127,7 +127,7 @@ public static partial class Tokenizer
///
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
- public static StreamTokenizer GetGraphemes(this Stream stream, int minBufferBytes = 1024, byte[]? bufferStorage = null)
+ public static StreamTokenizer GetGraphemes(Stream stream, int minBufferBytes = 1024, byte[]? bufferStorage = null)
{
bufferStorage ??= new byte[minBufferBytes * 2];
var buffer = new Buffer(stream.Read, minBufferBytes, bufferStorage);
@@ -157,7 +157,7 @@ public static StreamTokenizer GetGraphemes(this Stream stream, int minBuff
///
/// An enumerator of graphemes. Use foreach (var grapheme in graphemes).
///
- public static StreamTokenizer GetGraphemes(this TextReader stream, int minBufferChars = 1024, char[]? bufferStorage = null)
+ public static StreamTokenizer GetGraphemes(TextReader stream, int minBufferChars = 1024, char[]? bufferStorage = null)
{
bufferStorage ??= new char[minBufferChars * 2];
var buffer = new Buffer(stream.Read, minBufferChars, bufferStorage);
diff --git a/uax29/Tokenizer.Sentences.cs b/uax29/Tokenizer.Sentences.cs
index 1bd5669..6442bca 100644
--- a/uax29/Tokenizer.Sentences.cs
+++ b/uax29/Tokenizer.Sentences.cs
@@ -11,7 +11,7 @@ public static partial class Tokenizer
///
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
- public static Tokenizer GetSentences(this Span input) => new(input, Sentences.SplitUtf8Bytes);
+ public static Tokenizer GetSentences(Span input) => new(input, Sentences.SplitUtf8Bytes);
///
/// Split the sentences in the given of UTF-8 encoded bytes, according to the Unicode UAX #29 spec. https://unicode.org/reports/tr29/
@@ -20,7 +20,7 @@ public static partial class Tokenizer
///
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
- public static Tokenizer GetSentences(this ReadOnlySpan input) => new(input, Sentences.SplitUtf8Bytes);
+ public static Tokenizer GetSentences(ReadOnlySpan input) => new(input, Sentences.SplitUtf8Bytes);
///
/// Split the sentences in the given of UTF-8 encoded bytes.
@@ -29,7 +29,7 @@ public static partial class Tokenizer
///
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
- public static Tokenizer GetSentences(this Memory input) => new(input.Span, Sentences.SplitUtf8Bytes);
+ public static Tokenizer GetSentences(Memory input) => new(input.Span, Sentences.SplitUtf8Bytes);
///
/// Split the sentences in the given of UTF-8 encoded bytes.
@@ -38,7 +38,7 @@ public static partial class Tokenizer
///
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
- public static Tokenizer GetSentences(this ReadOnlyMemory input) => new(input.Span, Sentences.SplitUtf8Bytes);
+ public static Tokenizer GetSentences(ReadOnlyMemory input) => new(input.Span, Sentences.SplitUtf8Bytes);
///
/// Split the sentences in the given array of UTF-8 encoded bytes.
@@ -47,7 +47,7 @@ public static partial class Tokenizer
///
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
- public static Tokenizer GetSentences(this byte[] input) => new(input.AsSpan(), Sentences.SplitUtf8Bytes);
+ public static Tokenizer GetSentences(byte[] input) => new(input.AsSpan(), Sentences.SplitUtf8Bytes);
///
/// Split the sentences in the given string.
@@ -56,7 +56,7 @@ public static partial class Tokenizer
///
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
- public static Tokenizer GetSentences(this string input) => new(input.AsSpan(), Sentences.SplitChars);
+ public static Tokenizer GetSentences(string input) => new(input.AsSpan(), Sentences.SplitChars);
///
/// Split the sentences in the given string.
@@ -65,7 +65,7 @@ public static partial class Tokenizer
///
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
- public static Tokenizer GetSentences(this char[] input) => new(input.AsSpan(), Sentences.SplitChars);
+ public static Tokenizer GetSentences(char[] input) => new(input.AsSpan(), Sentences.SplitChars);
///
/// Split the sentences in the given of .
@@ -75,7 +75,7 @@ public static partial class Tokenizer
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
///
- public static Tokenizer GetSentences(this Span input) => new(input, Sentences.SplitChars);
+ public static Tokenizer GetSentences(Span input) => new(input, Sentences.SplitChars);
///
/// Split the sentences in the given of .
@@ -84,7 +84,7 @@ public static partial class Tokenizer
///
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
- public static Tokenizer GetSentences(this ReadOnlySpan input) => new(input, Sentences.SplitChars);
+ public static Tokenizer GetSentences(ReadOnlySpan input) => new(input, Sentences.SplitChars);
///
/// Split the sentences in the given of .
@@ -93,7 +93,7 @@ public static partial class Tokenizer
///
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
- public static Tokenizer GetSentences(this Memory input) => new(input.Span, Sentences.SplitChars);
+ public static Tokenizer GetSentences(Memory input) => new(input.Span, Sentences.SplitChars);
///
/// Split the sentences in the given of .
@@ -102,7 +102,7 @@ public static partial class Tokenizer
///
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
- public static Tokenizer GetSentences(this ReadOnlyMemory input) => new(input.Span, Sentences.SplitChars);
+ public static Tokenizer GetSentences(ReadOnlyMemory input) => new(input.Span, Sentences.SplitChars);
@@ -129,7 +129,7 @@ public static partial class Tokenizer
///
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
- public static StreamTokenizer GetSentences(this Stream stream, int minBufferBytes = 1024, byte[]? bufferStorage = null)
+ public static StreamTokenizer GetSentences(Stream stream, int minBufferBytes = 1024, byte[]? bufferStorage = null)
{
bufferStorage ??= new byte[minBufferBytes * 2];
var buffer = new Buffer(stream.Read, minBufferBytes, bufferStorage);
@@ -159,7 +159,7 @@ public static StreamTokenizer GetSentences(this Stream stream, int minBuff
///
/// An enumerator of sentences. Use foreach (var sentence in sentences).
///
- public static StreamTokenizer GetSentences(this TextReader stream, int minBufferChars = 1024, char[]? bufferStorage = null)
+ public static StreamTokenizer GetSentences(TextReader stream, int minBufferChars = 1024, char[]? bufferStorage = null)
{
bufferStorage ??= new char[minBufferChars * 2];
var buffer = new Buffer(stream.Read, minBufferChars, bufferStorage);
diff --git a/uax29/Tokenizer.Test.cs b/uax29/Tokenizer.Test.cs
index 2ffedf3..650517a 100644
--- a/uax29/Tokenizer.Test.cs
+++ b/uax29/Tokenizer.Test.cs
@@ -85,7 +85,6 @@ static int ExpectedOverloads()
expected++; // Stream
expected++; // TextReader
- expected *= 2; // One regular call, one extension call
expected *= 3; // Words, Graphemes, Sentences
return expected;
@@ -108,30 +107,23 @@ public void Overloads()
{
// chars
- input.GetWords(); got++;
Tokenizer.GetWords(input); got++;
var array = input.ToCharArray();
- array.GetWords(); got++;
Tokenizer.GetWords(array); got++;
var span = new Span(array);
- span.GetWords(); got++;
Tokenizer.GetWords(span); got++;
ReadOnlySpan rspan = input.AsSpan();
- rspan.GetWords(); got++;
Tokenizer.GetWords(rspan); got++;
var mem = new Memory(array);
- mem.GetWords(); got++;
Tokenizer.GetWords(mem); got++;
ReadOnlyMemory rmem = input.AsMemory();
- rmem.GetWords(); got++;
Tokenizer.GetWords(rmem); got++;
- reader.GetWords(); got++;
Tokenizer.GetWords(reader); got++;
}
@@ -139,30 +131,23 @@ public void Overloads()
{
// chars
- input.GetGraphemes(); got++;
Tokenizer.GetGraphemes(input); got++;
var array = input.ToCharArray();
- array.GetGraphemes(); got++;
Tokenizer.GetGraphemes(array); got++;
var span = new Span(array);
- span.GetGraphemes(); got++;
Tokenizer.GetGraphemes(span); got++;
ReadOnlySpan rspan = input.AsSpan();
- rspan.GetGraphemes(); got++;
Tokenizer.GetGraphemes(rspan); got++;
var mem = new Memory(array);
- mem.GetGraphemes(); got++;
Tokenizer.GetGraphemes(mem); got++;
ReadOnlyMemory rmem = input.AsMemory();
- rmem.GetGraphemes(); got++;
Tokenizer.GetGraphemes(rmem); got++;
- reader.GetGraphemes(); got++;
Tokenizer.GetGraphemes(reader); got++;
}
@@ -170,56 +155,43 @@ public void Overloads()
{
// chars
- input.GetSentences(); got++;
Tokenizer.GetSentences(input); got++;
var array = input.ToCharArray();
- array.GetSentences(); got++;
Tokenizer.GetSentences(array); got++;
var span = new Span(array);
- span.GetSentences(); got++;
Tokenizer.GetSentences(span); got++;
ReadOnlySpan rspan = input.AsSpan();
- rspan.GetSentences(); got++;
Tokenizer.GetSentences(rspan); got++;
var mem = new Memory(array);
- mem.GetSentences(); got++;
Tokenizer.GetSentences(mem); got++;
ReadOnlyMemory rmem = input.AsMemory();
- rmem.GetSentences(); got++;
Tokenizer.GetSentences(rmem); got++;
- reader.GetSentences(); got++;
Tokenizer.GetSentences(reader); got++;
}
{
// bytes
- bytes.GetWords(); got++;
Tokenizer.GetWords(bytes); got++;
Span span = bytes.AsSpan();
- span.GetWords(); got++;
Tokenizer.GetWords(span); got++;
ReadOnlySpan rspan = bytes.AsSpan();
- rspan.GetWords(); got++;
Tokenizer.GetWords(rspan); got++;
Memory mem = bytes.AsMemory();
- mem.GetWords(); got++;
Tokenizer.GetWords(mem); got++;
ReadOnlyMemory rmem = bytes.AsMemory();
- rmem.GetWords(); got++;
Tokenizer.GetWords(rmem); got++;
- stream.GetWords(); got++;
Tokenizer.GetWords(stream); got++;
}
@@ -227,26 +199,20 @@ public void Overloads()
{
// bytes
- bytes.GetGraphemes(); got++;
Tokenizer.GetGraphemes(bytes); got++;
Span span = bytes.AsSpan();
- span.GetGraphemes(); got++;
Tokenizer.GetGraphemes(span); got++;
ReadOnlySpan rspan = bytes.AsSpan();
- rspan.GetGraphemes(); got++;
Tokenizer.GetGraphemes(rspan); got++;
Memory mem = bytes.AsMemory();
- mem.GetGraphemes(); got++;
Tokenizer.GetGraphemes(mem); got++;
ReadOnlyMemory rmem = bytes.AsMemory();
- rmem.GetGraphemes(); got++;
Tokenizer.GetGraphemes(rmem); got++;
- stream.GetGraphemes(); got++;
Tokenizer.GetGraphemes(stream); got++;
}
@@ -254,26 +220,20 @@ public void Overloads()
{
// bytes
- bytes.GetSentences(); got++;
Tokenizer.GetSentences(bytes); got++;
Span span = bytes.AsSpan();
- span.GetSentences(); got++;
Tokenizer.GetSentences(span); got++;
ReadOnlySpan rspan = bytes.AsSpan();
- rspan.GetSentences(); got++;
Tokenizer.GetSentences(rspan); got++;
Memory mem = bytes.AsMemory();
- mem.GetSentences(); got++;
Tokenizer.GetSentences(mem); got++;
ReadOnlyMemory rmem = bytes.AsMemory();
- rmem.GetSentences(); got++;
Tokenizer.GetSentences(rmem); got++;
- stream.GetSentences(); got++;
Tokenizer.GetSentences(stream); got++;
}
@@ -286,7 +246,7 @@ public void Enumerator()
var input = "Hello, how are you?";
var mem = input.AsMemory();
var bytes = Encoding.UTF8.GetBytes(input);
- mem.GetWords();
+ Tokenizer.GetWords(mem);
var tokens = Tokenizer.GetWords(input);
var first = new List();
diff --git a/uax29/Tokenizer.Words.cs b/uax29/Tokenizer.Words.cs
index 4ae14df..ff4ee9c 100644
--- a/uax29/Tokenizer.Words.cs
+++ b/uax29/Tokenizer.Words.cs
@@ -11,7 +11,7 @@ public static partial class Tokenizer
///
/// An enumerator of words. Use foreach (var word in words).
///
- public static Tokenizer GetWords(this Span input) => new(input, Words.SplitUtf8Bytes);
+ public static Tokenizer GetWords(Span input) => new(input, Words.SplitUtf8Bytes);
///
/// Split the words in the given of UTF-8 encoded bytes, according to the Unicode UAX #29 spec. https://unicode.org/reports/tr29/
@@ -20,7 +20,7 @@ public static partial class Tokenizer
///
/// An enumerator of words. Use foreach (var word in words).
///
- public static Tokenizer GetWords(this ReadOnlySpan input) => new(input, Words.SplitUtf8Bytes);
+ public static Tokenizer GetWords(ReadOnlySpan input) => new(input, Words.SplitUtf8Bytes);
///
/// Split the words in the given of UTF-8 encoded bytes.
@@ -29,7 +29,7 @@ public static partial class Tokenizer
///
/// An enumerator of words. Use foreach (var word in words).
///
- public static Tokenizer GetWords(this Memory input) => new(input.Span, Words.SplitUtf8Bytes);
+ public static Tokenizer GetWords(Memory input) => new(input.Span, Words.SplitUtf8Bytes);
///
/// Split the words in the given of UTF-8 encoded bytes.
@@ -38,7 +38,7 @@ public static partial class Tokenizer
///
/// An enumerator of words. Use foreach (var word in words).
///
- public static Tokenizer GetWords(this ReadOnlyMemory input) => new(input.Span, Words.SplitUtf8Bytes);
+ public static Tokenizer GetWords(ReadOnlyMemory input) => new(input.Span, Words.SplitUtf8Bytes);
///
/// Split the words in the given array of UTF-8 encoded bytes.
@@ -47,7 +47,7 @@ public static partial class Tokenizer
///
/// An enumerator of words. Use foreach (var word in words).
///
- public static Tokenizer GetWords(this byte[] input) => new(input.AsSpan(), Words.SplitUtf8Bytes);
+ public static Tokenizer GetWords(byte[] input) => new(input.AsSpan(), Words.SplitUtf8Bytes);
///
/// Split the words in the given string.
@@ -56,7 +56,7 @@ public static partial class Tokenizer
///
/// An enumerator of words. Use foreach (var word in words).
///
- public static Tokenizer GetWords(this string input) => new(input.AsSpan(), Words.SplitChars);
+ public static Tokenizer GetWords(string input) => new(input.AsSpan(), Words.SplitChars);
///
/// Split the words in the given string.
@@ -65,7 +65,7 @@ public static partial class Tokenizer
///
/// An enumerator of words. Use foreach (var word in words).
///
- public static Tokenizer GetWords(this char[] input) => new(input.AsSpan(), Words.SplitChars);
+ public static Tokenizer GetWords(char[] input) => new(input.AsSpan(), Words.SplitChars);
///
/// Split the words in the given of .
@@ -75,7 +75,7 @@ public static partial class Tokenizer
/// An enumerator of words. Use foreach (var word in words).
///
///
- public static Tokenizer GetWords(this Span input) => new(input, Words.SplitChars);
+ public static Tokenizer GetWords(Span input) => new(input, Words.SplitChars);
///
/// Split the words in the given of .
@@ -84,7 +84,7 @@ public static partial class Tokenizer
///
/// An enumerator of words. Use foreach (var word in words).
///
- public static Tokenizer GetWords(this ReadOnlySpan input) => new(input, Words.SplitChars);
+ public static Tokenizer GetWords(ReadOnlySpan input) => new(input, Words.SplitChars);
///
/// Split the words in the given of .
@@ -93,7 +93,7 @@ public static partial class Tokenizer
///
/// An enumerator of words. Use foreach (var word in words).
///
- public static Tokenizer GetWords(this Memory input) => new(input.Span, Words.SplitChars);
+ public static Tokenizer GetWords(Memory input) => new(input.Span, Words.SplitChars);
///
/// Split the words in the given of .
@@ -102,7 +102,7 @@ public static partial class Tokenizer
///
/// An enumerator of words. Use foreach (var word in words).
///
- public static Tokenizer GetWords(this ReadOnlyMemory input) => new(input.Span, Words.SplitChars);
+ public static Tokenizer GetWords(ReadOnlyMemory input) => new(input.Span, Words.SplitChars);
///
/// Split the words in the given of UTF-8 encoded bytes.
@@ -127,7 +127,7 @@ public static partial class Tokenizer
///
/// An enumerator of words. Use foreach (var word in words).
///
- public static StreamTokenizer GetWords(this Stream stream, int minBufferBytes = 1024, byte[]? bufferStorage = null)
+ public static StreamTokenizer GetWords(Stream stream, int minBufferBytes = 1024, byte[]? bufferStorage = null)
{
bufferStorage ??= new byte[minBufferBytes * 2];
var buffer = new Buffer(stream.Read, minBufferBytes, bufferStorage);
@@ -157,7 +157,7 @@ public static StreamTokenizer GetWords(this Stream stream, int minBufferBy
///
/// An enumerator of words. Use foreach (var word in words).
///
- public static StreamTokenizer GetWords(this TextReader stream, int minBufferChars = 1024, char[]? bufferStorage = null)
+ public static StreamTokenizer GetWords(TextReader stream, int minBufferChars = 1024, char[]? bufferStorage = null)
{
bufferStorage ??= new char[minBufferChars * 2];
var buffer = new Buffer(stream.Read, minBufferChars, bufferStorage);