diff --git a/cmd/jargon/main.go b/cmd/jargon/main.go index b7daf07..1e0bf81 100644 --- a/cmd/jargon/main.go +++ b/cmd/jargon/main.go @@ -34,6 +34,7 @@ func main() { lemmas := flag.Bool("lemmas", false, "only return tokens that have been changed by a filter (lemmatized)") count := flag.Bool("count", false, "count the tokens") + lines := flag.Bool("lines", false, "add a line break between all tokens") flag.Parse() @@ -51,6 +52,7 @@ func main() { HTML: *html, Lemmas: *lemmas, Count: *count, + Lines: *lines, } // @@ -112,6 +114,7 @@ type config struct { HTML bool Count bool + Lines bool Lemmas bool Filters []jargon.Filter @@ -273,6 +276,7 @@ func execute(c *config) error { } for _, f := range c.Filters { + fmt.Println(f) tokens = f(tokens) } @@ -289,9 +293,25 @@ func execute(c *config) error { return nil } - if _, err := tokens.WriteTo(c.Writer); err != nil { + // Write all + for tokens.Scan() { + token := tokens.Token() + _, err := c.Writer.WriteString(token.String()) + if err != nil { + return err + } + + if c.Lines { + _, err := c.Writer.WriteRune('\n') + if err != nil { + return err + } + } + } + if err := tokens.Err(); err != nil { return err } + if err := c.Writer.Flush(); err != nil { return err } diff --git a/tokenstream.go b/tokenstream.go index 7d1182d..cf62c8c 100644 --- a/tokenstream.go +++ b/tokenstream.go @@ -109,7 +109,7 @@ func (stream *TokenStream) WriteTo(w io.Writer) (int64, error) { } if err := stream.Err(); err != nil { - return written, stream.Err() + return written, err } return written, nil