diff --git a/api/internal/humiographql/filter-alerts.go b/api/internal/humiographql/filter-alerts.go index bd458c09..0407231a 100644 --- a/api/internal/humiographql/filter-alerts.go +++ b/api/internal/humiographql/filter-alerts.go @@ -1,6 +1,8 @@ package humiographql -import graphql "github.com/cli/shurcooL-graphql" +import ( + graphql "github.com/cli/shurcooL-graphql" +) type FilterAlert struct { ID graphql.String `graphql:"id"` diff --git a/api/internal/humiographql/parsers.go b/api/internal/humiographql/parsers.go new file mode 100644 index 00000000..03110ccd --- /dev/null +++ b/api/internal/humiographql/parsers.go @@ -0,0 +1,79 @@ +package humiographql + +import ( + graphql "github.com/cli/shurcooL-graphql" +) + +type UpdateParserScriptInput struct { + Script graphql.String `json:"script"` +} + +type ParserTestEventInput struct { + RawString graphql.String `json:"rawString"` +} + +type FieldHasValueInput struct { + FieldName graphql.String `json:"fieldName"` + ExpectedValue graphql.String `json:"expectedValue"` +} + +type ParserTestCaseOutputAssertionsInput struct { + FieldsNotPresent []graphql.String `json:"fieldsNotPresent"` + FieldsHaveValues []FieldHasValueInput `json:"fieldsHaveValues"` +} + +type ParserTestCaseAssertionsForOutputInput struct { + OutputEventIndex graphql.Int `json:"outputEventIndex"` + Assertions ParserTestCaseOutputAssertionsInput `json:"assertions"` +} + +type ParserTestCaseInput struct { + Event ParserTestEventInput `json:"event"` + OutputAssertions []ParserTestCaseAssertionsForOutputInput `json:"outputAssertions"` +} + +type ParserTestEvent struct { + RawString graphql.String `graphql:"rawString"` +} + +type FieldHasValue struct { + FieldName graphql.String `graphql:"fieldName"` + ExpectedValue graphql.String `graphql:"expectedValue"` +} + +type ParserTestCaseOutputAssertions struct { + FieldsNotPresent []string `graphql:"fieldsNotPresent"` + FieldsHaveValues []FieldHasValue `graphql:"fieldsHaveValues"` +} + +type ParserTestCaseAssertionsForOutput struct { + OutputEventIndex graphql.Int `graphql:"outputEventIndex"` + Assertions ParserTestCaseOutputAssertions `graphql:"assertions"` +} + +type ParserTestCase struct { + Event ParserTestEvent `graphql:"event"` // can we move this from api pkg? + OutputAssertions []ParserTestCaseAssertionsForOutput `graphql:"outputAssertions"` +} + +type Parser struct { + ID graphql.String `graphql:"id"` + Name graphql.String `graphql:"name"` + DisplayName graphql.String `graphql:"displayName"` + Description graphql.String `graphql:"description""` + IsBuiltIn graphql.Boolean `graphql:"isBuiltIn"` + Script graphql.String `graphql:"script"` + FieldsToTag []graphql.String `graphql:"fieldsToTag"` + FieldsToBeRemovedBeforeParsing []graphql.String `graphql:"fieldsToBeRemovedBeforeParsing"` + TestCases []ParserTestCase `graphql:"testCases"` +} + +type CreateParserInputV2 struct { + Name graphql.String `json:"name"` + Script graphql.String `json:"script""` + TestCases []ParserTestCaseInput `json:"testCases"` + RepositoryName RepoOrViewName `json:"repositoryName"` + FieldsToTag []graphql.String `json:"fieldsToTag"` + FieldsToBeRemovedBeforeParsing []graphql.String `json:"fieldsToBeRemovedBeforeParsing"` + AllowOverwritingExistingParser graphql.Boolean `json:"allowOverwritingExistingParser"` +} diff --git a/api/parsers.go b/api/parsers.go index cf82a29f..876c0629 100644 --- a/api/parsers.go +++ b/api/parsers.go @@ -1,19 +1,35 @@ package api -import graphql "github.com/cli/shurcooL-graphql" +import ( + "fmt" + graphql "github.com/cli/shurcooL-graphql" + "github.com/humio/cli/api/internal/humiographql" +) + +const LogScaleVersionWithParserAPIv2 = "1.129.0" + +type ParserTestEvent struct { + RawString string `json:"rawString" yaml:"rawString"` +} + +type ParserTestCaseAssertions struct { + OutputEventIndex int `json:"outputEventIndex" yaml:"outputEventIndex"` + FieldsNotPresent []string `json:"fieldsNotPresent" yaml:"fieldsNotPresent"` + FieldsHaveValues map[string]string `json:"fieldsHaveValues" yaml:"fieldsHaveValues"` +} type ParserTestCase struct { - Input string - Output map[string]string + Event ParserTestEvent `json:"event" yaml:"event"` + Assertions []ParserTestCaseAssertions `json:"assertions" yaml:"assertions"` } type Parser struct { - ID string - Name string - Tests []string `yaml:",omitempty"` - Example string `yaml:",omitempty"` - Script string `yaml:",flow"` - TagFields []string `yaml:",omitempty"` + ID string + Name string + Script string `json:"script" yaml:",flow"` + TestCases []ParserTestCase `json:"testCases" yaml:"testCases"` + FieldsToTag []string `json:"tagFields" yaml:"tagFields"` + FieldsToBeRemovedBeforeParsing []string `json:"fieldsToBeRemovedBeforeParsing,omitempty" yaml:"fieldsToBeRemovedBeforeParsing"` } type Parsers struct { @@ -47,12 +63,39 @@ func (p *Parsers) List(repositoryName string) ([]ParserListItem, error) { return parsers, err } -func (p *Parsers) Remove(repositoryName string, parserName string) error { +func (p *Parsers) Delete(repositoryName string, parserName string) error { + status, err := p.client.Status() + if err != nil { + return err + } + + atLeast, err := status.AtLeast(LogScaleVersionWithParserAPIv2) + if !atLeast { + var mutation struct { + RemoveParser struct { + // We have to make a selection, so just take __typename + Typename graphql.String `graphql:"__typename"` + } `graphql:"removeParser(input: { id: $id, repositoryName: $repositoryName })"` + } + + parser, err := p.client.Parsers().Get(repositoryName, parserName) + if err != nil { + return err + } + + variables := map[string]interface{}{ + "repositoryName": graphql.String(repositoryName), + "id": graphql.String(parser.ID), + } + + return p.client.Mutate(&mutation, variables) + } + var mutation struct { - RemoveParser struct { + DeleteParser struct { // We have to make a selection, so just take __typename Typename graphql.String `graphql:"__typename"` - } `graphql:"removeParser(input: { id: $id, repositoryName: $repositoryName })"` + } `graphql:"deleteParser(input: { id: $id, repositoryName: $repositoryName })"` } parser, err := p.client.Parsers().Get(repositoryName, parserName) @@ -68,73 +111,204 @@ func (p *Parsers) Remove(repositoryName string, parserName string) error { return p.client.Mutate(&mutation, variables) } -func (p *Parsers) Add(repositoryName string, parser *Parser, force bool) error { +func (p *Parsers) Add(repositoryName string, newParser *Parser, allowOverwritingExistingParser bool) (*Parser, error) { + if newParser == nil { + return nil, fmt.Errorf("newFilterAlert must not be nil") + } - var mutation struct { - CreateParser struct { - // We have to make a selection, so just take __typename - Typename graphql.String `graphql:"__typename"` - } `graphql:"createParser(input: { name: $name, repositoryName: $repositoryName, testData: $testData, tagFields: $tagFields, sourceCode: $sourceCode, force: $force})"` + status, err := p.client.Status() + if err != nil { + return nil, err } - tagFieldsGQL := make([]graphql.String, len(parser.TagFields)) + atLeast, err := status.AtLeast(LogScaleVersionWithParserAPIv2) + if !atLeast { + var mutation struct { + CreateParser struct { + // We have to make a selection, so just take __typename + Parser humiographql.Parser `graphql:"parser"` + } `graphql:"createParser(input: { name: $name, repositoryName: $repositoryName, testData: $testData, tagFields: $tagFields, sourceCode: $sourceCode, force: $force})"` + } + + testDataGQL := make([]graphql.String, len(newParser.TestCases)) + for i := range newParser.TestCases { + testDataGQL[i] = graphql.String(newParser.TestCases[i].Event.RawString) + } + tagFieldsGQL := make([]graphql.String, len(newParser.FieldsToTag)) + for i := range newParser.FieldsToTag { + tagFieldsGQL[i] = graphql.String(newParser.FieldsToTag[i]) + } + + variables := map[string]interface{}{ + "name": graphql.String(newParser.Name), + "sourceCode": graphql.String(newParser.Script), + "repositoryName": graphql.String(repositoryName), + "testData": testDataGQL, + "tagFields": tagFieldsGQL, + "force": graphql.Boolean(allowOverwritingExistingParser), + } + + err = p.client.Mutate(&mutation, variables) + if err != nil { + return nil, err + } + + parser := mapHumioGraphqlParserToParser(mutation.CreateParser.Parser) + + return &parser, nil + } - for i, field := range parser.TagFields { - tagFieldsGQL[i] = graphql.String(field) + var mutation struct { + humiographql.Parser `graphql:"createParserV2(input: $input)"` } - testsGQL := make([]graphql.String, len(parser.Tests)) + fieldsToTagGQL := make([]graphql.String, len(newParser.FieldsToTag)) + for i, field := range newParser.FieldsToTag { + fieldsToTagGQL[i] = graphql.String(field) + } + fieldsToBeRemovedBeforeParsingGQL := make([]graphql.String, len(newParser.FieldsToBeRemovedBeforeParsing)) + for i, field := range newParser.FieldsToBeRemovedBeforeParsing { + fieldsToBeRemovedBeforeParsingGQL[i] = graphql.String(field) + } + testCasesGQL := make([]humiographql.ParserTestCaseInput, len(newParser.TestCases)) + for i := range newParser.TestCases { + testCasesGQL[i] = mapParserTestCaseToInput(newParser.TestCases[i]) + } - for i, field := range parser.Tests { - testsGQL[i] = graphql.String(field) + createParser := humiographql.CreateParserInputV2{ + Name: graphql.String(newParser.Name), + Script: graphql.String(newParser.Script), + TestCases: testCasesGQL, + RepositoryName: humiographql.RepoOrViewName(repositoryName), + FieldsToTag: fieldsToTagGQL, + FieldsToBeRemovedBeforeParsing: fieldsToBeRemovedBeforeParsingGQL, + AllowOverwritingExistingParser: graphql.Boolean(allowOverwritingExistingParser), } variables := map[string]interface{}{ - "name": graphql.String(parser.Name), - "sourceCode": graphql.String(parser.Script), - "repositoryName": graphql.String(repositoryName), - "testData": testsGQL, - "tagFields": tagFieldsGQL, - "force": graphql.Boolean(force), + "input": createParser, } - return p.client.Mutate(&mutation, variables) + err = p.client.Mutate(&mutation, variables) + if err != nil { + return nil, err + } + + parser := mapHumioGraphqlParserToParser(mutation.Parser) + + return &parser, nil +} + +func mapParserTestCaseToInput(p ParserTestCase) humiographql.ParserTestCaseInput { + parserTestCaseAssertionsForOutputInput := make([]humiographql.ParserTestCaseAssertionsForOutputInput, len(p.Assertions)) + for i := range p.Assertions { + fieldsNotPresent := make([]graphql.String, len(p.Assertions[i].FieldsNotPresent)) + for i := range p.Assertions[i].FieldsNotPresent { + fieldsNotPresent[i] = graphql.String(p.Assertions[i].FieldsNotPresent[i]) + } + fieldsHaveValuesInput := make([]humiographql.FieldHasValueInput, len(p.Assertions[i].FieldsHaveValues)) + for field, value := range p.Assertions[i].FieldsHaveValues { + fieldsHaveValuesInput[i] = humiographql.FieldHasValueInput{ + FieldName: graphql.String(field), + ExpectedValue: graphql.String(value), + } + } + parserTestCaseAssertionsForOutputInput[i] = humiographql.ParserTestCaseAssertionsForOutputInput{ + OutputEventIndex: graphql.Int(p.Assertions[i].OutputEventIndex), + Assertions: humiographql.ParserTestCaseOutputAssertionsInput{ + FieldsNotPresent: fieldsNotPresent, + FieldsHaveValues: fieldsHaveValuesInput, + }, + } + } + return humiographql.ParserTestCaseInput{ + Event: humiographql.ParserTestEventInput{RawString: graphql.String(p.Event.RawString)}, + OutputAssertions: parserTestCaseAssertionsForOutputInput, + } } func (p *Parsers) Get(repositoryName string, parserName string) (*Parser, error) { + status, err := p.client.Status() + if err != nil { + return nil, err + } + + atLeast, err := status.AtLeast(LogScaleVersionWithParserAPIv2) + if !atLeast { + var query struct { + Repository struct { + Parser *struct { + ID string + Name string + SourceCode string + TestData []string + TagFields []string + } `graphql:"parser(name: $parserName)"` + } `graphql:"repository(name: $repositoryName)"` + } + + variables := map[string]interface{}{ + "parserName": graphql.String(parserName), + "repositoryName": graphql.String(repositoryName), + } + + err := p.client.Query(&query, variables) + if err != nil { + return nil, err + } + + if query.Repository.Parser == nil { + return nil, ParserNotFound(parserName) + } + + parser := Parser{ + ID: query.Repository.Parser.ID, + Name: query.Repository.Parser.Name, + Script: query.Repository.Parser.SourceCode, + FieldsToTag: query.Repository.Parser.TagFields, + } + parser.TestCases = make([]ParserTestCase, len(query.Repository.Parser.TestData)) + for i := range query.Repository.Parser.TestData { + parser.TestCases[i] = ParserTestCase{ + Event: ParserTestEvent{RawString: query.Repository.Parser.TestData[i]}, + } + } + + return &parser, nil + } + + parserList, err := p.List(repositoryName) + if err != nil { + return nil, err + } + parserID := "" + for i := range parserList { + if parserList[i].Name == parserName { + parserID = parserList[i].ID + break + } + } + if parserID == "" { + return nil, ParserNotFound(parserName) + } + var query struct { Repository struct { - Parser *struct { - ID string - Name string - SourceCode string - TestData []string - TagFields []string - } `graphql:"parser(name: $parserName)"` + Parser *humiographql.Parser `graphql:"parser(id: $parserID)"` } `graphql:"repository(name: $repositoryName)"` } variables := map[string]interface{}{ - "parserName": graphql.String(parserName), + "parserID": graphql.String(parserID), "repositoryName": graphql.String(repositoryName), } - err := p.client.Query(&query, variables) + err = p.client.Query(&query, variables) if err != nil { return nil, err } - if query.Repository.Parser == nil { - return nil, ParserNotFound(parserName) - } - - parser := Parser{ - ID: query.Repository.Parser.ID, - Name: query.Repository.Parser.Name, - Tests: query.Repository.Parser.TestData, - Script: query.Repository.Parser.SourceCode, - TagFields: query.Repository.Parser.TagFields, - } + parser := mapHumioGraphqlParserToParser(*query.Repository.Parser) return &parser, nil } @@ -167,3 +341,48 @@ func (p *Parsers) Export(repositoryName string, parserName string) (string, erro return query.Repository.Parser.YamlTemplate, nil } + +func mapHumioGraphqlParserToParser(input humiographql.Parser) Parser { + var fieldsToTag = make([]string, len(input.FieldsToTag)) + for i := range input.FieldsToTag { + fieldsToTag[i] = string(input.FieldsToTag[i]) + } + + var fieldsToBeRemovedBeforeParsing = make([]string, len(input.FieldsToBeRemovedBeforeParsing)) + for i := range input.FieldsToBeRemovedBeforeParsing { + fieldsToBeRemovedBeforeParsing[i] = string(input.FieldsToBeRemovedBeforeParsing[i]) + } + + var testCases = make([]ParserTestCase, len(input.TestCases)) + for i := range input.TestCases { + var assertions = make([]ParserTestCaseAssertions, len(input.TestCases[i].OutputAssertions)) + for j := range input.TestCases[i].OutputAssertions { + var fieldsHaveValues = make(map[string]string, len(input.TestCases[i].OutputAssertions[j].Assertions.FieldsHaveValues)) + for k := range input.TestCases[i].OutputAssertions[j].Assertions.FieldsHaveValues { + fieldName := string(input.TestCases[i].OutputAssertions[j].Assertions.FieldsHaveValues[k].FieldName) + expectedValue := string(input.TestCases[i].OutputAssertions[j].Assertions.FieldsHaveValues[k].ExpectedValue) + fieldsHaveValues[fieldName] = expectedValue + } + + assertions[j] = ParserTestCaseAssertions{ + OutputEventIndex: int(input.TestCases[i].OutputAssertions[j].OutputEventIndex), + FieldsNotPresent: input.TestCases[i].OutputAssertions[j].Assertions.FieldsNotPresent, + FieldsHaveValues: fieldsHaveValues, + } + } + + testCases[i] = ParserTestCase{ + Event: ParserTestEvent{RawString: string(input.TestCases[i].Event.RawString)}, + Assertions: assertions, + } + } + + return Parser{ + ID: string(input.ID), + Name: string(input.Name), + Script: string(input.Script), + TestCases: testCases, + FieldsToTag: fieldsToTag, + FieldsToBeRemovedBeforeParsing: fieldsToBeRemovedBeforeParsing, + } +} diff --git a/api/status.go b/api/status.go index 80cf0aa3..3b1eb975 100644 --- a/api/status.go +++ b/api/status.go @@ -5,6 +5,9 @@ import ( "fmt" "io" "net/http" + "strings" + + "github.com/Masterminds/semver/v3" ) type StatusResponse struct { @@ -16,6 +19,21 @@ func (s StatusResponse) IsDown() bool { return s.Status != "OK" && s.Status != "WARN" } +func (s StatusResponse) AtLeast(ver string) (bool, error) { + assumeLatest := true + version := strings.Split(s.Version, "-") + constraint, err := semver.NewConstraint(fmt.Sprintf(">= %s", ver)) + if err != nil { + return assumeLatest, fmt.Errorf("could not parse constraint of `%s`: %w", fmt.Sprintf(">= %s", ver), err) + } + semverVersion, err := semver.NewVersion(version[0]) + if err != nil { + return assumeLatest, fmt.Errorf("could not parse version of `%s`: %w", version[0], err) + } + + return constraint.Check(semverVersion), nil +} + func (c *Client) Status() (*StatusResponse, error) { resp, err := c.HTTPRequest(http.MethodGet, "api/v1/status", nil) diff --git a/cmd/humioctl/parsers.go b/cmd/humioctl/parsers.go index 21eb1c25..827386dc 100644 --- a/cmd/humioctl/parsers.go +++ b/cmd/humioctl/parsers.go @@ -28,6 +28,7 @@ func newParsersCmd() *cobra.Command { cmd.AddCommand(newParsersListCmd()) cmd.AddCommand(newParsersRemoveCmd()) cmd.AddCommand(newParsersExportCmd()) + cmd.AddCommand(newParsersShowCmd()) return cmd } diff --git a/cmd/humioctl/parsers_get.go b/cmd/humioctl/parsers_get.go new file mode 100644 index 00000000..e79a871b --- /dev/null +++ b/cmd/humioctl/parsers_get.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "github.com/humio/cli/cmd/internal/format" + "github.com/spf13/cobra" + "strings" +) + +func newParsersShowCmd() *cobra.Command { + cmd := cobra.Command{ + Use: "show ", + Short: "Show details for a parser in a repository.", + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + repoName := args[0] + parserName := args[1] + client := NewApiClient(cmd) + + parser, err := client.Parsers().Get(repoName, parserName) + exitOnError(cmd, err, "Error fetching parser") + + details := [][]format.Value{ + {format.String("ID"), format.String(parser.ID)}, + {format.String("Name"), format.String(parser.Name)}, + {format.String("Script"), format.String(parser.Script)}, + {format.String("TagFields"), format.String(strings.Join(parser.FieldsToTag, "\n"))}, + {format.String("FieldsToBeRemovedBeforeParsing"), format.String(strings.Join(parser.FieldsToBeRemovedBeforeParsing, "\n"))}, + {format.String("TestCases"), format.String(fmt.Sprintf("%+v", parser.TestCases))}, + } + + printDetailsTable(cmd, details) + }, + } + + return &cmd +} diff --git a/cmd/humioctl/parsers_install.go b/cmd/humioctl/parsers_install.go index 654b08b8..bb727bc9 100644 --- a/cmd/humioctl/parsers_install.go +++ b/cmd/humioctl/parsers_install.go @@ -25,7 +25,7 @@ import ( ) func newParsersInstallCmd() *cobra.Command { - var force bool + var allowOverwritingExistingParser bool var filePath, url, name string cmd := cobra.Command{ @@ -70,12 +70,12 @@ Use the --force flag to update existing parsers with conflicting names. parser.Name = name } - err = client.Parsers().Add(repositoryName, &parser, force) + _, err = client.Parsers().Add(repositoryName, &parser, allowOverwritingExistingParser) exitOnError(cmd, err, "Error installing parser") }, } - cmd.Flags().BoolVarP(&force, "force", "f", false, "Overrides any parser with the same name. This can be used for updating parser that are already installed. (See --name)") + cmd.Flags().BoolVar(&allowOverwritingExistingParser, "allow-overwriting-existing-parser", false, "Overrides any parser with the same name. This can be used for updating parser that are already installed. (See --name)") cmd.Flags().StringVar(&filePath, "file", "", "The local file path to the parser to install.") cmd.Flags().StringVar(&url, "url", "", "A URL to fetch the parser file from.") cmd.Flags().StringVarP(&name, "name", "n", "", "Install the parser under a specific name, ignoring the `name` attribute in the parser file.") diff --git a/cmd/humioctl/parsers_remove.go b/cmd/humioctl/parsers_remove.go index 4f137e73..66122ea1 100644 --- a/cmd/humioctl/parsers_remove.go +++ b/cmd/humioctl/parsers_remove.go @@ -30,7 +30,7 @@ func newParsersRemoveCmd() *cobra.Command { parser := args[1] client := NewApiClient(cmd) - err := client.Parsers().Remove(repo, parser) + err := client.Parsers().Delete(repo, parser) exitOnError(cmd, err, "Error removing parser") fmt.Fprintf(cmd.OutOrStdout(), "Successfully removed parser %q from repository %q\n", parser, repo) diff --git a/go.mod b/go.mod index 09676075..73edf16c 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/humio/cli go 1.22 require ( + github.com/Masterminds/semver/v3 v3.2.1 github.com/cli/shurcooL-graphql v0.0.4 github.com/gofrs/uuid v3.2.0+incompatible github.com/hpcloud/tail v1.0.0 diff --git a/go.sum b/go.sum index a72ade7f..edd52469 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/cli/shurcooL-graphql v0.0.4 h1:6MogPnQJLjKkaXPyGqPRXOI2qCsQdqNfUY1QSJu2GuY= github.com/cli/shurcooL-graphql v0.0.4/go.mod h1:3waN4u02FiZivIV+p1y4d0Jo1jc6BViMA73C+sZo2fk= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=