Skip to content

Commit

Permalink
Remove filters field from source config
Browse files Browse the repository at this point in the history
  • Loading branch information
@kaidaguerre
kaidaguerre committed Jan 3, 2025
1 parent 5f1fa72 commit 07b4473
Show file tree
Hide file tree
Showing 4 changed files with 143 additions and 152 deletions.
57 changes: 27 additions & 30 deletions artifact_source_config/artifact_source_config_base.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
package artifact_source_config

import (
"fmt"
"github.com/hashicorp/hcl/v2"
"github.com/turbot/go-kit/helpers"
gokithelpers "github.com/turbot/go-kit/helpers"
"github.com/turbot/pipe-fittings/filter"
helpers2 "github.com/turbot/tailpipe-plugin-sdk/helpers"
)

type ArtifactSourceConfigBase struct {
Expand All @@ -18,39 +16,37 @@ type ArtifactSourceConfigBase struct {
// grok patterns to add to the grok parser used to parse the layout
Patterns map[string]string `hcl:"patterns,optional"`

// list of filters to apply to the path segments
// note: each filter must refer to a single property only
// filters are ANDED together
Filters []string `hcl:"filters,optional"`

// map of parsed filters, keyed by target property
FilterMap map[string]*filter.SqlFilter
}

func (b *ArtifactSourceConfigBase) Validate() error {
// #TODO https://github.com/turbot/tailpipe/issues/97
// once filters are pushed down from the CLI, we can populate the filter map
// parse filters and put into map keyed by property name
filterMap, err := helpers2.BuildFilterMap(b.Filters)
if err != nil {
return err
}
b.FilterMap = filterMap

// validate the filters - if filters are set, file layout must be set
if len(b.Filters) > 0 {
if b.FileLayout == nil {
return fmt.Errorf("filters are set, but file_layout is not set")
}

// validate all fields referred to in the filters exist in the filter layout
metadataProperties := helpers.SliceToLookup(helpers2.ExtractNamedGroupsFromGrok(*b.FileLayout))
// we have already pulled out the property names in the map keys
for k := range b.FilterMap {
if _, ok := metadataProperties[k]; !ok {
return fmt.Errorf("filter %s refers to a property not in the file layout", k)
}
}
}
//filterMap, err := helpers.BuildFilterMap(b.Filters)
//if err != nil {
// return err
//}
//b.FilterMap = map[string]filterMap
//// validate the filters - if filters are set, file layout must be set
//if len(b.Filters) > 0 {
// if b.FileLayout == nil {
// return fmt.Errorf("filters are set, but file_layout is not set")
// }
//
// // validate all fields referred to in the filters exist in the filter layout
// metadataProperties := gokithelpers.SliceToLookup(helpers.ExtractNamedGroupsFromGrok(*b.FileLayout))
// // we have already pulled out the property names in the map keys
// for k := range b.FilterMap {
// if _, ok := metadataProperties[k]; !ok {
// return fmt.Errorf("filter %s refers to a property not in the file layout", k)
// }
// }
//}
//

b.FilterMap = map[string]*filter.SqlFilter{}
return nil
}

Expand All @@ -61,12 +57,13 @@ func (b *ArtifactSourceConfigBase) Identifier() string {
func (b *ArtifactSourceConfigBase) GetFileLayout() *string {
return b.FileLayout
}

func (b *ArtifactSourceConfigBase) GetPatterns() map[string]string {
return b.Patterns
}

func (b *ArtifactSourceConfigBase) DefaultTo(other ArtifactSourceConfig) {
if helpers.IsNil(other) {
if gokithelpers.IsNil(other) {
return
}

Expand Down
234 changes: 114 additions & 120 deletions artifact_source_config/artifact_source_config_base_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,122 +1,116 @@
package artifact_source_config

import (
"github.com/hashicorp/hcl/v2"
"github.com/turbot/pipe-fittings/filter"
"github.com/turbot/pipe-fittings/utils"
"testing"
)

func TestArtifactSourceConfigBase_Validate(t *testing.T) {
type fields struct {
Remain hcl.Body
FileLayout *string
Filters []string
FilterMap map[string]*filter.SqlFilter
}
tests := []struct {
name string
fields fields
wantErr bool
}{
{
name: "Valid filters - single filter",
fields: fields{
FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"),
Filters: []string{"org = 'org1'"},
},
wantErr: false,
},
{
name: "Valid filters - multiple filters",
fields: fields{
FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail/%{WORD:region}/%{NOTSPACE:file_name}.%{WORD:ext}"),
Filters: []string{"org = 'org1'", "region = 'us-east-1'"},
},
wantErr: false,
},
{
name: "Filter refer to field not in FileLayout",
fields: fields{
FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail/%{NOTSPACE:file_name}.%{WORD:ext}"),
Filters: []string{"org = 'org1'", "region = 'us-east-1'"},
},
wantErr: true,
},
{
name: "Invalid filter - no LHS property",
fields: fields{
FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"),
Filters: []string{"= 'org1'"},
},
wantErr: true,
},
{
name: "Invalid filter - multiple LHS properties",
fields: fields{
FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"),
Filters: []string{"org = 'org1' AND account = '123'"},
},
wantErr: true,
},
{
name: "Empty filters",
fields: fields{
Filters: []string{},
},
wantErr: false,
},
{
name: "Nil filters",
fields: fields{
Filters: nil,
},
wantErr: false,
},
{
name: "Invalid filter syntax",
fields: fields{
FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"),
Filters: []string{"org =="},
},
wantErr: true,
},
{
name: "Duplicate filters for the same field",
fields: fields{
FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"),
Filters: []string{"org = 'org1'", "org != 'org2'"},
},
wantErr: false,
},
{
name: "Valid filters with FileLayout",
fields: fields{
FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/%{NUMBER:account_id}/CloudTrail"),
Filters: []string{"org = 'org1'", "account_id = '123'"},
},
wantErr: false,
},
{
name: "Empty FileLayout and Filters",
fields: fields{
FileLayout: nil,
Filters: []string{},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := &ArtifactSourceConfigBase{
Remain: tt.fields.Remain,
FileLayout: tt.fields.FileLayout,
Filters: tt.fields.Filters,
FilterMap: tt.fields.FilterMap,
}
if err := b.Validate(); (err != nil) != tt.wantErr {
t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
// #TODO reenable once we support filter pushdown https://github.com/turbot/tailpipe/issues/97
//func TestArtifactSourceConfigBase_Validate(t *testing.T) {
// type fields struct {
// Remain hcl.Body
// FileLayout *string
// Filters []string
// FilterMap map[string]*filter.SqlFilter
// }
// tests := []struct {
// name string
// fields fields
// wantErr bool
// }{
// {
// name: "Valid filters - single filter",
// fields: fields{
// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"),
// Filters: []string{"org = 'org1'"},
// },
// wantErr: false,
// },
// {
// name: "Valid filters - multiple filters",
// fields: fields{
// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail/%{WORD:region}/%{NOTSPACE:file_name}.%{WORD:ext}"),
// Filters: []string{"org = 'org1'", "region = 'us-east-1'"},
// },
// wantErr: false,
// },
// {
// name: "Filter refer to field not in FileLayout",
// fields: fields{
// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail/%{NOTSPACE:file_name}.%{WORD:ext}"),
// Filters: []string{"org = 'org1'", "region = 'us-east-1'"},
// },
// wantErr: true,
// },
// {
// name: "Invalid filter - no LHS property",
// fields: fields{
// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"),
// Filters: []string{"= 'org1'"},
// },
// wantErr: true,
// },
// {
// name: "Invalid filter - multiple LHS properties",
// fields: fields{
// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"),
// Filters: []string{"org = 'org1' AND account = '123'"},
// },
// wantErr: true,
// },
// {
// name: "Empty filters",
// fields: fields{
// Filters: []string{},
// },
// wantErr: false,
// },
// {
// name: "Nil filters",
// fields: fields{
// Filters: nil,
// },
// wantErr: false,
// },
// {
// name: "Invalid filter syntax",
// fields: fields{
// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"),
// Filters: []string{"org =="},
// },
// wantErr: true,
// },
// {
// name: "Duplicate filters for the same field",
// fields: fields{
// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"),
// Filters: []string{"org = 'org1'", "org != 'org2'"},
// },
// wantErr: false,
// },
// {
// name: "Valid filters with FileLayout",
// fields: fields{
// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/%{NUMBER:account_id}/CloudTrail"),
// Filters: []string{"org = 'org1'", "account_id = '123'"},
// },
// wantErr: false,
// },
// {
// name: "Empty FileLayout and Filters",
// fields: fields{
// FileLayout: nil,
// Filters: []string{},
// },
// wantErr: false,
// },
// }
// for _, tt := range tests {
// t.Run(tt.name, func(t *testing.T) {
// b := &ArtifactSourceConfigBase{
// Remain: tt.fields.Remain,
// FileLayout: tt.fields.FileLayout,
// Filters: tt.fields.Filters,
// FilterMap: tt.fields.FilterMap,
// }
// if err := b.Validate(); (err != nil) != tt.wantErr {
// t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr)
// }
// })
// }
//}
2 changes: 1 addition & 1 deletion grpc/proto/plugin.pb.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion grpc/proto/plugin_grpc.pb.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 07b4473

Please sign in to comment.