diff --git a/artifact_source_config/artifact_source_config_base.go b/artifact_source_config/artifact_source_config_base.go index 5d74693..63191b4 100644 --- a/artifact_source_config/artifact_source_config_base.go +++ b/artifact_source_config/artifact_source_config_base.go @@ -1,11 +1,9 @@ package artifact_source_config import ( - "fmt" "github.com/hashicorp/hcl/v2" - "github.com/turbot/go-kit/helpers" + gokithelpers "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/filter" - helpers2 "github.com/turbot/tailpipe-plugin-sdk/helpers" ) type ArtifactSourceConfigBase struct { @@ -18,39 +16,37 @@ type ArtifactSourceConfigBase struct { // grok patterns to add to the grok parser used to parse the layout Patterns map[string]string `hcl:"patterns,optional"` - // list of filters to apply to the path segments - // note: each filter must refer to a single property only - // filters are ANDED together - Filters []string `hcl:"filters,optional"` - // map of parsed filters, keyed by target property FilterMap map[string]*filter.SqlFilter } func (b *ArtifactSourceConfigBase) Validate() error { + // #TODO https://github.com/turbot/tailpipe/issues/97 + // once filters are pushed down from the CLI, we can populate the filter map // parse filters and put into map keyed by property name - filterMap, err := helpers2.BuildFilterMap(b.Filters) - if err != nil { - return err - } - b.FilterMap = filterMap - - // validate the filters - if filters are set, file layout must be set - if len(b.Filters) > 0 { - if b.FileLayout == nil { - return fmt.Errorf("filters are set, but file_layout is not set") - } - - // validate all fields referred to in the filters exist in the filter layout - metadataProperties := helpers.SliceToLookup(helpers2.ExtractNamedGroupsFromGrok(*b.FileLayout)) - // we have already pulled out the property names in the map keys - for k := range b.FilterMap { - if _, ok := metadataProperties[k]; !ok { - return fmt.Errorf("filter %s refers to a property not in the file layout", k) - } - } - } + //filterMap, err := helpers.BuildFilterMap(b.Filters) + //if err != nil { + // return err + //} + //b.FilterMap = map[string]filterMap + //// validate the filters - if filters are set, file layout must be set + //if len(b.Filters) > 0 { + // if b.FileLayout == nil { + // return fmt.Errorf("filters are set, but file_layout is not set") + // } + // + // // validate all fields referred to in the filters exist in the filter layout + // metadataProperties := gokithelpers.SliceToLookup(helpers.ExtractNamedGroupsFromGrok(*b.FileLayout)) + // // we have already pulled out the property names in the map keys + // for k := range b.FilterMap { + // if _, ok := metadataProperties[k]; !ok { + // return fmt.Errorf("filter %s refers to a property not in the file layout", k) + // } + // } + //} // + + b.FilterMap = map[string]*filter.SqlFilter{} return nil } @@ -61,12 +57,13 @@ func (b *ArtifactSourceConfigBase) Identifier() string { func (b *ArtifactSourceConfigBase) GetFileLayout() *string { return b.FileLayout } + func (b *ArtifactSourceConfigBase) GetPatterns() map[string]string { return b.Patterns } func (b *ArtifactSourceConfigBase) DefaultTo(other ArtifactSourceConfig) { - if helpers.IsNil(other) { + if gokithelpers.IsNil(other) { return } diff --git a/artifact_source_config/artifact_source_config_base_test.go b/artifact_source_config/artifact_source_config_base_test.go index f4c04ae..71a4914 100644 --- a/artifact_source_config/artifact_source_config_base_test.go +++ b/artifact_source_config/artifact_source_config_base_test.go @@ -1,122 +1,116 @@ package artifact_source_config -import ( - "github.com/hashicorp/hcl/v2" - "github.com/turbot/pipe-fittings/filter" - "github.com/turbot/pipe-fittings/utils" - "testing" -) - -func TestArtifactSourceConfigBase_Validate(t *testing.T) { - type fields struct { - Remain hcl.Body - FileLayout *string - Filters []string - FilterMap map[string]*filter.SqlFilter - } - tests := []struct { - name string - fields fields - wantErr bool - }{ - { - name: "Valid filters - single filter", - fields: fields{ - FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"), - Filters: []string{"org = 'org1'"}, - }, - wantErr: false, - }, - { - name: "Valid filters - multiple filters", - fields: fields{ - FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail/%{WORD:region}/%{NOTSPACE:file_name}.%{WORD:ext}"), - Filters: []string{"org = 'org1'", "region = 'us-east-1'"}, - }, - wantErr: false, - }, - { - name: "Filter refer to field not in FileLayout", - fields: fields{ - FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail/%{NOTSPACE:file_name}.%{WORD:ext}"), - Filters: []string{"org = 'org1'", "region = 'us-east-1'"}, - }, - wantErr: true, - }, - { - name: "Invalid filter - no LHS property", - fields: fields{ - FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"), - Filters: []string{"= 'org1'"}, - }, - wantErr: true, - }, - { - name: "Invalid filter - multiple LHS properties", - fields: fields{ - FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"), - Filters: []string{"org = 'org1' AND account = '123'"}, - }, - wantErr: true, - }, - { - name: "Empty filters", - fields: fields{ - Filters: []string{}, - }, - wantErr: false, - }, - { - name: "Nil filters", - fields: fields{ - Filters: nil, - }, - wantErr: false, - }, - { - name: "Invalid filter syntax", - fields: fields{ - FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"), - Filters: []string{"org =="}, - }, - wantErr: true, - }, - { - name: "Duplicate filters for the same field", - fields: fields{ - FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"), - Filters: []string{"org = 'org1'", "org != 'org2'"}, - }, - wantErr: false, - }, - { - name: "Valid filters with FileLayout", - fields: fields{ - FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/%{NUMBER:account_id}/CloudTrail"), - Filters: []string{"org = 'org1'", "account_id = '123'"}, - }, - wantErr: false, - }, - { - name: "Empty FileLayout and Filters", - fields: fields{ - FileLayout: nil, - Filters: []string{}, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := &ArtifactSourceConfigBase{ - Remain: tt.fields.Remain, - FileLayout: tt.fields.FileLayout, - Filters: tt.fields.Filters, - FilterMap: tt.fields.FilterMap, - } - if err := b.Validate(); (err != nil) != tt.wantErr { - t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} +// #TODO reenable once we support filter pushdown https://github.com/turbot/tailpipe/issues/97 +//func TestArtifactSourceConfigBase_Validate(t *testing.T) { +// type fields struct { +// Remain hcl.Body +// FileLayout *string +// Filters []string +// FilterMap map[string]*filter.SqlFilter +// } +// tests := []struct { +// name string +// fields fields +// wantErr bool +// }{ +// { +// name: "Valid filters - single filter", +// fields: fields{ +// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"), +// Filters: []string{"org = 'org1'"}, +// }, +// wantErr: false, +// }, +// { +// name: "Valid filters - multiple filters", +// fields: fields{ +// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail/%{WORD:region}/%{NOTSPACE:file_name}.%{WORD:ext}"), +// Filters: []string{"org = 'org1'", "region = 'us-east-1'"}, +// }, +// wantErr: false, +// }, +// { +// name: "Filter refer to field not in FileLayout", +// fields: fields{ +// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail/%{NOTSPACE:file_name}.%{WORD:ext}"), +// Filters: []string{"org = 'org1'", "region = 'us-east-1'"}, +// }, +// wantErr: true, +// }, +// { +// name: "Invalid filter - no LHS property", +// fields: fields{ +// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"), +// Filters: []string{"= 'org1'"}, +// }, +// wantErr: true, +// }, +// { +// name: "Invalid filter - multiple LHS properties", +// fields: fields{ +// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"), +// Filters: []string{"org = 'org1' AND account = '123'"}, +// }, +// wantErr: true, +// }, +// { +// name: "Empty filters", +// fields: fields{ +// Filters: []string{}, +// }, +// wantErr: false, +// }, +// { +// name: "Nil filters", +// fields: fields{ +// Filters: nil, +// }, +// wantErr: false, +// }, +// { +// name: "Invalid filter syntax", +// fields: fields{ +// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"), +// Filters: []string{"org =="}, +// }, +// wantErr: true, +// }, +// { +// name: "Duplicate filters for the same field", +// fields: fields{ +// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/CloudTrail"), +// Filters: []string{"org = 'org1'", "org != 'org2'"}, +// }, +// wantErr: false, +// }, +// { +// name: "Valid filters with FileLayout", +// fields: fields{ +// FileLayout: utils.ToPointer("AWSLogs/%{WORD:org}/%{NUMBER:account_id}/CloudTrail"), +// Filters: []string{"org = 'org1'", "account_id = '123'"}, +// }, +// wantErr: false, +// }, +// { +// name: "Empty FileLayout and Filters", +// fields: fields{ +// FileLayout: nil, +// Filters: []string{}, +// }, +// wantErr: false, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// b := &ArtifactSourceConfigBase{ +// Remain: tt.fields.Remain, +// FileLayout: tt.fields.FileLayout, +// Filters: tt.fields.Filters, +// FilterMap: tt.fields.FilterMap, +// } +// if err := b.Validate(); (err != nil) != tt.wantErr { +// t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) +// } +// }) +// } +//} diff --git a/grpc/proto/plugin.pb.go b/grpc/proto/plugin.pb.go index 56ee966..34bb045 100644 --- a/grpc/proto/plugin.pb.go +++ b/grpc/proto/plugin.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v5.28.3 +// protoc v5.29.2 // source: plugin.proto package proto diff --git a/grpc/proto/plugin_grpc.pb.go b/grpc/proto/plugin_grpc.pb.go index 64f0ebc..87dc33f 100644 --- a/grpc/proto/plugin_grpc.pb.go +++ b/grpc/proto/plugin_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v5.28.3 +// - protoc v5.29.2 // source: plugin.proto package proto