refactor: organize files (#212)

cmd/config.go

@@ -6,7 +6,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/checkmarx/2ms/lib"
+	"github.com/checkmarx/2ms/lib/utils"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
@@ -17,8 +17,8 @@ func initialize() {
 	if err != nil {
 		cobra.CheckErr(err)
 	}
-	cobra.CheckErr(lib.LoadConfig(vConfig, configFilePath))
-	cobra.CheckErr(lib.BindFlags(rootCmd, vConfig, envPrefix))
+	cobra.CheckErr(utils.LoadConfig(vConfig, configFilePath))
+	cobra.CheckErr(utils.BindFlags(rootCmd, vConfig, envPrefix))
 
 	logLevel := zerolog.InfoLevel
 	switch strings.ToLower(logLevelVar) {

cmd/main.go

@@ -4,10 +4,11 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/checkmarx/2ms/config"
+	"github.com/checkmarx/2ms/engine"
+	"github.com/checkmarx/2ms/lib/config"
+	"github.com/checkmarx/2ms/lib/reporting"
+	"github.com/checkmarx/2ms/lib/secrets"
 	"github.com/checkmarx/2ms/plugins"
-	"github.com/checkmarx/2ms/reporting"
-	"github.com/checkmarx/2ms/secrets"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -40,7 +41,7 @@ var (
 	customRegexRuleVar []string
 	ignoreVar          []string
 	ignoreOnExitVar    = ignoreOnExitNone
-	engineConfigVar    secrets.EngineConfig
+	engineConfigVar    engine.EngineConfig
 	validateVar        bool
 )
 
@@ -94,7 +95,7 @@ func Execute() (int, error) {
 	rootCmd.PersistentFlags().IntVar(&engineConfigVar.MaxTargetMegabytes, maxTargetMegabytesFlagName, 0, "files larger than this will be skipped.\nOmit or set to 0 to disable this check.")
 	rootCmd.PersistentFlags().BoolVar(&validateVar, validate, false, "trigger additional validation to check if discovered secrets are active or revoked")
 
-	rootCmd.AddCommand(secrets.GetRulesCommand(&engineConfigVar))
+	rootCmd.AddCommand(engine.GetRulesCommand(&engineConfigVar))
 
 	group := "Commands"
 	rootCmd.AddGroup(&cobra.Group{Title: group, ID: group})
@@ -124,7 +125,7 @@ func preRun(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	engine, err := secrets.Init(engineConfigVar)
+	engine, err := engine.Init(engineConfigVar)
 	if err != nil {
 		return err
 	}

cmd/workers.go

@@ -3,10 +3,10 @@ package cmd
 import (
 	"sync"
 
-	"github.com/checkmarx/2ms/secrets"
+	"github.com/checkmarx/2ms/engine"
 )
 
-func processItems(engine *secrets.Engine) {
+func processItems(engine *engine.Engine) {
 	defer channels.WaitGroup.Done()
 
 	wgItems := &sync.WaitGroup{}
@@ -32,7 +32,7 @@ func processSecrets() {
 	close(validationChan)
 }
 
-func processValidation(engine *secrets.Engine) {
+func processValidation(engine *engine.Engine) {
 	defer channels.WaitGroup.Done()
 
 	wgValidation := &sync.WaitGroup{}

secrets/engine.go → engine/engine.go

@@ -1,4 +1,4 @@
-package secrets
+package engine
 
 import (
 	"crypto/sha1"
@@ -9,8 +9,10 @@ import (
 	"sync"
 	"text/tabwriter"
 
+	"github.com/checkmarx/2ms/engine/rules"
+	"github.com/checkmarx/2ms/engine/validation"
+	"github.com/checkmarx/2ms/lib/secrets"
 	"github.com/checkmarx/2ms/plugins"
-	"github.com/checkmarx/2ms/secrets/rules"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/zricethezav/gitleaks/v8/config"
@@ -21,7 +23,7 @@ import (
 type Engine struct {
 	rules     map[string]config.Rule
 	detector  detect.Detector
-	validator Validator
+	validator validation.Validator
 }
 
 const customRegexRuleIdFormat = "custom-regex-%d"
@@ -55,19 +57,19 @@ func Init(engineConfig EngineConfig) (*Engine, error) {
 	return &Engine{
 		rules:     rulesToBeApplied,
 		detector:  *detector,
-		validator: *NewValidator(),
+		validator: *validation.NewValidator(),
 	}, nil
 }
 
-func (s *Engine) Detect(item plugins.Item, secretsChannel chan *Secret, wg *sync.WaitGroup, ignoredIds []string) {
+func (s *Engine) Detect(item plugins.Item, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup, ignoredIds []string) {
 	defer wg.Done()
 
 	fragment := detect.Fragment{
 		Raw: item.Content,
 	}
 	for _, value := range s.detector.Detect(fragment) {
 		itemId := getFindingId(item, value)
-		secret := &Secret{
+		secret := &secrets.Secret{
 			ID:          itemId,
 			Source:      item.Source,
 			RuleID:      value.RuleID,
@@ -102,7 +104,7 @@ func (s *Engine) AddRegexRules(patterns []string) error {
 	return nil
 }
 
-func (s *Engine) RegisterForValidation(secret *Secret, wg *sync.WaitGroup) {
+func (s *Engine) RegisterForValidation(secret *secrets.Secret, wg *sync.WaitGroup) {
 	defer wg.Done()
 	s.validator.RegisterForValidation(secret)
 }
@@ -117,7 +119,7 @@ func getFindingId(item plugins.Item, finding report.Finding) string {
 	return fmt.Sprintf("%x", sha)
 }
 
-func isSecretIgnored(secret *Secret, ignoredIds *[]string) bool {
+func isSecretIgnored(secret *secrets.Secret, ignoredIds *[]string) bool {
 	for _, ignoredId := range *ignoredIds {
 		if secret.ID == ignoredId {
 			return true
@@ -151,7 +153,7 @@ func GetRulesCommand(engineConfig *EngineConfig) *cobra.Command {
 					rule.Rule.RuleID,
 					rule.Rule.Description,
 					strings.Join(rule.Tags, ","),
-					canValidateDisplay[isCanValidateRule(rule.Rule.RuleID)],
+					canValidateDisplay[validation.IsCanValidateRule(rule.Rule.RuleID)],
 				)
 			}
 			if err := tab.Flush(); err != nil {

secrets/engine_test.go → engine/engine_test.go

@@ -1,12 +1,13 @@
-package secrets
+package engine
 
 import (
 	"fmt"
 	"sync"
 	"testing"
 
+	"github.com/checkmarx/2ms/engine/rules"
+	"github.com/checkmarx/2ms/lib/secrets"
 	"github.com/checkmarx/2ms/plugins"
-	"github.com/checkmarx/2ms/secrets/rules"
 )
 
 func Test_Init(t *testing.T) {
@@ -61,7 +62,7 @@ func Test_Init(t *testing.T) {
 }
 
 func TestSecrets(t *testing.T) {
-	secrets := []struct {
+	secretsCases := []struct {
 		Content    string
 		Name       string
 		ShouldFind bool
@@ -115,14 +116,14 @@ func TestSecrets(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, secret := range secrets {
+	for _, secret := range secretsCases {
 		name := secret.Name
 		if name == "" {
 			name = secret.Content
 		}
 		t.Run(name, func(t *testing.T) {
 			fmt.Printf("Start test %s", name)
-			secretsChan := make(chan *Secret, 1)
+			secretsChan := make(chan *secrets.Secret, 1)
 			wg := &sync.WaitGroup{}
 			wg.Add(1)
 			detector.Detect(plugins.Item{Content: secret.Content}, secretsChan, wg, nil)

secrets/rules/rule_test.go → engine/rules/rule_test.go

@@ -3,7 +3,7 @@ package rules_test
 import (
 	"testing"
 
-	"github.com/checkmarx/2ms/secrets/rules"
+	"github.com/checkmarx/2ms/engine/rules"
 	"github.com/zricethezav/gitleaks/v8/config"
 )
 

secrets/alibaba.go → engine/validation/alibaba.go

@@ -1,4 +1,4 @@
-package secrets
+package validation
 
 import (
 	"crypto/hmac"
@@ -11,19 +11,20 @@ import (
 	"strings"
 	"time"
 
+	"github.com/checkmarx/2ms/lib/secrets"
 	"github.com/rs/zerolog/log"
 )
 
 // https://www.alibabacloud.com/help/en/sdk/alibaba-cloud-api-overview
 // https://www.alibabacloud.com/help/en/sdk/product-overview/rpc-mechanism#sectiondiv-y9b-x9s-wvp
 
-func validateAlibaba(secrets pairsByRuleId) {
+func validateAlibaba(secretsPairs pairsByRuleId) {
 
-	accessKeys := secrets["alibaba-access-key-id"]
-	secretKeys := secrets["alibaba-secret-key"]
+	accessKeys := secretsPairs["alibaba-access-key-id"]
+	secretKeys := secretsPairs["alibaba-secret-key"]
 
 	for _, accessKey := range accessKeys {
-		accessKey.ValidationStatus = Unknown
+		accessKey.ValidationStatus = secrets.UnknownResult
 
 		for _, secretKey := range secretKeys {
 			status, err := alibabaRequest(accessKey.Value, secretKey.Value)
@@ -32,17 +33,17 @@ func validateAlibaba(secrets pairsByRuleId) {
 			}
 
 			secretKey.ValidationStatus = status
-			if accessKey.ValidationStatus.CompareTo(status) == second {
+			if accessKey.ValidationStatus.CompareTo(status) > 0 {
 				accessKey.ValidationStatus = status
 			}
 		}
 	}
 }
 
-func alibabaRequest(accessKey, secretKey string) (validationResult, error) {
+func alibabaRequest(accessKey, secretKey string) (secrets.ValidationResult, error) {
 	req, err := http.NewRequest("GET", "https://ecs.aliyuncs.com/", nil)
 	if err != nil {
-		return Unknown, err
+		return secrets.UnknownResult, err
 	}
 
 	// Workaround for gitleaks returns the key ends with "
@@ -70,20 +71,20 @@ func alibabaRequest(accessKey, secretKey string) (validationResult, error) {
 	client := &http.Client{}
 	resp, err := client.Do(req)
 	if err != nil {
-		return Unknown, err
+		return secrets.UnknownResult, err
 	}
 	log.Debug().Str("service", "alibaba").Int("status_code", resp.StatusCode)
 
 	// If the access key is invalid, the response will be 404
 	// If the secret key is invalid, the response will be 400 along with other signautre Errors
 	if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusBadRequest {
-		return Revoked, nil
+		return secrets.RevokedResult, nil
 	}
 
 	if resp.StatusCode == http.StatusOK {
-		return Valid, nil
+		return secrets.ValidResult, nil
 	}
 
 	err = fmt.Errorf("unexpected status code: %d", resp.StatusCode)
-	return Unknown, err
+	return secrets.UnknownResult, err
 }

engine/validation/github.go

@@ -0,0 +1,32 @@
+package validation
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/checkmarx/2ms/lib/secrets"
+	"github.com/rs/zerolog/log"
+)
+
+func validateGithub(s *secrets.Secret) secrets.ValidationResult {
+	const githubURL = "https://api.github.com/"
+
+	req, err := http.NewRequest("GET", githubURL, nil)
+	if err != nil {
+		log.Warn().Err(err).Msg("Failed to validate secret")
+		return secrets.UnknownResult
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("token %s", s.Value))
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Warn().Err(err).Msg("Failed to validate secret")
+		return secrets.UnknownResult
+	}
+
+	if resp.StatusCode == http.StatusOK {
+		return secrets.ValidResult
+	}
+	return secrets.RevokedResult
+}

secrets/pairs.go → engine/validation/pairs.go

@@ -1,10 +1,12 @@
-package secrets
+package validation
 
 import (
 	"sync"
+
+	"github.com/checkmarx/2ms/lib/secrets"
 )
 
-type pairsByRuleId map[string][]*Secret
+type pairsByRuleId map[string][]*secrets.Secret
 type pairsBySource map[string]pairsByRuleId
 type pairsByGeneralKey map[string]pairsBySource
 
@@ -16,7 +18,7 @@ func newPairsCollector() *pairsCollector {
 	return &pairsCollector{pairs: make(pairsByGeneralKey)}
 }
 
-func (p *pairsCollector) addIfNeeded(secret *Secret) bool {
+func (p *pairsCollector) addIfNeeded(secret *secrets.Secret) bool {
 	generalKey, ok := ruleToGeneralKey[secret.RuleID]
 	if !ok {
 		return false
@@ -29,7 +31,7 @@ func (p *pairsCollector) addIfNeeded(secret *Secret) bool {
 		p.pairs[generalKey][secret.Source] = make(pairsByRuleId)
 	}
 	if _, ok := p.pairs[generalKey][secret.Source][secret.RuleID]; !ok {
-		p.pairs[generalKey][secret.Source][secret.RuleID] = make([]*Secret, 0)
+		p.pairs[generalKey][secret.Source][secret.RuleID] = make([]*secrets.Secret, 0)
 	}
 
 	p.pairs[generalKey][secret.Source][secret.RuleID] = append(p.pairs[generalKey][secret.Source][secret.RuleID], secret)