Skip to content

Commit

Permalink
feat: option to load arguments from config file (#107)
Browse files Browse the repository at this point in the history
- bind flags from viper to cobra command
- add tests
- test for Execute
- support multiple config formats
- update readme with help message
- document configuration env and file

Closes #21

**Proposed Changes**
- feat: read arguments from Environment Variables
- feat: read arguments from JSON/YAML file
- docs: auto-update the README when the `help` message was changed
- docs: how to use arguments from Environment Variables or config file

**Limitations**
- You still can run only one plugin at a time.
- You have to add the plugin command to the CLI command.
- Positional arguments are not yet supported.

---------

Co-authored-by: Jossef Harush Kadouri <jossef12@gmail.com>
  • Loading branch information
Baruch Odem (Rothkoff) and jossef authored Jun 29, 2023
1 parent 514e07a commit 127e190
Show file tree
Hide file tree
Showing 6 changed files with 938 additions and 14 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/pr-validation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,17 @@ jobs:
with:
go-version: "^1.20"

- name: go mod tidy
run: |
go mod tidy
git diff --exit-code
- name: Go Linter
run: docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.52.0 golangci-lint run -v -E gofmt --timeout=5m

- name: Go Test
run: go test -v ./...

- name: go mod tidy
run: |
go mod tidy
git diff --exit-code
build:
runs-on: ubuntu-latest
steps:
Expand Down
73 changes: 68 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,75 @@ docker run -v path/to/my/repo:/repo checkmarx/2ms git /repo

## Getting started

### Command line arguments (wip, see [#20](https://github.com/Checkmarx/2ms/discussions/20))
```
2ms Secrets Detection: A tool to detect secrets in public websites and communication services.
Usage:
2ms [command]
Commands
confluence Scan Confluence server
discord Scan Discord server
filesystem Scan local folder
git Scan Git repository
paligo Scan Paligo instance
slack Scan Slack team
Additional Commands:
completion Generate the autocompletion script for the specified shell
help Help about any command
Flags:
--config string YAML config file path
-h, --help help for 2ms
--log-level string log level (trace, debug, info, warn, error, fatal) (default "info")
--regex stringArray custom regexes to apply to the scan, must be valid Go regex
--report-path strings path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif)
--stdout-format string stdout output format, available formats are: json, yaml, sarif (default "yaml")
--tags strings select rules to be applied (default [all])
-v, --version version for 2ms
Use "2ms [command] --help" for more information about a command.
```

| :warning: Using configuration env or file |
| :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Please note that even using configuration file or environment variables, you still need to specify the subcommand name in the CLI arguments. Also, positional arguments are not yet supported. |

### Environment Variables

To use a flag as an environment variable, see the following rules:

- Replace `-` with `_`
- Start with `2MS_`
- Prefer uppercase
- Append the subcommand name(s) (if any) with `_`

- `--confluence` The URL of the Confluence instance to scan.
- `--confluence-spaces` A comma-separated list of Confluence spaces to scan.
- `--confluence-username` confluence username or email
- `--confluence-token` confluence token
Examples:

- `--log-level` -> `2MS_LOG_LEVEL`
- `paligo instance` -> `2MS_PALIGO_INSTANCE`

### Configuration File

You can use `--config` flag to specify a configuration file. The configuration file is a YAML/JSON file with the following structure:

```yaml
# global flags that will be applied to all commands
log-level: info
report-path:
- ./report.yaml
- ./report.json
- ./report.sarif

# the subcommand will be selected from the CLI arguments
# the flags below will be applied to the selected subcommand
paligo:
instance: your-instance
username: your-username
# you can combine config file and Environment Variables
# token: your-token
```

---

Expand Down
26 changes: 24 additions & 2 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"strings"

"github.com/checkmarx/2ms/config"
"github.com/checkmarx/2ms/lib"

"sync"
"time"
Expand All @@ -18,6 +19,7 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)

var Version = "0.0.0"
Expand All @@ -27,6 +29,7 @@ const (
jsonFormat = "json"
yamlFormat = "yaml"
sarifFormat = "sarif"
configFileFlag = "config"

logLevelFlagName = "log-level"
reportPathFlagName = "report-path"
Expand All @@ -52,6 +55,11 @@ var rootCmd = &cobra.Command{
Version: Version,
}

const envPrefix = "2MS"

var configFilePath string
var vConfig = viper.New()

var allPlugins = []plugins.IPlugin{
&plugins.ConfluencePlugin{},
&plugins.DiscordPlugin{},
Expand All @@ -70,7 +78,16 @@ var channels = plugins.Channels{
var report = reporting.Init()
var secretsChan = make(chan reporting.Secret)

func initLog() {
func initialize() {
zerolog.SetGlobalLevel(zerolog.InfoLevel)

configFilePath, err := rootCmd.Flags().GetString(configFileFlag)
if err != nil {
cobra.CheckErr(err)
}
cobra.CheckErr(lib.LoadConfig(vConfig, configFilePath))
cobra.CheckErr(lib.BindFlags(rootCmd, vConfig, envPrefix))

switch strings.ToLower(logLevelVar) {
case "trace":
zerolog.SetGlobalLevel(zerolog.TraceLevel)
Expand All @@ -90,7 +107,12 @@ func initLog() {
}

func Execute() {
cobra.OnInitialize(initLog)
vConfig.SetEnvPrefix(envPrefix)
vConfig.AutomaticEnv()

cobra.OnInitialize(initialize)
rootCmd.PersistentFlags().StringVar(&configFilePath, configFileFlag, "", "config file path")
cobra.CheckErr(rootCmd.MarkPersistentFlagFilename(configFileFlag, "yaml", "yml", "json"))
rootCmd.PersistentFlags().StringVar(&logLevelVar, logLevelFlagName, "info", "log level (trace, debug, info, warn, error, fatal)")
rootCmd.PersistentFlags().StringSliceVar(&reportPathVar, reportPathFlagName, []string{}, "path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif)")
rootCmd.PersistentFlags().StringVar(&stdoutFormatVar, stdoutFormatFlagName, "yaml", "stdout output format, available formats are: json, yaml, sarif")
Expand Down
7 changes: 5 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ require (
github.com/rs/zerolog v1.29.0
github.com/slack-go/slack v0.12.2
github.com/spf13/cobra v1.6.1
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.15.0
github.com/stretchr/testify v1.8.1
github.com/zricethezav/gitleaks/v8 v8.16.1
golang.org/x/time v0.1.0
gopkg.in/yaml.v2 v2.4.0
Expand All @@ -16,6 +19,7 @@ require (
require (
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/charmbracelet/lipgloss v0.7.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fatih/semgroup v1.2.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
Expand All @@ -33,12 +37,11 @@ require (
github.com/muesli/termenv v0.15.1 // indirect
github.com/pelletier/go-toml/v2 v2.0.7 // indirect
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.15.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
golang.org/x/sync v0.1.0 // indirect
Expand Down
101 changes: 101 additions & 0 deletions lib/flags.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package lib

import (
"fmt"
"path/filepath"
"strings"

"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/spf13/viper"
)

func LoadConfig(v *viper.Viper, configFilePath string) error {
if configFilePath == "" {
return nil
}

configType := strings.TrimPrefix(filepath.Ext(configFilePath), ".")

v.SetConfigType(configType)
v.SetConfigFile(configFilePath)
return v.ReadInConfig()
}

// TODO: can be a package

// BindFlags fill flags values with config file or environment variables data
func BindFlags(cmd *cobra.Command, v *viper.Viper, envPrefix string) error {
commandHierarchy := getCommandHierarchy(cmd)

bindFlag := func(f *pflag.Flag) {
fullFlagName := fmt.Sprintf("%s%s", commandHierarchy, f.Name)
bindEnvVarIntoViper(v, fullFlagName, envPrefix)

if f.Changed {
return
}

if v.IsSet(fullFlagName) {
val := v.Get(fullFlagName)
applyViperFlagToCommand(f, val, cmd)
}
}
cmd.PersistentFlags().VisitAll(bindFlag)
cmd.Flags().VisitAll(bindFlag)

for _, subCmd := range cmd.Commands() {
if err := BindFlags(subCmd, v, envPrefix); err != nil {
return err
}
}

return nil
}

func bindEnvVarIntoViper(v *viper.Viper, fullFlagName, envPrefix string) {
envVarSuffix := strings.ToUpper(strings.ReplaceAll(strings.ReplaceAll(fullFlagName, "-", "_"), ".", "_"))
envVarName := fmt.Sprintf("%s_%s", envPrefix, envVarSuffix)

if err := v.BindEnv(fullFlagName, envVarName, strings.ToLower(envVarName)); err != nil {
log.Err(err).Msg("Failed to bind Viper flags")
}
}

func applyViperFlagToCommand(flag *pflag.Flag, val interface{}, cmd *cobra.Command) {
switch t := val.(type) {
case []interface{}:
var paramSlice []string
for _, param := range t {
paramSlice = append(paramSlice, param.(string))
}
valStr := strings.Join(paramSlice, ",")
if err := flag.Value.Set(valStr); err != nil {
log.Err(err).Msg("Failed to set Viper flags")
}
default:
newVal := fmt.Sprintf("%v", val)
if err := flag.Value.Set(newVal); err != nil {
log.Err(err).Msg("Failed to set Viper flags")
}
}
flag.Changed = true
}

func getCommandHierarchy(cmd *cobra.Command) string {
names := []string{}
if !cmd.HasParent() {
return ""
}

for parent := cmd; parent.HasParent() && parent.Name() != ""; parent = parent.Parent() {
names = append([]string{parent.Name()}, names...)
}

if len(names) == 0 {
return ""
}

return strings.Join(names, ".") + "."
}
Loading

0 comments on commit 127e190

Please sign in to comment.