interpret certain values in provisioning files as templates

This provides a convenient way of creating provisioning files that can
be controlled by parameters.

Author: JoelColledge

cmd/image_build.go

@@ -13,7 +13,7 @@ import (
 func imageBuildCommand() *cobra.Command {
 	var vmID uint
 	var provisionFile string
-	var provisionValues []string
+	var provisionOverrides []string
 
 	var vcpus uint
 
@@ -74,8 +74,8 @@ step, and then committing the resulting volume.`,
 			}
 
 			provOpt := virter.ProvisionOption{
-				FilePath: provisionFile,
-				Values:   provisionValues,
+				FilePath:  provisionFile,
+				Overrides: provisionOverrides,
 			}
 
 			provisionConfig, err := virter.NewProvisionConfig(provOpt)
@@ -113,7 +113,7 @@ step, and then committing the resulting volume.`,
 	}
 
 	buildCmd.Flags().StringVarP(&provisionFile, "provision", "p", "", "name of toml file containing provisioning steps")
-	buildCmd.Flags().StringSliceVarP(&provisionValues, "set", "s", []string{}, "set/override provisioning steps")
+	buildCmd.Flags().StringSliceVarP(&provisionOverrides, "set", "s", []string{}, "set/override provisioning steps")
 	buildCmd.Flags().UintVarP(&vmID, "id", "", 0, "ID for VM which determines the IP address")
 	buildCmd.Flags().UintVar(&vcpus, "vcpus", 1, "Number of virtual CPUs to allocate for the VM")
 

cmd/vm_exec.go

@@ -4,16 +4,17 @@ import (
 	"context"
 	"strings"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/LINBIT/virter/internal/virter"
 	"github.com/LINBIT/virter/pkg/netcopy"
-	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
 )
 
 func vmExecCommand() *cobra.Command {
 	var provisionFile string
-	var provisionValues []string
+	var provisionOverrides []string
 
 	execCmd := &cobra.Command{
 		Use:   "exec vm_name [vm_name...]",
@@ -22,8 +23,8 @@ func vmExecCommand() *cobra.Command {
 		Args:  cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
 			provOpt := virter.ProvisionOption{
-				FilePath: provisionFile,
-				Values:   provisionValues,
+				FilePath:  provisionFile,
+				Overrides: provisionOverrides,
 			}
 			if err := execProvision(provOpt, args); err != nil {
 				log.Fatal(err)
@@ -32,7 +33,7 @@ func vmExecCommand() *cobra.Command {
 	}
 
 	execCmd.Flags().StringVarP(&provisionFile, "provision", "p", "", "name of toml file containing provisioning steps")
-	execCmd.Flags().StringSliceVarP(&provisionValues, "set", "s", []string{}, "set/override provisioning steps")
+	execCmd.Flags().StringSliceVarP(&provisionOverrides, "set", "s", []string{}, "set/override provisioning steps")
 
 	return execCmd
 }

doc/provisioning.md

@@ -16,39 +16,38 @@ It can also be applied to one or multiple already running VMs:
 $ virter vm exec -p provisioning.toml centos-1 centos-2 centos-3
 ```
 
+## Provisioning types
+
 The following provisioning types are supported.
 
-## `docker`
+### `docker`
 A `docker` provisioning step allows specifying a Docker image to run provisioning steps in. This image will be executed on the host, so it will need to connect to the target VM and run its provisioning commands over SSH (or use a provisioning tool such as Ansible).
 
-### Configuration Options
 The Docker provisioning step can be parameterized using the following configuration options:
-* `image` is the Docker image used to provision the VM. It follows the standard Docker format of `<repository>/<image>:<tag>`.
-* `env` is a map of environment variables to be passed to the Docker container, in `KEY=value` format.
+* `image` is the Docker image used to provision the VM. It follows the standard Docker format of `<repository>/<image>:<tag>`. This is a Go template.
+* `env` is a map of environment variables to be passed to the Docker container, in `KEY=value` format. The values are Go templates.
 
   Note that Virter already passes two environment variables by default:
   * `TARGETS` is a comma separated list of all VMs to run the provisioning on.
   * `SSH_PRIVATE_KEY` is the SSH private key Virter uses to connect to the machine as `root`.
 
-## Shell
+### Shell
 The `shell` provisioning step allows running arbitrary commands on the target VM over SSH. This is easier to use than the `docker` step, but also less flexible.
 
-### Configuration Options
 The `shell` provisioning step accepts the following parameters:
 * `script` is a string containing the command(s) to be run.
   It can be either a single line string to run only a single command, or a multi-line string (as defined by toml), in which case every line of the string will be considered a separate command to run.
-* `env` is a map of environment variables to be set in the target VM, in `KEY=value` format.
+* `env` is a map of environment variables to be set in the target VM, in `KEY=value` format. The values are Go templates.
 
-## rsync
+### rsync
 
 The `rsync` provisioning step can be used to distribute files from the host to the guest machines using the `rsync` utility.
 
 **NOTE**: This step requires that the `rsync` program is installed both on the host and on the guest machine. The user is responsible for making sure that this requirement is met.
 
-### Configuration Options
-
+The `rsync` provisioning step accepts the following parameters:
 * `source` is as a glob pattern of files on the host machine.
-  It will first be expanded by Go's [os.ExpandEnv](https://golang.org/pkg/os/#ExpandEnv) and
+  It will first be expanded as a Go template and
   then interpreted according to the rules of Go's [filepath.Match](https://golang.org/pkg/path/filepath/#Match)
   function, so refer to the Go documentation for details.
 * `dest` is the path on the guest machine(s) where the files should be copied to.
@@ -59,16 +58,30 @@ The glob-expanded `source` list of files and the `dest` path are passed verbatim
 
 There are also global options which can be set for all provisioning steps in a file.
 
-* `env` is a map of environment variables in `KEY=value` format. These will be set in all provisioning steps that support `env` by themselves.
+* `env` is a map of environment variables in `KEY=value` format. These will be set in all provisioning steps that support `env` by themselves. The values are Go templates.
+
+## Template values
+
+As documented for the various provisioning types above, many of the values in a provisioning file are interpreted as
+[Go templates](https://golang.org/pkg/text/template/).
+
+The data provided to these templates is the `[values]` section in the provisioning file. These values can be set or overridden with `--set`. For instance:
+```
+$ virter vm exec my-vm -p examples/hello-world/hello-world.toml --set values.Image=my-image-name
+```
 
 ## Example
 ```
+[values]
+Image = "virter-hello-text"
+
 [env]
 foo = "rck"
 
 [[steps]]
 [steps.docker]
-image = "virter-hello-text:latest"
+# Go templating can be used for many values
+image = "{{.Image}}"
 [steps.docker.env]
 TEXT = "foo"
 VAR_BAR = "hi"

examples/hello-world/hello-world.toml

@@ -0,0 +1,6 @@
+[values]
+Image = "virter-hello-world"
+
+[[steps]]
+[steps.docker]
+image = "{{.Image}}"

internal/virter/provision.go

@@ -1,8 +1,11 @@
 package virter
 
 import (
+	"bytes"
+	"fmt"
 	"io"
 	"os"
+	"text/template"
 
 	"github.com/BurntSushi/toml"
 	"github.com/helm/helm/pkg/strvals"
@@ -36,8 +39,9 @@ type ProvisionStep struct {
 
 // ProvisionConfig holds the configuration of the whole provisioning
 type ProvisionConfig struct {
-	Env   map[string]string `toml:"env"`
-	Steps []ProvisionStep   `toml:"steps"`
+	Values map[string]string `toml:"values"`
+	Env    map[string]string `toml:"env"`
+	Steps  []ProvisionStep   `toml:"steps"`
 }
 
 // NeedsDocker checks if there is a provision step that requires a docker client
@@ -77,8 +81,8 @@ func EnvmapToSlice(envMap map[string]string) []string {
 
 // ProvisionOption sumarizes all the options used for generating the final ProvisionConfig
 type ProvisionOption struct {
-	FilePath string
-	Values   []string
+	FilePath  string
+	Overrides []string
 }
 
 // NewProvisionConfig returns a ProvisionConfig from a ProvisionOption
@@ -116,11 +120,27 @@ func newProvisionConfigReader(provReader io.Reader, provOpt ProvisionOption) (Pr
 		return pc, err
 	}
 
-	for _, s := range pc.Steps {
+	for i, s := range pc.Steps {
 		if s.Docker != nil {
 			s.Docker.Env = mergeEnv(&pc.Env, &s.Docker.Env)
+
+			if s.Docker.Image, err = executeTemplate(s.Docker.Image, pc.Values); err != nil {
+				return pc, fmt.Errorf("failed to execute template for docker.image for step %d: %w", i, err)
+			}
+
+			if err := executeTemplates(s.Docker.Env, pc.Values); err != nil {
+				return pc, fmt.Errorf("failed to execute template for docker.env for step %d: %w", i, err)
+			}
 		} else if s.Shell != nil {
 			s.Shell.Env = mergeEnv(&pc.Env, &s.Shell.Env)
+
+			if err := executeTemplates(s.Shell.Env, pc.Values); err != nil {
+				return pc, fmt.Errorf("failed to execute template for shell.env for step %d: %w", i, err)
+			}
+		} else if s.Rsync != nil {
+			if s.Rsync.Source, err = executeTemplate(s.Rsync.Source, pc.Values); err != nil {
+				return pc, fmt.Errorf("failed to execute template for rsync.source for step %d: %w", i, err)
+			}
 		}
 	}
 
@@ -130,11 +150,37 @@ func newProvisionConfigReader(provReader io.Reader, provOpt ProvisionOption) (Pr
 func genValueMap(provOpt ProvisionOption) (map[string]interface{}, error) {
 	base := map[string]interface{}{}
 
-	for _, value := range provOpt.Values {
+	for _, value := range provOpt.Overrides {
 		if err := strvals.ParseInto(value, base); err != nil {
 			return base, err
 		}
 	}
 
 	return base, nil
 }
+
+func executeTemplates(templates map[string]string, templateData map[string]string) error {
+	for k, v := range templates {
+		result, err := executeTemplate(v, templateData)
+		if err != nil {
+			return err
+		}
+		templates[k] = result
+	}
+	return nil
+}
+
+func executeTemplate(templateText string, templateData map[string]string) (string, error) {
+	tmpl, err := template.New("").Option("missingkey=error").Parse(templateText)
+	if err != nil {
+		return "", err
+	}
+
+	var b bytes.Buffer
+	err = tmpl.Execute(&b, templateData)
+	if err != nil {
+		return "", err
+	}
+
+	return b.String(), nil
+}