diff --git a/.gitignore b/.gitignore index b775fa4..a5ceb0b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.tar -.DS_Store \ No newline at end of file +.DS_Store +.byebug_history \ No newline at end of file diff --git a/README.md b/README.md index 15ba717..e78a349 100644 --- a/README.md +++ b/README.md @@ -5,396 +5,370 @@ This lambda function is meant to allow you to execute InSpec profiles in a serverless fashion. It strives to be as similar as it can be to how you would normally run `inspec exec` on your CLI, while also adding some useful functionality specific to AWS. ## Table of Contents -- [How Can I Deploy this Lambda Function?](#how-can-i-deploy-this-lambda-function) +- [How can I deploy this lambda function?](#how-can-i-deploy-this-lambda-function) - [Scan Configuration Examples](#scan-configuration-examples) -- [Where Do The Results Go?](#where-do-the-results-go) -- [How Do I Store and Specify an SSH Key for a Scan?](#how-do-i-store-and-specify-an-ssh-key-for-a-scan) -- [How can I specify `--target`?](#how-can-i-specify---target) -- [How can I specify which profile to execute?](#how-can-i-specify-which-profile-to-execute) +- [What does the `results_buckets` event attribute do?](#what-does-the-results_buckets-event-attribute-do) +- [What does the `command` event attribute do?](#what-does-the-command-event-attribute-do) +- [What does the `resources` event attribute do?](#what-does-the-resources-event-attribute-do) +- [What does the `env` event attribute do?](#what-does-the-env-event-attribute-do) +- [What does the `eval_tags` event attribute do?](#what-does-the-eval_tags-event-attribute-do) +- [What does the `results_name` event attribute do?](#what-does-the-results_name-event-attribute-do) +- [What does the `tmp_ssm_ssh_key` event attribute do?](#what-does-the-tmp_ssm_ssh_key-event-attribute-do) +- [What does the `ssm_port_forward` event attribute do?](#what-does-the-ssm_port_forward-event-attribute-do) - [How can I run profiles with dependencies in an offline environment?](#how-can-i-run-profiles-with-dependencies-in-an-offline-environment) -- [How can I specify `--input-file`?](#how-can-i-specify---input-file) -- [How can I Specify `--input`?](#how-can-i-specify---input) -- [What other kinds of configurations can I specify?](#what-other-kinds-of-configurations-can-i-specify) -- [Where Can I Read More about the InSpec Config?](#where-can-i-read-more-about-the-inspec-config) +- [How do I set up an SSM managed instance?](#how-do-i-set-up-an-ssm-managed-instance) - [Scheduling Recurring Scans](#scheduling-recurring-scans) -## How Can I Deploy this Lambda Function? -For instructions on how to configure and deploy this Lambda function, see [EXAMPLE.md](./EXAMPLE.md) +## How can I deploy this lambda function? +For instructions on how to configure and deploy this Lambda function with Terraform, see [TF_EXAMPLE.md](./TF_EXAMPLE.md) ## Scan Configuration Examples These are examples of the JSON that can be passed into the lambda event to obtain a successful scan. You can find more details on these configurations and additional configuration options in this README. -### What are the requirement differences between SSH, WinRM, SSH via SSM, SSM send command, etc? - -| | Must manage SSH keys | Must have network access to target | Must have SSM Agent installed on target | Both lambda and target must have network access to SSM | Example Target | -|:-----------------------------:|:--------------------:|:----------------------------------:|:---------------------------------------:|:------------------------------------------------------:|------------------------------------| -| SSH | ✅ | ✅ | ❌ | ❌ | ssh://root@domain.com | -| WinRM | ✅ | ✅ | ❌ | ❌ | winrm://domain.com | -| SSH via SSM | ✅ /❌ (optional) | ❌ | ✅ | ✅ | ssh://ec2-user@i-0e35ab216355084ee | -| WinRM via SSM Port Forwarding | ✅ | ❌ | ✅ | ✅ | winrm://i-0e35ab216355084ee | -| SSM Send Command | ❌ | ❌ | ✅ | ✅ | awsssm://i-0e35ab216355084ee | - - ### AWS Resource Scanning Note that if you are running InSpec AWS scans, then the lambda's IAM profile must have suffient permissions to analyze your environment. ```json { - "results_bucket": "inspec-results-bucket", - "profile": "https://github.com/mitre/aws-foundations-cis-baseline/archive/refs/heads/master.zip", - "profile_common_name": "demo-aws-baseline-master", - "config": { - "target": "aws://" - } + "command": "inspec exec https://github.com/mitre/aws-foundations-cis-baseline/archive/master.tar.gz -t aws://", + "results_name": "aws-foundations-cis-baseline", + "results_buckets": [ + "inspec-results-bucket-dev" + ], + "eval_tags": "ServerlessInspec,AwsCisBaseline,AWS" } ``` ### RedHat 7 STIG Baseline (SSH) ```json { - "results_bucket": "inspec-results-bucket", - "ssh_key_ssm_param": "/inspec/test-ssh-key", - "profile": { - "bucket": "inspec-profiles-bucket", - "key": "redhat-enterprise-linux-7-stig-baseline-master.zip" - }, - "profile_common_name": "redhat-enterprise-linux-7-stig-baseline-master", - "config": { - "target": "ssh://ec2-user@ec2-15-200.us-gov-west-1.compute.amazonaws.com", - "sudo": true, - "input_file": { - "bucket": "inspec-profiles-bucket-dev-28wd", - "key": "rhel7-stig-baseline-master-disable-slow-controls.yml" + "command": "inspec exec /tmp/redhat-enterprise-linux-7-stig-baseline-2.6.6.tar.gz -t ssh://ec2-user@ec2-15-200-235-74.us-gov-west-1.compute.amazonaws.com -i /tmp/id_rsa --sudo --input=disable_slow_controls=true", + "results_name": "redhat-enterprise-linux-7-stig-baseline-inspec-rhel7-test", + "results_buckets": [ + "inspec-results-bucket-dev" + ], + "eval_tags": "ServerlessInspec,RHEL7,inspec-rhel7-test,SSH", + "resources": [ + { + "local_file_path": "/tmp/redhat-enterprise-linux-7-stig-baseline-2.6.6.tar.gz", + "source_aws_s3_bucket": "inspec-profiles-bucket-dev", + "source_aws_s3_key": "redhat-enterprise-linux-7-stig-baseline-2.6.6.tar.gz" + }, + { + "local_file_path": "/tmp/id_rsa", + "source_aws_ssm_parameter_key": "/inspec/rhel-7-test/id_rsa" } - } + ] } ``` -### RedHat 7 STIG Baseline (SSH via SSM with managed SSH key) +### RedHat 7 STIG Baseline (SSH via SSM tunneled through SSM with a temporary SSH key) +The `--proxy_command` command line argument is tunneling the session through SSM. + +The `tmp_ssm_ssh_key` defines attributes around a temporary SSH key that will be generated and disposed of during the lifetime of the function execution, which provides temporary access (in a 60 second window) to the SSM managed instance for the function. ```json { - "results_bucket": "inspec-results-bucket", - "ssh_key_ssm_param": "/inspec/test-ssh-key", - "profile": "https://github.com/mitre/redhat-enterprise-linux-7-stig-baseline.git", - "profile_common_name": "redhat-enterprise-linux-7-stig-baseline-master", - "config": { - "target": "ssh://ec2-user@i-00f1868f8f3b4cc03", - "input": [ - "disable_slow_controls=true" - ], - "sudo": true + "command": "inspec exec https://github.com/mitre/redhat-enterprise-linux-7-stig-baseline/archive/master.tar.gz -t ssh://ssm-user@i-00f1868f8f3b4eb03 -i /tmp/tmp_ssh_key --input=disable_slow_controls=true --proxy-command='sh -c \"aws ssm start-session --target %h --document-name AWS-StartSSHSession --parameters portNumber=%p\"'", + "results_name": "redhat-enterprise-linux-7-stig-baseline-inspec-rhel7-test", + "results_buckets": [ + "inspec-results-bucket-dev" + ], + "eval_tags": "ServerlessInspec,RHEL7,inspec-rhel7-test,SSH-SSM", + "tmp_ssm_ssh_key": { + "host": "i-00f1868f8f3b4eb03", + "user": "ssm-user", + "key_name": "tmp_ssh_key" } } ``` -### RedHat 7 STIG Baseline (SSH via SSM without managed SSH key) +### RedHat 7 STIG Baseline (SSM Send Command with awsssm:// transport) ```json { - "results_bucket": "inspec-results-bucket", - "ssm_temp_ssh_key": true, - "profile": "https://github.com/mitre/redhat-enterprise-linux-7-stig-baseline.git", - "profile_common_name": "redhat-enterprise-linux-7-stig-baseline-master", - "config": { - "target": "ssh://root@i-00f1868f8f3b4cc03", - "input": [ - "disable_slow_controls=true" - ], - } + "command": "inspec exec https://github.com/mitre/redhat-enterprise-linux-7-stig-baseline/archive/master.tar.gz -t awsssm://i-00f1868f8f3b4eb03 --input=disable_slow_controls=true", + "results_name": "redhat-enterprise-linux-7-stig-baseline-inspec-rhel7-test", + "results_buckets": [ + "inspec-results-bucket-dev" + ], + "eval_tags": "ServerlessInspec,RHEL7,inspec-rhel7-test,AWSSSM" } ``` -### RedHat 7 STIG Baseline (SSM Send Command) +### Windows Server 2019 STIG Baseline (WinRM via SSM Port Forwarding) +Note that the target is set to `winrm://localhost` because port forwarding is being set up with the `ssm_port_forward` event property. ```json { - "results_bucket": "inspec-results-bucket", - "profile": "https://github.com/mitre/redhat-enterprise-linux-7-stig-baseline.git", - "profile_common_name": "redhat-enterprise-linux-7-stig-baseline-master", - "config": { - "target": "awsssm://i-00f1868f8f3b4cc03", - "input": [ - "disable_slow_controls=true" - ], - "sudo": true + "command": "inspec exec /tmp/microsoft-windows-server-2019-stig-baseline-1.3.10.tar.gz -t winrm://localhost --password $WIN_PASS", + "results_name": "windows-server-2019-stig-baseline-inspec-win2019-test", + "results_buckets": [ + "inspec-results-bucket-dev" + ], + "eval_tags": "ServerlessInspec,WinSvr2019,inspec-win2019-test,WinRM", + "resources": [ + { + "local_file_path": "/tmp/microsoft-windows-server-2019-stig-baseline-1.3.10.tar.gz", + "source_aws_s3_bucket": "inspec-profiles-bucket-dev", + "source_aws_s3_key": "microsoft-windows-server-2019-stig-baseline-1.3.10.tar.gz" + }, + { + "env_variable": "WIN_PASS", + "source_aws_secrets_manager_secret_name": "/inspec/inspec-win2019-test/password" + } + ], + "ssm_port_forward": { + "instance_id": "i-0e35ab216355084ee", + "ports": [5985, 5986] } } ``` -### PostgreSQL 12 STIG Baseline (TODO) -```json -"https://github.com/mitre/aws-rds-crunchy-data-postgresql-9-stig-baseline" -``` - -### Windows Server 2019 STIG Baseline (WinRM) +### Windows Server 2019 STIG Baseline (SSM Send Command with awsssm:// transport) ```json { - "results_bucket": "inspec-results-bucket", - "profile": "https://github.com/mitre/microsoft-windows-server-2019-stig-baseline.git", - "profile_common_name": "microsoft-windows-server-2019-stig-baseline", - "config": { - "target": "winrm://ec2-160.us-gov-west-1.compute.amazonaws.com", - "user": "Administrator", - "password": { - "instance_id": "i-00f1868f8f3b4cc03", - "launch_key": "/inspec/test-ssh-key" - } - } + "command": "inspec exec /tmp/microsoft-windows-server-2019-stig-baseline-1.3.10.tar.gz -t awsssm://i-00f1868f8f3b4eb03", + "results_name": "windows-server-2019-stig-baseline-inspec-win2019-test", + "results_buckets": [ + "inspec-results-bucket-dev" + ], + "eval_tags": "ServerlessInspec,WinSvr2019,inspec-win2019-test,AWSSSM", + "resources": [ + { + "local_file_path": "/tmp/microsoft-windows-server-2019-stig-baseline-1.3.10.tar.gz", + "source_aws_s3_bucket": "inspec-profiles-bucket-dev", + "source_aws_s3_key": "microsoft-windows-server-2019-stig-baseline-1.3.10.tar.gz" + }, + ], } ``` -### Windows Server 2019 STIG Baseline (WinRM via SSM Port Forwarding) +### Kubernetes (with k8s:// transport) ```json { - "results_bucket": "inspec-results-bucket-dev-28wd", - "profile": "https://github.com/mitre/microsoft-windows-server-2019-stig-baseline.git", - "profile_common_name": "microsoft-windows-server-2019-stig-baseline", - "config": { - "target": "winrm://i-0e35ab216355084ee", - "user": "Administrator", - "password": { - "instance_id": "i-0e35ab216355084ee", - "launch_key": "/inspec/test-ssh-key" + "command": "inspec exec https://gitlab.dsolab.io/scv-content/inspec/kubernetes/baselines/k8s-cluster-stig-baseline/-/archive/master/k8s-cluster-stig-baseline-master.tar.gz -t k8s://", + "results_name": "k8s-cluster-stig-baseline-dev-cluster", + "results_buckets": [ + "inspec-results-bucket-dev" + ], + "eval_tags": "ServerlessInspec,k8s", + "resources": [ + { + "local_file_path": "/tmp/kube/config", + "source_aws_s3_bucket": "inspec-profiles-bucket-dev", + "source_aws_s3_key": "kube-dev/config" + }, + { + "local_file_path": "/tmp/kube/client.crt", + "source_aws_ssm_parameter_key": "/inspec/kube-dev/client_crt" + }, + { + "local_file_path": "/tmp/kube/client.key", + "source_aws_ssm_parameter_key": "/inspec/kube-dev/client_key" + }, + { + "local_file_path": "/tmp/kube/ca.crt", + "source_aws_ssm_parameter_key": "/inspec/kube-dev/ca_crt" } + ], + "env": { + "KUBECONFIG": "/tmp/kube/config" } } ``` -### Windows Server 2019 STIG Baseline (SSM Send Command) +### PostgreSQL 12 STIG Baseline (TODO) +Database scans have not been tested yet with this lambda function. ```json -{ - "results_bucket": "inspec-results-bucket-dev-28wd", - "profile": "https://github.com/mitre/microsoft-windows-server-2019-stig-baseline.git", - "profile_common_name": "microsoft-windows-server-2019-stig-baseline", - "config": { - "target": "awsssm://i-0e35ab216355084ee" - } -} +"https://github.com/mitre/aws-rds-crunchy-data-postgresql-9-stig-baseline" ``` -## Where Do The Results Go? -If you DO NOT specify the `results_bucket` parameter in the lambda event, then the results will just be logged to CloudWatch. If you DO specify the `results_bucket` parameter in the lambda event, then the lambda will attempt to save the results JSON to the S3 bucket under `unprocessed/*`. The format of the JSON is meant to be a incomplete API call to push results to a Heimdall Server and looks like this: +## What does the `results_buckets` event attribute do? +The `results_buckets` event attribute defines S3 buckets that the function will push JSON results to. + +If you DO NOT specify the `results_buckets` parameter in the lambda event, then the results will just be logged to CloudWatch. If you DO specify the `results_buckets` parameter in the lambda event, then the lambda will attempt to save the results JSON to the S3 bucket under `unprocessed/*`. The format of the JSON is meant to be a incomplete API call to push results to a Heimdall Server and looks like this: ```javascript { "data": {}, // this contains the HDF results - "eval_tags": "ServerlessInspec" + "eval_tags": "" } ``` -## How Do I Store and Specify an SSH Key for a Scan? -SSH keys for this lambda are expected to be stored in an Secure String parameter within Systems Manager's Parameter Store. Note that if you are trying to scan against an AWS-provided EC2 instance, then you will likely want to save the public key material to `/ec2-user/.ssh/authorized_keys` on the instance. - -If you are encrypting the Secure String parameter with something other than the default KMS key (this is recommended), then you will need to ensure that the lambda's IAM role has permissions to execute `kms:Decrypt` against your KMS key. - -## How can I specify `--target`? -If you omit the `config['target']` argument, then InSpec will attempt to execute the profile against the lambda itself. - -#### Connecting to SSM Managed EC2 Instance -One additional feature that this lambda provides on top of standard InSpec is that it allows you to establish an SSH or WinRM session for an InSpec scan tunneled through SSM. This is especially useful if the lambda does not have direct network access to its target, but can have a connection through SSM. You can read more about SSM Managed Instance sessions [here](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-sessions-start.html) - -You can make an EC2 instance a SSM managed instance using [this guide](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-setting-up.html). This also requires that your EC2 instance has the SSM agent software installed on it. Some AWS-provided images already have this installed, but if it is not already installed on you instance then you can use [this guide](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-setting-up.html) to get it installed. Note that tunneling through SSM for an SSH or WinRM session still requires that you have the credentials to authenticate to the instance. - -Note that you aren't limited to just scanning AWS resources, as long as the lambda has access to the internet, then it can scan any resource that you would scan with a normal `inspec exec` command. - -### SSH +The `results_bucket` event parameter format looks like the following: ```json { "...": "...", - "config": { - "target": "ssh://ec2-user@somednsname.aws.com" - } + "results_buckets": [ + "inspec-results-bucket-dev" + ] } ``` -### SSM Send Command -SSM Send Command is enabled with the [train-awsssm](https://github.com/tecracer-chef/train-awsssm) gem. Use of this target method allows you to use the SSM provided SendCommand functionality to interact with the target. This requires that SSM agent be installed on the target, as well as both the lamdba and the target having network access to an AWS SSM Endpoint. The key benefit to this method is that it does not require you to manage SSH keys because AWS SSM will handle that work for you. You are technically allowed to specify the domain name or the instance ID after `awsssm://*`, however, it will likely be more convenient to use the instance ID in most cases. -```json -{ - "...": "...", - "config": { - "target": "awsssm://i-00f1868f8f3b4cc03" - } -} -``` +## What does the `command` event attribute do? +The `command` event attribute defines the inspec exec command that the lambda function will execute. Note that ` --show-progress --reporter cli json:` will be appended to the end of the `command` attribute before execution. + +This __MUST__ be an `inspec exec` command. [Read more about inspec exec here](https://docs.chef.io/inspec/cli/#exec) -### SSH (Tunneled Through SSM) -The difference with this example and the one above is that the `target` is the instance ID of the EC2 instance. This tells the lambda to use a [SSM SSH connection](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-getting-started-enable-ssh-connections.html) to tunnel the SSH connection. This is particularly useful if there is not direct network access to the EC2 isntance, but both the lambda and EC2 instance have access to SSM. Note that this also requires that your EC2 instance be a SSM managed instance. -#### With Your Own Managed Keys ```json { - "...": "...", - "config": { - "target": "ssh://ec2-user@i-00f1868f8f3b4cc03" - } + "command": "inspec exec /tmp/redhat-enterprise-linux-7-stig-baseline-2.6.6.tar.gz -t ssh://root@host -i /tmp/id_rsa --sudo --input=disable_slow_controls=true", + "...": "..." } ``` -#### Without Your Own Managed Keys -This method of InSpec scanning works with the following sequence of events: -1. Generate a SSH key pair within the lambda function -2. Use the [train-awsssm](https://github.com/tecracer-chef/train-awsssm) plugin to send the public key material to `~/.ssh/authorized_keys` using [SSM Send Command](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SSM/Client.html#send_command-instance_method) -3. Immedately queue another SSM Send Command to remove the key from `~/.ssh/authorized_keys` after 60 seconds -4. Start an SSH session using the generated key pair and execute the InSpec scan over SSH +## What does the `resources` event attribute do? +The `resources` event attribute defines what files and/or environment variables are needed prior to executing the InSpec command. This might be a tar.gz of an InSpec profile stored in S3, a Windows password stored in SSM Parameter store, etc. -Assumptions with this method: -- Scanning linux-based instances (i.e. not Windows) -- The instance has the following commands installed: `su`, `mkdir`, `touch`, `echo`, `sleep`, `grep`, `mv` -- The user that runs "SSM Send Command" commands is priviledged to write to any user's `~/.ssh` directory (this should default to root unless explicitly changed) +You may define a resources as needing to be downloaded as a local file on disk (must be located within `/tmp/`), or as needing have their contents stored in an environment variable. Downloaded files and environment variable resources will be usable by your `inspec exec ...` command. -This method is advantageous over the "SSM Send Command" method mentioned above because invoking all InSpec commands over SSM Send Command is significantly slower than over SSH, and it shares advantage of relieving the need to manually manage SSH keys. +### Local File Resource ```json { - "...": "...", - "ssm_temp_ssh_key": true, - "config": { - "target": "ssh://ec2-use@i-00f1868f8f3b4cc03" - } + "resources": [ + { + "local_file_path": "/tmp/microsoft-windows-server-2019-stig-baseline-1.3.10.tar.gz", + "...": "...", + } + ], } ``` -### WinRM +### Environment Variable Resource ```json { - "...": "...", - "config": { - "target": "ssh://ec2-user@somednsname.aws.com" - } + "resources": [ + { + "env_variable": "WIN_PASS", + "...": "..." + } + ], } ``` -### WinRM (Tunneled Through SSM) -The difference with this example and the one above is that the `target` is the instance ID of the EC2 instance. This tells the lambda to use [SSM port forwarding](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-sessions-start.html) to tunnel the WinRM connection. This is particularly useful if there is not direct network access to the EC2 isntance, but both the lambda and EC2 instance have access to SSM. Note that this also requires that your EC2 instance be a SSM managed instance. +The possible sources for the lamdba's `resources` are defined below: + +### S3 Resources +Resources may be downloaded from an S3 bucket. Ensure that you lambda's IAM role has `s3:getObject` permissions for the desired bucket/object. ```json { - "...": "...", - "config": { - "target": "ssh://ec2-user@i-00f1868f8f3b4cc03" - } + "resources": [ + { + "...": "...", + "source_aws_s3_bucket": "inspec-profiles-bucket-dev", + "source_aws_s3_key": "microsoft-windows-server-2019-stig-baseline-1.3.10.tar.gz" + }, + ] } ``` -### AWS +### AWS SSM Parameter Store Resources +Resources may be fetched from AWS SSM Parameter Store. Ensure that you lambda's IAM role has `kms:Decrypt` permissions for the secrets's KMS key and has `ssm:GetParameter` permissions for the parameter. + ```json { - "...": "...", - "config": { - "target": "aws://" - } + "resources": [ + { + "...": "...", + "source_aws_ssm_parameter_key": "/inspec/kube-dev/client_crt" + } + ] } ``` -## How can I specify which profile to execute? -Profile sources are documented by InSpec [here](https://docs.chef.io/inspec/cli/#exec). - -### Zipped folder on S3 Bucket -In addition to what is already allowed by the vanilla InSpec exec command, you are able to specify a file from an AWS bucket that may be private that the lambda has permissions to access via the AWS API. - -If the bucket is not public, you must provide the proper permissions to the lambda's IAM role! This also supports `tar.gz` format. +### AWS Secrets Manager Resources +Resources may be fetched from AWS Secrets Manager. Ensure that you lambda's IAM role has `kms:Decrypt` permissions for the secrets's KMS key and `secretsmanager:GetSecretValue` permissions for the secret. ```json { - "...": "...", - "profile": { - "bucket": "inspec-profiles-bucket", - "key": "profiles/inspec-profile.zip" - } + "resources": [ + { + "...": "...", + "source_aws_secrets_manager_secret_name": "/inspec/kube-dev/client_crt" + } + ] } ``` -### GitHub Repository +## What does the `env` event attribute do? +The `env` event attribute allows definition of static envrionment variables that are needed by the InSpec command. Note that you may not overwrite an existing environment varaible. ```json { "...": "...", - "profile": "https://github.com/mitre/demo-aws-baseline.git" + "env": { + "KUBECONFIG": "/tmp/kube/config", + "OTHER_ENV": "value" + } } ``` -### Web hosted +## What does the `eval_tags` event attribute do? +The `eval_tags` event attribute allows definition of comma separated Heimdall `eval_tags` that will be passed through to the results file. ```json { "...": "...", - "profile": "https://username:password@webserver/linux-baseline.tar.gz" + "eval_tags": "hostname,profile,etc" } ``` -### Chef Supermarket -(This hasn't been tested yet!) +## What does the `results_name` event attribute do? +The `results_name` event attribute defines the filename for generated InSpec scan results. If this is not set, then the `results_name` will default to `unnamed_profile`. ```json { "...": "...", - "profile": "supermarket://username/linux-baseline" + "results_name": "human-readable-name-of-my-results" } ``` -## How can I run profiles with dependencies in an offline environment? -The recommendation for offline environments is to save vendored InSpec profiles to an S3 bucket. +## What does the `tmp_ssm_ssh_key` event attribute do? +The `tmp_ssm_ssh_key` event attribute allows the function to push temporary SSH keys to a linux-based SSM managed instance. These keys are generated as needed and are disposed of on the target machine after 60 seconds. -```bash -git clone git@github.com:mitre/aws-foundations-cis-baseline.git -inspec vendor ./aws-foundations-cis-baseline && inspec archive ./aws-foundations-cis-baseline -# Then upload ./aws-foundations-cis-baseline.tar.gz to you S3 bucket -``` +This method of InSpec scanning works with the following sequence of events: +1. Generate a SSH key pair within the lambda function +2. Use the [train-awsssm](https://github.com/tecracer-chef/train-awsssm) plugin to send the public key material to `~/.ssh/authorized_keys` using [SSM Send Command](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SSM/Client.html#send_command-instance_method) +3. Immedately queue another SSM Send Command to remove the key from `~/.ssh/authorized_keys` after 60 seconds +4. Start an SSH session using the generated key pair and execute the InSpec scan over SSH -## How can I specify `--input-file`? -You can read more about InSpec inputs [here](https://docs.chef.io/inspec/inputs/) +Assumptions with this method: +- Scanning linux-based instances (i.e. not Windows) +- The instance has the following commands installed: `su`, `mkdir`, `touch`, `echo`, `sleep`, `grep`, `mv` +- The user that runs "SSM Send Command" commands is priviledged to write to any user's `~/.ssh` directory (this should default to root unless explicitly changed) -### File on S3 Bucket -Note that you must ensure that the lambda's IAM role has permissions to get objects for the specified bucket. +This method is advantageous over using the `awsssm://` transport by itself because invoking all InSpec commands over SSM Send Command is significantly slower than over SSH. ```json { "...": "...", - "config": { - "bucket": "inspec-profiles-bucket", - "key": "input_files/custom-inspec.yml" + "tmp_ssm_ssh_key": { + "host": "i-00f1868f8f3b4eb03", + "user": "ssm-user", + "key_name": "tmp_ssh_key" } } ``` -### SecureString SSM Parameter -Note that you must ensure that the lambda's IAM role has permission to the parameter as well as its KMS key to properly fetch & decrypt. -```json -{ - "...": "...", - "config": { - "input_file": { - "ssm_secure_string": "inspec/input_file/param" - } - } -} -``` +## What does the `ssm_port_forward` event attribute do? +The `tmp_ssm_ssh_key` event attribute defines local ports to be forwarded to a specific SSM managed instance. This is useful if the lambda function does not have direct network access to the machine, but both the lambda and machine have access to SSM. -## How can I Specify `--input`? +Note that forwarding local ports will mean that you will need to connect to `localhost` for your inspec exec command (e.g., `inspec exec ... -t winrm://localhost`) ```json { "...": "...", - "config": { - "input": [ - "disable_slow_controls=true", - "other_input=value" - ] + "ssm_port_forward": { + "instance_id": "i-0e35ab216355084ee", + "ports": [5985, 5986] } } ``` -## What other kinds of configurations can I specify? +## How can I run profiles with dependencies in an offline environment? +The recommendation for offline environments is to save vendored InSpec profiles to an S3 bucket. -```javascript -{ - "ssh_key_ssm_param": "/inspec/test-ssh-key", // --key-files / -ia - "config": { - "user": "username", // --user - "self_signed": true, // --self-signed - "sudo": true, // --sudo - "bastion_host": "BASTION_HOST", // --bastion-host - "bastion_port": "BASTION_PORT", // --bastion-port - "bastion_user": "BASTION_USER" // --bastion-user - } -} +```bash +git clone git@github.com:mitre/aws-foundations-cis-baseline.git +inspec vendor ./aws-foundations-cis-baseline && inspec archive ./aws-foundations-cis-baseline +# Then upload ./aws-foundations-cis-baseline.tar.gz to you S3 bucket ``` -## Where Can I Read More about the InSpec Config? -You can read more about InSpec configuraitons [here](https://docs.chef.io/inspec/config/) and about InSpec reporters [here](https://docs.chef.io/inspec/reporters/). There are some configuration items that are always overridden so that the lambda can work properly - like the reporter, logger, and type. - -InSpec doesn't necessarily document the configuration futher than this (to aid easier use of InSpec from Ruby code and not the CLI). The workaround for this was to add an interactive debugger (or even just a `puts conf` statement) to the InSpec Runner source code on a local develeopment machine (found under `//inspec-core-4.37.17/lib/inspec/runner.rb#initialize`). Once the interactive debugger is in place, you can specify InSpec CLI commands as you normally would and view how the configuration is affected. You can find the location of the inspec gem source by running `gem which inspec`. +## How Do I Set Up an SSM Managed Instance? +You can make an EC2 instance a SSM managed instance using [this guide](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-setting-up.html). This also requires that your EC2 instance has the SSM agent software installed on it. Some AWS-provided images already have this installed, but if it is not already installed on you instance then you can use [this guide](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-setting-up.html) to get it installed. ## Scheduling Recurring Scans The recommended way to set up recurring scans is to create an Event Rule within AWS CloudWatch. @@ -422,3 +396,4 @@ MITRE hereby grants express written permission to use, reproduce, distribute, mo This software was produced for the U. S. Government under Contract Number W56KGU-18-D-0004, and is subject to Federal Acquisition Regulation Clause 52.227-14, Rights in Data-General. No other use other than that granted to the U. S. Government, or to those acting on behalf of the U. S. Government under that Clause is authorized without the express written permission of The MITRE Corporation. + diff --git a/EXAMPLE.md b/TF_EXAMPLE.md similarity index 100% rename from EXAMPLE.md rename to TF_EXAMPLE.md diff --git a/src/Dockerfile b/src/Dockerfile index a3bf461..60e1d59 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -39,6 +39,8 @@ RUN yum install -y gcc make gcc-c++ git unzip &&\ git init &&\ # Install gem dependencies with bundler bundle install --path vendor/bundle/ &&\ + # Install k8s train plugin + inspec plugin install train-kubernetes &&\ # Install the AWS CLI curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" &&\ unzip awscliv2.zip &&\ diff --git a/src/Gemfile b/src/Gemfile index 6adcef1..6e13c5a 100644 --- a/src/Gemfile +++ b/src/Gemfile @@ -4,6 +4,7 @@ source 'https://rubygems.org' gem 'aws-sdk-lambda', '~> 1' gem 'aws-sdk-s3', '~> 1' +gem 'aws-sdk-secretsmanager', '~> 1' gem 'aws-sdk-ssm', '~> 1' # net-ssh requires the following gems for ed25519 support: @@ -13,3 +14,4 @@ gem 'ed25519', '>= 1.2', '< 2.0' gem 'inspec' gem 'inspec-bin' gem 'train-awsssm' +gem 'train-kubernetes', '>=0.1.6' diff --git a/src/Gemfile.lock b/src/Gemfile.lock index 7bd17a5..f135e9d 100644 --- a/src/Gemfile.lock +++ b/src/Gemfile.lock @@ -242,6 +242,32 @@ GEM multi_json domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) + dry-configurable (0.12.1) + concurrent-ruby (~> 1.0) + dry-core (~> 0.5, >= 0.5.0) + dry-container (0.8.0) + concurrent-ruby (~> 1.0) + dry-configurable (~> 0.1, >= 0.1.3) + dry-core (0.7.1) + concurrent-ruby (~> 1.0) + dry-equalizer (0.3.0) + dry-inflector (0.2.1) + dry-logic (0.6.1) + concurrent-ruby (~> 1.0) + dry-core (~> 0.2) + dry-equalizer (~> 0.2) + dry-struct (0.5.1) + dry-core (~> 0.4, >= 0.4.3) + dry-equalizer (~> 0.2) + dry-types (~> 0.13) + ice_nine (~> 0.11) + dry-types (0.13.4) + concurrent-ruby (~> 1.0) + dry-container (~> 0.3) + dry-core (~> 0.4, >= 0.4.4) + dry-equalizer (~> 0.2) + dry-inflector (~> 0.1, >= 0.1.2) + dry-logic (~> 0.4, >= 0.4.2) ed25519 (1.2.4) erubi (1.10.0) excon (0.85.0) @@ -285,12 +311,14 @@ GEM ffi (>= 1.0.1) gyoku (1.3.1) builder (>= 2.1.2) + hashdiff (1.0.1) hashie (4.1.0) http-cookie (1.0.4) domain_name (~> 0.5) httpclient (2.8.3) i18n (1.8.10) concurrent-ruby (~> 1.0) + ice_nine (0.11.2) inifile (3.0.0) inspec (4.41.2) faraday_middleware (>= 0.12.2, < 1.1) @@ -327,7 +355,19 @@ GEM tty-table (~> 0.10) jmespath (1.4.0) json (2.5.1) + jsonpath (0.9.9) + multi_json + to_regexp (~> 0.2.1) jwt (2.2.3) + k8s-ruby (0.10.5) + dry-struct (~> 0.5.0) + dry-types (~> 0.13.0) + excon (~> 0.71) + hashdiff (~> 1.0.0) + jsonpath (~> 0.9.5) + recursive-open-struct (~> 1.1.0) + yajl-ruby (~> 1.4.0) + yaml-safe_load_stream (~> 0.1) license-acceptance (2.1.13) pastel (~> 0.7) tomlrb (>= 1.2, < 3.0) @@ -372,6 +412,7 @@ GEM coderay (~> 1.1) method_source (~> 1.0) public_suffix (4.0.6) + recursive-open-struct (1.1.3) representable (3.1.1) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) @@ -411,6 +452,7 @@ GEM strings-ansi (0.2.0) thor (1.1.0) timeliness (0.3.10) + to_regexp (0.2.1) tomlrb (1.3.0) trailblazer-option (0.1.1) train (3.8.1) @@ -500,6 +542,9 @@ GEM net-scp (>= 1.2, < 4.0) net-ssh (>= 2.9, < 7.0) train-habitat (0.2.22) + train-kubernetes (0.1.6) + k8s-ruby (~> 0.10) + train (~> 3.0) train-winrm (0.2.12) winrm (>= 2.3.6, < 3.0) winrm-elevated (~> 1.2.2) @@ -549,6 +594,8 @@ GEM rubyzip (~> 2.0) winrm (~> 2.0) wisper (2.0.1) + yajl-ruby (1.4.1) + yaml-safe_load_stream (0.1.1) zeitwerk (2.4.2) PLATFORMS @@ -557,12 +604,14 @@ PLATFORMS DEPENDENCIES aws-sdk-lambda (~> 1) aws-sdk-s3 (~> 1) + aws-sdk-secretsmanager (~> 1) aws-sdk-ssm (~> 1) bcrypt_pbkdf (>= 1.0, < 2.0) ed25519 (>= 1.2, < 2.0) inspec inspec-bin train-awsssm + train-kubernetes (>= 0.1.6) BUNDLED WITH 2.2.11 diff --git a/src/lambda_function.rb b/src/lambda_function.rb index b55633b..5e4cff5 100644 --- a/src/lambda_function.rb +++ b/src/lambda_function.rb @@ -5,25 +5,16 @@ require 'aws-sdk-s3' require 'json' require 'inspec' +require 'inspec/cli' require 'logger' +require 'shellwords' require 'train-awsssm' puts "RUBY_VERSION: #{RUBY_VERSION}" $logger = Logger.new($stdout) -## -# The vanilla `dig` method will throw an exception if it hits a non-hash object -# and still has more levels to dig - for example: { a: 'test' }.dig(:a, :b, :c). -# -# The `safe_dig` method will just return nil if `dig` throws an exception. -# -class Hash - def safe_dig(*args) - dig(*args) - rescue StandardError - nil - end -end +Encoding.default_external = Encoding::UTF_8 +Encoding.default_internal = Encoding::UTF_8 ## # The current train-awsssm gem does not implement file_via_connection. @@ -41,361 +32,266 @@ def file_via_connection(path) end ## -# Entrypoint for the Serverless InSpec lambda functoin +# Entrypoint for the Serverless InSpec lambda function # # See the README for more information # # rubocop:disable Lint/UnusedMethodArgument def lambda_handler(event:, context:) # Set export filename - filename, file_path = generate_json_file(event['profile_common_name'] || 'unnamed_profile') + filename, file_path = generate_json_file(event['results_name'] || 'unnamed_profile') $logger.info("Will write JSON at #{file_path}") - # Build the config we will use when executing InSpec - config = build_config(event, file_path) + # Make modifcations to the InSpec command + inspec_cmd = event['command'] + inspec_cmd = inspec_cmd.strip + inspec_cmd = inspec_cmd.sub('inspec ', '') if inspec_cmd.start_with?('inspec ') + inspec_cmd = inspec_cmd.strip + inspec_cmd = "exec #{inspec_cmd}" unless inspec_cmd.start_with?('exec ') + inspec_cmd += " --show-progress --reporter cli json:#{file_path}" - # Define InSpec Runner - $logger.info('Building InSpec runner.') - runner = Inspec::Runner.new(config) + # Resources and ENV setup + configure_event_env(event['env']) unless event['env'].nil? + fetch_resources(event['resources']) unless event['resources'].nil? + push_tmp_ssh_key_to_instance(event['tmp_ssm_ssh_key']) unless event['tmp_ssm_ssh_key'].nil? + setup_ssm_port_forward(event['ssm_port_forward']) unless event['ssm_port_forward'].nil? - # Set InSpec Target - $logger.info('Adding InSpec target.') - runner.add_target(event['profile']) + # ENV replacement in inspec_cmd + env_inspec_cmd = inspec_cmd + ENV.each { |key, value| env_inspec_cmd.gsub!("$#{key}", value) } - # Trigger InSpec Scan - $logger.info('Running InSpec.') - runner.run + # Execute InSpec + # https://ruby-doc.org/core-2.3.0/Kernel.html#method-i-system + $logger.info("Executing InSpec command: #{inspec_cmd}") + system('inspec', *Shellwords.split(env_inspec_cmd)) + $logger.info('InSpec exec completed!') - return if event['results_bucket'].nil? + return if event['results_buckets'].nil? || event['results_buckets'].empty? # Push the results to S3 # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html # Consider allowing passing additional eval_tags through the event # Consider tagging with the account ID - s3_client = Aws::S3::Client.new - s3_client.put_object( - { - body: StringIO.new({ - 'data' => JSON.parse(File.read(file_path)), - 'eval_tags' => 'ServerlessInspec' - }.to_json), - bucket: event['results_bucket'], - key: "unprocessed/#{filename}" - } - ) + event['results_buckets'].each do |bucket| + $logger.info("Pushing results to S3 bucket: #{bucket}") + s3_client = Aws::S3::Client.new + s3_client.put_object( + { + body: StringIO.new({ + 'data' => JSON.parse(File.read(file_path)), + 'eval_tags' => event['eval_tags'] || 'ServerlessInspec' + }.to_json), + bucket: bucket, + key: "unprocessed/#{filename}" + } + ) + end end # rubocop:enable Lint/UnusedMethodArgument -def get_account_id(context) - aws_account_id = context.invoked_function_arn.split(':')[4] - /^\d{12}$/.match?(aws_account_id) ? aws_account_id : nil +def configure_event_env(env) + $logger.info('Configuring ENV defined in event..') + env.each { |k, v| set_env(k, v) } end ## -# Temporarily add a pubic key to a target managed instance for use over SSH. +# Simple helper to re-use logic that we should not overwrite ENV variables that already exist # -# params: -# - host (string) The host ip or ID such as 'i-0e35ab216355084ee' -# - pub_key (string) The public key material -# - rm_wait (int) How long to keep the key active on the target system -# -def add_tmp_ssh_key(host, user, pub_key, rm_wait = 60) - $logger.info('Adding temporary SSH key pair to instance') - pub_key = pub_key.strip - train = Train.create( - 'awsssm', - { host: host, logger: Logger.new($stdout, level: :info), execution_timeout: rm_wait + 30 } - ) - conn = train.connection - - home_dir = conn.run_command("sudo -u #{user} sh -c 'echo $HOME'").stdout.strip +def set_env(name, value) + is_env_already_set = !ENV[name].nil? + raise(StandardError, "Could overwrite existing ENV variable: #{name}") if is_env_already_set - put_cmd = "mkdir -p #{home_dir}/.ssh;"\ - " touch #{home_dir}/.ssh/authorized_keys;"\ - " echo '#{pub_key}' >> #{home_dir}/.ssh/authorized_keys;" - - rm_cmd = "sleep #{rm_wait};"\ - " grep -vF \"#{pub_key}\" #{home_dir}/.ssh/authorized_keys > #{home_dir}/.ssh/authorized_keys.tmp;"\ - " mv #{home_dir}/.ssh/authorized_keys.tmp #{home_dir}/.ssh/authorized_keys" - - put_result = conn.run_command(put_cmd) - puts "cmd result: #{put_result}" - Thread.new do - _ = conn.run_command(rm_cmd) - conn.close - end + ENV[name] = value end -## -# Generate an SSH key pair and return the path to the public and private key files -# -def generate_key_pair - $logger.info('Generating SSH key pair') - `rm -f /tmp/id_rsa*` - priv_key_path = '/tmp/id_rsa' - pub_key_path = '/tmp/id_rsa.pub' - # shell out to ssh-keygen - `ssh-keygen -f #{priv_key_path} -N ''` - - [priv_key_path, pub_key_path] +def fetch_resources(resources) + $logger.info('Fetching all resources defined in event...') + resources.map { |r| fetch_resource(r) } end -## -# Generates the configuration that will be used for the InSpec execution -# -def build_config(event, file_path) - # Call all builder helpers for various special configuration cases - handle_winrm_password(event) - handle_s3_profile(event) - handle_s3_input_file(event) - handle_secure_string_input_file(event) - - # Start with a default config and merge in the config that was passed into the lambda - config = default_config.merge(event['config'] || {}).merge(forced_config(file_path)) - - # Add private key to config if it is present - ssh_key = fetch_ssh_key(event['ssh_key_ssm_param']) - config['key_files'] = [ssh_key] unless ssh_key.nil? - - # SSH via SSM - if %r{ssh://.+@m?i-[a-z0-9]{17}}.match? config['target'] - $logger.info('Using proxy SSM session to SSH to managed EC2 instance.') - config['proxy_command'] = - 'sh -c "aws ssm start-session --target %h --document-name AWS-StartSSHSession --parameters portNumber=%p"' - - # SSH via SSM without managed keys - if event['ssm_temp_ssh_key'] - $logger.info('SSH via SSM will use a temporary key pair.') - priv_key_path, pub_key_path = generate_key_pair - # Parse the instance Id and pass to add_tmp_ssh_key - instance_id = config['target'][/m?i-[a-z0-9]{17}/] - user = config['target'][%r{ssh://.+@}][6..-2] - add_tmp_ssh_key(instance_id, user, File.read(pub_key_path)) - # If the config does not have keys yet, then initialize it before adding the key path to it - config['key_files'] ||= [] - config['key_files'] << priv_key_path - end +def fetch_resource(resource) + # Determine the destination resource type + is_file_download_dest = !resource['local_file_path'].nil? + is_env_variable_dest = !resource['env_variable'].nil? + + local_file_path = force_tmp_local_file_path(resource['local_file_path']) if is_file_download_dest + + # Determine the source resource type + is_s3_bucket_resource = !(resource['source_aws_s3_bucket'].nil? || resource['source_aws_s3_key'].nil?) + is_ssm_parameter_resource = !resource['source_aws_ssm_parameter_key'].nil? + is_secrets_manager_resource = !resource['source_aws_secrets_manager_secret_name'].nil? + + # Perform the fetch + if is_s3_bucket_resource + fetch_s3_bucket_resource(resource, local_file_path, is_file_download_dest, is_env_variable_dest) + elsif is_ssm_parameter_resource + fetch_ssm_parameter_resource(resource, local_file_path, is_file_download_dest, is_env_variable_dest) + elsif is_secrets_manager_resource + fetch_secrets_manager_resource(resource, local_file_path, is_file_download_dest, is_env_variable_dest) + else + raise(StandardError, "Could not fetch invalid resource definition: #{resource}") end +end - if %r{winrm://m?i-[a-z0-9]{17}}.match? config['target'] - $logger.info('Using port forwarded SSM session to WINRM to managed EC2 instance.') - instance_id = /m?i-[a-z0-9]{17}/.match(config['target'])[0] - Process.detach( - spawn( - "aws ssm start-session --target #{instance_id} --document-name AWS-StartPortForwardingSession"\ - " --parameters '{\"portNumber\":[\"5985\"], \"localPortNumber\":[\"5985\"]}'" - ) - ) - Process.detach( - spawn( - "aws ssm start-session --target #{instance_id} --document-name AWS-StartPortForwardingSession"\ - " --parameters '{\"portNumber\":[\"5986\"], \"localPortNumber\":[\"5986\"]}'" - ) - ) - config['target'] = 'winrm://localhost' - $logger.info('Waiting 30 seconds.') - sleep(30) +def fetch_ssm_parameter_resource(resource, local_file_path, is_file_download_dest, is_env_variable_dest) + $logger.info('Fetching SSM resource...') + # Either use the default client or a specified endpoint + ssm_client = nil + if ENV['SSM_ENDPOINT'].nil? + $logger.info('Using default SSM Parameter Store endpoint.') + ssm_client = Aws::SSM::Client.new + else + endpoint = "https://#{/vpce.+/.match(ENV['SSM_ENDPOINT'])[0]}" + $logger.info("Using SSM Parameter Store endpoint: #{endpoint}") + ssm_client = Aws::SSM::Client.new(endpoint: endpoint) end - $logger.info("Built config: #{config}") - config -end + # Fetch and return the parameter + resp = ssm_client.get_parameter( + { + name: resource['source_aws_ssm_parameter_key'], + with_decryption: true + } + ) -## -# AWS EC2 Windows instances may have their password saved and encrypted with an SSH key. -# -# If the config password is a hash with 'instance_id' and 'launch_key' atttributes, then -# this method will attempt to fetch and decrypt the password, then set the password -# attribute properly. -# -def handle_winrm_password(event) - instance_id = event.safe_dig('config', 'password', 'instance_id') - launch_key = event.safe_dig('config', 'password', 'launch_key') - return if instance_id.nil? || launch_key.nil? - - $logger.info('Fetching winrm password for authentication.') - file_path = '/tmp/launch_key' - File.write(file_path, fetch_ssm_param(launch_key)) - - password = JSON.parse(`aws ec2 get-password-data --instance-id #{instance_id} --priv-launch-key #{file_path}`)['PasswordData'] - event['config']['password'] = password -rescue StandardError - nil + if is_file_download_dest + File.write(local_file_path, resp.parameter.value) + elsif is_env_variable_dest + set_env(resource['env_variable'], resp.parameter.value) + else + raise(StandardError, "Could not determine local destination for resource definition: #{resource}") + end end -## -# If "profile" is a zip from an S3 bucket (notated by "profile" being a hash) -# then we need to fetch the file and download it to /tmp/ -# -# https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html -# -def handle_s3_profile(event) - bucket = event.safe_dig('profile', 'bucket') - key = event.safe_dig('profile', 'key') - return if bucket.nil? || key.nil? - - unless key.end_with?('.zip') || key.end_with?('.tar.gz') - $logger.error 'InSpec profiles from S3 are only supported as *.zip or *.tar.gz files!' - exit 1 +def fetch_secrets_manager_resource(resource, local_file_path, is_file_download_dest, is_env_variable_dest) + $logger.info('Fetching Secrets Manager resource...') + secrets_manager_client = nil + if ENV['SECRETS_MANAGER_ENDPOINT'].nil? + $logger.info('Using default Secrets Manager endpoint.') + secrets_manager_client = Aws::SecretsManager::Client.new + else + endpoint = "https://#{/vpce.+/.match(ENV['SECRETS_MANAGER_ENDPOINT'])[0]}" + $logger.info("Using Secrets Manager endpoint: #{ENV['SECRETS_MANAGER_ENDPOINT']}") + secrets_manager_client = Aws::SecretsManager::Client.new(endpoint: endpoint) end + resp = secrets_manager_client.get_secret_value({ secret_id: resource['source_aws_secrets_manager_secret_name'] }) - profile_download_path = "/tmp/inspec-profile#{key.end_with?('.zip') ? '.zip' : '.tar.gz'}" - $logger.info("Downloading InSpec profile to #{profile_download_path}") - s3_client = Aws::S3::Client.new - s3_client.get_object({ bucket: bucket, key: key }, target: profile_download_path) + # Write to the destination + if is_file_download_dest + File.write(local_file_path, resp.secret_string) + elsif is_env_variable_dest + set_env(resource['env_variable'], resp.secret_string) + else + raise(StandardError, "Could not determine local destination for resource definition: #{resource}") + end +end - event['profile'] = profile_download_path +def fetch_s3_bucket_resource(resource, local_file_path, is_file_download_dest, is_env_variable_dest) + $logger.info('Fetching S3 resource...') + s3_client = Aws::S3::Client.new + if is_file_download_dest + s3_client.get_object({ bucket: resource['source_aws_s3_bucket'], key: resource['source_aws_s3_key'] }, + target: local_file_path) + elsif is_env_variable_dest + resp = s3.get_object(bucket: resource['source_aws_s3_bucket'], key: resource['source_aws_s3_key']) + set_env(resource['env_variable'], resp.body.read) + else + raise(StandardError, "Could not determine local destination for resource definition: #{resource}") + end end ## -# If "input_file" is located in an S3 bucket -# (notated by "bucket" and "key" being present in the "input_file" parameter), -# then we need to fetch the file and download it to /tmp/ -# -# https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html +# In lambda functions we expect /tmp to be the only writable directory on the filesystem. +# This method should ensure that the path only be withi /tmp/ # -def handle_s3_input_file(event) - bucket = event.safe_dig('config', 'input_file', 'bucket') - key = event.safe_dig('config', 'input_file', 'key') - return if bucket.nil? || key.nil? +def force_tmp_local_file_path(local_file_path) + local_file_path = File.expand_path(local_file_path) + # Ensure `dir_name` starts with "/tmp/" + local_file_path = "/tmp/#{dir_name}" unless local_file_path.start_with?('/tmp/') - input_file_download_path = '/tmp/input_file.yml' - $logger.info("Downloading InSpec input_file to #{input_file_download_path}") - s3_client = Aws::S3::Client.new - s3_client.get_object( - { bucket: event['config']['input_file']['bucket'], key: event['config']['input_file']['key'] }, - target: input_file_download_path - ) + # Ensure that any subdir of "/tmp/" exists + FileUtils.mkdir_p(File.dirname(local_file_path)) - event['config']['input_file'] = [input_file_download_path] + local_file_path end -## -# If "input_file" is located inside of an SSM SecureString parameter -# (notated by "ssm_secure_string" being present in the "input_file" parameter), -# then we need to fetch & decrypt the parameter and save it to /tmp/input_file.yml -# -# https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SSM/Client.html -# -def handle_secure_string_input_file(event) - param = event.safe_dig('config', 'input_file', 'ssm_secure_string') - return if param.nil? +def generate_json_file(name) + filename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%S')}_#{name}.json" + file_path = "/tmp/#{filename}" + [filename, file_path] +end - file_path = '/tmp/input_file.yml' - File.write(file_path, fetch_ssm_param(param)) +def setup_ssm_port_forward(ssm_port_forward) + $logger.info("Using port forwarded SSM session for #{ssm_port_forward['instance_id']} on ports #{ssm_port_forward['ports']}.") + ssm_port_forward['ports'].each do |port| + Process.detach( + spawn( + "aws ssm start-session --target #{ssm_port_forward['instance_id']} --document-name AWS-StartPortForwardingSession"\ + " --parameters '{\"portNumber\":[\"#{port}\"], \"localPortNumber\":[\"#{port}\"]}'" + ) + ) + end + $logger.info('Waiting 15 seconds to ensure forwarded ports take effect.') + sleep(15) +end - # Update the event with the input_file - event['config']['input_file'] = [file_path] +def push_tmp_ssh_key_to_instance(tmp_ssm_ssh_key) + $logger.info('SSH via SSM will use a temporary key pair.') + _, pub_key_path = generate_key_pair(tmp_ssm_ssh_key['key_name']) + add_tmp_ssh_key(tmp_ssm_ssh_key, File.read(pub_key_path)) end ## -# Helper to fetch and return an SSM parameter. +# Generate an SSH key pair and return the path to the public and private key files # -def fetch_ssm_param(param) - # Either use the default client or a specified endpoint - ssm_client = nil - if ENV['SSM_ENDPOINT'].nil? - $logger.info('Using default SSM Parameter Store endpoint.') - ssm_client = Aws::SSM::Client.new - else - endpoint = "https://#{/vpce.+/.match(ENV['SSM_ENDPOINT'])[0]}" - $logger.info("Using SSM Parameter Store endpoint: #{endpoint}") - ssm_client = Aws::SSM::Client.new(endpoint: endpoint) - end +def generate_key_pair(key_name = nil) + $logger.info("Generating SSH key pair at /tmp/#{key_name}") + key_name ||= 'id_rsa' + `rm -f /tmp/#{key_name}*` + priv_key_path = "/tmp/#{key_name}" + pub_key_path = "/tmp/#{key_name}.pub" + # shell out to ssh-keygen + `ssh-keygen -f #{priv_key_path} -N ''` - # Fetch and return the parameter - resp = ssm_client.get_parameter({ - name: param, - with_decryption: true - }) - $logger.info("Successfully fetched #{param} SSM Parameter.") - resp.parameter.value + [priv_key_path, pub_key_path] end ## -# Fetch the SSH key from SSM Parameter Store if the function execution requires it -# -# If ENV['SSM_ENDPOINT'] is set, then it will use that VPC endpoint to reach SSM. -# -# Params: -# - ssh_key_ssm_param:String The SSM Parameter identifier to fetch +# Temporarily add a pubic key to a target managed instance for use over SSH. # -# Returns: -# - nil if no key has been fetched, or path to key if downloaded. +# params: +# - host (string) The host ip or ID such as 'i-0e35ab216355084ee' +# - pub_key (string) The public key material +# - rm_wait (int) How long to keep the key active on the target system # -def fetch_ssh_key(ssh_key_ssm_param) - if ssh_key_ssm_param.nil? || ssh_key_ssm_param.empty? - $logger.info('ssh_key_ssm_param is blank. Will not fetch SSH key.') - return nil - end +def add_tmp_ssh_key(tmp_ssm_ssh_key, pub_key) + $logger.info('Adding temporary SSH key pair to instance') + user = tmp_ssm_ssh_key['user'] + host = tmp_ssm_ssh_key['host'] + rm_wait = 60 + exec_timeout = rm_wait + 30 + pub_key = pub_key.strip + train = Train.create( + 'awsssm', + { host: host, logger: Logger.new($stdout, level: :info), execution_timeout: exec_timeout } + ) + conn = train.connection - ssm_client = nil - if ENV['SSM_ENDPOINT'].nil? - $logger.info('Using default SSM Parameter Store endpoint.') - ssm_client = Aws::SSM::Client.new - else - endpoint = "https://#{/vpce.+/.match(ENV['SSM_ENDPOINT'])[0]}" - $logger.info("Using SSM Parameter Store endpoint: #{endpoint}") - ssm_client = Aws::SSM::Client.new(endpoint: endpoint) - end - resp = ssm_client.get_parameter({ - name: ssh_key_ssm_param, - with_decryption: true - }) - file_path = '/tmp/id_rsa' - File.write(file_path, resp.parameter.value) - file_path -end + home_dir = conn.run_command("sudo -u #{user} sh -c 'echo $HOME'").stdout.strip -## -# This is the configuration that is absolutely necessary -# for the lambda to function properly -# -def forced_config(file_path) - { - 'logger' => Logger.new(nil), - 'type' => :exec, - 'reporter' => { - 'cli' => { - 'stdout' => true - }, - 'json' => { - 'file' => file_path, - 'stdout' => false - } - } - } -end + put_cmd = "mkdir -p #{home_dir}/.ssh;"\ + " touch #{home_dir}/.ssh/authorized_keys;"\ + " echo '#{pub_key}' >> #{home_dir}/.ssh/authorized_keys;" -## -# This is the configuration that is NOT absolutely necessary -# and can be overridden by configuration passed to the lambda -# -def default_config - { - 'version' => '1.1', - 'cli_options' => { - 'color' => 'true' - }, - 'show_progress' => false, - 'color' => true, - 'create_lockfile' => true, - 'backend_cache' => true, - 'enable_telemetry' => false, - 'winrm_transport' => 'negotiate', - 'insecure' => false, - 'winrm_shell_type' => 'powershell', - 'distinct_exit' => true, - 'diff' => true, - 'sort_results_by' => 'file', - 'filter_empty_profiles' => false, - 'reporter_include_source' => false - - } -end + rm_cmd = "sleep #{rm_wait};"\ + " grep -vF \"#{pub_key}\" #{home_dir}/.ssh/authorized_keys > #{home_dir}/.ssh/authorized_keys.tmp;"\ + " mv #{home_dir}/.ssh/authorized_keys.tmp #{home_dir}/.ssh/authorized_keys" -def generate_json_file(name) - filename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%S')}_#{name}.json" - file_path = "/tmp/#{filename}" - [filename, file_path] + put_result = conn.run_command(put_cmd) + puts "cmd result: #{put_result}" + Thread.new do + _ = conn.run_command(rm_cmd) + conn.close + $logger.info('Removed temporary SSH key pair from instance.') + end end diff --git a/src/run_lambda_locally.rb b/src/run_lambda_locally.rb index aa6b3e4..e82d7a1 100644 --- a/src/run_lambda_locally.rb +++ b/src/run_lambda_locally.rb @@ -11,17 +11,164 @@ lambda_handler( event: { - 'results_bucket' => 'inspec-results-bucket-dev-f9wg', - 'profile' => 'https://github.com/mitre/redhat-enterprise-linux-7-stig-baseline/archive/master.tar.gz', - 'profile_common_name' => 'rhel7-stig-testing', - 'ssm_temp_ssh_key' => true, - 'config' => { - 'target' => 'ssh://ssm-user@i-00f1868f8f3b4eb03', - 'input' => [ - 'disable_slow_controls=true' - ], - 'sudo' => true + 'command' => 'inspec exec https://gitlab.dsolab.io/scv-content/inspec/kubernetes/baselines/k8s-cluster-stig-baseline/-/archive/master/k8s-cluster-stig-baseline-master.tar.gz'\ + ' -t k8s://', + 'results_name' => 'k8s-cluster-stig-baseline-dev-cluster', + 'results_buckets' => [ + 'inspec-results-bucket-dev' + ], + 'eval_tags' => 'ServerlessInspec,k8s', + 'resources' => [ + { + 'local_file_path' => '/tmp/kube/config', + 'source_aws_s3_bucket' => 'inspec-profiles-bucket-dev', + 'source_aws_s3_key' => 'kube-dev/config' + }, + { + 'local_file_path' => '/tmp/kube/client.crt', + 'source_aws_ssm_parameter_key' => '/inspec/kube-dev/client_crt' + }, + { + 'local_file_path' => '/tmp/kube/client.key', + 'source_aws_ssm_parameter_key' => '/inspec/kube-dev/client_key' + }, + { + 'local_file_path' => '/tmp/kube/ca.crt', + 'source_aws_ssm_parameter_key' => '/inspec/kube-dev/ca_crt' + } + ], + 'env' => { + 'KUBECONFIG' => '/tmp/kube/config' } }, context: nil ) + +# SSH command +_ = { + 'command' => 'inspec exec /tmp/redhat-enterprise-linux-7-stig-baseline-2.6.6.tar.gz'\ + ' -t ssh://ec2-user@ec2-15-200-235-74.us-gov-west-1.compute.amazonaws.com'\ + ' --sudo'\ + ' --input=disable_slow_controls=true', + 'results_name' => 'redhat-enterprise-linux-7-stig-baseline-inspec-rhel7-test', + 'results_buckets' => [ + 'inspec-results-bucket-dev' + ], + 'eval_tags' => 'ServerlessInspec,RHEL7,inspec-rhel7-test,SSH', + 'resources' => [ + { + 'local_file_path' => '/tmp/redhat-enterprise-linux-7-stig-baseline-2.6.6.tar.gz', + 'source_aws_s3_bucket' => 'inspec-profiles-bucket-dev', + 'source_aws_s3_key' => 'redhat-enterprise-linux-7-stig-baseline-2.6.6.tar.gz' + } + ] +} + +# SSH via SSM command (with tmp key) +_ = { + 'command' => 'inspec exec https://github.com/mitre/redhat-enterprise-linux-7-stig-baseline/archive/master.tar.gz'\ + ' -t ssh://ssm-user@i-00f1868f8f3b4eb03'\ + ' -i /tmp/tmp_ssh_key'\ + ' --input=\'disable_slow_controls=true\''\ + ' --proxy-command=\'sh -c "aws ssm start-session --target %h --document-name AWS-StartSSHSession --parameters portNumber=%p"\'', + 'results_name' => 'redhat-enterprise-linux-7-stig-baseline-inspec-rhel7-test', + 'results_buckets' => [ + 'inspec-results-bucket-dev' + ], + 'eval_tags' => 'ServerlessInspec,RHEL7,inspec-rhel7-test,SSH-SSM', + 'tmp_ssm_ssh_key' => { + 'host' => 'i-00f1868f8f3b4eb03', + 'user' => 'ssm-user', + 'key_name' => 'tmp_ssh_key' + } +} + +# AWSSSM command +_ = { + 'command' => 'inspec exec https://github.com/mitre/redhat-enterprise-linux-7-stig-baseline/archive/master.tar.gz'\ + ' -t awsssm://i-00f1868f8f3b4eb03'\ + ' --input=\'disable_slow_controls=true\'', + 'results_name' => 'redhat-enterprise-linux-7-stig-baseline-inspec-rhel7-test', + 'results_buckets' => [ + 'inspec-results-bucket-dev' + ], + 'eval_tags' => 'ServerlessInspec,RHEL7,inspec-rhel7-test,AWSSSM' +} + +# WinRM command +_ = { + 'command' => 'inspec exec /tmp/microsoft-windows-server-2019-stig-baseline-1.3.10.tar.gz'\ + ' -t winrm://localhost'\ + ' --password $WIN_PASS', + 'results_name' => 'windows-server-2019-stig-baseline-inspec-win2019-test', + 'results_buckets' => [ + 'inspec-results-bucket-dev' + ], + 'eval_tags' => 'ServerlessInspec,WinSvr2019,inspec-win2019-test,WinRM', + 'resources' => [ + { + 'local_file_path' => '/tmp/microsoft-windows-server-2019-stig-baseline-1.3.10.tar.gz', + 'source_aws_s3_bucket' => 'inspec-profiles-bucket-dev', + 'source_aws_s3_key' => 'microsoft-windows-server-2019-stig-baseline-1.3.10.tar.gz' + }, + { + 'env_variable' => 'WIN_PASS', + 'source_aws_ssm_parameter_key' => '/inspec/inspec-win2019-test/password' + } + ], + 'ssm_port_forward' => { + 'instance_id' => 'i-0e35ab216355084ee', + 'ports' => [5985, 5986] + } +} + +# AWS CIS Baseline command +_ = { + 'command' => 'inspec exec /tmp/aws-foundations-cis-baseline-1.2.2.tar.gz'\ + ' -t aws://', + 'results_name' => 'aws-foundations-cis-baseline-6756-0937-9314', + 'results_buckets' => [ + 'inspec-results-bucket-dev' + ], + 'eval_tags' => 'ServerlessInspec,AwsCisBaseline,6756-0937-9314,AWS', + 'resources' => [ + { + 'local_file_path' => '/tmp/aws-foundations-cis-baseline-1.2.2.tar.gz', + 'source_aws_s3_bucket' => 'inspec-profiles-bucket-dev', + 'source_aws_s3_key' => 'aws-foundations-cis-baseline-1.2.2.tar.gz' + } + ] +} + +# k8s command +_ = { + 'command' => 'inspec exec https://gitlab.dsolab.io/scv-content/inspec/kubernetes/baselines/k8s-cluster-stig-baseline/-/archive/master/k8s-cluster-stig-baseline-master.tar.gz'\ + ' -t k8s://', + 'results_name' => 'k8s-cluster-stig-baseline-dev-cluster', + 'results_buckets' => [ + 'inspec-results-bucket-dev' + ], + 'eval_tags' => 'ServerlessInspec,k8s', + 'resources' => [ + { + 'local_file_path' => '/tmp/kube/config', + 'source_aws_s3_bucket' => 'inspec-profiles-bucket-dev', + 'source_aws_s3_key' => 'kube-dev/config' + }, + { + 'local_file_path' => '/tmp/kube/client.crt', + 'source_aws_ssm_parameter_key' => '/inspec/kube-dev/client_crt' + }, + { + 'local_file_path' => '/tmp/kube/client.key', + 'source_aws_ssm_parameter_key' => '/inspec/kube-dev/client_key' + }, + { + 'local_file_path' => '/tmp/kube/ca.crt', + 'source_aws_ssm_parameter_key' => '/inspec/kube-dev/ca_crt' + } + ], + 'env' => { + 'KUBECONFIG' => '/tmp/kube/config' + } +} diff --git a/version b/version index 0548fb4..7092c7c 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.14.0 \ No newline at end of file +0.15.0 \ No newline at end of file