From 604143e8b63981e2769cceb78a8f7ac951c6dd72 Mon Sep 17 00:00:00 2001 From: diodonfrost Date: Thu, 27 Jun 2019 21:54:49 +0200 Subject: [PATCH 01/10] catch warning when spot instance is scheduled --- package/autoscaling_handler.py | 38 +++++++++++++------ package/ec2_handler.py | 67 +++++++++++++++++++++------------- 2 files changed, 68 insertions(+), 37 deletions(-) diff --git a/package/autoscaling_handler.py b/package/autoscaling_handler.py index 01e57a71..e9e8220d 100644 --- a/package/autoscaling_handler.py +++ b/package/autoscaling_handler.py @@ -31,12 +31,19 @@ def autoscaling_schedule(schedule_action, tag_key, tag_value): except ClientError as e: logging.error("Unexpected error: %s" % e) - # Stop all instances in autoscaling group - try: - ec2.stop_instances(InstanceIds=instance_list) - print("Stop autoscaling instances {0}".format(instance_list)) - except ClientError as e: - logging.error("Unexpected error: %s" % e) + for ec2_instance in instance_list: + # Stop all instances in autoscaling group + try: + ec2.stop_instances(InstanceIds=[ec2_instance]) + print("Stop autoscaling instances {0}".format(ec2_instance)) + except ClientError as e: + error_code = e.response["Error"]["Code"] + if error_code == "UnsupportedOperation": + logging.warning( + "%s is a spot instance and can not be stopped", ec2_instance + ) + else: + logging.error("Unexpected error: %s" % e) # Resume autoscaling group elif schedule_action == 'start': @@ -49,12 +56,19 @@ def autoscaling_schedule(schedule_action, tag_key, tag_value): except ClientError as e: logging.error("Unexpected error: %s" % e) - # Start all instances in autoscaling group - try: - ec2.start_instances(InstanceIds=instance_list) - print("Start autoscaling instances {0}".format(instance_list)) - except ClientError as e: - logging.error("Unexpected error: %s" % e) + for ec2_instance in instance_list: + # Start all instances in autoscaling group + try: + ec2.start_instances(InstanceIds=[ec2_instance]) + print("Start autoscaling instances {0}".format(ec2_instance)) + except ClientError as e: + error_code = e.response["Error"]["Code"] + if error_code == "UnsupportedOperation": + logging.warning( + "%s is a spot instance and can not be started", ec2_instance + ) + else: + logging.error("Unexpected error: %s" % e) def autoscaling_list_groups(tag_key, tag_value): diff --git a/package/ec2_handler.py b/package/ec2_handler.py index e5d2fc69..6aa04a4e 100644 --- a/package/ec2_handler.py +++ b/package/ec2_handler.py @@ -12,26 +12,40 @@ def ec2_schedule(schedule_action, tag_key, tag_value): """ # Define the connection - ec2 = boto3.client('ec2') + ec2 = boto3.client("ec2") # Retrieve instance list ec2_instance_list = ec2_list_instances(tag_key, tag_value) - # Stop ec2 instances in list - if schedule_action == 'stop': - try: - ec2.stop_instances(InstanceIds=ec2_instance_list) - print("Stop instances {0}".format(ec2_instance_list)) - except ClientError as e: - logging.error("Unexpected error: %s" % e) + for ec2_instance in ec2_instance_list: - # Start ec2 instances in list - elif schedule_action == 'start': - try: - ec2.start_instances(InstanceIds=ec2_instance_list) - print("Start instances {0}".format(ec2_instance_list)) - except ClientError as e: - logging.error("Unexpected error: %s" % e) + # Stop ec2 instances in list + if schedule_action == "stop": + try: + ec2.stop_instances(InstanceIds=[ec2_instance]) + print("Stop instances {0}".format(ec2_instance)) + except ClientError as e: + error_code = e.response["Error"]["Code"] + if error_code == "UnsupportedOperation": + logging.warning( + "%s is a spot instance and can not be stopped", ec2_instance + ) + else: + logging.error("Unexpected error: %s" % e) + + # Start ec2 instances in list + elif schedule_action == "start": + try: + ec2.start_instances(InstanceIds=[ec2_instance]) + print("Start instances {0}".format(ec2_instance)) + except ClientError as e: + error_code = e.response["Error"]["Code"] + if error_code == "UnsupportedOperation": + logging.warning( + "%s is a spot instance and can not be started", ec2_instance + ) + else: + logging.error("Unexpected error: %s" % e) def ec2_list_instances(tag_key, tag_value): @@ -41,25 +55,28 @@ def ec2_list_instances(tag_key, tag_value): """ # Define the connection - ec2 = boto3.client('ec2') - paginator = ec2.get_paginator('describe_instances') + ec2 = boto3.client("ec2") + paginator = ec2.get_paginator("describe_instances") page_iterator = paginator.paginate( - Filters=[{'Name': 'tag:'+tag_key, 'Values': [tag_value]}, - {'Name': 'instance-state-name', 'Values': ['pending', - 'running', - 'stopping', - 'stopped']}]) + Filters=[ + {"Name": "tag:" + tag_key, "Values": [tag_value]}, + { + "Name": "instance-state-name", + "Values": ["pending", "running", "stopping", "stopped"], + }, + ] + ) # Initialize instance list instance_list = [] # Retrieve ec2 instances for page in page_iterator: - for reservation in page['Reservations']: - for instance in reservation['Instances']: + for reservation in page["Reservations"]: + for instance in reservation["Instances"]: # Retrieve ec2 instance id and add in list - instance_id = instance['InstanceId'] + instance_id = instance["InstanceId"] instance_list.insert(0, instance_id) return instance_list From d3394ca8978ee72737e7c5c11b0dd76777502363 Mon Sep 17 00:00:00 2001 From: diodonfrost Date: Fri, 28 Jun 2019 23:01:39 +0200 Subject: [PATCH 02/10] add spot instance scheduler support --- main.tf | 25 +++++++++++++++- package/main.py | 8 +++++- package/spot_handler.py | 63 +++++++++++++++++++++++++++++++++++++++++ variables.tf | 15 ++++++---- 4 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 package/spot_handler.py diff --git a/main.tf b/main.tf index 7007480e..87985654 100644 --- a/main.tf +++ b/main.tf @@ -63,6 +63,27 @@ EOF } # Create custom policy for manage ec2 +resource "aws_iam_role_policy" "schedule_spot" { + name = "${var.name}-spot-custom-policy-scheduler" + role = "${aws_iam_role.scheduler_lambda.id}" + + policy = < Date: Fri, 28 Jun 2019 23:03:04 +0200 Subject: [PATCH 03/10] update terraform example test_fixture --- examples/test_fixture/main.tf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/test_fixture/main.tf b/examples/test_fixture/main.tf index cde86cff..a85946d1 100644 --- a/examples/test_fixture/main.tf +++ b/examples/test_fixture/main.tf @@ -5,9 +5,10 @@ module "aws-stop-friday" { name = "stop-aws" cloudwatch_schedule_expression = "cron(0 23 ? * FRI *)" schedule_action = "stop" + autoscaling_schedule = "true" + spot_schedule = "terminate" ec2_schedule = "true" rds_schedule = "true" - autoscaling_schedule = "true" resources_tag = { key = "tostop" @@ -20,9 +21,10 @@ module "aws-start-monday" { name = "start-aws" cloudwatch_schedule_expression = "cron(0 07 ? * MON *)" schedule_action = "start" + autoscaling_schedule = "true" + spot_schedule = "false" ec2_schedule = "true" rds_schedule = "true" - autoscaling_schedule = "true" resources_tag = { key = "tostop" From 46d09464819bf169e80a59122a467d8d565c24ec Mon Sep 17 00:00:00 2001 From: diodonfrost Date: Fri, 28 Jun 2019 23:56:55 +0200 Subject: [PATCH 04/10] disable scheduler spot instances by default --- package/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/main.py b/package/main.py index 38fc8698..e52f92e9 100644 --- a/package/main.py +++ b/package/main.py @@ -16,7 +16,7 @@ def lambda_handler(event, context): tag_key = os.getenv('TAG_KEY', 'tostop') tag_value = os.getenv('TAG_VALUE', 'true') autoscaling_schedule = os.getenv('AUTOSCALING_SCHEDULE', 'true') - spot_schedule = os.getenv('SPOT_SCHEDULE', 'terminate') + spot_schedule = os.getenv('SPOT_SCHEDULE', 'false') ec2_schedule = os.getenv('EC2_SCHEDULE', 'true') rds_schedule = os.getenv('RDS_SCHEDULE', 'true') From ccc7e081a45b069db40f209c8f6640955846d4ff Mon Sep 17 00:00:00 2001 From: diodonfrost Date: Sat, 29 Jun 2019 12:20:30 +0200 Subject: [PATCH 05/10] linting comments in python spot handler --- package/spot_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package/spot_handler.py b/package/spot_handler.py index c16a2a43..c617c331 100644 --- a/package/spot_handler.py +++ b/package/spot_handler.py @@ -7,8 +7,8 @@ def spot_schedule(schedule_action, tag_key, tag_value): """ - Aws ec2 spot instance scheduler function, stop or - start ec2 spot instances by using the tag defined. + Aws spot instance scheduler function, + terminate spot instances by using the tag defined. """ # Define the connection @@ -19,7 +19,7 @@ def spot_schedule(schedule_action, tag_key, tag_value): for spot_instance in spot_instance_list: - # Stop spot instances in list + # Terminate spot instances in list if schedule_action == "stop": try: ec2.terminate_instances(InstanceIds=[spot_instance]) From 3dc404a120e163b78ed6c8c80e4a38161af25ef3 Mon Sep 17 00:00:00 2001 From: diodonfrost Date: Sat, 29 Jun 2019 12:20:58 +0200 Subject: [PATCH 06/10] add terraform spot scheduler example --- examples/spot-schedule/main.tf | 127 ++++++++++++++++++++++++++++++ examples/spot-schedule/outputs.tf | 9 +++ 2 files changed, 136 insertions(+) create mode 100644 examples/spot-schedule/main.tf create mode 100644 examples/spot-schedule/outputs.tf diff --git a/examples/spot-schedule/main.tf b/examples/spot-schedule/main.tf new file mode 100644 index 00000000..b7095e09 --- /dev/null +++ b/examples/spot-schedule/main.tf @@ -0,0 +1,127 @@ +# Terraform spot instance with lambda scheduler + +data "aws_ami" "ubuntu" { + most_recent = true + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + owners = ["099720109477"] # Canonical +} + +# Create vpc use by asg +resource "aws_vpc" "main" { + cidr_block = "10.0.0.0/16" +} + +# Create subnet use bt asg +resource "aws_subnet" "main" { + vpc_id = "${aws_vpc.main.id}" + cidr_block = "10.0.1.0/24" +} + +# Run spot instances that will be scheduled +resource "aws_launch_template" "scheduled" { + name_prefix = "spot-scheduled" + image_id = "${data.aws_ami.ubuntu.id}" + instance_type = "t2.micro" +} + +resource "aws_autoscaling_group" "scheduled" { + desired_capacity = 3 + max_size = 3 + min_size = 3 + vpc_zone_identifier = ["${aws_subnet.main.id}"] + + mixed_instances_policy { + instances_distribution { + on_demand_percentage_above_base_capacity = "0" + } + + launch_template { + launch_template_specification { + launch_template_id = "${aws_launch_template.scheduled.id}" + } + + override { + instance_type = "t2.micro" + } + + override { + instance_type = "t2.nano" + } + } + } + + tag { + key = "tostop" + value = "true" + propagate_at_launch = true + } +} + +# Run spot instances that will be not scheduled +resource "aws_launch_template" "not_scheduled" { + name_prefix = "spot-not_scheduled" + image_id = "${data.aws_ami.ubuntu.id}" + instance_type = "t2.micro" +} + +resource "aws_autoscaling_group" "not_scheduled" { + desired_capacity = 2 + max_size = 2 + min_size = 2 + vpc_zone_identifier = ["${aws_subnet.main.id}"] + + mixed_instances_policy { + instances_distribution { + on_demand_percentage_above_base_capacity = "0" + } + + launch_template { + launch_template_specification { + launch_template_id = "${aws_launch_template.scheduled.id}" + } + + override { + instance_type = "t2.micro" + } + + override { + instance_type = "t2.nano" + } + } + } + + tag { + key = "tostop" + value = "false" + propagate_at_launch = true + } +} + + +### Terraform module ## + +module "spot-terminate-friday" { + source = "../../" + name = "terminate-spot" + cloudwatch_schedule_expression = "cron(0 23 ? * FRI *)" + schedule_action = "stop" + spot_schedule = "terminate" + ec2_schedule = "false" + rds_schedule = "false" + autoscaling_schedule = "false" + + resources_tag = { + key = "tostop" + value = "true" + } +} diff --git a/examples/spot-schedule/outputs.tf b/examples/spot-schedule/outputs.tf new file mode 100644 index 00000000..80bc09ce --- /dev/null +++ b/examples/spot-schedule/outputs.tf @@ -0,0 +1,9 @@ +# Terraform ex2-schedule outputs + +output "lambda_stop_name" { + value = "${module.spot-terminate-friday.scheduler_lambda_name}" +} + +output "lambda_stop_arn" { + value = "${module.spot-terminate-friday.scheduler_lambda_arn}" +} From c64bfe310ae79f98e9a3632a79721678a0d58a36 Mon Sep 17 00:00:00 2001 From: diodonfrost Date: Sat, 29 Jun 2019 12:21:11 +0200 Subject: [PATCH 07/10] update terraform examples --- examples/autoscaling-schedule/main.tf | 5 +++-- examples/ec2-schedule/main.tf | 12 +++++++++--- examples/rds-schedule/main.tf | 6 ++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/examples/autoscaling-schedule/main.tf b/examples/autoscaling-schedule/main.tf index 61ec4223..ff045fc3 100644 --- a/examples/autoscaling-schedule/main.tf +++ b/examples/autoscaling-schedule/main.tf @@ -74,7 +74,7 @@ resource "aws_autoscaling_group" "not_scheduled" { ### Terraform modules ### module "autoscaling-stop-friday" { - source = "diodonfrost/lambda-scheduler-stop-start/aws" + source = "../../" name = "stop-autoscaling" cloudwatch_schedule_expression = "cron(0 23 ? * FRI *)" schedule_action = "stop" @@ -89,10 +89,11 @@ module "autoscaling-stop-friday" { } module "autoscaling-start-monday" { - source = "diodonfrost/lambda-scheduler-stop-start/aws" + source = "../../" name = "start-autoscaling" cloudwatch_schedule_expression = "cron(0 07 ? * MON *)" schedule_action = "start" + spot_schedule = "false" ec2_schedule = "false" rds_schedule = "false" autoscaling_schedule = "true" diff --git a/examples/ec2-schedule/main.tf b/examples/ec2-schedule/main.tf index d21c547c..31994089 100644 --- a/examples/ec2-schedule/main.tf +++ b/examples/ec2-schedule/main.tf @@ -22,7 +22,11 @@ resource "aws_instance" "scheduled" { instance_type = "t2.micro" tags = { - tostop = "true" + Name = "shceduled" + tostop = "true" + team = "flarf" + lint = "yes" + provider = "terraform" } } @@ -39,10 +43,11 @@ resource "aws_instance" "not_scheduled" { ### Terraform modules ### module "ec2-stop-friday" { - source = "diodonfrost/lambda-scheduler-stop-start/aws" + source = "../../" name = "stop-ec2" cloudwatch_schedule_expression = "cron(0 23 ? * FRI *)" schedule_action = "stop" + spot_schedule = "false" ec2_schedule = "true" rds_schedule = "false" autoscaling_schedule = "false" @@ -54,10 +59,11 @@ module "ec2-stop-friday" { } module "ec2-start-monday" { - source = "diodonfrost/lambda-scheduler-stop-start/aws" + source = "../../" name = "start-ec2" cloudwatch_schedule_expression = "cron(0 07 ? * MON *)" schedule_action = "start" + spot_schedule = "false" ec2_schedule = "true" rds_schedule = "false" autoscaling_schedule = "false" diff --git a/examples/rds-schedule/main.tf b/examples/rds-schedule/main.tf index de6d517e..b5ff3690 100644 --- a/examples/rds-schedule/main.tf +++ b/examples/rds-schedule/main.tf @@ -86,10 +86,11 @@ resource "aws_db_instance" "mysql_not_scheduled" { ### Terraform modules ### module "rds-stop-friday" { - source = "diodonfrost/lambda-scheduler-stop-start/aws" + source = "../../" name = "stop-rds" cloudwatch_schedule_expression = "cron(0 23 ? * FRI *)" schedule_action = "stop" + spot_schedule = "false" ec2_schedule = "false" rds_schedule = "true" autoscaling_schedule = "false" @@ -101,10 +102,11 @@ module "rds-stop-friday" { } module "rds-start-monday" { - source = "diodonfrost/lambda-scheduler-stop-start/aws" + source = "../../" name = "start-rds" cloudwatch_schedule_expression = "cron(0 07 ? * MON *)" schedule_action = "start" + spot_schedule = "false" ec2_schedule = "false" rds_schedule = "true" autoscaling_schedule = "false" From c7b92d611ce4e4bf624693be0489b625fae75cd4 Mon Sep 17 00:00:00 2001 From: diodonfrost Date: Sat, 29 Jun 2019 12:37:53 +0200 Subject: [PATCH 08/10] update comments in variables.tf --- variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/variables.tf b/variables.tf index 623248fa..3d5300ac 100644 --- a/variables.tf +++ b/variables.tf @@ -33,7 +33,7 @@ variable "autoscaling_schedule" { } variable "spot_schedule" { - description = "Enable scheduling on spot instances resources" + description = "Enable scheduling on spot instance resources" default = "false" } From e55c47706fb632c343b96f28117a16aa19151edb Mon Sep 17 00:00:00 2001 From: diodonfrost Date: Sat, 29 Jun 2019 12:38:10 +0200 Subject: [PATCH 09/10] update README.md --- README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 79d8785e..18181a8e 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,17 @@ Stop and start ec2, rds resources and autoscaling groups with lambda function. ## Features -* Aws lambda runtine Python 3.6 +* Aws lambda runtine Python 3.7 * ec2 instances scheduling +* spot instances scheduling * rds clusters scheduling * rds instances scheduling * autoscalings scheduling * Aws cloudWatch logs for lambda +### Caveats +You can't stop and start an [Amazon Spot instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/how-spot-instances-work.html) (only the Spot service can stop and start a Spot Instance), but you can reboot or terminate a Spot Instance. That why this module support only scheduler action `terminate` for spot instance. + ## Usage ```hcl module "stop_ec2_instance" { @@ -20,6 +24,8 @@ module "stop_ec2_instance" { name = "ec2_stop" cloudwatch_schedule_expression = "cron(0 00 ? * FRI *)" schedule_action = "stop" + autoscaling_schedule = "false" + spot_schedule = "terminate" ec2_schedule = "true" rds_schedule = "false" autoscaling_schedule = "false" @@ -32,9 +38,10 @@ module "stop_ec2_instance" { ## Examples -* [EC2 scheduler](https://github.com/diodonfrost/terraform-aws-lambda-scheduler-stop-start/tree/master/examples/ec2-schedule) - Create lamnda functions to stop ec2 with tag `tostop = true` on Friday at 23:00 Gmt and start them on Monday at 07:00 GMT -* [Rds aurora - mariadb scheduler](https://github.com/diodonfrost/terraform-aws-lambda-scheduler-stop-start/tree/master/examples/rds-schedule) - Create lamnda functions to stop rds mariadb and aurora cluster with tag `tostop = true` on Friday at 23:00 Gmt and start them on Monday at 07:00 GMT -* [Autoscaling scheduler](https://github.com/diodonfrost/terraform-aws-lambda-scheduler-stop-start/tree/master/examples/autoscaling-schedule) - Create lamnda functions to suspend autoscaling group with tag `tostop = true` and terminate its ec2 instances on Friday at 23:00 Gmt and start them on Monday at 07:00 GMT +* [Autoscaling scheduler](https://github.com/diodonfrost/terraform-aws-lambda-scheduler-stop-start/tree/master/examples/autoscaling-schedule) - Create lambda functions to suspend autoscaling group with tag `tostop = true` and terminate its ec2 instances on Friday at 23:00 Gmt and start them on Monday at 07:00 GMT +* [Spot scheduler](https://github.com/diodonfrost/terraform-aws-lambda-scheduler-stop-start/tree/master/examples/spot-schedule) - Create lambda functions to stop spot instance with tag `tostop = true` on Friday at 23:00 Gmt +* [EC2 scheduler](https://github.com/diodonfrost/terraform-aws-lambda-scheduler-stop-start/tree/master/examples/ec2-schedule) - Create lambda functions to stop ec2 with tag `tostop = true` on Friday at 23:00 Gmt and start them on Monday at 07:00 GMT +* [Rds aurora - mariadb scheduler](https://github.com/diodonfrost/terraform-aws-lambda-scheduler-stop-start/tree/master/examples/rds-schedule) - Create lambda functions to stop rds mariadb and aurora cluster with tag `tostop = true` on Friday at 23:00 Gmt and start them on Monday at 07:00 GMT * [test fixture](https://github.com/diodonfrost/terraform-aws-lambda-scheduler-stop-start/tree/master/examples/test_fixture) - Deploy environment for testing module @@ -47,9 +54,10 @@ module "stop_ec2_instance" { | cloudwatch_schedule_expression | The scheduling expression | string | `"cron(0 22 ? * MON-FRI *)"` | yes | | schedule_action | Define schedule action to apply on resources | string | `"stop"` | yes | | resources_tag | Set the tag use for identify resources to stop or start | map | { tostop = "true" } | yes | -| ec2_schedule | Enable scheduling on ec2 resources | string | `"false"` | no | -| rds_schedule | Enable scheduling on rds resources | string | `"false"` | no | | autoscaling_schedule | Enable scheduling on autoscaling resources | string | `"false"` | no | +| spot_schedule | Enable scheduling on spot instance resources | string | `"false"` | no | +| ec2_schedule | Enable scheduling on ec2 instance resources | string | `"false"` | no | +| rds_schedule | Enable scheduling on rds resources | string | `"false"` | no | ## Outputs From e07db133418730f389f151692f145c13ca0067af Mon Sep 17 00:00:00 2001 From: diodonfrost Date: Sat, 29 Jun 2019 12:56:21 +0200 Subject: [PATCH 10/10] remove Gemfile --- Gemfile | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 Gemfile diff --git a/Gemfile b/Gemfile deleted file mode 100644 index d220a80b..00000000 --- a/Gemfile +++ /dev/null @@ -1,7 +0,0 @@ -source "https://rubygems.org" - -gem 'aws-sdk', '~> 3.0.1' -gem 'awspec', '~> 1.17.3' -gem 'kitchen-terraform', '~> 3.3.1' -gem 'kitchen-verifier-awspec', '~> 0.1.2' -gem 'rhcl', '~> 0.1.0'