From c214325021a7cf7841d110ccc04e7e369b0f6cb4 Mon Sep 17 00:00:00 2001 From: ejether Date: Fri, 3 Jan 2020 08:01:18 -0800 Subject: [PATCH] adding create namespace fxnality to reckoner to replace it in helm3 (#165) * adding create namespace fxnality to reckoner to replace it in helm3 * fixing lack of install of kuberentes module, adding a test for the new option * adding docs * testing things and stuff * phew, finally found that errror * removing debug line * making suggested chnanges --- CHANGELOG.md | 1 + docs/README.md | 31 +++++++------ end_to_end_testing/run_end_to_end.sh | 23 ++++++++++ end_to_end_testing/setup_helm2.sh | 2 +- end_to_end_testing/setup_helm3.sh | 10 +---- end_to_end_testing/test_create_namespace.yml | 18 ++++++++ reckoner/chart.py | 21 +++++++++ reckoner/cli.py | 6 ++- reckoner/kube.py | 47 ++++++++++++++++++++ reckoner/reckoner.py | 3 +- reckoner/tests/test_chart.py | 10 +++++ reckoner/tests/test_course.py | 8 ++++ setup.py | 1 + tests/test_reckoner.py | 13 +++++- 14 files changed, 166 insertions(+), 28 deletions(-) create mode 100644 end_to_end_testing/test_create_namespace.yml create mode 100644 reckoner/kube.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ee3e4022..a05e1cfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased ### Fixed - bug where the helm client version check would fail for helm2 and never proceed to check helm3 +- added create namespace functionality because they removed it from helm3 ## [2.2.0] ### Changes diff --git a/docs/README.md b/docs/README.md index 8ad724ed..74c55202 100644 --- a/docs/README.md +++ b/docs/README.md @@ -154,23 +154,26 @@ Commands: You can add `--help` to any `Command` and get output like the one below: ```text $> reckoner plot --help - Usage: reckoner plot [OPTIONS] COURSE_FILE Install charts with given arguments as listed in yaml file argument Options: - --dry-run Pass --dry-run to helm so no action is taken. - Implies --debug and skips hooks. - --debug DEPRECATED - use --dry-run instead, or pass - to --helm-args - -o, --only, --heading Only run a specific chart by name - --helm-args TEXT Passes the following arg on to helm, can be - used more than once. WARNING: Setting this - will completely override any helm_args in the - course. Also cannot be used for configuring - how helm connects to tiller. - --continue-on-error Attempt to install all charts in the course, - even if any charts or hooks fail to run. - --help Show this message and exit. + --dry-run Pass --dry-run to helm so no action is + taken. Implies --debug and skips hooks. + --debug DEPRECATED - use --dry-run instead, or pass + to --helm-args + -o, --only, --heading Only run a specific chart by name + --helm-args TEXT Passes the following arg on to helm, can be + used more than once. WARNING: Setting this + will completely override any helm_args in + the course. Also cannot be used for + configuring how helm connects to tiller. + --continue-on-error Attempt to install all charts in the course, + even if any charts or hooks fail to run. + --create-namespace / --no-create-namespace + Will create the specified nameaspace if it + does not already exist. Replaces + functionality lost in Helm3 + --help Show this message and exit. ``` diff --git a/end_to_end_testing/run_end_to_end.sh b/end_to_end_testing/run_end_to_end.sh index 14980653..f2cd2c04 100755 --- a/end_to_end_testing/run_end_to_end.sh +++ b/end_to_end_testing/run_end_to_end.sh @@ -15,6 +15,7 @@ set -o errtrace E2E_FAILED_TESTS=false E2E_FAILED_MESSAGES=() E2E_SKIPPED_MESSAGES=() +HELM_VERSION="${HELM_VERSION:-2}" function print_status_end_exit() { if [ "${#E2E_SKIPPED_MESSAGES[@]}" -gt 0 ]; then echo -e "* * *\nSkipped Tests:"; fi @@ -151,6 +152,28 @@ function helm_release_key_value_is_type() { fi } +function e2e_test_namespace_creation_flag_on_chart_install() { + if [ "${HELM_VERSION}" -eq "3" ]; then + if reckoner plot --no-create-namespace test_create_namespace.yml; then + mark_failed "${FUNCNAME[0]}" "With --no-create-namespace set, this should have failed" + fi + + if helm_has_release_name_in_namespace "namespace-test" "farglebargle"; then + mark_failed "${FUNCNAME[0]}" "Found namespace_test in farglebargle namespace after install and should not have" + fi + + if ! reckoner plot test_create_namespace.yml; then + mark_failed "${FUNCNAME[0]}" "Without --no-create-namespace set, this should not have failed" + fi + + if ! helm_has_release_name_in_namespace "namespace-test" "farglebargle"; then + mark_failed "${FUNCNAME[0]}" "Did not find namespace_test in farglebargle namespace after install." + fi + fi + + +} + function e2e_test_basic_chart_install() { if ! reckoner plot test_basic.yml; then mark_failed "${FUNCNAME[0]}" "Plot had a bad exit code" diff --git a/end_to_end_testing/setup_helm2.sh b/end_to_end_testing/setup_helm2.sh index 2378df18..0c79fde5 100755 --- a/end_to_end_testing/setup_helm2.sh +++ b/end_to_end_testing/setup_helm2.sh @@ -1,7 +1,7 @@ source "$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/setup_common.sh echo "Installing Helm" -curl -sL https://storage.googleapis.com/kubernetes-helm/helm-v2.14.2-linux-amd64.tar.gz | tar xzv linux-amd64/helm +curl -sL https://storage.googleapis.com/kubernetes-helm/helm-v2.16.1-linux-amd64.tar.gz | tar xzv linux-amd64/helm sudo mv linux-amd64/helm /usr/local/bin/helm rm -rf linux-amd64 helm version --client diff --git a/end_to_end_testing/setup_helm3.sh b/end_to_end_testing/setup_helm3.sh index 7be35aac..94a0703b 100755 --- a/end_to_end_testing/setup_helm3.sh +++ b/end_to_end_testing/setup_helm3.sh @@ -1,17 +1,9 @@ source "$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/setup_common.sh echo "Installing Helm" -curl -sL https://get.helm.sh/helm-v3.0.1-linux-amd64.tar.gz | tar xzv linux-amd64/helm +curl -sL https://get.helm.sh/helm-v3.0.2-linux-amd64.tar.gz | tar xzv linux-amd64/helm sudo mv linux-amd64/helm /usr/local/bin/helm rm -rf linux-amd64 helm version -kubectl create namespace infra -kubectl create namespace test -kubectl create namespace testing -kubectl create namespace polaris -kubectl create namespace another-polaris -kubectl create namespace a-different-one -kubectl create namespace redis-test-namespace - helm repo add stable https://kubernetes-charts.storage.googleapis.com diff --git a/end_to_end_testing/test_create_namespace.yml b/end_to_end_testing/test_create_namespace.yml new file mode 100644 index 00000000..1d0b1f15 --- /dev/null +++ b/end_to_end_testing/test_create_namespace.yml @@ -0,0 +1,18 @@ +namespace: farglebargle #namespace to install the chart in, defaults to 'kube-system' +repositories: + test_repo: + url: https://kubernetes-charts.storage.googleapis.com + incubator: + url: https://kubernetes-charts-incubator.storage.googleapis.com + fairwinds-stable: + url: https://charts.fairwinds.com/stable + fairwinds-incubator: + url: https://charts.fairwinds.com/incubator +minimum_versions: #set minimum version requirements here + helm: 0.0.0 + reckoner: 0.0.0 +charts: + namespace-test: + repository: stable + chart: nginx-ingress + namespace: farglebargle diff --git a/reckoner/chart.py b/reckoner/chart.py index b445f333..df28bafc 100644 --- a/reckoner/chart.py +++ b/reckoner/chart.py @@ -16,6 +16,8 @@ import logging import os + +from .kube import create_namespace, list_namespace_names from tempfile import NamedTemporaryFile as tempfile from .yaml.handler import Handler as yaml_handler @@ -26,10 +28,12 @@ from .repository import Repository from .command_line_caller import call + default_repository = {'name': 'stable', 'url': 'https://kubernetes-charts.storage.googleapis.com'} class ChartResult: + def __init__(self, name: str, failed: bool, error_reason: str): self.name = name self.failed = failed @@ -238,6 +242,21 @@ def update_dependencies(self): except ReckonerCommandException as error: logging.warn("Unable to update chart dependencies: {}".format(error.stderr)) + def create_namespace_if_needed(self): + """ Creates the charts specified namespace if it does not already exist + Requires `self.config.create_namespace` to true. Caches the existing namespace list + in the self.config option to avoid going back to the api for each chart. + """ + + if self.config.create_namespace: + if self.config.cluster_namespaces is None: + self.config.cluster_namespaces = list_namespace_names() + + if self.namespace not in self.config.cluster_namespaces and not self.dryrun: + if create_namespace(self.namespace): + logging.info('Namespace {} not found. Creating it now.'.format(self.namespace)) + self.config.cluster_namespaces.append(self.namespace) + def install(self, namespace=None, context=None) -> None: """ Description: @@ -257,6 +276,8 @@ def install(self, namespace=None, context=None) -> None: # Try to run the install process for mark the result as failed try: + self.create_namespace_if_needed() + # Fire the pre_install_hook self.pre_install_hook() diff --git a/reckoner/cli.py b/reckoner/cli.py index 647bd859..b6a562ec 100644 --- a/reckoner/cli.py +++ b/reckoner/cli.py @@ -48,14 +48,16 @@ def cli(ctx, log_level, *args, **kwargs): 'configuring how helm connects to tiller.', multiple=True) @click.option("--continue-on-error", is_flag=True, default=False, help="Attempt to install all charts in the course, even if any charts or hooks fail to run.") -def plot(ctx, course_file=None, dry_run=False, debug=False, only=None, helm_args=None, continue_on_error=False): +@click.option("--create-namespace/--no-create-namespace", default=True, + help="Will create the specified nameaspace if it does not already exist. Replaces functionality lost in Helm3") +def plot(ctx, course_file=None, dry_run=False, debug=False, only=None, helm_args=None, continue_on_error=False, create_namespace=True): """ Install charts with given arguments as listed in yaml file argument """ try: # Check Schema of Course FileA with open(course_file.name, 'rb') as course_file_stream: validate_course_file(course_file_stream) # Load Reckoner - r = Reckoner(course_file=course_file, dryrun=dry_run, debug=debug, helm_args=helm_args, continue_on_error=continue_on_error) + r = Reckoner(course_file=course_file, dryrun=dry_run, debug=debug, helm_args=helm_args, continue_on_error=continue_on_error, create_namespace=create_namespace) # Convert tuple to list only = list(only) r.install(only) diff --git a/reckoner/kube.py b/reckoner/kube.py new file mode 100644 index 00000000..852b6e29 --- /dev/null +++ b/reckoner/kube.py @@ -0,0 +1,47 @@ + +import logging +import traceback + +from kubernetes import client, config + + +def create_namespace(namespace): + """ Create a namespace in the configured kubernetes cluster if it does not already exist + + Arguments: + + namespace: The namespace to create + + Returns True on success + Raises error in case of failure + + """ + try: + config.load_kube_config() + v1 = client.CoreV1Api() + response = v1.create_namespace( + client.V1Namespace( + metadata=client.V1ObjectMeta(name=namespace) + ) + ) + return True + except Exception as e: + logging.error("Unable to create namespace in cluster! {}".format(e)) + logging.debug(traceback.format_exc()) + raise e + + +def list_namespace_names(): + """ Lists namespaces in the configured kubernetes cluster. + No arguments + Returns list + """ + try: + config.load_kube_config() + v1 = client.CoreV1Api() + namespaces = v1.list_namespace() + return [namespace.metadata.name for namespace in namespaces.items] + except Exception as e: + logging.error("Unable to get namespaces in cluster! {}".format(e)) + logging.debug(traceback.format_exc()) + raise e diff --git a/reckoner/reckoner.py b/reckoner/reckoner.py index f765cc76..465ca5fb 100644 --- a/reckoner/reckoner.py +++ b/reckoner/reckoner.py @@ -63,13 +63,14 @@ class Reckoner(object): """ - def __init__(self, course_file: BufferedReader = None, dryrun=False, debug=False, helm_args=None, continue_on_error=False): + def __init__(self, course_file: BufferedReader = None, dryrun=False, debug=False, helm_args=None, continue_on_error=False, create_namespace=True): self.config = Config() self.results = ReckonerInstallResults() self.config.dryrun = dryrun self.config.debug = debug self.config.helm_args = helm_args self.config.continue_on_error = continue_on_error + self.config.create_namespace = create_namespace if course_file: self.config.course_path = course_file.name diff --git a/reckoner/tests/test_chart.py b/reckoner/tests/test_chart.py index 63ec8f4d..03be25c8 100644 --- a/reckoner/tests/test_chart.py +++ b/reckoner/tests/test_chart.py @@ -29,6 +29,7 @@ # would be more easily mockable @mock.patch('reckoner.chart.call') class TestChartHooks(unittest.TestCase): + def get_chart(self, *args): chart = Chart( {'name': { @@ -217,6 +218,8 @@ def test_interpolation_of_env_vars_kube_deploy_spec(self, environMock, chartConf chart._check_env_vars() self.assertEqual(chart.args[0], 'thing=$(environVar)') + @mock.patch('reckoner.chart.create_namespace', mock.MagicMock(return_value=True)) + @mock.patch('reckoner.chart.list_namespace_names', mock.MagicMock(return_value=[])) @mock.patch('reckoner.chart.Config', autospec=True) @mock.patch('reckoner.chart.Repository') def test_chart_install(self, repositoryMock, chartConfigMock): @@ -228,6 +231,8 @@ def test_chart_install(self, repositoryMock, chartConfigMock): chartConfig = chartConfigMock() chartConfig.course_base_directory = '.' chartConfig.dryrun = False + chartConfig.create_namespace = True + chartConfig.cluster_namespaces = [] debug_args = mock.PropertyMock(debug_args=['fake']) type(chart).debug_args = debug_args @@ -236,6 +241,8 @@ def test_chart_install(self, repositoryMock, chartConfigMock): upgrade_call = helm_client_mock.upgrade.call_args self.assertEqual(upgrade_call[0][0], ['nameofchart', '', '--namespace', 'fakenamespace']) + @mock.patch('reckoner.chart.create_namespace', mock.MagicMock(return_value=True)) + @mock.patch('reckoner.chart.list_namespace_names', mock.MagicMock(return_value=[])) @mock.patch('reckoner.chart.Config', autospec=True) @mock.patch('reckoner.chart.Repository') def test_chart_install_with_plugin(self, repositoryMock, chartConfigMock): @@ -247,6 +254,8 @@ def test_chart_install_with_plugin(self, repositoryMock, chartConfigMock): chartConfig = chartConfigMock() chartConfig.course_base_directory = '.' chartConfig.dryrun = False + chartConfig.create_namespace = True + chartConfig.cluster_namespaces = [] debug_args = mock.PropertyMock(debug_args=['fake']) type(chart).debug_args = debug_args @@ -258,6 +267,7 @@ def test_chart_install_with_plugin(self, repositoryMock, chartConfigMock): class TestChartResult(unittest.TestCase): + def test_initialize(self): c = ChartResult( name="fake-result", diff --git a/reckoner/tests/test_course.py b/reckoner/tests/test_course.py index fa7cdc8a..729faf0c 100644 --- a/reckoner/tests/test_course.py +++ b/reckoner/tests/test_course.py @@ -26,6 +26,7 @@ @mock.patch('reckoner.course.get_helm_client', autospec=True) @mock.patch('reckoner.course.Config', autospec=True) class TestMinVersion(unittest.TestCase): + def test_init_error_fails_min_version_reckoner(self, configMock, helmClientMock, yamlLoadMock, sysMock, repoMock): """Tests that minimum version will throw an exit.""" c = configMock() @@ -76,6 +77,9 @@ def test_init_error_fails_min_version_helm(self, configMock, helmClientMock, yam class TestIntegrationWithChart(unittest.TestCase): + + @mock.patch('reckoner.chart.create_namespace', mock.MagicMock(return_value=True)) + @mock.patch('reckoner.chart.list_namespace_names', mock.MagicMock(return_value=[])) @mock.patch('reckoner.chart.Config', autospec=True) @mock.patch('reckoner.chart.call', autospec=True) @mock.patch('reckoner.repository.Repository', autospec=True) @@ -92,6 +96,9 @@ def test_failed_pre_install_hooks_fail_chart_installation(self, configMock, helm chartConfig = chartConfigMock() chartConfig.course_base_directory = '.' chartConfig.dryrun = False + chartConfig.create_namespace = True + chartConfig.cluster_namespaces = [] + h = helmClientMock(c.helm_args) h.client_version = '0.0.1' @@ -119,6 +126,7 @@ def test_failed_pre_install_hooks_fail_chart_installation(self, configMock, helm @mock.patch('reckoner.course.yaml_handler', autospec=True) @mock.patch('reckoner.course.get_helm_client', autospec=True) class TestCourse(unittest.TestCase): + def setUp(self): self.course_yaml = { 'charts': { diff --git a/setup.py b/setup.py index 48157d45..de21a9a3 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ "semver>=2.8.1", "ruamel.yaml>=0.16.0", "jsonschema>=3.0.2", + "kubernetes==10.0.1" ], entry_points=''' #for click integration [console_scripts] diff --git a/tests/test_reckoner.py b/tests/test_reckoner.py index 4ea1fda2..136c71f6 100644 --- a/tests/test_reckoner.py +++ b/tests/test_reckoner.py @@ -51,8 +51,9 @@ def reset_mock(self): def tearDown(self): self.subprocess_mock_patch.stop() - # Test properties of the mock + + @mock.patch('reckoner.reckoner.Config', autospec=True) @mock.patch('reckoner.reckoner.Course', autospec=True) @mock.patch.object(Helm2Client, 'tiller_version') @@ -73,7 +74,10 @@ def test_course(self, *args): self.assertTrue(hasattr(reckoner_instance, 'course')) +@mock.patch('reckoner.chart.create_namespace', mock.MagicMock(return_value=True)) +@mock.patch('reckoner.chart.list_namespace_names', mock.MagicMock(return_value=[])) class TestCourseMocks(unittest.TestCase): + @mock.patch('reckoner.course.yaml_handler', autospec=True) @mock.patch('reckoner.course.get_helm_client', autospec=True) def test_raises_errors_when_missing_heading(self, mock_helm, mock_yaml): @@ -235,6 +239,8 @@ def tearDownModule(): shutil.rmtree(test_files_path) +@mock.patch('reckoner.chart.create_namespace', mock.MagicMock(return_value=True)) +@mock.patch('reckoner.chart.list_namespace_names', mock.MagicMock(return_value=[])) class TestReckoner(TestBase): name = "test-pentagon-base" @@ -258,6 +264,8 @@ def test_install(self, mock_git_lib): self.assertEqual(self.a.install(), None) +@mock.patch('reckoner.chart.create_namespace', mock.MagicMock(return_value=True)) +@mock.patch('reckoner.chart.list_namespace_names', mock.MagicMock(return_value=[])) class TestCourse(TestBase): @mock.patch('reckoner.course.get_helm_client', autospec=True) @@ -291,6 +299,8 @@ def test_plot_course(self): self.assertEqual(self.c._charts_to_install, self.c.charts) +@mock.patch('reckoner.chart.create_namespace', mock.MagicMock(return_value=True)) +@mock.patch('reckoner.chart.list_namespace_names', mock.MagicMock(return_value=[])) class TestChart(TestBase): @mock.patch('reckoner.course.get_helm_client', autospec=True) @@ -398,6 +408,7 @@ def test_chart_install(self, mock_git_lib): class TestRepository(TestBase): + @mock.patch('reckoner.repository.git', autospec=True) def test_git_repository(self, mock_git_lib): helm_mock = mock.Mock()