From 292849b3dbad0d207aa572a038a2b181e6ae8f0d Mon Sep 17 00:00:00 2001 From: damwu1 Date: Mon, 13 May 2024 20:03:59 +1000 Subject: [PATCH] feat: [KTD-3526] add pitfile not create directory location path checking logic plus unit tests update --- k8s-deployer/src/deployer.ts | 36 +++- k8s-deployer/src/k8s.ts | 6 +- k8s-deployer/src/test-suite-handler.ts | 14 +- .../test/pitfile/pitfile-loader.spec.ts | 4 +- k8s-deployer/test/test-suite-handler.spec.ts | 204 +++++++++++++++--- 5 files changed, 224 insertions(+), 40 deletions(-) diff --git a/k8s-deployer/src/deployer.ts b/k8s-deployer/src/deployer.ts index 5361432..bdbe172 100644 --- a/k8s-deployer/src/deployer.ts +++ b/k8s-deployer/src/deployer.ts @@ -36,6 +36,27 @@ const isExecutable = async (filePath: string) => { } } +/* + Description: function isDirectoryExists: the purpose of this function is for checking the application directory is whether exists or not + + Usage example: + // Assuming we have a project root directory /workspace, and we have 1 sub directory under /workspace, which is /workspace/project-directory + // eg: /workspace + // - project-directory + + const isApplicationDirectoryExists = await isDirectoryExists('/workspace/project-directory') // return as true and it means application directory exists and accessible + const isApplicationDirectoryExists = await isDirectoryExists('/workspace/some-other-directory') // return as false because some-other-directory does not exist and it means application directory does not exist and not accessible +*/ +const isDirectoryExists = async (filePath: string): Promise => { + try { + await fs.promises.access(filePath, fs.constants.F_OK) + return true + } catch (e) { + logger.error(`There is no ${filePath} or it is not accessible.`, { cause: e }) + return false + } +} + export const cloneFromGit = async (appId: string, location: Schema.Location, targetDirectory: string): Promise => { logger.info("The '%s' will be copied from '%s' into %s' using '%s'", appId, location.gitRepository, targetDirectory, location.gitRef) const fullCommand = `k8s-deployer/scripts/git-co.sh ${ location.gitRepository } ${ location.gitRef } ${ targetDirectory }` @@ -164,6 +185,15 @@ const undeployApplication = async ( } } +// setAppDirectory is used to set app directory value based on the result of isDirectoryExists function +const setAppDirectory = async (appDir: string, specId: string): Promise => { + if (!appDir || !specId) throw new Error('appDir or specId value is missing') + + const isApplicationDirectoryExists = await isDirectoryExists(appDir) + + return isApplicationDirectoryExists ? specId : '.' +} + export const deployLockManager = async (config: Config, workspace: string, namespace: Namespace) => { const spec = getLockManagerConfig() const appName = spec.id @@ -215,7 +245,8 @@ export const deployComponent = async ( appDir = spec.location.path logger.info("The application directory will be taken from 'location.path' attribute: '%s' of '%s'", appDir, spec.name) } else { - appDir = spec.id + appDir = await setAppDirectory(appDir, spec.id) + logger.info("The application directory will be taken from 'id' attribute: '%s' of '%s'", appDir, spec.name) } commitSha = await Shell.exec(`cd ${ appDir } && git log --pretty=format:"%h" -1`) @@ -236,7 +267,8 @@ export const undeployComponent = async (workspace: string, namespace: Namespace, appDir = spec.location.path logger.info("The application directory will be taken from 'location.path' attribute: '%s' of '%s'", appDir, spec.name) } else { - appDir = spec.id + appDir = await setAppDirectory(appDir, spec.id) + logger.info("The application directory will be taken from 'id' attribute: '%s' of '%s'", appDir, spec.name) } } diff --git a/k8s-deployer/src/k8s.ts b/k8s-deployer/src/k8s.ts index 168a454..a481752 100644 --- a/k8s-deployer/src/k8s.ts +++ b/k8s-deployer/src/k8s.ts @@ -1,7 +1,7 @@ -import * as Shell from "./shell-facade.js" +import * as cfg from "./config.js" import { logger } from "./logger.js" import { Namespace } from "./model.js" -import * as cfg from "./config.js" +import * as Shell from "./shell-facade.js" const pad = (v: string | number, len: number = 2): string => { let result = `${ v }` @@ -29,8 +29,10 @@ export const createNamespace = async (workspace: string, rootNamespace: Namespac const logFile = `${ workspace }/logs/create-ns-${ namespace }.log` logger.info("Creating namespace: '%s'", namespace) + const command = `k8s-deployer/scripts/k8s-manage-namespace.sh ${ rootNamespace } create "${ namespace }" ${ timeoutSeconds }` const timeoutMs = timeoutSeconds * 1_000 + await Shell.exec(command, { logFileName: logFile, tailTarget: logger.info, timeoutMs }) logger.info("Namespace created: '%s'", namespace) diff --git a/k8s-deployer/src/test-suite-handler.ts b/k8s-deployer/src/test-suite-handler.ts index 57618bd..b3e27bb 100644 --- a/k8s-deployer/src/test-suite-handler.ts +++ b/k8s-deployer/src/test-suite-handler.ts @@ -1,14 +1,14 @@ import * as fs from "fs" +import { Config } from "./config.js" +import * as Deployer from "./deployer.js" +import * as K8s from "./k8s.js" import { LOG_SEPARATOR_LINE, logger } from "./logger.js" import { DeployedComponent, DeployedTestSuite, GraphDeploymentResult, Namespace, Prefix, Schema } from "./model.js" -import * as Deployer from "./deployer.js" -import { Config } from "./config.js" import * as PifFileLoader from "./pitfile/pitfile-loader.js" -import * as K8s from "./k8s.js" -import * as TestRunner from "./test-app-client/test-runner.js" -import * as Shell from "./shell-facade.js" import { PodLogTail } from "./pod-log-tail.js" +import * as Shell from "./shell-facade.js" +import * as TestRunner from "./test-app-client/test-runner.js" export const generatePrefix = (env: string): Prefix => { return generatePrefixByDate(new Date(), env) @@ -137,7 +137,7 @@ const deployLocal = async ( logger.info("process.env.MOCK_NS=%s", process.env.MOCK_NS) } - logger.info("NAMEPSACE IN USE=%s, process.env.MOCK_NS=%s", ns, process.env.MOCK_NS) + logger.info("NAMESPACE IN USE=%s, process.env.MOCK_NS=%s", ns, process.env.MOCK_NS) await deployLockManager(config, workspace, ns, pitfile.lockManager.enabled, testSuite.id) logger.info("") @@ -234,7 +234,7 @@ export const processTestSuite = async ( // By default assume processing strategy to be "deploy all then run tests one by one" logger.info("") - logger.info("--------------- Processig %s ---------------", testSuite.id) + logger.info("--------------- Processing %s ---------------", testSuite.id) logger.info("") const list = await deployAll(prefix, config, pitfile, seqNumber, testSuite) diff --git a/k8s-deployer/test/pitfile/pitfile-loader.spec.ts b/k8s-deployer/test/pitfile/pitfile-loader.spec.ts index ff15b88..2cf7272 100644 --- a/k8s-deployer/test/pitfile/pitfile-loader.spec.ts +++ b/k8s-deployer/test/pitfile/pitfile-loader.spec.ts @@ -1,6 +1,6 @@ -import * as sinon from "sinon" import * as chai from "chai" import chaiAsPromised from 'chai-as-promised' +import * as sinon from "sinon" chai.use(chaiAsPromised) import * as PifFileLoader from "../../src/pitfile/pitfile-loader.js" @@ -47,7 +47,7 @@ describe("Loads pitfile from disk", () => { }) it("should throw if location has no gitRemote", async () => { - const errorMessage = `Invalid configiuration for 'suite-1'. The 'location.gitRepository' is required when location.type is REMOTE` + const errorMessage = `Invalid configuration for 'suite-1'. The 'location.gitRepository' is required when location.type is REMOTE` await chai.expect(PifFileLoader.loadFromFile("dist/test/pitfile/test-pitfile-valid-3-invalid.yml")).eventually.rejectedWith(errorMessage) }) diff --git a/k8s-deployer/test/test-suite-handler.spec.ts b/k8s-deployer/test/test-suite-handler.spec.ts index fec6ff3..db43a6c 100644 --- a/k8s-deployer/test/test-suite-handler.spec.ts +++ b/k8s-deployer/test/test-suite-handler.spec.ts @@ -1,17 +1,17 @@ -import esmock from "esmock" -import * as sinon from "sinon" import * as chai from "chai" import { SpawnOptions } from "child_process" +import esmock from "esmock" import { RequestInit } from "node-fetch" +import * as sinon from "sinon" import { logger } from "../src/logger.js" -import * as webapi from "../src/test-app-client/web-api/schema-v1.js" -import { ShellOptions } from "../src/shell-facade.js" +import { Config, DEFAULT_SUB_NAMESPACE_PREFIX, SUB_NAMESPACE_GENERATOR_TYPE_DATE } from "../src/config.js" import { LocationType } from "../src/pitfile/schema-v1.js" import { SchemaVersion } from "../src/pitfile/version.js" -import { Config, DEFAULT_SUB_NAMESPACE_PREFIX, SUB_NAMESPACE_GENERATOR_TYPE_DATE } from "../src/config.js" +import { ShellOptions } from "../src/shell-facade.js" import { ScalarMetric, TestOutcomeType, TestStream } from "../src/test-app-client/report/schema-v1.js" +import * as webapi from "../src/test-app-client/web-api/schema-v1.js" import { generatePrefixByDate } from "../src/test-suite-handler.js" describe("Helper functions", () => { @@ -24,8 +24,8 @@ describe("Helper functions", () => { }) describe("Deployment happy path", async () => { - const prefix = "12345" - const testSuiteId = "t1" + let prefix = "12345" + let testSuiteId = "t1" const namespace = "nsChild" const workspace = `${ prefix }_${ testSuiteId }` const k8DeployerConfig: Config = { @@ -209,74 +209,224 @@ describe("Deployment happy path", async () => { // check verification for expected directories and shell executables - chai.expect(fsAccessStubs.callCount).eq(7) - - // check for presense of workspace direectory on the disk + chai.expect(fsAccessStubs.callCount).eq(9) + // check for presense of workspace directory on the disk chai.expect(fsAccessStubs.getCall(0).calledWith(workspace)).be.true chai.expect(fsAccessStubs.getCall(1).calledWith("lock-manager/deployment/pit/deploy.sh")).be.true chai.expect(fsAccessStubs.getCall(2).calledWith("lock-manager/deployment/pit/is-deployment-ready.sh")).be.true - chai.expect(fsAccessStubs.getCall(3).calledWith("comp-1/deployment/pit/deploy.sh")).be.true - chai.expect(fsAccessStubs.getCall(4).calledWith("comp-1/deployment/pit/is-deployment-ready.sh")).be.true - chai.expect(fsAccessStubs.getCall(5).calledWith("comp-1-test-app/deployment/pit/deploy.sh")).be.true - chai.expect(fsAccessStubs.getCall(6).calledWith("comp-1-test-app/deployment/pit/is-deployment-ready.sh")).be.true + chai.expect(fsAccessStubs.getCall(3).calledWith("12345_t1/comp-1")).be.true + chai.expect(fsAccessStubs.getCall(4).calledWith("comp-1/deployment/pit/deploy.sh")).be.true + chai.expect(fsAccessStubs.getCall(5).calledWith("comp-1/deployment/pit/is-deployment-ready.sh")).be.true + chai.expect(fsAccessStubs.getCall(6).calledWith("12345_t1/comp-1-test-app")).be.true + chai.expect(fsAccessStubs.getCall(7).calledWith("comp-1-test-app/deployment/pit/deploy.sh")).be.true + chai.expect(fsAccessStubs.getCall(8).calledWith("comp-1-test-app/deployment/pit/is-deployment-ready.sh")).be.true // check shell calls // debugging tools - // for (let i = 8; i < execStub.callCount; i++) { + // for (let i = 12; i < execStub.callCount; i++) { // logger.info("%s) %s", i, execStub.getCall(i).args) // } - // - // chai.expect(execStub.getCall(9).args[0]).eq(`deployment/pit/deploy.sh nsChild t1`) + + // chai.expect(execStub.getCall(9).args[0]).eq("deployment/pit/deploy.sh nsChild t1") // chai.expect(execStub.getCall(9).args[1]).eq(param2) + // chai.expect(execStub.getCall(5).args[1]).eq(undefined) chai.expect(execStub.callCount).eq(11) - chai.expect(execStub.getCall(0).calledWith(`mkdir -p 12345_t1/logs`)).be.true - chai.expect(execStub.getCall(1).calledWith(`mkdir -p 12345_t1/reports`)).be.true + + chai.expect(execStub.getCall(0).calledWith("mkdir -p 12345_t1/logs")).be.true + + chai.expect(execStub.getCall(1).calledWith("mkdir -p 12345_t1/reports")).be.true chai.expect(execStub.getCall(2).calledWith( `k8s-deployer/scripts/k8s-manage-namespace.sh nsParent create "nsChild" 2`, - { logFileName: `12345_t1/logs/create-ns-nsChild.log`, timeoutMs: 2000, tailTarget: sinon.match.any } + { logFileName: "12345_t1/logs/create-ns-nsChild.log", timeoutMs: 2000, tailTarget: sinon.match.any } )).be.true chai.expect(execStub.getCall(3).calledWith( - `deployment/pit/deploy.sh nsChild lock-manager`, + "deployment/pit/deploy.sh nsChild lock-manager", { homeDir: "lock-manager", logFileName: `12345_t1/logs/deploy-nsChild-lock-manager.log`, tailTarget: sinon.match.any }) ).be.true chai.expect(execStub.getCall(4).calledWith( - `deployment/pit/is-deployment-ready.sh nsChild`, + "deployment/pit/is-deployment-ready.sh nsChild", { homeDir: "lock-manager" }) ).be.true chai.expect(execStub.getCall(5).calledWith(`cd comp-1 && git log --pretty=format:"%h" -1`)).be.true chai.expect(execStub.getCall(6).calledWith( - `deployment/pit/deploy.sh nsChild`, - { homeDir: "comp-1", logFileName: `12345_t1/logs/deploy-nsChild-comp-1.log`, tailTarget: sinon.match.any }) + "deployment/pit/deploy.sh nsChild", + { homeDir: "comp-1", logFileName: "12345_t1/logs/deploy-nsChild-comp-1.log", tailTarget: sinon.match.any }) ).be.true chai.expect(execStub.getCall(7).calledWith( - `deployment/pit/is-deployment-ready.sh nsChild`, + "deployment/pit/is-deployment-ready.sh nsChild", { homeDir: "comp-1" } )).be.true chai.expect(execStub.getCall(8).calledWith(`cd comp-1-test-app && git log --pretty=format:"%h" -1`)).be.true + chai.expect(execStub.getCall(9).calledWith( - `deployment/pit/deploy.sh nsChild t1`, + "deployment/pit/deploy.sh nsChild t1", { homeDir: "comp-1-test-app", logFileName: `12345_t1/logs/deploy-nsChild-comp-1-test-app.log`, tailTarget: sinon.match.any }) ).be.true chai.expect(execStub.getCall(10).calledWith( - `deployment/pit/is-deployment-ready.sh nsChild`, + "deployment/pit/is-deployment-ready.sh nsChild", { homeDir: "comp-1-test-app" }) ).be.true // assert that log tailing was invoked chai.expect(nodeShellSpawnStub.callCount).eq(1) + chai.expect(nodeShellSpawnStub.getCall(0).calledWith( "k8s-deployer/scripts/tail-container-log.sh", [ namespace, "comp-1-test-app" ] )).be.true }) + + it ("processTestSuite with a different workspace", async () => { + prefix = '23456' + testSuiteId = 't2' + const testSuite2 = JSON.parse(JSON.stringify(testSuite)) + testSuite2.id = 't2' + testSuite2.name = 't2-name' + + const report = { + executedScenarios: [ + new webapi.ExecutedTestScenario( + "t2-sc2", + new Date(), + new Date(new Date().getTime() + 20_000), + [ new TestStream("t2-sc2-stream1", [ new ScalarMetric("tps", 100) ], [ new ScalarMetric("tps", 101) ], TestOutcomeType.PASS) ], + ["comp-2"] + ) + ] + } + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // Mock the environment + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + const execStub = sinon.stub() + const shellImportMock = { + exec: async (command: string, options?: ShellOptions) => { + if (options) { + logger.info("mock::shell-exec('%s', %s)", command, options) + } else { + logger.info("mock::shell-exec('%s')", command) + } + execStub(command, options) + } + } + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + const fsAccessStubs = sinon.stub() + // checking the presense of workspace + fsAccessStubs.withArgs(`${prefix}_${testSuiteId}/${testSuite2.deployment.graph.components[0].id}`).throws(new Error("path is not writable")) + // all other access checks + fsAccessStubs.returns(true) + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + const httpClientStub = sinon.stub() + httpClientStub.withArgs(sinon.match(`\/${ namespace }\.${ testSuiteId }\/start`), sinon.match.any).returns({ + ok: true, + json: async () => new webapi.StartResponse("session-2", testSuiteId) + }) + const respStatusRunning = { + ok: true, + json: async () => new webapi.StatusResponse("session-2", testSuiteId, webapi.TestStatus.RUNNING) + } + const respStatusCompleted = { ok: true, json: async () => new webapi.StatusResponse("session-2", testSuiteId, webapi.TestStatus.COMPLETED)} + httpClientStub.withArgs(sinon.match(`\/${ namespace }\.${ testSuiteId }\/status\?sessionId=session-2`), sinon.match.any) + .onFirstCall().returns(respStatusRunning) + .onSecondCall().returns(respStatusRunning) + .onThirdCall().returns(respStatusCompleted) + + httpClientStub.withArgs(sinon.match(`\/${ namespace }\.${ testSuiteId }\/reports\?sessionId=session-2`), sinon.match.any) + .returns( + { + ok: true, + json: async () => { return new webapi.ReportResponse("session-2", testSuiteId, webapi.TestStatus.COMPLETED, report) } + } + ) + + const httpImportMock = async (url: string, options: RequestInit) => { + let debugOpts = options + if (debugOpts.body) debugOpts = { ...options, body: JSON.parse(options.body as string) } + logger.info("mock::fetch(%s, %s)", url, JSON.stringify(debugOpts)) + return await httpClientStub(url, options) + } + + const K8s = await esmock("../src/k8s.js", { "../src/shell-facade.js": shellImportMock }) + + // mock log tailing logic + + const killChilldStub = sinon.stub() + killChilldStub.returns(true) + + const tailerInstanceStub = { + pid: 2222, + kill: (signal: string) => killChilldStub(signal) + } + + const nodeShellSpawnStub = sinon.stub() + nodeShellSpawnStub.returns(tailerInstanceStub) + + const PodLogTail = await esmock( + "../src/pod-log-tail.js", + { + "fs": { openSync: (a: string, b: string): number => { return 0 } }, + "child_process": { + spawn: (script: string, args: string[], opts: SpawnOptions[]) => { + // delegate spawining calls to stub and record them for later assertion + return nodeShellSpawnStub(script, args, opts) } + } + } + ) + + const SuiteHandler = await esmock( + "../src/test-suite-handler.js", + { + "../src/k8s.js": { ...K8s, generateNamespaceName:() => namespace } , + "../src/pod-log-tail.js": { ...PodLogTail } + }, + { + "../src/logger.js": { logger: { debug: () => {}, info: () => {}, warn: (s: string, a: any) => { logger.warn(s, a) }, error: (s: string, a: any) => { logger.error(s, a) } } }, + "node-fetch": httpImportMock, + "../src/shell-facade.js": shellImportMock, + "fs": { promises: { access: async (path: string, mode: number) => await fsAccessStubs(path, mode) }} + }, + ) + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // End of mocking + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + const pitfile = { + projectName: "TestPitFile", + version: SchemaVersion.VERSION_1_0, + lockManager: { enabled: true }, + testSuites: [ testSuite2 ] + } + + await SuiteHandler.processTestSuite(prefix, k8DeployerConfig, pitfile, testSuiteNumber, testSuite2) + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // Assert interactions with external services + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // check verification for expected directories and shell executables + chai.expect(fsAccessStubs.callCount).eq(9) + // check for presense of workspace directory on the disk + chai.expect(fsAccessStubs.getCall(0).calledWith('23456_t2')).be.true + chai.expect(fsAccessStubs.getCall(1).calledWith("lock-manager/deployment/pit/deploy.sh")).be.true + chai.expect(fsAccessStubs.getCall(2).calledWith("lock-manager/deployment/pit/is-deployment-ready.sh")).be.true + chai.expect(fsAccessStubs.getCall(3).calledWith("23456_t2/comp-1")).be.true + chai.expect(fsAccessStubs.getCall(4).calledWith("./deployment/pit/deploy.sh")).be.true + chai.expect(fsAccessStubs.getCall(5).calledWith("./deployment/pit/is-deployment-ready.sh")).be.true + chai.expect(fsAccessStubs.getCall(6).calledWith("23456_t2/comp-1-test-app")).be.true + chai.expect(fsAccessStubs.getCall(7).calledWith("comp-1-test-app/deployment/pit/deploy.sh")).be.true + chai.expect(fsAccessStubs.getCall(8).calledWith("comp-1-test-app/deployment/pit/is-deployment-ready.sh")).be.true + }) })