diff --git a/lib/infra/infra-stack.ts b/lib/infra/infra-stack.ts index 8cf8585052f..ef4fb6c856f 100644 --- a/lib/infra/infra-stack.ts +++ b/lib/infra/infra-stack.ts @@ -103,8 +103,12 @@ export interface InfraProps extends StackProps { readonly dataNodeStorage?: number, /** EBS block storage size for ml nodes */ readonly mlNodeStorage?: number, + /** EBS block storage size for ingest nodes */ + readonly ingestNodeStorage?: number, /** EC2 instance type for data nodes */ readonly dataInstanceType?: InstanceType, + /** EC2 instance type for ingest nodes */ + readonly ingestInstanceType?: InstanceType, /** EC2 instance type for ML nodes */ readonly mlInstanceType?: InstanceType, /** Whether to use 50% heap */ @@ -170,8 +174,12 @@ export class InfraStack extends Stack { private mlNodeStorage: number; + private ingestNodeStorage: number; + private dataInstanceType: InstanceType; + private ingestInstanceType: InstanceType; + private mlInstanceType: InstanceType; private use50PercentHeap: boolean; @@ -241,6 +249,7 @@ export class InfraStack extends Stack { this.dashboardsUrl = `${props?.dashboardsUrl ?? scope.node.tryGetContext('dashboardsUrl')}`; const dataInstanceType: InstanceType | string = `${props?.dataInstanceType ?? scope.node.tryGetContext('dataInstanceType')}`; const mlInstanceType: InstanceType | string = `${props?.mlInstanceType ?? scope.node.tryGetContext('mlInstanceType')}`; + const ingestInstanceType: InstanceType | string = `${props?.ingestInstanceType ?? scope.node.tryGetContext('ingestInstanceType')}`; this.cpuArch = `${props?.cpuArch ?? scope.node.tryGetContext('cpuArch')}`; if (this.cpuArch === 'undefined') { @@ -251,10 +260,12 @@ export class InfraStack extends Stack { instanceCpuType = AmazonLinuxCpuType.X86_64; this.dataInstanceType = getInstanceType(dataInstanceType, this.cpuArch.toString()); this.mlInstanceType = getInstanceType(mlInstanceType, this.cpuArch.toString()); + this.ingestInstanceType = getInstanceType(ingestInstanceType, this.cpuArch.toString()); } else { instanceCpuType = AmazonLinuxCpuType.ARM_64; this.dataInstanceType = getInstanceType(dataInstanceType, this.cpuArch.toString()); this.mlInstanceType = getInstanceType(mlInstanceType, this.cpuArch.toString()); + this.ingestInstanceType = getInstanceType(ingestInstanceType, this.cpuArch.toString()); } } else { throw new Error('Please provide a valid cpu architecture. The valid value can be either x64 or arm64'); @@ -320,6 +331,13 @@ export class InfraStack extends Stack { this.mlNodeStorage = parseInt(mlStorage, 10); } + const ingestStorage = `${props?.ingestNodeStorage ?? scope.node.tryGetContext('ingestNodeStorage')}`; + if (ingestStorage === 'undefined') { + this.ingestNodeStorage = 100; + } else { + this.ingestNodeStorage = parseInt(ingestStorage, 10); + } + this.jvmSysProps = `${props?.jvmSysProps ?? scope.node.tryGetContext('jvmSysProps')}`; const additionalConf = `${props?.additionalConfig ?? scope.node.tryGetContext('additionalConfig')}`; @@ -660,6 +678,37 @@ export class InfraStack extends Stack { Tags.of(mlNodeAsg).add('role', 'ml-node'); } + if (this.ingestNodeCount > 0) { + const ingestNodeAsg = new AutoScalingGroup(this, 'ingestNodeAsg', { + vpc: props.vpc, + instanceType: this.ingestInstanceType, + machineImage: MachineImage.latestAmazonLinux2023({ + cpuType: instanceCpuType, + }), + role: this.instanceRole, + maxCapacity: this.ingestNodeCount, + minCapacity: this.ingestNodeCount, + desiredCapacity: this.ingestNodeCount, + vpcSubnets: { + subnetType: SubnetType.PRIVATE_WITH_EGRESS, + }, + securityGroup: props.securityGroup, + blockDevices: [{ + deviceName: '/dev/xvda', + volume: BlockDeviceVolume.ebs(this.ingestNodeStorage, { deleteOnTermination: true, volumeType: this.storageVolumeType }), + }], + init: CloudFormationInit.fromElements(...this.getCfnInitElement(this, clusterLogGroup, 'ingest')), + initOptions: { + ignoreFailures: false, + }, + requireImdsv2: true, + signals: Signals.waitForAll(), + }); + Tags.of(ingestNodeAsg).add('role', 'ingest'); + } else { + Tags.of(dataNodeAsg).add('role', 'dualRole'); + } + opensearchListener.addTargets('opensearchTarget', { port: 9200, protocol: Protocol.TCP, diff --git a/lib/opensearch-config/node-config.ts b/lib/opensearch-config/node-config.ts index d192945eb74..0c1e56236f9 100644 --- a/lib/opensearch-config/node-config.ts +++ b/lib/opensearch-config/node-config.ts @@ -14,10 +14,18 @@ nodeConfig.set('manager', { 'node.roles': ['cluster_manager'], }); -nodeConfig.set('data', { +nodeConfig.set('dualRole', { 'node.roles': ['data', 'ingest'], }); +nodeConfig.set('data', { + 'node.roles': ['data'], +}); + +nodeConfig.set('ingest', { + 'node.roles': ['ingest'], +}); + nodeConfig.set('seed-manager', { 'node.name': 'seed', 'node.roles': ['cluster_manager'],