import * as applicationautoscaling from '@aws-cdk/aws-applicationautoscaling'; import *…
- User
import * as applicationautoscaling from '@aws-cdk/aws-applicationautoscaling'; import * as cloudWatch from '@aws-cdk/aws-cloudwatch'; import * as cw_actions from '@aws-cdk/aws-cloudwatch-actions'; import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as kms from '@aws-cdk/aws-kms'; import * as logs from '@aws-cdk/aws-logs'; import * as msk from '@aws-cdk/aws-msk'; import * as secretsmanager from '@aws-cdk/aws-secretsmanager'; import { Topic } from '@aws-cdk/aws-sns'; import * as ssm from '@aws-cdk/aws-ssm'; import * as cdk from '@aws-cdk/core'; import * as cr from '@aws-cdk/custom-resources'; import * as globalConfig from '../environment/global.json'; import { Config, EnvironmentUtils } from '../utils/EnvironmentUtils'; import { KafkaSecrets } from './kafka-secrets/kafka-secrets'; import configHelper from './utilities/config-helper';
// Create an interface to hold the vpc information. interface IwtNueMskInfraStackProps extends cdk.StackProps { monitoringTopic: Topic; }
export class IwtNueMskInfraStack extends cdk.Stack { readonly mskClusterArn: string; readonly kmsKeyArn: string; readonly mskClusterName: string; readonly vpnNuerenberg: string; readonly vpnHamburg: string; readonly officeZagreb: string; private mskSecurityGroupId: string;
constructor(scope: cdk.App, id: string, props: IwtNueMskInfraStackProps) { super(scope, id, props); const config = EnvironmentUtils.readEnvFile(); this.mskClusterName = `${config.Cluster.clusterName}-${config.Stage}`; this.vpnNuerenberg = globalConfig.VpnNuerenberg; this.vpnHamburg = globalConfig.VpnHamburg; this.officeZagreb = globalConfig.OfficeZagreb; const vpc = ec2.Vpc.fromLookup(this, 'VPC', { vpcName: config.VPC.vpcName, subnetGroupNameTag: 'SubnetGroupName', }); const kmsKey = new kms.Key(this, 'keyForKafka', { description: 'KMS key for MSK to access secretsmanager', }); kmsKey.addAlias('alias/msk'); this.kmsKeyArn = kmsKey.keyArn; const bastionSg = this.buildBastionSg(vpc); const bastionHost = new ec2.BastionHostLinux(this, 'BastionHost', { vpc: vpc, subnetSelection: { subnetType: ec2.SubnetType.PUBLIC }, securityGroup: bastionSg, }); this.addPoliciesToBastionHost(bastionHost); const kafka = this.buildKafka(config, vpc, bastionSg); this.mskClusterArn = kafka.ref; this.buildKafkaBrokerStorageAutoscaling(config, kafka); // set clients per stage with default dev const clients = configHelper.getClientConfig().clients; this.buildKafkaAdminSecret(kmsKey, config); this.buildSecrets(clients, kmsKey, config); this.buildMetrics(config, kafka, props); this.buildCfnOutput(kafka, bastionHost); this.buildSSMParameters(); } /** * Generate /msk/clusterArn and /msk/kmsKeyArn as SSM parameters. * This can be used by other dependent stacks that aren't part of this repo, * but still hosted in the same account, e.g. iwt-stream-feeder */ buildSSMParameters = () => { new ssm.StringParameter(this, 'SSMKafkaClusterArn', { parameterName: '/msk/clusterArn', stringValue: this.mskClusterArn, }); new ssm.StringParameter(this, 'SSMKMSKeyId', { parameterName: '/msk/kmsKeyArn', stringValue: this.kmsKeyArn, }); new ssm.StringParameter(this, 'SSMKafkaSecurityGroupId', { parameterName: '/msk/securityGroupId', stringValue: this.mskSecurityGroupId, }); } buildBastionSg = (vpc: ec2.IVpc) => { const bastionSg = new ec2.SecurityGroup(this, 'BastionSecGroup', { vpc: vpc, securityGroupName: 'bastion-host', allowAllOutbound: true, description: 'Security group for bastion host', }); bastionSg.addIngressRule(ec2.Peer.ipv4(this.vpnHamburg), ec2.Port.tcp(22), 'allow SSH access from ODN (includes Hamburg VPN)'); bastionSg.addIngressRule(ec2.Peer.ipv4(this.vpnNuerenberg), ec2.Port.tcp(22), 'allow SSH access from the Nuerenberg VPN'); bastionSg.addIngressRule(ec2.Peer.ipv4('3.120.181.40/29'), ec2.Port.tcp(22), 'allow SSH access for EC2_INSTANCE_CONNECT'); return bastionSg; } buildSecurityGroup = (config: Config, vpc: ec2.IVpc, bastionSg: ec2.SecurityGroup) => { const sg = new ec2.SecurityGroup(this, 'MSKSecGroup', { vpc: vpc, allowAllOutbound: true, description: 'Control traffic to the MSK Cluster', }); sg.addIngressRule(ec2.Peer.ipv4('10.64.0.0/10'), ec2.Port.tcp(9096), 'allow access to the brokers from immowelt AWS accounts'); sg.addIngressRule(ec2.Peer.ipv4('10.128.0.0/10'), ec2.Port.tcp(9096), 'allow access to the brokers from immowelt AWS accounts'); sg.addIngressRule(ec2.Peer.ipv4(this.vpnHamburg), ec2.Port.tcp(9096), 'allow access to the brokers from ODN (includes Hamburg VPN)'); sg.addIngressRule(ec2.Peer.ipv4(this.vpnNuerenberg), ec2.Port.tcp(9096), 'allow access from the Nuerenberg VPN'); sg.addIngressRule(ec2.Peer.ipv4(this.officeZagreb), ec2.Port.tcp(9096), 'allow access from the Zagreb Office'); sg.addIngressRule(bastionSg, ec2.Port.tcp(2181), 'allow access to the zookeeper nodes'); Object.entries(config.VPC.additionalIPv4IngressRuleRanges).forEach( ([key, range]) => { sg.addIngressRule(ec2.Peer.ipv4(range), ec2.Port.tcp(9096), `allow access from ${key}`); }, ); return sg; } buildIngressSecurityGroup = (vpc: ec2.IVpc, sg: ec2.SecurityGroup) => { const ingressSg = new ec2.SecurityGroup(this, 'MSKIngressSecGroup', { vpc: vpc, allowAllOutbound: true, description: 'Ingress group to access to the MSK cluster', }); sg.addIngressRule(ingressSg, ec2.Port.tcp(9096), 'allow access to the brokers'); return ingressSg; } buildCfnOutput = (kafka: msk.CfnCluster, bastionHost: ec2.BastionHostLinux) => { new cdk.CfnOutput(this, 'bastionHostInstanceId', { description: 'Instance ID of bastion host', value: bastionHost.instanceId, }); new cdk.CfnOutput(this, 'bastionHostPublicDnsName', { description: 'Public DNS name of bastion host', value: bastionHost.instancePublicDnsName, }); new cdk.CfnOutput(this, 'bastionHostAvailabilityZone', { description: 'Availability zone of bastion host', value: bastionHost.instanceAvailabilityZone, }); new cdk.CfnOutput(this, 'mskClusterArn', { description: 'ARN of the MSK cluster', value: kafka.ref, }); } buildKafka = (config: Config, vpc: ec2.IVpc, bastionSg: ec2.SecurityGroup, cfnClusterId?: string) => { const mskConfigArn = ssm.StringParameter.valueFromLookup(this, '/msk/configurationArn'); const mskConfigRevisionParameter = new cr.AwsCustomResource(this, 'GetConfigRevision', { onUpdate: { service: 'SSM', action: 'getParameter', parameters: { Name: '/msk/configurationRevision', }, physicalResourceId: cr.PhysicalResourceId.of(Date.now().toString()), // Update physical id to always fetch the latest version }, policy: cr.AwsCustomResourcePolicy.fromSdkCalls({ resources: cr.AwsCustomResourcePolicy.ANY_RESOURCE }), }); const mskConfigRevision = mskConfigRevisionParameter.getResponseField('Parameter.Value'); const subnets = vpc.selectSubnets({ subnetGroupName: 'private' }); const sg = this.buildSecurityGroup(config, vpc, bastionSg); const ingressSg = this.buildIngressSecurityGroup(vpc, sg); this.mskSecurityGroupId = ingressSg.securityGroupId; const kafka = new msk.CfnCluster(this, cfnClusterId ?? 'Kafka', { clusterName: `${config.Cluster.clusterName}-${config.Stage}`, brokerNodeGroupInfo: { securityGroups: [sg.securityGroupId], storageInfo: { ebsStorageInfo: { volumeSize: config.Cluster.volumeSize } }, clientSubnets: subnets.subnetIds, instanceType: config.Cluster.instanceType, }, numberOfBrokerNodes: config.Cluster.numberOfBrokerNodes, kafkaVersion: config.Cluster.kafkaVersion, configurationInfo: { arn: mskConfigArn, revision: cdk.Token.asNumber(mskConfigRevision), }, encryptionInfo: { //encryptionAtRest: {dataVolumeKmsKeyId: ''}, encryptionInTransit: { clientBroker: 'TLS', inCluster: true }, }, enhancedMonitoring: config.Cluster.enhancedMonitoring, loggingInfo: { brokerLogs: { cloudWatchLogs: { enabled: true, logGroup: new logs.LogGroup(this, 'KafkaLogGrp', { retention: logs.RetentionDays.TWO_WEEKS, removalPolicy: cdk.RemovalPolicy.DESTROY }).logGroupName, }, firehose: { enabled: false }, s3: { enabled: false }, }, }, }); kafka.addPropertyOverride('ClientAuthentication.Sasl', { Scram: { Enabled: true } }); return kafka; } buildKafkaBrokerStorageAutoscaling = (config: any, kafka: msk.CfnCluster) => { const mskAutoscalingTarget = new applicationautoscaling.ScalableTarget(this, 'mskAutoscalingTarget', { serviceNamespace: applicationautoscaling.ServiceNamespace.KAFKA, maxCapacity: 500, // provisioned maximal volume size (in GiB) minCapacity: 1, // provisioned minimal volume size (in GiB) resourceId: kafka.ref, scalableDimension: 'kafka:broker-storage:VolumeSize', }); // add scaling policy to scalable target const mskAutoscalingPolicy = new applicationautoscaling.TargetTrackingScalingPolicy(this, 'mskAutoscalingPolicy', { scalingTarget: mskAutoscalingTarget, targetValue: 80, policyName: `${config.Cluster.clusterName}-${config.Stage}-autoscaling-policy`, disableScaleIn: true, // must be KafkaBrokerStorageUtilization, const not in cdk yet, will be overriden below predefinedMetric: applicationautoscaling.PredefinedMetric.DYANMODB_WRITE_CAPACITY_UTILIZATION, }); const mskAutoscalingPolicyNode = mskAutoscalingPolicy.node.defaultChild as applicationautoscaling.CfnScalingPolicy; mskAutoscalingPolicyNode.addPropertyOverride('TargetTrackingScalingPolicyConfiguration.PredefinedMetricSpecification.PredefinedMetricType', 'KafkaBrokerStorageUtilization'); } buildMetrics = (config: any, kafka: msk.CfnCluster, props: any) => { for (let i = 1; i <= config.Cluster.numberOfBrokerNodes; i++) { const metric = new cloudWatch.Metric({ namespace: 'AWS/Kafka', metricName: 'CpuIdle', period: cdk.Duration.minutes(5), dimensions: { 'Cluster Name': kafka.clusterName, 'Broker ID': i.toString() }, }); const cpuBusyAlert = metric.createAlarm(this, `kafka-cpu-busy-alert-${i.toString()}`, { alarmName: `kafka-cpu-busy-alert-${i.toStrin Assistant
Here's a summary of the key features and configurations in the given code:
- MSK Cluster Creation:
- The MSK cluster is created using the
msk.CfnClusterresource in AWS CloudFormation. - The cluster's name and configuration are specified in the
config.Clusterobject. - The cluster's security group is configured to allow access from specific IP ranges and VPCs.
- The cluster's storage and instance type are configured based on the
config.Clusterobject. - The cluster's Kafka version is also specified in the configuration.
- Encryption at rest and in transit are enabled.
- Enhanced monitoring and logging are enabled.
- The MSK cluster is created using the
- KMS Key Creation:
- A KMS key is created and stored in the
config.Stageenvironment. - The KMS key is used to securely store secrets using the
KafkaSecretsresource.
- A KMS key is created and stored in the
- Kafka Admin Secret Creation:
- A secret containing the MSK cluster's admin credentials is created using the
KafkaSecretsresource. - This allows authorized users to access the MSK cluster's administration console.
- A secret containing the MSK cluster's admin credentials is created using the
- Secrets Creation:
- Additional secrets containing specific client configurations are created using the
KafkaSecretsresource. - These secrets can be accessed by authorized users to connect to the MSK cluster using their specific configurations.
- Additional secrets containing specific client configurations are created using the
- CloudWatch Alarm Creation:
- CloudWatch alarms are created to monitor the CPU usage of the MSK cluster's brokers.
- These alarms are configured based on the
config.Clusterobject.
- CloudWatch Logging:
- CloudWatch logs are configured to store the logs generated by the MSK cluster's brokers.
- This allows for easy access and analysis of the logs.
- SSM Parameter Creation:
- Parameters containing the MSK cluster's ARN, KMS key ID, and security group ID are created in the AWS Systems Manager (SSM).
- These parameters can be accessed by other AWS services or stacks that are part of the same account but not this repo.
- Bastion Host Creation:
- A bastion host is created using the
ec2.BastionHostLinuxresource in AWS EC2. - This allows for secure access to the MSK cluster and other AWS services.
- A bastion host is created using the
- Custom Resource Creation:
- A custom resource is created to allow for dynamic configuration of the MSK cluster's storage capacity.
- This allows for automatic scaling of the cluster's storage capacity based on usage.
- Security Group Creation:
- Two security groups (one for the MSK cluster and one for ingress) are created using the
ec2.SecurityGroupresource in AWS EC2. - These security groups allow for the specific IP ranges and VPCs to access the MSK cluster.
- CloudFormation Stack Creation:
- The MSK cluster, Kafka admin secret, secrets, CloudWatch alarms, CloudWatch logs, SSM parameters, bastion host, and custom resource are all created as part of a single CloudFormation stack.
- This allows for easy deployment and management of the MSK cluster and its associated resources.
- Kafka Authentication:
- SASL authentication is enabled for Kafka clients to ensure secure communication.
- Secrets Management:
- The
KafkaSecretsresource is used to securely store the MSK cluster's admin credentials and client configurations. - This allows for easy access to the secrets by authorized users.
- CloudWatch Monitoring:
- CloudWatch alarms are created to monitor the CPU usage of the MSK cluster's brokers.
- These alarms are configured based on the
config.Clusterobject.
- CloudWatch Logging:
- CloudWatch logs are configured to store the logs generated by the MSK cluster's brokers.
- This allows for easy access and analysis of the logs.
- SSM Parameter Creation:
- Parameters containing the MSK cluster's ARN, KMS key ID, and security group ID are created in the AWS Systems Manager (SSM).
- These parameters can be accessed by other AWS services or stacks that are part of the same account but not this repo.
- Bastion Host Creation:
- A bastion host is created using the
ec2.BastionHostLinuxresource in AWS EC2
- MSK Cluster Creation:
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.