Skip to content
The Chat Archive
zephyr-7b-beta Arena Human Preference 55K

buildKafka = (config: Config, vpc: ec2.IVpc, bastionSg: ec2.SecurityGroup,…

  1. User

    buildKafka = (config: Config, vpc: ec2.IVpc, bastionSg: ec2.SecurityGroup, cfnClusterId?: string) => {

        const mskConfigArn = ssm.StringParameter.valueFromLookup(this, '/msk/configurationArn');
        const mskConfigRevisionParameter = new cr.AwsCustomResource(this, 'GetConfigRevision', {
            onUpdate: {
                service: 'SSM',
                action: 'getParameter',
                parameters: {
                    Name: '/msk/configurationRevision',
                },
                physicalResourceId: cr.PhysicalResourceId.of(Date.now().toString()), // Update physical id to always fetch the latest version
            },
            policy: cr.AwsCustomResourcePolicy.fromSdkCalls({ resources: cr.AwsCustomResourcePolicy.ANY_RESOURCE }),
        });
        const mskConfigRevision = mskConfigRevisionParameter.getResponseField('Parameter.Value');
        const subnets = vpc.selectSubnets({ subnetGroupName: 'private' });
    
        const sg = this.buildSecurityGroup(config, vpc, bastionSg);
        const ingressSg = this.buildIngressSecurityGroup(vpc, sg);
        this.mskSecurityGroupId = ingressSg.securityGroupId;
    
        const kafka = new msk.CfnCluster(this,
            cfnClusterId ?? 'Kafka',
            {
                clusterName: `${config.Cluster.clusterName}-${config.Stage}`,
                brokerNodeGroupInfo:
                    {
                        securityGroups: [sg.securityGroupId],
                        storageInfo: { ebsStorageInfo: { volumeSize: config.Cluster.volumeSize } },
                        clientSubnets: subnets.subnetIds,
                        instanceType: config.Cluster.instanceType,
                    },
                numberOfBrokerNodes: config.Cluster.numberOfBrokerNodes,
                kafkaVersion: config.Cluster.kafkaVersion,
                configurationInfo: {
                    arn: mskConfigArn,
                    revision: cdk.Token.asNumber(mskConfigRevision),
                },
                encryptionInfo: {
                    //encryptionAtRest: {dataVolumeKmsKeyId: ''},
                    encryptionInTransit: { clientBroker: 'TLS', inCluster: true },
                },
                enhancedMonitoring: config.Cluster.enhancedMonitoring,
                loggingInfo: {
                    brokerLogs: {
                        cloudWatchLogs: {
                            enabled: true,
                            logGroup: new logs.LogGroup(this, 'KafkaLogGrp', { retention: logs.RetentionDays.TWO_WEEKS, removalPolicy: cdk.RemovalPolicy.DESTROY }).logGroupName,
                        },
                        firehose: { enabled: false },
                        s3: { enabled: false },
                    },
                },
            });
        kafka.addPropertyOverride('ClientAuthentication.Sasl', { Scram: { Enabled: true } });
    
        return kafka;
    }
    

    can I add somewhere in the code to enable multi-VPC connectivity?

  2. Assistant

    To enable multi-VPC connectivity for your Kafka cluster, you can follow these steps:

    1. Create additional security groups for each VPC where you want to enable connectivity to the Kafka cluster.
    2. Add the necessary ingress rules to these security groups to allow communication from the client subnets in each VPC to the Kafka broker nodes.
    3. Update the buildSecurityGroup function to create a new security group for each VPC and pass it as an argument to the buildKafka function.
    4. Update the subnets variable to select subnets from all the VPCs where you want to enable connectivity to the Kafka cluster.
    5. Update the clientSubnets property of the brokerNodeGroupInfo object in the buildKafka function to use the subnets variable instead of a specific subnet group.
    6. Update the sg and ingressSg variables in the buildKafka function to use the appropriate security groups for the current VPC.
    7. Create a new CfnCluster object for each VPC where you want to enable connectivity to the Kafka cluster, passing in the appropriate security groups and subnet groups.
    8. Add each CfnCluster object to the Stack.

    Here's an updated version of the buildKafka function with these modifications:

    import * as msk from 'aws-cdk-lib/aws-msk';
    import * as ec2 from 'aws-cdk-lib/aws-ec2';
    import * as cr from 'aws-cdk-lib/custom-resources';
    import * as logs from 'aws-cdk-lib/aws-logs';
    import * as ssm from 'aws-cdk-lib/aws-ssm';
    import * as cdk from 'aws-cdk-lib';
    import * as cfn from 'aws-cdk-lib/aws-cloudformation';
    
    interface KafkaConfig {
      Stage: string;
      Cluster: {
        clusterName: string;
        instanceType: string;
        numberOfBrokerNodes: number;
        volumeSize: number;
        kafkaVersion: string;
        enhancedMonitoring: boolean;
      };
    }
    
    interface KafkaProps {
      config: KafkaConfig;
      vpc: ec2.IVpc;
      bastionSg: ec2.SecurityGroup;
      cfnClusterId?: string;
    }
    
    class KafkaStack extends cdk.Stack {
      constructor(scope: cdk.Construct, id: string, props: KafkaProps) {
        super(scope, id, {
          stackName: 'Kafka',
          env: {
            account: cdk.Account.FROM_ENV,
            region: cdk.Region.CURRENT,
          },
        });
    
        const mskConfigArn = ssm.StringParameter.valueFromLookup(this, '/msk/configurationArn');
        const mskConfigRevisionParameter = new cr.AwsCustomResource(this, 'GetConfigRevision', {
          onUpdate: {
            service: 'SSM',
            action: 'getParameter',
            parameters: {
              Name: '/msk/configurationRevision',
            },
            physicalResourceId: cr.PhysicalResourceId.of(Date.now().toString()), // Update physical id to always fetch the latest version
          },
          policy: cr.AwsCustomResourcePolicy.fromSdkCalls({ resources: cr.AwsCustomResourcePolicy.ANY_RESOURCE }),
        });
        const mskConfigRevision = mskConfigRevisionParameter.getResponseField('Parameter.Value');
        const subnets = props.vpc.selectSubnets({ subnetGroupName: 'private' });
    
        const mskSecurityGroups: { [key: string]: ec2.SecurityGroup } = {};
        const ingressSg: ec2.SecurityGroup = new ec2.SecurityGroup(this, 'IngressSecurityGroup', {
          vpc: props.vpc,
          enableResourcePropertyConvention: false,
        });
        const sg = this.buildSecurityGroup(props.config, props.vpc, props.bastionSg);
        mskSecurityGroups[props.vpc.availabilityZones[0]] = sg;
        mskSecurityGroups[props.vpc.availabilityZones[1]] = sg; // assume there are
    

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.