Skip to content
TheChatArchiveThe Chat Archive
Hugging Facezephyr-7b-beta· Hugging FaceHugging Face Arena Human Preference 55K

import * as applicationautoscaling from '@aws-cdk/aws-applicationautoscaling'; import *…

From a Chatbot Arena battleThe same person sent this prompt to two anonymous models at once. Compare it with OpenChatopenchat-3.5-0106’s answer.
  1. User

    import * as applicationautoscaling from '@aws-cdk/aws-applicationautoscaling'; import * as cloudWatch from '@aws-cdk/aws-cloudwatch'; import * as cw_actions from '@aws-cdk/aws-cloudwatch-actions'; import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as kms from '@aws-cdk/aws-kms'; import * as logs from '@aws-cdk/aws-logs'; import * as msk from '@aws-cdk/aws-msk'; import * as secretsmanager from '@aws-cdk/aws-secretsmanager'; import { Topic } from '@aws-cdk/aws-sns'; import * as ssm from '@aws-cdk/aws-ssm'; import * as cdk from '@aws-cdk/core'; import * as cr from '@aws-cdk/custom-resources'; import * as globalConfig from '../environment/global.json'; import { Config, EnvironmentUtils } from '../utils/EnvironmentUtils'; import { KafkaSecrets } from './kafka-secrets/kafka-secrets'; import configHelper from './utilities/config-helper';

    // Create an interface to hold the vpc information. interface IwtNueMskInfraStackProps extends cdk.StackProps { monitoringTopic: Topic; }

    export class IwtNueMskInfraStack extends cdk.Stack { readonly mskClusterArn: string; readonly kmsKeyArn: string; readonly mskClusterName: string; readonly vpnNuerenberg: string; readonly vpnHamburg: string; readonly officeZagreb: string; private mskSecurityGroupId: string;

    constructor(scope: cdk.App, id: string, props: IwtNueMskInfraStackProps) {
        super(scope, id, props);
    
        const config = EnvironmentUtils.readEnvFile();
    
        this.mskClusterName = `${config.Cluster.clusterName}-${config.Stage}`;
    
        this.vpnNuerenberg = globalConfig.VpnNuerenberg;
        this.vpnHamburg = globalConfig.VpnHamburg;
        this.officeZagreb = globalConfig.OfficeZagreb;
    
        const vpc = ec2.Vpc.fromLookup(this, 'VPC', {
            vpcName: config.VPC.vpcName,
            subnetGroupNameTag: 'SubnetGroupName',
        });
    
        const kmsKey = new kms.Key(this, 'keyForKafka', {
            description: 'KMS key for MSK to access secretsmanager',
        });
        kmsKey.addAlias('alias/msk');
        this.kmsKeyArn = kmsKey.keyArn;
    
        const bastionSg = this.buildBastionSg(vpc);
        const bastionHost = new ec2.BastionHostLinux(this, 'BastionHost', {
            vpc: vpc,
            subnetSelection: { subnetType: ec2.SubnetType.PUBLIC },
            securityGroup: bastionSg,
        });
        this.addPoliciesToBastionHost(bastionHost);
    
        const kafka = this.buildKafka(config, vpc, bastionSg);
        this.mskClusterArn = kafka.ref;
    
        this.buildKafkaBrokerStorageAutoscaling(config, kafka);
    
        // set clients per stage with default dev
        const clients = configHelper.getClientConfig().clients;
        this.buildKafkaAdminSecret(kmsKey, config);
        this.buildSecrets(clients, kmsKey, config);
        this.buildMetrics(config, kafka, props);
        this.buildCfnOutput(kafka, bastionHost);
        this.buildSSMParameters();
    }
    
    /**
     * Generate /msk/clusterArn and /msk/kmsKeyArn as SSM parameters.
     * This can be used by other dependent stacks that aren't part of this repo,
     * but still hosted in the same account, e.g. iwt-stream-feeder
     */
    buildSSMParameters = () => {
        new ssm.StringParameter(this, 'SSMKafkaClusterArn', {
            parameterName: '/msk/clusterArn',
            stringValue: this.mskClusterArn,
        });
    
        new ssm.StringParameter(this, 'SSMKMSKeyId', {
            parameterName: '/msk/kmsKeyArn',
            stringValue: this.kmsKeyArn,
        });
        new ssm.StringParameter(this, 'SSMKafkaSecurityGroupId', {
            parameterName: '/msk/securityGroupId',
            stringValue: this.mskSecurityGroupId,
        });
    }
    
    
    buildBastionSg = (vpc: ec2.IVpc) => {
        const bastionSg = new ec2.SecurityGroup(this, 'BastionSecGroup',
            {
                vpc: vpc,
                securityGroupName: 'bastion-host',
                allowAllOutbound: true,
                description: 'Security group for bastion host',
            });
        bastionSg.addIngressRule(ec2.Peer.ipv4(this.vpnHamburg), ec2.Port.tcp(22), 'allow SSH access from ODN (includes Hamburg VPN)');
        bastionSg.addIngressRule(ec2.Peer.ipv4(this.vpnNuerenberg), ec2.Port.tcp(22), 'allow SSH access from the Nuerenberg VPN');
        bastionSg.addIngressRule(ec2.Peer.ipv4('3.120.181.40/29'), ec2.Port.tcp(22), 'allow SSH access for EC2_INSTANCE_CONNECT');
    
        return bastionSg;
    }
    
    buildSecurityGroup = (config: Config, vpc: ec2.IVpc, bastionSg: ec2.SecurityGroup) => {
        const sg = new ec2.SecurityGroup(this, 'MSKSecGroup',
            {
                vpc: vpc,
                allowAllOutbound: true,
                description: 'Control traffic to the MSK Cluster',
            });
    
        sg.addIngressRule(ec2.Peer.ipv4('10.64.0.0/10'), ec2.Port.tcp(9096), 'allow access to the brokers from immowelt AWS accounts');
        sg.addIngressRule(ec2.Peer.ipv4('10.128.0.0/10'), ec2.Port.tcp(9096), 'allow access to the brokers from immowelt AWS accounts');
        sg.addIngressRule(ec2.Peer.ipv4(this.vpnHamburg), ec2.Port.tcp(9096), 'allow access to the brokers from ODN (includes Hamburg VPN)');
        sg.addIngressRule(ec2.Peer.ipv4(this.vpnNuerenberg), ec2.Port.tcp(9096), 'allow access from the Nuerenberg VPN');
        sg.addIngressRule(ec2.Peer.ipv4(this.officeZagreb), ec2.Port.tcp(9096), 'allow access from the Zagreb Office');
    
        sg.addIngressRule(bastionSg, ec2.Port.tcp(2181), 'allow access to the zookeeper nodes');
    
        Object.entries(config.VPC.additionalIPv4IngressRuleRanges).forEach(
            ([key, range]) => {
                sg.addIngressRule(ec2.Peer.ipv4(range), ec2.Port.tcp(9096), `allow access from ${key}`);
            },
        );
    
        return sg;
    }
    
    buildIngressSecurityGroup = (vpc: ec2.IVpc, sg: ec2.SecurityGroup) => {
        const ingressSg = new ec2.SecurityGroup(this, 'MSKIngressSecGroup', {
            vpc: vpc,
            allowAllOutbound: true,
            description: 'Ingress group to access to the MSK cluster',
        });
        sg.addIngressRule(ingressSg, ec2.Port.tcp(9096), 'allow access to the brokers');
        return ingressSg;
    }
    
    buildCfnOutput = (kafka: msk.CfnCluster, bastionHost: ec2.BastionHostLinux) => {
        new cdk.CfnOutput(this, 'bastionHostInstanceId', {
            description: 'Instance ID of bastion host',
            value: bastionHost.instanceId,
        });
    
        new cdk.CfnOutput(this, 'bastionHostPublicDnsName', {
            description: 'Public DNS name of bastion host',
            value: bastionHost.instancePublicDnsName,
        });
    
        new cdk.CfnOutput(this, 'bastionHostAvailabilityZone', {
            description: 'Availability zone of bastion host',
            value: bastionHost.instanceAvailabilityZone,
        });
    
        new cdk.CfnOutput(this, 'mskClusterArn', {
            description: 'ARN of the MSK cluster',
            value: kafka.ref,
        });
    }
    
    buildKafka = (config: Config, vpc: ec2.IVpc, bastionSg: ec2.SecurityGroup, cfnClusterId?: string) => {
    
        const mskConfigArn = ssm.StringParameter.valueFromLookup(this, '/msk/configurationArn');
        const mskConfigRevisionParameter = new cr.AwsCustomResource(this, 'GetConfigRevision', {
            onUpdate: {
                service: 'SSM',
                action: 'getParameter',
                parameters: {
                    Name: '/msk/configurationRevision',
                },
                physicalResourceId: cr.PhysicalResourceId.of(Date.now().toString()), // Update physical id to always fetch the latest version
            },
            policy: cr.AwsCustomResourcePolicy.fromSdkCalls({ resources: cr.AwsCustomResourcePolicy.ANY_RESOURCE }),
        });
        const mskConfigRevision = mskConfigRevisionParameter.getResponseField('Parameter.Value');
        const subnets = vpc.selectSubnets({ subnetGroupName: 'private' });
    
        const sg = this.buildSecurityGroup(config, vpc, bastionSg);
        const ingressSg = this.buildIngressSecurityGroup(vpc, sg);
        this.mskSecurityGroupId = ingressSg.securityGroupId;
    
        const kafka = new msk.CfnCluster(this,
            cfnClusterId ?? 'Kafka',
            {
                clusterName: `${config.Cluster.clusterName}-${config.Stage}`,
                brokerNodeGroupInfo:
                    {
                        securityGroups: [sg.securityGroupId],
                        storageInfo: { ebsStorageInfo: { volumeSize: config.Cluster.volumeSize } },
                        clientSubnets: subnets.subnetIds,
                        instanceType: config.Cluster.instanceType,
                    },
                numberOfBrokerNodes: config.Cluster.numberOfBrokerNodes,
                kafkaVersion: config.Cluster.kafkaVersion,
                configurationInfo: {
                    arn: mskConfigArn,
                    revision: cdk.Token.asNumber(mskConfigRevision),
                },
                encryptionInfo: {
                    //encryptionAtRest: {dataVolumeKmsKeyId: ''},
                    encryptionInTransit: { clientBroker: 'TLS', inCluster: true },
                },
                enhancedMonitoring: config.Cluster.enhancedMonitoring,
                loggingInfo: {
                    brokerLogs: {
                        cloudWatchLogs: {
                            enabled: true,
                            logGroup: new logs.LogGroup(this, 'KafkaLogGrp', { retention: logs.RetentionDays.TWO_WEEKS, removalPolicy: cdk.RemovalPolicy.DESTROY }).logGroupName,
                        },
                        firehose: { enabled: false },
                        s3: { enabled: false },
                    },
                },
            });
        kafka.addPropertyOverride('ClientAuthentication.Sasl', { Scram: { Enabled: true } });
    
        return kafka;
    }
    
    buildKafkaBrokerStorageAutoscaling = (config: any, kafka: msk.CfnCluster) => {
        const mskAutoscalingTarget = new applicationautoscaling.ScalableTarget(this, 'mskAutoscalingTarget', {
            serviceNamespace: applicationautoscaling.ServiceNamespace.KAFKA,
            maxCapacity: 500, // provisioned maximal volume size (in GiB)
            minCapacity: 1, // provisioned minimal volume size (in GiB)
            resourceId: kafka.ref,
            scalableDimension: 'kafka:broker-storage:VolumeSize',
        });
    
        // add scaling policy to scalable target
        const mskAutoscalingPolicy = new applicationautoscaling.TargetTrackingScalingPolicy(this, 'mskAutoscalingPolicy', {
            scalingTarget: mskAutoscalingTarget,
            targetValue: 80,
            policyName: `${config.Cluster.clusterName}-${config.Stage}-autoscaling-policy`,
            disableScaleIn: true,
            // must be KafkaBrokerStorageUtilization, const not in cdk yet, will be overriden below
            predefinedMetric: applicationautoscaling.PredefinedMetric.DYANMODB_WRITE_CAPACITY_UTILIZATION,
        });
        const mskAutoscalingPolicyNode = mskAutoscalingPolicy.node.defaultChild as applicationautoscaling.CfnScalingPolicy;
        mskAutoscalingPolicyNode.addPropertyOverride('TargetTrackingScalingPolicyConfiguration.PredefinedMetricSpecification.PredefinedMetricType', 'KafkaBrokerStorageUtilization');
    }
    
    buildMetrics = (config: any, kafka: msk.CfnCluster, props: any) => {
        for (let i = 1; i <= config.Cluster.numberOfBrokerNodes; i++) {
            const metric = new cloudWatch.Metric({
                namespace: 'AWS/Kafka',
                metricName: 'CpuIdle',
                period: cdk.Duration.minutes(5),
                dimensions: { 'Cluster Name': kafka.clusterName, 'Broker ID': i.toString() },
            });
    
            const cpuBusyAlert = metric.createAlarm(this, `kafka-cpu-busy-alert-${i.toString()}`, {
                alarmName: `kafka-cpu-busy-alert-${i.toStrin
    
  2. Hugging Face Assistant

    Here's a summary of the key features and configurations in the given code:

    1. MSK Cluster Creation:
      • The MSK cluster is created using the msk.CfnCluster resource in AWS CloudFormation.
      • The cluster's name and configuration are specified in the config.Cluster object.
      • The cluster's security group is configured to allow access from specific IP ranges and VPCs.
      • The cluster's storage and instance type are configured based on the config.Cluster object.
      • The cluster's Kafka version is also specified in the configuration.
      • Encryption at rest and in transit are enabled.
      • Enhanced monitoring and logging are enabled.
    2. KMS Key Creation:
      • A KMS key is created and stored in the config.Stage environment.
      • The KMS key is used to securely store secrets using the KafkaSecrets resource.
    3. Kafka Admin Secret Creation:
      • A secret containing the MSK cluster's admin credentials is created using the KafkaSecrets resource.
      • This allows authorized users to access the MSK cluster's administration console.
    4. Secrets Creation:
      • Additional secrets containing specific client configurations are created using the KafkaSecrets resource.
      • These secrets can be accessed by authorized users to connect to the MSK cluster using their specific configurations.
    5. CloudWatch Alarm Creation:
      • CloudWatch alarms are created to monitor the CPU usage of the MSK cluster's brokers.
      • These alarms are configured based on the config.Cluster object.
    6. CloudWatch Logging:
      • CloudWatch logs are configured to store the logs generated by the MSK cluster's brokers.
      • This allows for easy access and analysis of the logs.
    7. SSM Parameter Creation:
      • Parameters containing the MSK cluster's ARN, KMS key ID, and security group ID are created in the AWS Systems Manager (SSM).
      • These parameters can be accessed by other AWS services or stacks that are part of the same account but not this repo.
    8. Bastion Host Creation:
      • A bastion host is created using the ec2.BastionHostLinux resource in AWS EC2.
      • This allows for secure access to the MSK cluster and other AWS services.
    9. Custom Resource Creation:
      • A custom resource is created to allow for dynamic configuration of the MSK cluster's storage capacity.
      • This allows for automatic scaling of the cluster's storage capacity based on usage.
    10. Security Group Creation:
    • Two security groups (one for the MSK cluster and one for ingress) are created using the ec2.SecurityGroup resource in AWS EC2.
    • These security groups allow for the specific IP ranges and VPCs to access the MSK cluster.
    1. CloudFormation Stack Creation:
    • The MSK cluster, Kafka admin secret, secrets, CloudWatch alarms, CloudWatch logs, SSM parameters, bastion host, and custom resource are all created as part of a single CloudFormation stack.
    • This allows for easy deployment and management of the MSK cluster and its associated resources.
    1. Kafka Authentication:
    • SASL authentication is enabled for Kafka clients to ensure secure communication.
    1. Secrets Management:
    • The KafkaSecrets resource is used to securely store the MSK cluster's admin credentials and client configurations.
    • This allows for easy access to the secrets by authorized users.
    1. CloudWatch Monitoring:
    • CloudWatch alarms are created to monitor the CPU usage of the MSK cluster's brokers.
    • These alarms are configured based on the config.Cluster object.
    1. CloudWatch Logging:
    • CloudWatch logs are configured to store the logs generated by the MSK cluster's brokers.
    • This allows for easy access and analysis of the logs.
    1. SSM Parameter Creation:
    • Parameters containing the MSK cluster's ARN, KMS key ID, and security group ID are created in the AWS Systems Manager (SSM).
    • These parameters can be accessed by other AWS services or stacks that are part of the same account but not this repo.
    1. Bastion Host Creation:
    • A bastion host is created using the ec2.BastionHostLinux resource in AWS EC2

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.