Skip to content
TheChatArchiveThe Chat Archive
gpt-3.5-turbo-0301· openaiHugging Face Suzhen/CodeChat

java实现sparkStreaming读取kafka中的单词数据并进行wordcount计算代码?

  1. User

    java实现sparkStreaming读取kafka中的单词数据并进行wordcount计算代码?

  2. Assistant
    import org.apache.spark.SparkConf;
    import org.apache.spark.streaming.Durations;
    import org.apache.spark.streaming.api.java.JavaPairDStream;
    import org.apache.spark.streaming.api.java.JavaStreamingContext;
    import org.apache.spark.streaming.kafka010.KafkaUtils;
    import org.apache.spark.streaming.kafka010.LocationStrategies;
    import org.apache.spark.streaming.kafka010.ConsumerStrategies;
    import scala.Tuple2;
    
    import java.util.Arrays;
    import java.util.Collection;
    import java.util.HashMap;
    import java.util.Map;
    
    public class SparkStreamingKafkaWordCount {
    
        public static void main(String[] args) throws Exception {
            // 创建 Spark 配置
            SparkConf conf = new SparkConf().setAppName("SparkStreamingKafkaWordCount");
    
            // 创建 StreamingContext
            JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(5));
    
            // 设置 Kafka 消费者配置
            Map<String, Object> kafkaParams = new HashMap<>();
            kafkaParams.put("bootstrap.servers", "localhost:9092");
            kafkaParams.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
            kafkaParams.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
            kafkaParams.put("group.id", "test");
    
            // 定义需要消费的 Kafka 主题
            Collection<String> topics = Arrays.asList("test");
    
            // 从 Kafka 中读取数据
            JavaPairDStream<String, String> stream =
                    KafkaUtils.createDirectStream(
                            jssc,
                            LocationStrategies.PreferConsistent(),
                            ConsumerStrategies.<String, String>Subscribe(topics, kafkaParams))
                            .mapToPair(record -> new Tuple2<>(record.key(), record.value()));
    
            // 对读入的数据进行处理,计算 wordcount
            JavaPairDStream<String, Integer> wordCounts =
                    stream.flatMap(tuple2 -> Arrays.asList(tuple2._2().split(" ")).iterator())
                            .mapToPair(word -> new Tuple2<>(word, 1))
                            .reduceByKey((x, y) -> x + y);
    
            // 输出结果
            wordCounts.print();
    
            // 启动 Spark Streaming
            jssc.start();
    
            // 等待处理结束
            jssc.awaitTermination();
        }
    
    }