gpt-3.5-turbo-0301· openai
Suzhen/CodeChat
java实现sparkStreaming读取kafka中的单词数据并进行wordcount计算代码?
- User
java实现sparkStreaming读取kafka中的单词数据并进行wordcount计算代码?
- Assistant
import org.apache.spark.SparkConf; import org.apache.spark.streaming.Durations; import org.apache.spark.streaming.api.java.JavaPairDStream; import org.apache.spark.streaming.api.java.JavaStreamingContext; import org.apache.spark.streaming.kafka010.KafkaUtils; import org.apache.spark.streaming.kafka010.LocationStrategies; import org.apache.spark.streaming.kafka010.ConsumerStrategies; import scala.Tuple2; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Map; public class SparkStreamingKafkaWordCount { public static void main(String[] args) throws Exception { // 创建 Spark 配置 SparkConf conf = new SparkConf().setAppName("SparkStreamingKafkaWordCount"); // 创建 StreamingContext JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(5)); // 设置 Kafka 消费者配置 Map<String, Object> kafkaParams = new HashMap<>(); kafkaParams.put("bootstrap.servers", "localhost:9092"); kafkaParams.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); kafkaParams.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); kafkaParams.put("group.id", "test"); // 定义需要消费的 Kafka 主题 Collection<String> topics = Arrays.asList("test"); // 从 Kafka 中读取数据 JavaPairDStream<String, String> stream = KafkaUtils.createDirectStream( jssc, LocationStrategies.PreferConsistent(), ConsumerStrategies.<String, String>Subscribe(topics, kafkaParams)) .mapToPair(record -> new Tuple2<>(record.key(), record.value())); // 对读入的数据进行处理,计算 wordcount JavaPairDStream<String, Integer> wordCounts = stream.flatMap(tuple2 -> Arrays.asList(tuple2._2().split(" ")).iterator()) .mapToPair(word -> new Tuple2<>(word, 1)) .reduceByKey((x, y) -> x + y); // 输出结果 wordCounts.print(); // 启动 Spark Streaming jssc.start(); // 等待处理结束 jssc.awaitTermination(); } }