Skip to content
TheChatArchiveThe Chat Archive
gpt-3.5-turbo-0301· OpenAIAi2 WildChat-4.8M (AI2)

Flink 集群 sink调用 mybatisplus IService 代码实现

  1. User

    Flink 集群 sink调用 mybatisplus IService 代码实现

  2. Assistant
    1. 引入依赖

    在 pom.xml 文件中添加如下依赖:

    <dependency>
        <groupId>com.baomidou</groupId>
        <artifactId>mybatis-plus-boot-starter</artifactId>
        <version>最新版本号</version>
    </dependency>
    
    1. 创建实体类

    以用户实体类为例,创建如下 User 类:

    @Data
    @NoArgsConstructor
    @AllArgsConstructor
    @TableName("user")
    public class User implements Serializable {
        private static final long serialVersionUID = 1L;
    
        @TableId(value = "id", type = IdType.AUTO)
        private Integer id;
    
        @TableField(value = "name")
        private String name;
    
        @TableField(value = "age")
        private Integer age;
    }
    
    1. 创建 Mapper 接口

    以用户 Mapper 接口为例,创建如下 UserMapper 接口:

    public interface UserMapper extends BaseMapper<User> {
    }
    
    1. 创建 Service 接口和实现类

    以用户 Service 接口为例,创建如下 UserService 接口:

    public interface UserService extends IService<User> {
    }
    

    创建 UserServiceImpl 类实现 UserService 接口:

    @Service
    public class UserServiceImpl extends ServiceImpl<UserMapper, User> implements UserService {
    }
    
    1. Flink 集群 Sink 调用 Service 代码实现

    在 Flink 程序中:

    public class FlinkMybatisSinkDemo {
        public static void main(String[] args) throws Exception {
            StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
            env.setParallelism(1);
    
            // 模拟数据源
            DataStream<User> userStream = env.fromCollection(Arrays.asList(
                    new User(null, "lily", 18),
                    new User(null, "lucy", 20),
                    new User(null, "tom", 25)
            ));
    
            // 构造 MybatisPlusSink
            MybatisPlusSink<User> mybatisSink = new MybatisPlusSink<>("UserMapper.insert", new UserServiceImpl());
    
            // 将数据发送到 MybatisPlusSink 中
            userStream.addSink(mybatisSink);
    
            env.execute("FlinkMybatisSinkDemo");
        }
    }
    

    构造 MybatisPlusSink 的方式有多种,可以参考以下示例:

    // 构造 MybatisPlusRecordSinkProvider,并注册到 MybatisPlusSink 中
    MybatisPlusRecordSinkProvider<User> mybatisPlusSinkProvider = new MybatisPlusRecordSinkProvider<>("UserMapper.insert", new UserServiceImpl());
    MybatisPlusSink<User> mybatisSink = new MybatisPlusSink<>(mybatisPlusSinkProvider);
    
    // 或者直接使用类名和 Bean 对象构造 MybatisPlusSink
    MybatisPlusSink<User> mybatisSink = new MybatisPlusSink<>("UserMapper.insert", new UserServiceImpl());
    
  3. User

    MybatisPlusSink 代码实现

  4. Assistant
    1. 定义 MybatisPlusSink
    public class MybatisPlusSink<T> extends RichSinkFunction<T> {
        private static final Logger LOGGER = LoggerFactory.getLogger(MybatisPlusSink.class);
    
        private final MybatisPlusRecordSinkProvider<T> provider;
        private transient MybatisPlusRecordSink<T> sink;
    
        public MybatisPlusSink(MybatisPlusRecordSinkProvider<T> provider) {
            this.provider = provider;
        }
    
        /**
         * 构造方法
         *
         * @param statement          Mapper 方法名
         * @param service            BaseService 实现类
         */
        public MybatisPlusSink(String statement, IService<T> service) {
            this.provider = new MybatisPlusRecordSinkProvider<>(statement, service);
        }
    
        /**
         * 初始化方法,获取 MybatisPlusRecordSink 实例
         *
         * @param parameters 参数
         * @throws Exception 异常
         */
        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
            sink = provider.createRecordSink();
        }
    
        /**
         * 调用 MybatisPlusRecordSink 写入数据
         *
         * @param value 数据
         * @param context 上下文
         * @throws Exception 异常
         */
        @Override
        public void invoke(T value, Context context) throws Exception {
            sink.writeRecord(value);
        }
    
        /**
         * 关闭方法,释放资源
         *
         * @throws Exception 异常
         */
        @Override
        public void close() throws Exception {
            super.close();
            if (sink != null) {
                sink.close();
            }
        }
    }
    
    1. 定义 MybatisPlusRecordSinkProvider
    public class MybatisPlusRecordSinkProvider<T> implements RecordSinkProvider<T> {
        private final String statement;
        private transient IService<T> service;
    
        public MybatisPlusRecordSinkProvider(String statement, IService<T> service) {
            this.statement = statement;
            this.service = service;
        }
    
        /**
         * 获取 MybatisPlusRecordSink 实例
         *
         * @return MybatisPlusRecordSink 实例
         * @throws IOException 异常
         */
        @Override
        public RecordSink<T> createRecordSink() throws IOException {
            return new MybatisPlusRecordSink<>(statement, service);
        }
    }
    
    1. 定义 MybatisPlusRecordSink
    public class MybatisPlusRecordSink<T> implements RecordSink<T> {
        private static final Logger LOGGER = LoggerFactory.getLogger(MybatisPlusRecordSink.class);
    
        private final String statement;
        private transient IService<T> service;
        private transient SqlSession sqlSession;
    
        public MybatisPlusRecordSink(String statement, IService<T> service) {
            this.statement = statement;
            this.service = service;
        }
    
        /**
         * 写入数据到数据库
         *
         * @param record 数据
         * @throws IOException 异常
         */
        @Override
        public void writeRecord(T record) throws IOException {
            try {
                if (sqlSession == null) {
                    sqlSession = SqlSessionFactoryUtils.getSqlSession();
                }
    
                service.save(record);
                sqlSession.commit();
            } catch (Exception e) {
                LOGGER.error("Failed to write record, record={}", record, e);
                sqlSession.rollback();
                throw new IOException("Failed to write record, record=" + record, e);
            }
        }
    
        /**
         * 关闭 sqlSession 和 service
         *
         * @throws IOException 异常
         */
        @Override
        public void close() throws IOException {
            if (sqlSession != null) {
                sqlSession.close();
                sqlSession = null;
            }
            if (service != null) {
                service = null;
            }
        }
    }
    
    1. SqlSessionFactoryUtils

    这是 MybatisPlusRecordSink 内部使用的工具类,用于获取单例的 SqlSession 实例。

    public class SqlSessionFactoryUtils {
        private static final SqlSessionFactory SESSION_FACTORY;
    
        static {
            String resource = "mybatis-config.xml";
            InputStream inputStream = null;
            try {
                inputStream = Resources.getResourceAsStream(resource);
            } catch (IOException e) {
                LOGGER.error("Failed to load mybatis config, resource={}", resource, e);
            }
            SESSION_FACTORY = new SqlSessionFactoryBuilder().build(inputStream);
        }
    
        /**
         * 获取 SqlSession 实例
         *
         * @return SqlSession 实例
         */
        public static SqlSession getSqlSession() {
            return SESSION_FACTORY.openSession();
        }
    }
    

    需要注意的是,此处是使用 Mybatis 的 SqlSession 进行操作的,因此需要先定义一个 Mybatis 的配置文件 mybatis-config.xml,示例如下:

    <?xml version="1.0" encoding="UTF-8" ?>
    <!DOCTYPE configuration
            PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
            "http://mybatis.org/dtd/mybatis-3-config.dtd">
    <configuration>
        <typeAliases>
            <typeAlias type="包名.实体类名" alias="别名" />
        </typeAliases>
    </configuration>
    

    其中,typeAlias 中的包名、实体类名和别名需要根据实际情况进行填写。

  5. User

    Flink 集群 调用mybatisplus service

  6. Assistant

    在 Flink 程序中,可以通过创建 MybatisPlusSink 对象来调用 MybatisPlus 中的 Service 方法。

    以用户实体类为例,创建如下 User 类:

    @Data
    @NoArgsConstructor
    @AllArgsConstructor
    @TableName("user")
    public class User implements Serializable {
        private static final long serialVersionUID = 1L;
    
        @TableId(value = "id", type = IdType.AUTO)
        private Integer id;
    
        @TableField(value = "name")
        private String name;
    
        @TableField(value = "age")
        private Integer age;
    }
    

    创建 UserService 接口和实现类:

    public interface UserService extends IService<User> {
    }
    
    @Service
    public class UserServiceImpl extends ServiceImpl<UserMapper, User> implements UserService {
    }
    

    创建 Flink 程序,使用 MybatisPlusSink 调用 UserServiceImpl 中的方法,向数据库中写入数据。

    public class FlinkMybatisPlusSinkDemo {
        public static void main(String[] args) throws Exception {
            StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
            env.setParallelism(1);
            
            // 模拟数据源
            DataStream<User> userStream = env.fromCollection(Arrays.asList(
                    new User(null, "lily", 18),
                    new User(null, "lucy", 20),
                    new User(null, "tom", 25)
            ));
    
            // 构造 MybatisPlusSink
            MybatisPlusSink<User> mybatisSink = new MybatisPlusSink<>("UserMapper.insert", new UserServiceImpl());
    
            // 将数据发送到 MybatisPlusSink 中
            userStream.addSink(mybatisSink);
    
            env.execute("FlinkMybatisPlusSinkDemo");
        }
    }
    

    需要注意的是,在 Flink 程序中调用 MybatisPlus 中的 Service 方法,需要保证 Service 实现类是线程安全的,否则会出现并发问题。通常情况下,可以通过在 Service 实现类上添加 @Service 注解来保证线程安全。

  7. User

    MybatisPlusSink 代码实现

  8. Assistant
    1. 定义 MybatisPlusSink
    public class MybatisPlusSink<T> extends RichSinkFunction<T> {
        private static final Logger LOGGER = LoggerFactory.getLogger(MybatisPlusSink.class);
    
        private final MybatisPlusRecordSinkProvider<T> provider;
        private transient MybatisPlusRecordSink<T> sink;
    
        public MybatisPlusSink(MybatisPlusRecordSinkProvider<T> provider) {
            this.provider = provider;
        }
    
        /**
         * 构造方法
         *
         * @param statement          Mapper 方法名
         * @param service            BaseService 实现类
         */
        public MybatisPlusSink(String statement, IService<T> service) {
            this.provider = new MybatisPlusRecordSinkProvider<>(statement, service);
        }
    
        /**
         * 初始化方法,获取 MybatisPlusRecordSink 实例
         *
         * @param parameters 参数
         * @throws Exception 异常
         */
        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
            sink = provider.createRecordSink();
        }
    
        /**
         * 调用 MybatisPlusRecordSink 写入数据
         *
         * @param value 数据
         * @param context 上下文
         * @throws Exception 异常
         */
        @Override
        public void invoke(T value, Context context) throws Exception {
            sink.writeRecord(value);
        }
    
        /**
         * 关闭方法,释放资源
         *
         * @throws Exception 异常
         */
        @Override
        public void close() throws Exception {
            super.close();
            if (sink != null) {
                sink.close();
            }
        }
    }
    
    1. 定义 MybatisPlusRecordSinkProvider
    public class MybatisPlusRecordSinkProvider<T> implements RecordSinkProvider<T> {
        private final String statement;
        private transient IService<T> service;
    
        public MybatisPlusRecordSinkProvider(String statement, IService<T> service) {
            this.statement = statement;
            this.service = service;
        }
    
        /**
         * 获取 MybatisPlusRecordSink 实例
         *
         * @return MybatisPlusRecordSink 实例
         * @throws IOException 异常
         */
        @Override
        public RecordSink<T> createRecordSink() throws IOException {
            return new MybatisPlusRecordSink<>(statement, service);
        }
    }
    
    1. 定义 MybatisPlusRecordSink
    public class MybatisPlusRecordSink<T> implements RecordSink<T> {
        private static final Logger LOGGER = LoggerFactory.getLogger(MybatisPlusRecordSink.class);
    
        private final String statement;
        private transient IService<T> service;
        private transient SqlSession sqlSession;
    
        public MybatisPlusRecordSink(String statement, IService<T> service) {
            this.statement = statement;
            this.service = service;
        }
    
        /**
         * 写入数据到数据库
         *
         * @param record 数据
         * @throws IOException 异常
         */
        @Override
        public void writeRecord(T record) throws IOException {
            try {
                if (sqlSession == null) {
                    sqlSession = SqlSessionFactoryUtils.getSqlSession();
                }
    
                service.save(record);
                sqlSession.commit();
            } catch (Exception e) {
                LOGGER.error("Failed to write record, record={}", record, e);
                sqlSession.rollback();
                throw new IOException("Failed to write record, record=" + record, e);
            }
        }
    
        /**
         * 关闭 sqlSession 和 service
         *
         * @throws IOException 异常
         */
        @Override
        public void close() throws IOException {
            if (sqlSession != null) {
                sqlSession.close();
                sqlSession = null;
            }
            if (service != null) {
                service = null;
            }
        }
    }
    
    1. SqlSessionFactoryUtils

    这是 MybatisPlusRecordSink 内部使用的工具类,用于获取单例的 SqlSession 实例。

    public class SqlSessionFactoryUtils {
        private static final SqlSessionFactory SESSION_FACTORY;
    
        static {
            String resource = "mybatis-config.xml";
            InputStream inputStream = null;
            try {
                inputStream = Resources.getResourceAsStream(resource);
            } catch (IOException e) {
                LOGGER.error("Failed to load mybatis config, resource={}", resource, e);
            }
            SESSION_FACTORY = new SqlSessionFactoryBuilder().build(inputStream);
        }
    
        /**
         * 获取 SqlSession 实例
         *
         * @return SqlSession 实例
         */
        public static SqlSession getSqlSession() {
            return SESSION_FACTORY.openSession();
        }
    }
    

    需要注意的是,此处是使用 Mybatis 的 SqlSession 进行操作的,因此需要先定义一个 Mybatis 的配置文件 mybatis-config.xml,示例如下:

    <?xml version="1.0" encoding="UTF-8" ?>
    <!DOCTYPE configuration
            PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
            "http://mybatis.org/dtd/mybatis-3-config.dtd">
    <configuration>
        <typeAliases>
            <typeAlias type="包名.实体类名" alias="别名" />
        </typeAliases>
    </configuration>
    

    其中,typeAlias 中的包名、实体类名和别名需要根据实际情况进行填写。

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.