DATAX

DataX DataX 是阿里巴巴集团内被广泛使用的离线数据同步工具/平台，实现包括 MySQL、Oracle、SqlServer、Postgre、HDFS、Hive、ADS、HBase、TableStore(OTS)、MaxCompute(ODPS)、DRDS 等各种异构数据源之间高效的数据同步功能。

下载安装datax.tar，下载链接https://github.com/alibaba/DataX/blob/master/mysqlwriter/doc/mysqlwriter.md

MysqlWriter 插件实现了写入数据到 Mysql 主库的目的表的功能。在底层实现上， MysqlWriter 通过 JDBC 连接远程 Mysql 数据库，并执行相应的 insert into ... 或者 ( replace into ...) 的 sql 语句将数据写入 Mysql，内部会分批次提交入库，需要数据库本身采用 innodb 引擎。

MysqlWriter 面向ETL开发工程师，他们使用 MysqlWriter 从数仓导入数据到 Mysql。同时 MysqlWriter 亦可以作为数据迁移工具为DBA等用户提供服务。

[root@scripts]# tar -xvf datax.tar

[root@datax]# ls bin conf job lib log log_perf plugin script tmp

[root@job]# ls job.json

[root@job]# vi mysql2mysql.json

[root@job]# cat mysql2mysql.json  { "job": { "setting": { "speed": { "channel": 8 }, "errorLimit": { "record": 0, "percentage": 0.02 } }, "content": [ { "reader": { "name": "mysqlreader", "parameter": { "username": "username", "password": "password", "column": [ "order_id", "parent_order_id", "chartered_id", "multi_order_id", "carpool_order_id", "order_no", "city_id", "type", "order_service_type", "booking_user_id", "booking_phone", "rider_phone", "booking_start_time", "booking_start_addr", "booking_end_addr", "booking_start_short_addr", "booking_end_short_addr", "driver_id", "license_plates", "order_status", "channels_num", "is_display", "dissent_flag", "is_carpool", "create_date", "update_date", "sub_order_count", "sub_order_flag", "push_driver_type", "buyout_flag", "is_order_others", "auto_level_up", "finished_date", "order_create_date", "order_update_date", "cancel_type", "business_id", "order_type", "fact_date", "fact_start_addr", "fact_end_addr", "cancelorder_penalty" ], "connection": [ { "table": [ "yoon_table" #表名 ], "jdbcUrl": [ "jdbc:mysql://192.168.11.111:3306/yoon" #连接地址、端口、数据库 ] } ], "where": "order_create_date >='2019-05-01 00:00:00' and order_create_date < '2019-06-01 00:00:00'" #根据where条件传输数据，全表导的话就把where条件去掉即可 } }, "writer": { "name": "mysqlwriter", "parameter": { "writeMode": "replace", "username": "username", "password": "password", "column": [ "order_id", "parent_order_id", "chartered_id", "multi_order_id", "carpool_order_id", "order_no", "city_id", "type", "order_service_type", "booking_user_id", "booking_phone", "rider_phone", "booking_start_time", "booking_start_addr", "booking_end_addr", "booking_start_short_addr", "booking_end_short_addr", "driver_id", "license_plates", "order_status", "channels_num", "is_display", "dissent_flag", "is_carpool", "create_date", "update_date", "sub_order_count", "sub_order_flag", "push_driver_type", "buyout_flag", "is_order_others", "auto_level_up", "finished_date", "order_create_date", "order_update_date", "cancel_type", "business_id", "order_type", "fact_date", "fact_start_addr", "fact_end_addr", "cancelorder_penalty" ], "connection": [ { "jdbcUrl": "jdbc:mysql://192.168.11.222:3306/hank?useUnicode=true&characterEncoding=utf8", #连接地址、端口、数据库 "table": [ "yoon_hank" #表名 ] } ] } } } ] } }