From c4ed5595c191cf93a44680cf3b4f76ffdeeccc6a Mon Sep 17 00:00:00 2001 From: huzekang <1040080742@qq.com> Date: Mon, 15 Apr 2019 15:44:01 +0800 Subject: [PATCH] =?UTF-8?q?change=EF=BC=9A=201.=E4=BF=AE=E6=94=B9readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- job-sample/oracle2mysql.json | 8 ++-- 动态参数完成增量抽取.md | 77 ++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 动态参数完成增量抽取.md diff --git a/job-sample/oracle2mysql.json b/job-sample/oracle2mysql.json index d29b5f86..54a8a5d3 100644 --- a/job-sample/oracle2mysql.json +++ b/job-sample/oracle2mysql.json @@ -2,7 +2,7 @@ "job": { "setting": { "speed": { - "channel": 5 + "channel": 16 } }, "content": [ @@ -10,6 +10,7 @@ "reader": { "name": "oraclereader", "parameter": { + "splitPk": "user_id", "username": "yibo", "password": "yibo123", "column": [ @@ -19,7 +20,7 @@ "connection": [ { "table": [ - "TB_CIS_OP_FEE_DETAIL" + "yibo.YIBO_SYS_USER" ], "jdbcUrl": [ "jdbc:oracle:thin:@192.168.1.120:1521:xe" @@ -40,11 +41,12 @@ "session": [ "set session sql_mode='ANSI'" ], + "batchSize": "4096", "connection": [ { "jdbcUrl": "jdbc:mysql://192.168.1.120:3306/yibo?useUnicode=true&characterEncoding=gbk", "table": [ - "TB_CIS_OP_FEE_DETAIL" + "yibo.YIBO_SYS_USER" ] } ] diff --git a/动态参数完成增量抽取.md b/动态参数完成增量抽取.md new file mode 100644 index 00000000..4d3cae49 --- /dev/null +++ b/动态参数完成增量抽取.md @@ -0,0 +1,77 @@ +# 根据创建日期进行增量抽取 + +## datax作业配置文件 + +datax.json + +```json +{ + "job": { + "setting": { + "speed": { + "channel": 16 + } + }, + "content": [ + { + "reader": { + "name": "oraclereader", + "parameter": { + "splitPk": "id", + "username": "yibo", + "password": "yibo123", + "column": [ + "*" + + ], + "connection": [ + { + + "jdbcUrl": [ + "jdbc:oracle:thin:@10.91.1.5:1521:gzfy" + ], + "querySql": [ + "select * from tb_lis_indicators where trunc(create_date) > to_date(${createdate}, 'yyyy/mm/dd') " + ] + } + ] + } + }, + "writer": { + "name": "mysqlwriter", + "parameter": { + + "username": "root", + "password": "123456", + "column": [ + "*" + ], + "batchSize": "4096", + "connection": [ + { + "jdbcUrl": "jdbc:mysql://10.91.1.4:3306/yibo", + "table": [ + "TB_LIS_INDICATORS" + ] + } + ] + } + } + } + ] + } +} +``` + +其中${createdate}即可以替换的变量。 + + + +## Datax启动命令 + +由于作业配置中存在变量,固需在启动任务时填充该变量 + +```shell +python datax.py -p "-Dcreatedate='2019/03/1'" datax.json +``` +