1. update MongoDB writer client init configuration
2. update readme.md ,add todo list
This commit is contained in:
huzekang 2019-05-05 20:32:47 +08:00
parent 2d42156793
commit 15d43a1bbf
3 changed files with 22 additions and 3 deletions

View File

@ -1,10 +1,23 @@
## preparation
only Java环境
jdk1.8
## introduction
- 使用springboot启动datax不再需要用python启动。
- 以restful接口启动datax作业
## TODO LIST
* [x] springboot重构项目
* [x] 通过restful接口调度datax完成抽取数据作业
* [ ] 通过restful接口传入job配置json生成临时文件根据文件配置调度datax执行该作业
* [ ] 实现datax分布式作业
* [ ] 网页端修改job配置的json
* [ ] 网页端实时查看抽取日志
* [ ] 网页端各种插件模板生成
* [ ] job配置持久化到db
* [ ] 精简assembly打包结构
## how to run
### 1. 在父工程目录下使用maven打包
```

View File

@ -39,7 +39,7 @@
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>3.2.2</version>
<version>3.4.2</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>

View File

@ -5,6 +5,7 @@ import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.plugin.writer.mongodbwriter.KeyConstant;
import com.alibaba.datax.plugin.writer.mongodbwriter.MongoDBWriterErrorCode;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;
@ -40,7 +41,12 @@ public class MongoUtil {
}
try {
MongoCredential credential = MongoCredential.createCredential(userName, database, password.toCharArray());
return new MongoClient(parseServerAddress(addressList), Arrays.asList(credential));
//当超过这个闲置时间客户端主动关闭连接下次使用时重新建立连接这样可以有效避免连接失效的问题
MongoClientOptions mongoClientOptions = new MongoClientOptions.Builder()
.maxConnectionIdleTime(1000)
.maxConnectionLifeTime(0)
.build();
return new MongoClient(parseServerAddress(addressList), Arrays.asList(credential),mongoClientOptions);
} catch (UnknownHostException e) {
throw DataXException.asDataXException(MongoDBWriterErrorCode.ILLEGAL_ADDRESS,"不合法的地址");