diff --git a/README.md b/README.md index 154615b9..1c312ebe 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,23 @@ ## preparation -only Java环境 +jdk1.8 ## introduction - 使用springboot启动datax,不再需要用python启动。 - 以restful接口启动datax作业 +## TODO LIST + +* [x] springboot重构项目 +* [x] 通过restful接口调度datax完成抽取数据作业 +* [ ] 通过restful接口传入job配置json生成临时文件,根据文件配置调度datax执行该作业 +* [ ] 实现datax分布式作业 +* [ ] 网页端修改job配置的json +* [ ] 网页端实时查看抽取日志 +* [ ] 网页端各种插件模板生成 +* [ ] job配置持久化到db +* [ ] 精简assembly打包结构 + + ## how to run ### 1. 在父工程目录下使用maven打包 ``` diff --git a/mongodbwriter/pom.xml b/mongodbwriter/pom.xml index 0360db2a..6c25aebb 100644 --- a/mongodbwriter/pom.xml +++ b/mongodbwriter/pom.xml @@ -39,7 +39,7 @@ org.mongodb mongo-java-driver - 3.2.2 + 3.4.2 com.google.guava diff --git a/mongodbwriter/src/main/java/com/alibaba/datax/plugin/writer/mongodbwriter/util/MongoUtil.java b/mongodbwriter/src/main/java/com/alibaba/datax/plugin/writer/mongodbwriter/util/MongoUtil.java index 17334be4..c65032f2 100644 --- a/mongodbwriter/src/main/java/com/alibaba/datax/plugin/writer/mongodbwriter/util/MongoUtil.java +++ b/mongodbwriter/src/main/java/com/alibaba/datax/plugin/writer/mongodbwriter/util/MongoUtil.java @@ -5,6 +5,7 @@ import com.alibaba.datax.common.util.Configuration; import com.alibaba.datax.plugin.writer.mongodbwriter.KeyConstant; import com.alibaba.datax.plugin.writer.mongodbwriter.MongoDBWriterErrorCode; import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; @@ -40,7 +41,12 @@ public class MongoUtil { } try { MongoCredential credential = MongoCredential.createCredential(userName, database, password.toCharArray()); - return new MongoClient(parseServerAddress(addressList), Arrays.asList(credential)); + //当超过这个闲置时间客户端主动关闭连接,下次使用时重新建立连接,这样可以有效避免连接失效的问题 + MongoClientOptions mongoClientOptions = new MongoClientOptions.Builder() + .maxConnectionIdleTime(1000) + .maxConnectionLifeTime(0) + .build(); + return new MongoClient(parseServerAddress(addressList), Arrays.asList(credential),mongoClientOptions); } catch (UnknownHostException e) { throw DataXException.asDataXException(MongoDBWriterErrorCode.ILLEGAL_ADDRESS,"不合法的地址");