From 58af580f84e12b2f29e2abe8dffaeb87c0c551dc Mon Sep 17 00:00:00 2001 From: huzekang <1040080742@qq.com> Date: Thu, 9 May 2019 00:35:17 +0800 Subject: [PATCH] add: 1.mybatis and swagger init --- README.md | 12 +++--- core/pom.xml | 5 +++ .../java/com/alibaba/datax/core/Engine.java | 4 +- datax-web/pom.xml | 31 ++++++++++++++ .../wugui/dataxweb/DataxWebApplication.java | 2 + .../config/MybatisMetaObjectHandler.java | 34 +++++++++++++++ .../dataxweb/config/MybatisPlusConfig.java | 39 ++++++++++++++++++ ...tentController.java => JobController.java} | 7 +++- .../wugui/dataxweb/dao/JobConfigMapper.java | 7 ++++ .../dataxweb/dao/mapping/JobConfigMapper.xml | 22 ++++++++++ .../com/wugui/dataxweb/entity/JobConfig.java | 37 +++++++++++++++++ .../dataxweb/service/JobConfigService.java | 10 +++++ .../src/main/resources/application.properties | 1 - datax-web/src/main/resources/application.yml | 41 +++++++++++++++++++ .../dataxweb/DataxWebApplicationTests.java | 9 ++++ 15 files changed, 251 insertions(+), 10 deletions(-) create mode 100644 datax-web/src/main/java/com/wugui/dataxweb/config/MybatisMetaObjectHandler.java create mode 100644 datax-web/src/main/java/com/wugui/dataxweb/config/MybatisPlusConfig.java rename datax-web/src/main/java/com/wugui/dataxweb/controller/{JobContentController.java => JobController.java} (95%) create mode 100644 datax-web/src/main/java/com/wugui/dataxweb/dao/JobConfigMapper.java create mode 100644 datax-web/src/main/java/com/wugui/dataxweb/dao/mapping/JobConfigMapper.xml create mode 100644 datax-web/src/main/java/com/wugui/dataxweb/entity/JobConfig.java create mode 100644 datax-web/src/main/java/com/wugui/dataxweb/service/JobConfigService.java delete mode 100644 datax-web/src/main/resources/application.properties create mode 100644 datax-web/src/main/resources/application.yml diff --git a/README.md b/README.md index 0df4dff0..80cbc345 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,13 @@ jdk1.8 * [x] springboot重构项目 * [x] 通过restful接口调度datax完成抽取数据作业 * [x] 通过restful接口传入job配置json生成临时文件，根据文件配置调度datax执行该作业 -* [ ] 集成swagger，方便调试 -* [ ] 实现datax分布式作业 -* [ ] 网页端修改job配置的json -* [ ] 网页端实时查看抽取日志 -* [ ] 网页端各种插件模板生成 -* [ ] job配置持久化到db +* [x] 集成swagger，方便调试 +* [x] 集成mybatis plus和h2数据库存放应用数据 +* [ ] 网页端修改并持久化job配置的json到数据库 +* [ ] 网页端实时查看抽取日志，类似Jenkins的日志控制台输出功能 +* [ ] 网页端各种读写插件模板生成，可以在页面组装使用 * [ ] 精简assembly打包结构 +* [ ] 实现datax分布式作业 ## how to run diff --git a/core/pom.xml b/core/pom.xml index abab6fac..bc0d1c9b 100755 --- a/core/pom.xml +++ b/core/pom.xml @@ -33,6 +33,11 @@ commons-configuration ${commons-configuration-version} + + com.h2database + h2 + LATEST + commons-cli commons-cli diff --git a/core/src/main/java/com/alibaba/datax/core/Engine.java b/core/src/main/java/com/alibaba/datax/core/Engine.java index 4e2a4b5b..f6d8d24f 100755 --- a/core/src/main/java/com/alibaba/datax/core/Engine.java +++ b/core/src/main/java/com/alibaba/datax/core/Engine.java @@ -251,7 +251,8 @@ public class Engine { } } - //todo 都是用同一个文件，是否需要考虑线程安全问题 + // todo 都是用同一个文件，是否需要考虑线程安全问题 + // 需要做成异步的，否则前端会一直loading等待完成作业 public static void startJobByJsonStr(String jobJson) { String tmpFilePath = "jobTmp.conf-"+ System.currentTimeMillis(); // 根据json写入到临时本地文件 @@ -285,7 +286,6 @@ public class Engine { } } - //删除临时文件 FileUtil.del(new File("jobTmp.conf")); } diff --git a/datax-web/pom.xml b/datax-web/pom.xml index b6247785..c0253f1b 100644 --- a/datax-web/pom.xml +++ b/datax-web/pom.xml @@ -16,9 +16,40 @@ 1.8 + 3.0.7.1 + 2.9.2 + + + + com.baomidou + mybatis-plus-boot-starter + ${mybatisplus.version} + + + com.baomidou + mybatis-plus-generator + + + + + com.baomidou + mybatis-plus + ${mybatisplus.version} + + + + io.springfox + springfox-swagger2 + ${swagger.version} + + + io.springfox + springfox-swagger-ui + ${swagger.version} + org.springframework.boot spring-boot-starter-web diff --git a/datax-web/src/main/java/com/wugui/dataxweb/DataxWebApplication.java b/datax-web/src/main/java/com/wugui/dataxweb/DataxWebApplication.java index a0650703..74eb2558 100644 --- a/datax-web/src/main/java/com/wugui/dataxweb/DataxWebApplication.java +++ b/datax-web/src/main/java/com/wugui/dataxweb/DataxWebApplication.java @@ -2,7 +2,9 @@ package com.wugui.dataxweb; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import springfox.documentation.swagger2.annotations.EnableSwagger2; +@EnableSwagger2 @SpringBootApplication public class DataxWebApplication{ diff --git a/datax-web/src/main/java/com/wugui/dataxweb/config/MybatisMetaObjectHandler.java b/datax-web/src/main/java/com/wugui/dataxweb/config/MybatisMetaObjectHandler.java new file mode 100644 index 00000000..e87438c2 --- /dev/null +++ b/datax-web/src/main/java/com/wugui/dataxweb/config/MybatisMetaObjectHandler.java @@ -0,0 +1,34 @@ +package com.wugui.dataxweb.config; + +import com.baomidou.mybatisplus.core.handlers.MetaObjectHandler; +import lombok.extern.slf4j.Slf4j; +import org.apache.ibatis.reflection.MetaObject; +import org.springframework.stereotype.Component; + +import java.util.Date; + +/** + * 通用的字段填充，如createBy createDate这些字段的自动填充 + * @author huzekang + */ +@Component +@Slf4j +public class MybatisMetaObjectHandler implements MetaObjectHandler { + + @Override + public void insertFill(MetaObject metaObject) { + log.debug("start insert fill ...."); + //todo createBy + Date date = new Date(); + this.setInsertFieldValByName("createDate", date, metaObject); + this.setInsertFieldValByName("updateDate", date, metaObject); + } + + @Override + public void updateFill(MetaObject metaObject) { + //todo updateBY + log.debug("start update fill ...."); + Date date = new Date(); + this.setUpdateFieldValByName("updateDate", date, metaObject); + } +} \ No newline at end of file diff --git a/datax-web/src/main/java/com/wugui/dataxweb/config/MybatisPlusConfig.java b/datax-web/src/main/java/com/wugui/dataxweb/config/MybatisPlusConfig.java new file mode 100644 index 00000000..c4289273 --- /dev/null +++ b/datax-web/src/main/java/com/wugui/dataxweb/config/MybatisPlusConfig.java @@ -0,0 +1,39 @@ +package com.wugui.dataxweb.config; + +import com.baomidou.mybatisplus.core.injector.ISqlInjector; +import com.baomidou.mybatisplus.extension.injector.LogicSqlInjector; +import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor; +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +/** + * MybatisPlus配置类 Spring boot方式 + * @author huzekang + */ +@EnableTransactionManagement +@Configuration +@MapperScan("com.wugui.dataxweb.dao") +public class MybatisPlusConfig { + + /** + * 分页插件 + */ + @Bean + public PaginationInterceptor paginationInterceptor() { + + PaginationInterceptor paginationInterceptor = new PaginationInterceptor(); + return paginationInterceptor.setOverflow(true); + } + + /** + * MyBatisPlus逻辑删除 ，需要在 yml 中配置开启 + * + * @return + */ + @Bean + public ISqlInjector sqlInjector() { + return new LogicSqlInjector(); + } +} diff --git a/datax-web/src/main/java/com/wugui/dataxweb/controller/JobContentController.java b/datax-web/src/main/java/com/wugui/dataxweb/controller/JobController.java similarity index 95% rename from datax-web/src/main/java/com/wugui/dataxweb/controller/JobContentController.java rename to datax-web/src/main/java/com/wugui/dataxweb/controller/JobController.java index 2fa69379..e86f3fce 100644 --- a/datax-web/src/main/java/com/wugui/dataxweb/controller/JobContentController.java +++ b/datax-web/src/main/java/com/wugui/dataxweb/controller/JobController.java @@ -1,6 +1,8 @@ package com.wugui.dataxweb.controller; import com.alibaba.datax.core.Engine; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -13,7 +15,8 @@ import org.springframework.web.bind.annotation.RestController; * @create: 2019-05-05 14:21 **/ @RestController -public class JobContentController { +@Api(tags = "datax作业接口") +public class JobController { @GetMapping("/testStartJob") public void testStartJob() { @@ -77,6 +80,7 @@ public class JobContentController { } @GetMapping("/mock_stream2stream") + @ApiOperation("提供stream2stream的配置") public String mock2() { return "{\n" + " \"job\": {\n" + @@ -121,6 +125,7 @@ public class JobContentController { * @param jobJson * @return */ + @ApiOperation("通过传入json配置启动一个datax作业") @PostMapping("/runJob") public String runJob(@RequestBody String jobJson) { Engine.startJobByJsonStr(jobJson); diff --git a/datax-web/src/main/java/com/wugui/dataxweb/dao/JobConfigMapper.java b/datax-web/src/main/java/com/wugui/dataxweb/dao/JobConfigMapper.java new file mode 100644 index 00000000..3cc0aab9 --- /dev/null +++ b/datax-web/src/main/java/com/wugui/dataxweb/dao/JobConfigMapper.java @@ -0,0 +1,7 @@ +package com.wugui.dataxweb.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.wugui.dataxweb.entity.JobConfig; + +public interface JobConfigMapper extends BaseMapper { +} \ No newline at end of file diff --git a/datax-web/src/main/java/com/wugui/dataxweb/dao/mapping/JobConfigMapper.xml b/datax-web/src/main/java/com/wugui/dataxweb/dao/mapping/JobConfigMapper.xml new file mode 100644 index 00000000..98ea1d01 --- /dev/null +++ b/datax-web/src/main/java/com/wugui/dataxweb/dao/mapping/JobConfigMapper.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + ID, CONFIG, CREATE_DATE, CREATE_BY, UPDATE_BY, UPDATE_DATE, "STATUS" + + \ No newline at end of file diff --git a/datax-web/src/main/java/com/wugui/dataxweb/entity/JobConfig.java b/datax-web/src/main/java/com/wugui/dataxweb/entity/JobConfig.java new file mode 100644 index 00000000..a4dc0693 --- /dev/null +++ b/datax-web/src/main/java/com/wugui/dataxweb/entity/JobConfig.java @@ -0,0 +1,37 @@ +package com.wugui.dataxweb.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import java.util.Date; +import lombok.Data; + +@Data +@TableName(value = "JOB_CONFIG") +public class JobConfig { + @TableId(value = "ID", type = IdType.AUTO) + private Integer id; + + @TableField(value = "CONFIG") + private String config; + + @TableField(value = "CREATE_DATE") + private Date createDate; + + @TableField(value = "CREATE_BY") + private Integer createBy; + + @TableField(value = "UPDATE_BY") + private Integer updateBy; + + @TableField(value = "UPDATE_DATE") + private Date updateDate; + + @TableField(value = "STATUS") + private Integer status; + + @TableField(value = "USER_ID") + private Integer userID; +} + diff --git a/datax-web/src/main/java/com/wugui/dataxweb/service/JobConfigService.java b/datax-web/src/main/java/com/wugui/dataxweb/service/JobConfigService.java new file mode 100644 index 00000000..7bd5ac14 --- /dev/null +++ b/datax-web/src/main/java/com/wugui/dataxweb/service/JobConfigService.java @@ -0,0 +1,10 @@ +package com.wugui.dataxweb.service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.wugui.dataxweb.dao.JobConfigMapper; +import com.wugui.dataxweb.entity.JobConfig; +import org.springframework.stereotype.Service; +@Service +public class JobConfigService extends ServiceImpl{ + +} diff --git a/datax-web/src/main/resources/application.properties b/datax-web/src/main/resources/application.properties deleted file mode 100644 index 8b137891..00000000 --- a/datax-web/src/main/resources/application.properties +++ /dev/null @@ -1 +0,0 @@ - diff --git a/datax-web/src/main/resources/application.yml b/datax-web/src/main/resources/application.yml new file mode 100644 index 00000000..baafccbb --- /dev/null +++ b/datax-web/src/main/resources/application.yml @@ -0,0 +1,41 @@ +spring: + #数据源 + datasource: + username: root + password: root + url: jdbc:h2:file:~/H2Data/datax-web;AUTO_SERVER=TRUE;DB_CLOSE_ON_EXIT=FALSE + driver-class-name: org.h2.Driver + h2: + console: + enabled: true + path: /h2-console + settings: + web-allow-others: true + + +mybatis-plus: + # mapper.xml文件扫描 + mapper-locations: classpath*:com/wugui/**/dao/mapping/*.xml + # 实体扫描，多个package用逗号或者分号分隔 + #typeAliasesPackage: com.yibo.essyncclient.*.entity + global-config: + # 数据库相关配置 + db-config: + # 主键类型 AUTO:"数据库ID自增", INPUT:"用户输入ID", ID_WORKER:"全局唯一ID (数字类型唯一ID)", UUID:"全局唯一ID UUID"; + id-type: AUTO + # 字段策略 IGNORED:"忽略判断",NOT_NULL:"非 NULL 判断"),NOT_EMPTY:"非空判断" + field-strategy: NOT_NULL + # 驼峰下划线转换 + column-underline: true + # 逻辑删除 + logic-delete-value: 0 + logic-not-delete-value: 1 + # 数据库类型 + db-type: h2 + banner: false + # mybatis原生配置 + configuration: + map-underscore-to-camel-case: true + cache-enabled: false + call-setters-on-nulls: true + jdbc-type-for-null: 'null' diff --git a/datax-web/src/test/java/com/wugui/dataxweb/DataxWebApplicationTests.java b/datax-web/src/test/java/com/wugui/dataxweb/DataxWebApplicationTests.java index 9fca373f..6756040a 100644 --- a/datax-web/src/test/java/com/wugui/dataxweb/DataxWebApplicationTests.java +++ b/datax-web/src/test/java/com/wugui/dataxweb/DataxWebApplicationTests.java @@ -1,7 +1,10 @@ package com.wugui.dataxweb; +import com.wugui.dataxweb.dao.JobConfigMapper; +import com.wugui.dataxweb.entity.JobConfig; import org.junit.Test; import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; @@ -9,8 +12,14 @@ import org.springframework.test.context.junit4.SpringRunner; @SpringBootTest public class DataxWebApplicationTests { + @Autowired + JobConfigMapper jobConfigMapper; + @Test public void contextLoads() { + JobConfig jobConfig = new JobConfig(); + jobConfig.setConfig("{}"); + jobConfigMapper.insert(jobConfig); } }