CodeCow/datax-web
1
0
Fork 0
mirror of https://github.com/WeiYe-Jing/datax-web.git synced 2026-07-03 21:08:58 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
zhouhongfa 2019-06-17 22:45:52 +08:00
commit c4ca5307c9
10 changed files with 148 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -24,6 +24,8 @@
* [ ] 实现部分写插件支持自动建表功能
## 前端项目
https://github.com/zhouhongfa/datax-vue-admin.git
## how to run
### 1. 在父工程目录下使用maven打包
```
@ -47,3 +49,7 @@ curl http://localhost:8080/startJob
```
可以看到成功跑完一个datax作业
![](https://raw.githubusercontent.com/peter1040080742/picbed/master/20190505162333.png)
### 5. 打开网页端启动作业
http://localhost:8080/index.html#/datax/job
![](https://raw.githubusercontent.com/huzekang/picbed/master/20190617120207.png)

8
core/pom.xml
View File

@ -155,6 +155,14 @@
<encoding>${project-sourceEncoding}</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>

68
core/src/main/java/com/alibaba/datax/core/Engine.java
View File

@ -1,6 +1,5 @@
package com.alibaba.datax.core;
import cn.hutool.core.io.FileUtil;
import com.alibaba.datax.common.element.ColumnCast;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.spi.ErrorCode;
@ -19,10 +18,6 @@ import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
@ -33,6 +28,8 @@ import java.util.regex.Pattern;
* Engine是DataX入口类，该类负责初始化Job或者Task的运行容器，并运行插件的Job或者Task逻辑
*/
public class Engine {
private static final Logger LOG = LoggerFactory.getLogger(Engine.class);
private static String RUNTIME_MODE;
@ -225,70 +222,9 @@ public class Engine {
System.exit(exitCode);
}
/**
* 测试使用springboot启动作业job
*
* @author: huzekang
* @Date: 2019-05-05
*/
public static void testStartJob(String jobPath) {
try {
Engine.entry(jobPath);
} catch (Throwable e) {
LOG.error("\n\n经DataX智能分析,该任务最可能的错误原因是:\n" + ExceptionTracker.trace(e));
if (e instanceof DataXException) {
DataXException tempException = (DataXException) e;
ErrorCode errorCode = tempException.getErrorCode();
if (errorCode instanceof FrameworkErrorCode) {
FrameworkErrorCode tempErrorCode = (FrameworkErrorCode) errorCode;
}
}
}
}
// todo 都是用同一个文件，是否需要考虑线程安全问题
// 需要做成异步的，否则前端会一直loading等待完成作业
public static void startJobByJsonStr(String jobJson) {
final String tmpFilePath = "jobTmp-"+System.currentTimeMillis()+".conf";
// 根据json写入到临时本地文件
PrintWriter writer = null;
try {
writer = new PrintWriter(tmpFilePath, "UTF-8");
writer.println(jobJson);
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} finally {
if (writer != null) {
writer.close();
}
}
// 使用临时本地文件启动datax作业
try {
Engine.entry(tmpFilePath);
} catch (Throwable e) {
LOG.error("\n\n经DataX智能分析,该任务最可能的错误原因是:\n" + ExceptionTracker.trace(e));
if (e instanceof DataXException) {
DataXException tempException = (DataXException) e;
ErrorCode errorCode = tempException.getErrorCode();
if (errorCode instanceof FrameworkErrorCode) {
FrameworkErrorCode tempErrorCode = (FrameworkErrorCode) errorCode;
}
}
}
//删除临时文件
FileUtil.del(new File(tmpFilePath));
}
}

5
datax-web/pom.xml
View File

@ -44,6 +44,11 @@
<artifactId>mybatis-plus</artifactId>
<version>${mybatisplus.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>27.0.1-jre</version>
</dependency>
<!-- 接口管理 -->
<dependency>
<groupId>io.springfox</groupId>

24
datax-web/src/main/java/com/wugui/dataxweb/config/WebConfig.java Normal file
View File

@ -0,0 +1,24 @@
package com.wugui.dataxweb.config;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
/**
* 前端静态资源访问
*
* @program: datax-all
* @author: huzekang
* @create: 2019-06-17 10:40
**/
@Configuration
public class WebConfig implements WebMvcConfigurer {
@Override
public void addResourceHandlers(ResourceHandlerRegistry registry) {
registry.addResourceHandler("/index.html").addResourceLocations("classpath:/static/index.html");
registry.addResourceHandler("/static/**").addResourceLocations("classpath:/static/static/");
}
}

14
datax-web/src/main/java/com/wugui/dataxweb/controller/JobController.java
View File

@ -1,8 +1,11 @@
package com.wugui.dataxweb.controller;
import com.alibaba.datax.core.Engine;
import com.baomidou.mybatisplus.extension.api.R;
import com.wugui.dataxweb.service.IDataxJobService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
/**
@ -16,11 +19,14 @@ import org.springframework.web.bind.annotation.*;
@Api(tags = "datax作业接口")
public class JobController {
@Autowired
IDataxJobService iDataxJobService;
@GetMapping("/testStartJob")
public void testStartJob() {
// 指定获取作业配置json的接口，此处用下面mock出来的接口提供
String jobPath = "http://localhost:8080/mock_stream2stream";
Engine.testStartJob(jobPath);
iDataxJobService.startJobByJsonStr(jobPath);
}
@GetMapping("/mock_oracle2mongodb")
@ -125,9 +131,9 @@ public class JobController {
*/
@ApiOperation("通过传入json配置启动一个datax作业")
@PostMapping("/runJob")
public String runJob(@RequestBody String jobJson) {
Engine.startJobByJsonStr(jobJson);
return "success";
public R<String> runJob(@RequestBody String jobJson) {
String result = iDataxJobService.startJobByJsonStr(jobJson);
return R.ok(result);
}

16
datax-web/src/main/java/com/wugui/dataxweb/service/IDataxJobService.java Normal file
View File

@ -0,0 +1,16 @@
package com.wugui.dataxweb.service;
/**
* @program: datax-all
* @author: huzekang
* @create: 2019-06-17 11:25
**/
public interface IDataxJobService {
/**
* 根据json字符串用线程池启动一个datax作业
*
* @author: huzekang
* @Date: 2019-06-17
*/
String startJobByJsonStr(String jobJson);
}

76
datax-web/src/main/java/com/wugui/dataxweb/service/impl/IDataxJobServiceImpl.java Normal file
View File

@ -0,0 +1,76 @@
package com.wugui.dataxweb.service.impl;
import cn.hutool.core.io.FileUtil;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.spi.ErrorCode;
import com.alibaba.datax.core.Engine;
import com.alibaba.datax.core.util.ExceptionTracker;
import com.alibaba.datax.core.util.FrameworkErrorCode;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.wugui.dataxweb.service.IDataxJobService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.concurrent.*;
/**
* @program: datax-all
* @author: huzekang
* @create: 2019-06-17 11:26
**/
@Slf4j
@Service
public class IDataxJobServiceImpl implements IDataxJobService {
private ThreadFactory namedThreadFactory = new ThreadFactoryBuilder().setNameFormat("datax-job-%d").build();
private ExecutorService jobPool = new ThreadPoolExecutor(5, 200, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(1024), namedThreadFactory, new ThreadPoolExecutor.AbortPolicy());
@Override
public String startJobByJsonStr(String jobJson) {
jobPool.submit(() -> {
final String tmpFilePath = "jobTmp-" + System.currentTimeMillis() + ".conf";
// 根据json写入到临时本地文件
PrintWriter writer = null;
try {
writer = new PrintWriter(tmpFilePath, "UTF-8");
writer.println(jobJson);
} catch (FileNotFoundException | UnsupportedEncodingException e) {
e.printStackTrace();
} finally {
if (writer != null) {
writer.close();
}
}
try {
// 使用临时本地文件启动datax作业
Engine.entry(tmpFilePath);
// 删除临时文件
FileUtil.del(new File(tmpFilePath));
} catch (Throwable e) {
log.error("\n\n经DataX智能分析,该任务最可能的错误原因是:\n" + ExceptionTracker.trace(e));
if (e instanceof DataXException) {
DataXException tempException = (DataXException) e;
ErrorCode errorCode = tempException.getErrorCode();
if (errorCode instanceof FrameworkErrorCode) {
FrameworkErrorCode tempErrorCode = (FrameworkErrorCode) errorCode;
}
}
}
});
return "success";
}
}

BIN
datax-web/src/main/resources/static/favicon.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 66 KiB

1
datax-web/src/main/resources/static/index.html Normal file
View File

@ -0,0 +1 @@
<!DOCTYPE html><html><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=viewport content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><link rel=icon href=/favicon.ico><title>datax管理</title><link href=/static/css/chunk-elementUI.18b11d0e.css rel=stylesheet><link href=/static/css/chunk-libs.887691f5.css rel=stylesheet><link href=/static/css/app.212f3c84.css rel=stylesheet></head><body><noscript><strong>We're sorry but datax管理 doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id=app></div><script src=/static/js/chunk-elementUI.9c03131d.js></script><script src=/static/js/chunk-libs.38c602cd.js></script><script>(function(e){function t(t){for(var r,o,a=t[0],i=t[1],f=t[2],l=0,d=[];l<a.length;l++)o=a[l],u[o]&&d.push(u[o][0]),u[o]=0;for(r in i)Object.prototype.hasOwnProperty.call(i,r)&&(e[r]=i[r]);s&&s(t);while(d.length)d.shift()();return c.push.apply(c,f||[]),n()}function n(){for(var e,t=0;t<c.length;t++){for(var n=c[t],r=!0,o=1;o<n.length;o++){var a=n[o];0!==u[a]&&(r=!1)}r&&(c.splice(t--,1),e=i(i.s=n[0]))}return e}var r={},o={runtime:0},u={runtime:0},c=[];function a(e){return i.p+"static/js/"+({}[e]||e)+"."+{"chunk-0163d01c":"dec7aa5d","chunk-0d403ef4":"39e450d7","chunk-52071f51":"e1d01a73","chunk-24b1b612":"432df031","chunk-57ebe89e":"13d9169d","chunk-6274770a":"2401449d"}[e]+".js"}function i(t){if(r[t])return r[t].exports;var n=r[t]={i:t,l:!1,exports:{}};return e[t].call(n.exports,n,n.exports,i),n.l=!0,n.exports}i.e=function(e){var t=[],n={"chunk-0163d01c":1,"chunk-0d403ef4":1,"chunk-52071f51":1,"chunk-24b1b612":1,"chunk-57ebe89e":1,"chunk-6274770a":1};o[e]?t.push(o[e]):0!==o[e]&&n[e]&&t.push(o[e]=new Promise(function(t,n){for(var r="static/css/"+({}[e]||e)+"."+{"chunk-0163d01c":"1d6bb21b","chunk-0d403ef4":"43dd6fbf","chunk-52071f51":"9a886743","chunk-24b1b612":"398cbdb8","chunk-57ebe89e":"ab2f8884","chunk-6274770a":"2ec66f7c"}[e]+".css",u=i.p+r,c=document.getElementsByTagName("link"),a=0;a<c.length;a++){var f=c[a],l=f.getAttribute("data-href")||f.getAttribute("href");if("stylesheet"===f.rel&&(l===r||l===u))return t()}var d=document.getElementsByTagName("style");for(a=0;a<d.length;a++){f=d[a],l=f.getAttribute("data-href");if(l===r||l===u)return t()}var s=document.createElement("link");s.rel="stylesheet",s.type="text/css",s.onload=t,s.onerror=function(t){var r=t&&t.target&&t.target.src||u,c=new Error("Loading CSS chunk "+e+" failed.\n("+r+")");c.code="CSS_CHUNK_LOAD_FAILED",c.request=r,delete o[e],s.parentNode.removeChild(s),n(c)},s.href=u;var h=document.getElementsByTagName("head")[0];h.appendChild(s)}).then(function(){o[e]=0}));var r=u[e];if(0!==r)if(r)t.push(r[2]);else{var c=new Promise(function(t,n){r=u[e]=[t,n]});t.push(r[2]=c);var f,l=document.createElement("script");l.charset="utf-8",l.timeout=120,i.nc&&l.setAttribute("nonce",i.nc),l.src=a(e),f=function(t){l.onerror=l.onload=null,clearTimeout(d);var n=u[e];if(0!==n){if(n){var r=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src,c=new Error("Loading chunk "+e+" failed.\n("+r+": "+o+")");c.type=r,c.request=o,n[1](c)}u[e]=void 0}};var d=setTimeout(function(){f({type:"timeout",target:l})},12e4);l.onerror=l.onload=f,document.head.appendChild(l)}return Promise.all(t)},i.m=e,i.c=r,i.d=function(e,t,n){i.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:n})},i.r=function(e){"undefined"!==typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.t=function(e,t){if(1&t&&(e=i(e)),8&t)return e;if(4&t&&"object"===typeof e&&e&&e.__esModule)return e;var n=Object.create(null);if(i.r(n),Object.defineProperty(n,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var r in e)i.d(n,r,function(t){return e[t]}.bind(null,r));return n},i.n=function(e){var t=e&&e.__esModule?function(){return e["default"]}:function(){return e};return i.d(t,"a",t),t},i.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},i.p="/",i.oe=function(e){throw console.error(e),e};var f=window["webpackJsonp"]=window["webpackJsonp"]||[],l=f.push.bind(f);f.push=t,f=f.slice();for(var d=0;d<f.length;d++)t(f[d]);var s=l;n()})([]);</script><script src=/static/js/app.516ef226.js></script></body></html>