- 6、jdbc添加hive数据源支持

- 7、json构建模块代码重构
- 8、json构建支持hive
- 9、添加数据源测试功能

README.md

@@ -49,7 +49,7 @@ DataX阿里的开源的时候并未提供任何可视化界面，我们在使用
 - 2、datax.job.executor.logpath(数据抽取日志文件保存路径)
 - 3、datax.executor.jsonpath(datax json临时文件保存路径)
 - 4、datax.pypath(datax/bin/datax.py)注意：是第一步中DataX打包好的，DataX启动文件的地址
-
+如果系统配置DataX环境变量（DATAX_HOME），2、3、4步可省略，log文件和临时json存放在环境变量路径下。
 ### 5.执行器配置(使用开源项目xxl-job)
 ![](https://github.com/WeiYe-Jing/datax-web/blob/master/doc/img/executor.png)
 - 1、"调度中心OnLine:"右侧显示在线的"调度中心"列表, 任务执行结束后, 将会以failover的模式进行回调调度中心通知执行结果, 避免回调的单点风险;
@@ -116,6 +116,11 @@ Quick Start操作完前四步之后
 - 3、数据源连接错误提醒功能
 - 4、任务模板创建
 - 5、构建JSON之后选择任务模板创建任务
+- 6、jdbc添加hive数据源支持
+- 7、json构建模块代码重构
+- 8、json构建支持hive
+- 9、添加数据源测试功能
+- 10、优先通过环境变量获取DataX文件目录
 ## TODO List
 - 1、从源表到目标端表的自动创建
 - 2、任务批量导入功能

datax-admin/src/main/java/com/wugui/datax/admin/controller/JdbcDatasourceQueryController.java

@@ -6,9 +6,7 @@ import com.wugui.datax.admin.service.JdbcDatasourceQueryService;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.bind.annotation.*;
 
 import java.util.List;
 

datax-admin/src/main/java/com/wugui/datax/admin/service/impl/JdbcDatasourceQueryServiceImpl.java

@@ -47,7 +47,7 @@ public class JdbcDatasourceQueryServiceImpl implements JdbcDatasourceQueryServic
             return Lists.newArrayList();
         }
         BaseQueryTool queryTool = QueryToolFactory.getByDbType(jdbcDatasource);
-        return queryTool.getColumnNames(tableName);
+        return queryTool.getColumnNames(tableName,jdbcDatasource.getJdbcDriverClass());
     }
 
     @Override

datax-admin/src/main/java/com/wugui/datax/admin/tool/query/BaseQueryTool.java

@@ -1,6 +1,7 @@
 package com.wugui.datax.admin.tool.query;
 
 import cn.hutool.core.util.StrUtil;
+import com.alibaba.druid.util.JdbcConstants;
 import com.alibaba.druid.util.JdbcUtils;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
@@ -11,6 +12,7 @@ import com.wugui.datax.admin.tool.database.DasColumn;
 import com.wugui.datax.admin.tool.database.TableInfo;
 import com.wugui.datax.admin.tool.meta.DatabaseInterface;
 import com.wugui.datax.admin.tool.meta.DatabaseMetaFactory;
+import com.wugui.datax.admin.util.Constant;
 import com.zaxxer.hikari.HikariDataSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -62,8 +64,6 @@ public abstract class BaseQueryTool implements QueryToolInterface {
             dataSource.setMaximumPoolSize(1);
             dataSource.setMinimumIdle(0);
             dataSource.setConnectionTimeout(30000);
-            //设为只读
-            dataSource.setReadOnly(true);
             this.datasource = dataSource;
             this.connection = this.datasource.getConnection();
         } else {
@@ -251,7 +251,7 @@ public abstract class BaseQueryTool implements QueryToolInterface {
     }
 
     @Override
-    public List<String> getColumnNames(String tableName) {
+    public List<String> getColumnNames(String tableName, String driverClass) {
 
         List<String> res = Lists.newArrayList();
         Statement stmt = null;
@@ -268,7 +268,17 @@ public abstract class BaseQueryTool implements QueryToolInterface {
 
             int columnCount = metaData.getColumnCount();
             for (int i = 1; i <= columnCount; i++) {
-                res.add(metaData.getColumnName(i));
+                String columnName = metaData.getColumnName(i);
+                if (JdbcConstants.HIVE_DRIVER.equals(driverClass)) {
+                    if (columnName.contains(Constant.SPLIT_POINT)) {
+                        res.add(columnName.substring(columnName.indexOf(Constant.SPLIT_POINT) + 1) + Constant.SPLIT_SCOLON + metaData.getColumnTypeName(i));
+                    } else {
+                        res.add(columnName + Constant.SPLIT_SCOLON + metaData.getColumnTypeName(i));
+                    }
+                } else {
+                    res.add(columnName);
+                }
+
             }
         } catch (SQLException e) {
             logger.error("[getColumnNames Exception] --> "
@@ -304,6 +314,19 @@ public abstract class BaseQueryTool implements QueryToolInterface {
         return tables;
     }
 
+    public Boolean dataSourceTest() {
+        try {
+            DatabaseMetaData metaData = connection.getMetaData();
+            if (metaData.getDatabaseProductName().length() > 0) {
+                return true;
+            }
+        } catch (SQLException e) {
+            logger.error("[dataSourceTest Exception] --> "
+                    + "the exception message is:" + e.getMessage());
+        }
+        return false;
+    }
+
     /**
      * 不需要其他参数的可不重写
      *

datax-admin/src/main/java/com/wugui/datax/admin/util/MysqlAndOracleTypeToHive.java

@@ -0,0 +1,41 @@
+package com.wugui.datax.admin.util;
+
+import java.util.ArrayList;
+
+import java.util.List;
+
+/**
+ * @Author: zhc
+ * @Data: 2019/12/30 16:19
+ * @Email: ah.zhanghaicheng@aisino.com
+ * @Description: mysql、oracle数据库类型和hive类型做适配，先实现功能，后期在考虑优化
+ */
+public class MysqlAndOracleTypeToHive {
+    public static List<String> changeType(List<String> type) {
+        List<String> hiveType = new ArrayList<>();
+//        System.out.println(type.get(0));
+        for (int i = 0; i < type.size(); i++) {
+//            System.out.println(type.get(i));
+            if ("CHAR".equals(type.get(i)) || "VARCHAR".equals(type.get(i)) || "TINYBLOB".equals(type.get(i))
+                    || "TINYTEXT".equals(type.get(i)) || "BLOB".equals(type.get(i)) || "TEXT".equals("type.get(i)")
+                    || "MEDIUMBLOB".equals(type.get(i)) || "MEDIUMTEXT".equals(type.get(i)) || "LONGBLOB".equals(type.get(i))
+                    || "LONGTEXT".equals(type.get(i)) || "VARCHAR2".equals(type.get(i)) || "NCHAR".equals(type.get(i)) || "NVARCHAR".equals(type.get(i)) || "NVARCHAR2".equals(type.get(i))
+                    || "RAW".equals(type.get(i)) || "LONG RAW".equals(type.get(i)) || "CLOB".equals("type.get(i)")
+                    || "NCLOB".equals(type.get(i)) || "BFILE".equals(type.get(i))) {
+                hiveType.add(i, "string");
+            } else if ("TINYINT".equals(type.get(i)) || "SMALLINT".equals(type.get(i)) || "INT".equals(type.get(i)) || "BIGINT".equals(type.get(i))
+                    || "FLOAT".equals(type.get(i)) || "DOUBLE".equals(type.get(i)) || "DECIMAL".equals(type.get(i)) || "DATE".equals(type.get(i)) || "TIMESTAMP".equals(type.get(i))) {
+                hiveType.add(i, type.get(i));
+            } else if ("LONG".equals(type.get(i))) {
+                hiveType.add(i, "bigint");
+            } else if ("DATETIME".equals(type.get(i))) {
+                hiveType.add(i, "timestamp");
+            } else if ("NUMBER".equals(type.get(i))) {
+                hiveType.add(i, "double");
+            } else {
+                hiveType.add(i, "string");
+            }
+        }
+        return hiveType;
+    }
+}

datax-admin/src/main/resources/static/static/css/chunk-868ab8de.53ac87fa.css

@@ -1 +0,0 @@
-.waves-ripple{position:absolute;border-radius:100%;background-color:rgba(0,0,0,.15);background-clip:padding-box;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transform:scale(0);transform:scale(0);opacity:1}.waves-ripple.z-active{opacity:0;-webkit-transform:scale(2);transform:scale(2);-webkit-transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out,-webkit-transform .6s ease-out}

datax-admin/src/main/resources/static/static/css/chunk-c4d95f1e.8920edb5.css

@@ -1 +1 @@
-.waves-ripple{position:absolute;border-radius:100%;background-color:rgba(0,0,0,.15);background-clip:padding-box;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transform:scale(0);transform:scale(0);opacity:1}.waves-ripple.z-active{opacity:0;-webkit-transform:scale(2);transform:scale(2);-webkit-transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out,-webkit-transform .6s ease-out}.log-container[data-v-1e37135b]{margin-bottom:20px;background:#f5f5f5;width:100%;height:500px;overflow:scroll}.log-container pre[data-v-1e37135b]{display:block;padding:10px;margin:0 0 10.5px;word-break:break-all;word-wrap:break-word;color:#334851;background-color:#f5f5f5;border-radius:1px}
+.waves-ripple{position:absolute;border-radius:100%;background-color:rgba(0,0,0,.15);background-clip:padding-box;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transform:scale(0);transform:scale(0);opacity:1}.waves-ripple.z-active{opacity:0;-webkit-transform:scale(2);transform:scale(2);-webkit-transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out,-webkit-transform .6s ease-out}.log-container[data-v-647c09c4]{margin-bottom:20px;background:#f5f5f5;width:100%;height:500px;overflow:scroll}.log-container pre[data-v-647c09c4]{display:block;padding:10px;margin:0 0 10.5px;word-break:break-all;word-wrap:break-word;color:#334851;background-color:#f5f5f5;border-radius:1px}

datax-admin/src/test/java/com/wugui/datax/admin/tool/datax/DataxJsonHelperTest.java

@@ -1,13 +1,6 @@
 package com.wugui.datax.admin.tool.datax;
 
-import com.google.common.collect.ImmutableList;
 import com.wugui.datax.admin.entity.JobJdbcDatasource;
-import com.wugui.datax.admin.tool.datax.writer.StreamWriter;
-import com.wugui.datax.admin.util.JSONUtils;
-import org.junit.Test;
-
-import java.util.List;
-import java.util.Map;
 
 public class DataxJsonHelperTest {
 
@@ -31,71 +24,4 @@ public class DataxJsonHelperTest {
         return writerDatasource;
     }
 
-    @Test
-    public void buildJob() {
-        DataxJsonHelper dataxJsonHelper = new DataxJsonHelper();
-
-        //表名
-        List<String> readerTables = ImmutableList.of("datax_plugin");
-        List<String> writerTables = ImmutableList.of("datax_plugin");
-
-        //抽取的字段
-        List<String> columns = ImmutableList.of("id");
-
-        dataxJsonHelper.initReader(getReaderDatasource(), readerTables, columns);
-        dataxJsonHelper.initWriter(getWriterDatasource(), writerTables, columns);
-        Map<String, Object> map = dataxJsonHelper.buildJob();
-        System.out.println(JSONUtils.formatJson(map));
-    }
-
-    @Test
-    public void builSetting() {
-        DataxJsonHelper dataxJsonHelper = new DataxJsonHelper();
-        Map<String, Object> map = dataxJsonHelper.buildSetting();
-        System.out.println(JSONUtils.formatJson(map));
-    }
-
-    @Test
-    public void buildContent() {
-        DataxJsonHelper dataxJsonHelper = new DataxJsonHelper();
-        dataxJsonHelper.initReader(getReaderDatasource(), ImmutableList.of("datax_plugin"), ImmutableList.of("id"));
-        dataxJsonHelper.initWriter(getWriterDatasource(), ImmutableList.of("datax_plugin"), ImmutableList.of("id"));
-        Map<String, Object> map = dataxJsonHelper.buildContent();
-        System.out.println(JSONUtils.formatJson(map));
-    }
-
-    @Test
-    public void buildReader() {
-        DataxJsonHelper dataxJsonHelper = new DataxJsonHelper();
-        dataxJsonHelper.initReader(getReaderDatasource(), ImmutableList.of("datax_plugin"), ImmutableList.of("id"));
-//        dataxJsonHelper.addWhereParams("1=1");
-        dataxJsonHelper.setQuerySql("select 1");
-        Map<String, Object> reader = dataxJsonHelper.buildReader();
-        System.out.println(JSONUtils.formatJson(reader));
-    }
-
-    @Test
-    public void buildWriter() {
-        DataxJsonHelper dataxJsonHelper = new DataxJsonHelper();
-        dataxJsonHelper.initWriter(getWriterDatasource(), ImmutableList.of("datax_plugin"), ImmutableList.of("id"));
-        dataxJsonHelper.setPreSql("delete from datax_prubin");
-        Map<String, Object> writer = dataxJsonHelper.buildWriter();
-        System.out.println(JSONUtils.formatJson(writer));
-    }
-
-    @Test
-    public void buildJobWithStreamWriter() {
-        DataxJsonHelper dataxJsonHelper = new DataxJsonHelper();
-        dataxJsonHelper.setWriterPlugin(new StreamWriter());
-        //表名
-        List<String> readerTables = ImmutableList.of("datax_plugin");
-
-        //抽取的字段
-        List<String> columns = ImmutableList.of("id");
-
-        dataxJsonHelper.initReader(getReaderDatasource(), readerTables, columns);
-
-        Map<String, Object> map = dataxJsonHelper.buildJob();
-        System.out.println(JSONUtils.formatJson(map));
-    }
-}
+}