datax and dataxweb installation deployment installation deployment
App version:
?datax-web V2.1.2
?datax_v202306
Original project address:
- datax https://github.com/alibaba/DataX
- Datax-web https://github.com/WeiYe-Jing/datax-web
- Dockerhub https://hub.docker.com/repository/docker/876745833/datax-admin/general
PS:
The project has fixed bugs in the original code and repackaged.
The modified package has not been uploaded yet, and will be uploaded later.
Deploy via docker-compose
The mirror image is datax-web + datax cluster deployment, including mysql. If there is already mysql, remove the deployment of mysql in yml by yourself
Create a network environment
docker network create -d bridge datax-net
Deploy and start docker-compose
## Arbitrary directory, create datax folder cd *** mkdir datax cd datax
Fill in the docker-compose file
vim docker-compose.yml ## Copy the script below to the yml file and save it
## execute command docker-compose up -d
version: '3.7'
services:
datax-mysql:
image: mysql:5.7
restart: always
environment:
MYSQL_ROOT_PASSWORD: admin
#MYSQL_USER: root
MYSQL_DATABASE: dataxweb
TZ: Asia/Shanghai
character-set-server: utf8mb4
collation-server: utf8mb4_unicode_ci
ports:
- 3306:3306
volumes:
- ./mysql/data:/var/lib/mysql
- ./mysql/conf:/etc/mysql/conf.d
- ./mysql/logs:/logs
- ./mysql/db:/docker-entrypoint-initdb.d/
networks:
-datax-net
container_name: datax-mysql
datax-admin:
#Local mirror, the mirror of the next build
image: 876745833/datax-admin:1.0
restart: always
ports:
-8080:8080
environment:
SERVER_PORT: 8080
DB_USERNAME: root
DB_PASSWORD: admin
DB_HOST: datax-mysql
DB_PORT: 3306
DB_DATABASE: dataxweb
MAIL_USERNAME: email
MAIL_PASSWORD: 123456
depends_on:
-datax-mysql
networks:
-datax-net
container_name: datax-admin
datax-executor:
#Local mirror, the mirror of the next build
image: 876745833/datax-executor:1.0
restart: always
volumes:
- ./datax_web/executor/python:/opt/datax-executor/python
- ./datax_web/executor/json:/opt/datax-executor/json
- ./datax_web/executor/data:/opt/datax-executor/data
ports:
-8081:8081
environment:
SERVER_PORT: 8081
DATAX_ADMIN_HOST: datax-admin
DATAX_ADMIN_PORT: 8080
EXECUTOR_PORT: 9999
DATA_PATH: ./data
FILE_SAVE: 'false'
#Check here whether dataxWebAdmin is started successfully, if container_name and container_name.SERVER_PORT are changed, please respond to the change with the following command
entrypoint: "/etc/init.d/wait-for-it.sh datax-admin:8080 --java com.wugui.datax.executor.DataXExecutorApplication"
depends_on:
-datax-mysql
-datax-admin
networks:
-datax-net
container_name: datax-executor
networks:
datax-net:
external: true
FAQ
Script not executed automatically
Execute the following script manually
/*
Navicat Premium Data Transfer
Source Server : localhost
Source Server Type : MySQL
Source Server Version : 50725
Source Host : localhost:3306
Source Schema : datax_web
Target Server Type : MySQL
Target Server Version : 50725
File Encoding: 65001
Date: 15/12/2019 22:27:10
*/
SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;
-- ----------------------------
-- Table structure for job_group
-- ----------------------------
DROP TABLE IF EXISTS `job_group`;
CREATE TABLE `job_group` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`app_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'Actuator AppName',
`title` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'executor name',
`order` int(11) NOT NULL DEFAULT 0 COMMENT 'order',
`address_type` tinyint(4) NOT NULL DEFAULT 0 COMMENT 'Actuator address type: 0=automatic registration, 1=manual entry',
`address_list` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'Executor address list, multiple addresses separated by commas',
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
-- ----------------------------
-- Records of job_group
-- ----------------------------
INSERT INTO `job_group` VALUES (1, 'datax-executor', 'datax-executor', 1, 0, NULL);
-- ----------------------------
-- Table structure for job_info
-- ----------------------------
DROP TABLE IF EXISTS `job_info`;
CREATE TABLE `job_info` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`job_group` int(11) NOT NULL COMMENT 'Actuator primary key ID',
`job_cron` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'job execution CRON',
`job_desc` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
`add_time` datetime(0) NULL DEFAULT NULL,
`update_time` datetime(0) NULL DEFAULT NULL,
`author` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'Author',
`alarm_email` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'alarm email',
`executor_route_strategy` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'executor routing strategy',
`executor_handler` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'executor task handler',
`executor_param` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'executor task parameter',
`executor_block_strategy` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'blocking processing strategy',
`executor_timeout` int(11) NOT NULL DEFAULT 0 COMMENT 'Task execution timeout, in seconds',
`executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT 'Number of failed retries',
`glue_type` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'GLUE type',
`glue_source` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL COMMENT 'GLUE source code',
`glue_remark` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'GLUE remark',
`glue_updatetime` datetime(0) NULL DEFAULT NULL COMMENT 'GLUE update time',
`child_jobid` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'Subtask IDs, separated by multiple commas',
`trigger_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT 'Scheduling status: 0-stop, 1-run',
`trigger_last_time` bigint(13) NOT NULL DEFAULT 0 COMMENT 'last scheduled time',
`trigger_next_time` bigint(13) NOT NULL DEFAULT 0 COMMENT 'next scheduling time',
`job_json` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT 'datax run script',
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 7 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
-- ----------------------------
-- Table structure for job_jdbc_datasource
-- ----------------------------
DROP TABLE IF EXISTS `job_jdbc_datasource`;
CREATE TABLE `job_jdbc_datasource` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'Auto-increment primary key',
`datasource_name` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'datasource name',
`datasource_group` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT 'Default' COMMENT 'data source group',
`jdbc_username` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'username',
`jdbc_password` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'password',
`jdbc_url` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'jdbc url',
`jdbc_driver_class` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'jdbc driver class',
`status` tinyint(1) NOT NULL DEFAULT 1 COMMENT 'status: 0 delete 1 enable 2 disable',
`create_by` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'Creator',
`create_date` datetime(0) NULL DEFAULT CURRENT_TIMESTAMP(0) COMMENT 'Create time',
`update_by` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'update by',
`update_date` datetime(0) NULL DEFAULT NULL COMMENT 'Update time',
`comments` varchar(1000) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'Comments',
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 6 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = 'jdbc data source configuration' ROW_FORMAT = Dynamic;
-- ----------------------------
-- Table structure for job_lock
-- ----------------------------
DROP TABLE IF EXISTS `job_lock`;
CREATE TABLE `job_lock` (
`lock_name` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'lock name',
PRIMARY KEY (`lock_name`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
-- ----------------------------
-- Records of job_lock
-- ----------------------------
INSERT INTO `job_lock` VALUES ('schedule_lock');
-- ----------------------------
-- Table structure for job_log
-- ----------------------------
DROP TABLE IF EXISTS `job_log`;
CREATE TABLE `job_log` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`job_group` int(11) NOT NULL COMMENT 'Actuator primary key ID',
`job_id` int(11) NOT NULL COMMENT 'job, primary key ID',
`job_desc` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`executor_address` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'executor address, the address of this execution',
`executor_handler` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'executor task handler',
`executor_param` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'executor task parameter',
`executor_sharding_param` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'Executor task sharding parameter, the format is like 1/2',
`executor_fail_retry_count` int(11) NULL DEFAULT 0 COMMENT 'Number of failed retries',
`trigger_time` datetime(0) NULL DEFAULT NULL COMMENT 'scheduling-time',
`trigger_code` int(11) NOT NULL COMMENT 'scheduling-result',
`trigger_msg` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL COMMENT 'scheduling-log',
`handle_time` datetime(0) NULL DEFAULT NULL COMMENT 'execution-time',
`handle_code` int(11) NOT NULL COMMENT 'execution-status',
`handle_msg` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL COMMENT 'execution-log',
`alarm_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT 'Alarm status: 0-default, 1-no alarm, 2-alarm success, 3-alarm failure',
`process_id` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'datax process Id',
`max_id` bigint(20) NULL DEFAULT NULL COMMENT 'increment table max id',
PRIMARY KEY (`id`) USING BTREE,
INDEX `I_trigger_time`(`trigger_time`) USING BTREE,
INDEX `I_handle_code`(`handle_code`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 0 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
-- ----------------------------
-- Table structure for job_log_report
-- ----------------------------
DROP TABLE IF EXISTS `job_log_report`;
CREATE TABLE `job_log_report` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`trigger_day` datetime(0) NULL DEFAULT NULL COMMENT 'schedule-time',
`running_count` int(11) NOT NULL DEFAULT 0 COMMENT 'Running - number of logs',
`suc_count` int(11) NOT NULL DEFAULT 0 COMMENT 'Execution succeeded - number of logs',
`fail_count` int(11) NOT NULL DEFAULT 0 COMMENT 'Execution failed - number of logs',
PRIMARY KEY (`id`) USING BTREE,
UNIQUE INDEX `i_trigger_day`(`trigger_day`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 28 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
-- ----------------------------
-- Records of job_log_report
-- ----------------------------
INSERT INTO `job_log_report` VALUES (20, '2019-12-07 00:00:00', 0, 0, 0);
INSERT INTO `job_log_report` VALUES (21, '2019-12-10 00:00:00', 77, 52, 23);
INSERT INTO `job_log_report` VALUES (22, '2019-12-11 00:00:00', 9, 2, 11);
INSERT INTO `job_log_report` VALUES (23, '2019-12-13 00:00:00', 9, 48, 74);
INSERT INTO `job_log_report` VALUES (24, '2019-12-12 00:00:00', 10, 8, 30);
INSERT INTO `job_log_report` VALUES (25, '2019-12-14 00:00:00', 78, 45, 66);
INSERT INTO `job_log_report` VALUES (26, '2019-12-15 00:00:00', 24, 76, 9);
INSERT INTO `job_log_report` VALUES (27, '2019-12-16 00:00:00', 23, 85, 10);
-- ----------------------------
-- Table structure for job_logglue
-- ----------------------------
DROP TABLE IF EXISTS `job_logglue`;
CREATE TABLE `job_logglue` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`job_id` int(11) NOT NULL COMMENT 'job, primary key ID',
`glue_type` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'GLUE type',
`glue_source` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL COMMENT 'GLUE source code',
`glue_remark` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'GLUE remark',
`add_time` datetime(0) NULL DEFAULT NULL,
`update_time` datetime(0) NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
-- ----------------------------
-- Table structure for job_registry
-- ----------------------------
DROP TABLE IF EXISTS `job_registry`;
CREATE TABLE `job_registry` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`registry_group` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
`registry_key` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
`registry_value` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
`update_time` datetime(0) NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `i_g_k_v`(`registry_group`, `registry_key`, `registry_value`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 26 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
-- ----------------------------
-- Table structure for job_user
-- ----------------------------
DROP TABLE IF EXISTS `job_user`;
CREATE TABLE `job_user` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`username` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'account',
`password` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'password',
`role` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'Role: 0-normal user, 1-administrator',
`permission` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'Permission: Executor ID list, separated by multiple commas',
PRIMARY KEY (`id`) USING BTREE,
UNIQUE INDEX `i_username`(`username`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 10 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
-- ----------------------------
-- Records of job_user
-- ----------------------------
INSERT INTO `job_user` VALUES (1, 'admin', '$2a$10$2KCqRbra0Yn2TwvkZxtfLuWuUP5KyCWsljO/ci5pLD27pqR3TV1vy', 'ROLE_ADMIN', NULL);
/**
v2.1.1 script update
*/
ALTER TABLE `job_info`
ADD COLUMN `replace_param` VARCHAR(100) NULL DEFAULT NULL COMMENT 'dynamic parameter' AFTER `job_json`,
ADD COLUMN `jvm_param` VARCHAR(200) NULL DEFAULT NULL COMMENT 'jvm parameter' AFTER `replace_param`,
ADD COLUMN `custom_param` VARCHAR(200) NULL DEFAULT NULL COMMENT 'custom parameter' AFTER `jvm_param`,
ADD COLUMN `time_offset` INT(11) NULL DEFAULT '0' COMMENT 'time offset' AFTER `custom_param`;
/**
Incremental revision script update
*/
ALTER TABLE `job_info` DROP COLUMN `time_offset`;
ALTER TABLE `job_info`
ADD COLUMN `inc_start_time` DATETIME NULL DEFAULT NULL COMMENT 'increment initial time' AFTER `jvm_param`;
-- ----------------------------
-- Table structure for job_template
-- ----------------------------
DROP TABLE IF EXISTS `job_template`;
CREATE TABLE `job_template` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`job_group` int(11) NOT NULL COMMENT 'Actuator primary key ID',
`job_cron` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'job execution CRON',
`job_desc` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
`add_time` datetime(0) NULL DEFAULT NULL,
`update_time` datetime(0) NULL DEFAULT NULL,
`user_id` int(11) NOT NULL COMMENT 'Modify user',
`alarm_email` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'alarm email',
`executor_route_strategy` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'executor routing strategy',
`executor_handler` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'executor task handler',
`executor_param` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'executor parameter',
`executor_block_strategy` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'blocking processing strategy',
`executor_timeout` int(11) NOT NULL DEFAULT 0 COMMENT 'Task execution timeout, in seconds',
`executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT 'Number of failed retries',
`glue_type` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'GLUE type',
`glue_source` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL COMMENT 'GLUE source code',
`glue_remark` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'GLUE remark',
`glue_updatetime` datetime(0) NULL DEFAULT NULL COMMENT 'GLUE update time',
`child_jobid` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'Subtask IDs, separated by multiple commas',
`trigger_last_time` bigint(13) NOT NULL DEFAULT 0 COMMENT 'last scheduled time',
`trigger_next_time` bigint(13) NOT NULL DEFAULT 0 COMMENT 'next scheduling time',
`job_json` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT 'datax run script',
`jvm_param` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'jvm parameter',
`custom_param` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'custom parameter',
`project_id` int(11) NULL DEFAULT NULL COMMENT 'Project Id',
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 22 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
/**
Add data source field
*/
ALTER TABLE `job_jdbc_datasource`
ADD COLUMN `datasource` VARCHAR(45) NOT NULL COMMENT 'datasource' AFTER `datasource_name`;
/**
Add partition field
*/
ALTER TABLE `job_info`
ADD COLUMN `partition_info` VARCHAR(100) NULL DEFAULT NULL COMMENT 'partition information' AFTER `inc_start_time`;
/**
Added in version 2.1.1 -------------------------------------------- --------------------------------------------------
*/
/**
last execution status
*/
ALTER TABLE `job_info`
ADD COLUMN `last_handle_code` INT(11) NULL DEFAULT '0' COMMENT 'last execution status' AFTER `partition_info`;
/**
zookeeper address
*/
ALTER TABLE `job_jdbc_datasource`
ADD COLUMN `zk_adress` VARCHAR(200) NULL DEFAULT NULL AFTER `jdbc_driver_class`;
ALTER TABLE `job_info`
CHANGE COLUMN `executor_timeout` `executor_timeout` INT(11) NOT NULL DEFAULT '0' COMMENT 'Task execution timeout, unit minutes' ;
/**
Username and password changed to non-required
*/
ALTER TABLE `job_jdbc_datasource`
CHANGE COLUMN `jdbc_username` `jdbc_username` VARCHAR(100) CHARACTER SET 'utf8mb4' NULL DEFAULT NULL COMMENT 'username' ,
CHANGE COLUMN `jdbc_password` `jdbc_password` VARCHAR(200) CHARACTER SET 'utf8mb4' NULL DEFAULT NULL COMMENT 'password' ;
/**
Add mongodb database name field
*/
ALTER TABLE `job_jdbc_datasource`
ADD COLUMN `database_name` VARCHAR(45) NULL DEFAULT NULL COMMENT 'database_name' AFTER `datasource_group`;
/**
Add executor resource field
*/
ALTER TABLE `job_registry`
ADD COLUMN `cpu_usage` DOUBLE NULL AFTER `registry_value`,
ADD COLUMN `memory_usage` DOUBLE NULL AFTER `cpu_usage`,
ADD COLUMN `load_average` DOUBLE NULL AFTER `memory_usage`;
-- ----------------------------
-- Table structure for job_permission
-- ----------------------------
DROP TABLE IF EXISTS `job_permission`;
CREATE TABLE `job_permission` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`name` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'permission name',
`description` varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'Permission description',
`url` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`pid` int(11) NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 3 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
ALTER TABLE `job_info`
ADD COLUMN `replace_param_type` varchar(255) NULL COMMENT 'Incremental time format' AFTER `last_handle_code`;
ALTER TABLE `job_info`
ADD COLUMN `project_id` int(11) NULL COMMENT 'the project id' AFTER `job_desc`;
ALTER TABLE `job_info`
ADD COLUMN `reader_table` VARCHAR(255) NULL COMMENT 'reader table name' AFTER `replace_param_type`,
ADD COLUMN `primary_key` VARCHAR(50) NULL COMMENT 'increment table primary key' AFTER `reader_table`,
ADD COLUMN `inc_start_id` VARCHAR(20) NULL COMMENT 'increment initial id' AFTER `primary_key`,
ADD COLUMN `increment_type` TINYINT(4) NULL COMMENT 'increment type' AFTER `inc_start_id`,
ADD COLUMN `datasource_id` BIGINT(11) NULL COMMENT 'datasource id' AFTER `increment_type`;
CREATE TABLE `job_project` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key',
`name` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'project name',
`description` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`user_id` int(11) NULL DEFAULT NULL COMMENT 'creator id',
`flag` tinyint(4) NULL DEFAULT 1 COMMENT '0 not available, 1 available',
`create_time` datetime(0) NULL DEFAULT CURRENT_TIMESTAMP(0) COMMENT 'create time',
`update_time` datetime(0) NULL DEFAULT CURRENT_TIMESTAMP(0) COMMENT 'update time',
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
ALTER TABLE `job_info`
CHANGE COLUMN `author` `user_id` INT(11) NOT NULL COMMENT 'Modify user';
ALTER TABLE `job_info`
CHANGE COLUMN `increment_type` `increment_type` TINYINT(4) NULL DEFAULT 0 COMMENT 'increment type' ;
Not automatically registered
In executor management, manually add machines
## Add machine ## The address is docker container name: 9999 or docker container ip ## Take the image name datax-executor as an example datax-executor:9999 ## How to get the docker container docker inspect datax-executor ## Find the IPAddress configuration under Networks, which is the IP address of the container. The configuration file is as follows 192.168.80.4:9999
"Networks": {<!-- -->
"datax-net": {<!-- -->
"IPAMConfig": null,
"Links": null,
"Aliases": [
"datax-executor",
"datax-executor",
"aa9289cb62a6"
],
"NetworkID": "224431e5fe146a1a87694a0d8c1922ba384658dbccd7d7e6ee9d83b472917f73",
"EndpointID": "ee5805fdc639c55b4a629f2f1339c34efb622a220cfd4bfeaaf0c607d8354649",
"Gateway": "192.168.80.1",
"IPAddress": "192.168.80.4",
"IPPrefixLen": 20,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:c0:a8:50:04",
"DriverOpts": null
}
}
Under the condition of total bps rate limit, the bps value of a single channel cannot be empty
Add the core.transport.cannel.speed.byte configuration in the same directory as the job
{
"job":
{},
"core":
{
"transport":
{
"channel":
{
"speed":
{
"byte": 1231412
}
}
}
}
}