getTypes() {
+// return TYPES;
+// }
+
+// /**
+// * Get the name of the Java class for the given value type.
+// *
+// * @param type the value type
+// * @return the class name
+// */
+// public static String getTypeClassName(int type) {
+// switch (type) {
+// case BOOLEAN:
+// // "java.lang.Boolean";
+// return Boolean.class.getName();
+// case BYTE:
+// // "java.lang.Byte";
+// return Byte.class.getName();
+// case SHORT:
+// // "java.lang.Short";
+// return Short.class.getName();
+// case INT:
+// // "java.lang.Integer";
+// return Integer.class.getName();
+// case LONG:
+// // "java.lang.Long";
+// return Long.class.getName();
+// case DECIMAL:
+// // "java.math.BigDecimal";
+// return BigDecimal.class.getName();
+// case TIME:
+// // "java.sql.Time";
+// return Time.class.getName();
+// case DATE:
+// // "java.sql.Date";
+// return Date.class.getName();
+// case TIMESTAMP:
+// // "java.sql.Timestamp";
+// return Timestamp.class.getName();
+// case STRING:
+// // case STRING_IGNORECASE:
+// // case STRING_FIXED:
+// case MULTI_VALUE:
+// // "java.lang.String";
+// return String.class.getName();
+// case DOUBLE:
+// // "java.lang.Double";
+// return Double.class.getName();
+// case FLOAT:
+// // "java.lang.Float";
+// return Float.class.getName();
+// // case NULL:
+// // return null;
+// default:
+// throw new IllegalArgumentException("type=" + type);
+// }
+// }
+
+ /**
+ * Get the data type object for the given value type.
+ *
+ * @param type the value type
+ * @return the data type object
+ */
+ public static ColumnDataType getDataType(int type) {
+ if (type < 0 || type >= TYPE_COUNT) {
+ throw new IllegalArgumentException("type=" + type);
+ }
+ ColumnDataType dt = TYPES_BY_VALUE_TYPE.get(type);
+ // if (dt == null) {
+ // dt = TYPES_BY_VALUE_TYPE.get(NULL);
+ // }
+ return dt;
+ }
+
+ /**
+ * Convert a value type to a SQL type.
+ *
+ * @param type the value type
+ * @return the SQL type
+ */
+ public static int convertTypeToSQLType(int type) {
+ return getDataType(type).sqlType;
+ }
+
+ /**
+ * Convert a SQL type to a value type.
+ *
+ * @param sqlType the SQL type
+ * @return the value type
+ */
+ public static int convertSQLTypeToValueType(int sqlType) {
+ switch (sqlType) {
+ // case Types.CHAR:
+ // case Types.NCHAR:
+ // return STRING_FIXED;
+ case Types.VARCHAR:
+ case Types.LONGVARCHAR:
+ case Types.NVARCHAR:
+ case Types.LONGNVARCHAR:
+ return STRING;
+ case Types.NUMERIC:
+ case Types.DECIMAL:
+ return DECIMAL;
+ case Types.BIT:
+ case Types.BOOLEAN:
+ return BOOLEAN;
+ case Types.INTEGER:
+ return INT;
+ case Types.SMALLINT:
+ return SHORT;
+ case Types.TINYINT:
+ return BYTE;
+ case Types.BIGINT:
+ return LONG;
+ case Types.REAL:
+ return FLOAT;
+ case Types.DOUBLE:
+ case Types.FLOAT:
+ return DOUBLE;
+ case Types.DATE:
+ return DATE;
+ case Types.TIME:
+ return TIME;
+ case Types.TIMESTAMP:
+ return TIMESTAMP;
+ // case Types.NULL:
+ // return NULL;
+ default:
+ throw new IllegalArgumentException("JDBC Type: " + sqlType);
+ }
+ }
+
+ /**
+ * Get the value type for the given Java class.
+ *
+ * @param x the Java class
+ * @return the value type
+ */
+ public static int getTypeFromClass(Class x) {
+ // if (x == null || Void.TYPE == x) {
+ // return NULL;
+ // }
+ if (x.isPrimitive()) {
+ x = getNonPrimitiveClass(x);
+ }
+ if (String.class == x) {
+ return STRING;
+ } else if (Integer.class == x) {
+ return INT;
+ } else if (Long.class == x) {
+ return LONG;
+ } else if (Boolean.class == x) {
+ return BOOLEAN;
+ } else if (Double.class == x) {
+ return DOUBLE;
+ } else if (Byte.class == x) {
+ return BYTE;
+ } else if (Short.class == x) {
+ return SHORT;
+ } else if (Float.class == x) {
+ return FLOAT;
+ // } else if (Void.class == x) {
+ // return NULL;
+ } else if (BigDecimal.class.isAssignableFrom(x)) {
+ return DECIMAL;
+ } else if (Date.class.isAssignableFrom(x)) {
+ return DATE;
+ } else if (Time.class.isAssignableFrom(x)) {
+ return TIME;
+ } else if (Timestamp.class.isAssignableFrom(x)) {
+ return TIMESTAMP;
+ } else if (java.util.Date.class.isAssignableFrom(x)) {
+ return TIMESTAMP;
+ } else {
+ throw new IllegalArgumentException("class=" + x);
+ }
+ }
+
+ /**
+ * ads getNonPrimitiveClass
+ * @param clazz
+ * @return
+ */
+ public static Class getNonPrimitiveClass(Class clazz) {
+ if (!clazz.isPrimitive()) {
+ return clazz;
+ } else if (clazz == boolean.class) {
+ // ads return "java.lang.Boolean";
+ return Boolean.class;
+ } else if (clazz == char.class) {
+ // ads return "java.lang.Character";
+ return Character.class;
+ } else if (clazz == byte.class) {
+ // ads return "java.lang.Byte";
+ return Byte.class;
+ } else if (clazz == double.class) {
+ // ads return "java.lang.Double";
+ return Double.class;
+ } else if (clazz == float.class) {
+ // ads return "java.lang.Float";
+ return Float.class;
+ } else if (clazz == int.class) {
+ // ads return "java.lang.Integer";
+ return Integer.class;
+ } else if (clazz == short.class) {
+ // ads return "java.lang.Short";
+ return Short.class;
+ } else if (clazz == long.class) {
+ // ads return "java.lang.Long";
+ return Long.class;
+ } else if (clazz == void.class) {
+ // ads return "java.lang.Void";
+ return Void.class;
+ }
+ return clazz;
+ }
+
+ /**
+ * Get a data type object from a type name.
+ *
+ * @param s the type name
+ * @return the data type object
+ */
+ public static ColumnDataType getTypeByName(String s) {
+ return TYPES_BY_NAME.get(s);
+ }
+
+ /**
+ * Check if the given value type is a String (VARCHAR,...).
+ *
+ * @param type the value type
+ * @return true if the value type is a String type
+ */
+ public static boolean isStringType(int type) {
+ if (type == STRING /* || type == STRING_FIXED || type == STRING_IGNORECASE */
+ || type == MULTI_VALUE) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * @return
+ */
+ public boolean supportsAdd() {
+ return supportsAdd(type);
+ }
+
+ /**
+ * Check if the given value type supports the add operation.
+ *
+ * @param type the value type
+ * @return true if add is supported
+ */
+ public static boolean supportsAdd(int type) {
+ switch (type) {
+ case BYTE:
+ case DECIMAL:
+ case DOUBLE:
+ case FLOAT:
+ case INT:
+ case LONG:
+ case SHORT:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * Get the data type that will not overflow when calling 'add' 2 billion times.
+ *
+ * @param type the value type
+ * @return the data type that supports adding
+ */
+ public static int getAddProofType(int type) {
+ switch (type) {
+ case BYTE:
+ return LONG;
+ case FLOAT:
+ return DOUBLE;
+ case INT:
+ return LONG;
+ case LONG:
+ return DECIMAL;
+ case SHORT:
+ return LONG;
+ default:
+ return type;
+ }
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnInfo.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnInfo.java
new file mode 100644
index 000000000..030ce35d1
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnInfo.java
@@ -0,0 +1,72 @@
+package com.alibaba.datax.plugin.writer.adswriter.ads;
+
+/**
+ * ADS column meta.
+ * select ordinal_position,column_name,data_type,type_name,column_comment
+ *
+ * @since 0.0.1
+ */
+public class ColumnInfo {
+
+ private int ordinal;
+ private String name;
+ private ColumnDataType dataType;
+ private boolean isDeleted;
+ private String comment;
+
+ public int getOrdinal() {
+ return ordinal;
+ }
+
+ public void setOrdinal(int ordinal) {
+ this.ordinal = ordinal;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public ColumnDataType getDataType() {
+ return dataType;
+ }
+
+ public void setDataType(ColumnDataType dataType) {
+ this.dataType = dataType;
+ }
+
+ public boolean isDeleted() {
+ return isDeleted;
+ }
+
+ public void setDeleted(boolean isDeleted) {
+ this.isDeleted = isDeleted;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("ColumnInfo [ordinal=").append(ordinal).append(", name=").append(name).append(", dataType=")
+ .append(dataType).append(", isDeleted=").append(isDeleted).append(", comment=").append(comment)
+ .append("]");
+ return builder.toString();
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/TableInfo.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/TableInfo.java
new file mode 100644
index 000000000..f2395d6b7
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/TableInfo.java
@@ -0,0 +1,69 @@
+package com.alibaba.datax.plugin.writer.adswriter.ads;
+
+import java.util.List;
+
+/**
+ * ADS table meta.
+ * select table_schema, table_name,comments
+ *
+ * select ordinal_position,column_name,data_type,type_name,column_comment
+ *
+ * @since 0.0.1
+ */
+public class TableInfo {
+
+ private String tableSchema;
+ private String tableName;
+ private List columns;
+ private String comments;
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("TableInfo [tableSchema=").append(tableSchema).append(", tableName=").append(tableName)
+ .append(", columns=").append(columns).append(", comments=").append(comments).append("]");
+ return builder.toString();
+ }
+
+ public String getTableSchema() {
+ return tableSchema;
+ }
+
+ public void setTableSchema(String tableSchema) {
+ this.tableSchema = tableSchema;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public List getColumns() {
+ return columns;
+ }
+
+ public void setColumns(List columns) {
+ this.columns = columns;
+ }
+
+ public String getComments() {
+ return comments;
+ }
+
+ public void setComments(String comments) {
+ this.comments = comments;
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/package-info.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/package-info.java
new file mode 100644
index 000000000..b396c49ff
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/package-info.java
@@ -0,0 +1,6 @@
+/**
+ * ADS meta and service.
+ *
+ * @since 0.0.1
+ */
+package com.alibaba.datax.plugin.writer.adswriter.ads;
\ No newline at end of file
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertProxy.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertProxy.java
new file mode 100644
index 000000000..bd01a1553
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertProxy.java
@@ -0,0 +1,286 @@
+package com.alibaba.datax.plugin.writer.adswriter.insert;
+
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.plugin.RecordReceiver;
+import com.alibaba.datax.common.plugin.TaskPluginCollector;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.rdbms.util.DBUtil;
+import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
+import com.alibaba.datax.plugin.writer.adswriter.util.Constant;
+import com.alibaba.datax.plugin.writer.adswriter.util.Key;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Triple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class AdsInsertProxy {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AdsInsertProxy.class);
+
+ private String table;
+ private List columns;
+ private TaskPluginCollector taskPluginCollector;
+ private Configuration configuration;
+ private Boolean emptyAsNull;
+
+ private Triple, List, List> resultSetMetaData;
+
+ public AdsInsertProxy(String table, List columns, Configuration configuration, TaskPluginCollector taskPluginCollector) {
+ this.table = table;
+ this.columns = columns;
+ this.configuration = configuration;
+ this.taskPluginCollector = taskPluginCollector;
+ this.emptyAsNull = configuration.getBool(Key.EMPTY_AS_NULL, false);
+ }
+
+ public void startWriteWithConnection(RecordReceiver recordReceiver,
+ Connection connection,
+ int columnNumber) {
+ //目前 ads 新建的表 如果未插入数据 不能通过select colums from table where 1=2，获取列信息。
+// this.resultSetMetaData = DBUtil.getColumnMetaData(connection,
+// this.table, StringUtils.join(this.columns, ","));
+
+ this.resultSetMetaData = AdsInsertUtil.getColumnMetaData(configuration, columns);
+
+ int batchSize = this.configuration.getInt(Key.BATCH_SIZE, Constant.DEFAULT_BATCH_SIZE);
+ List writeBuffer = new ArrayList(batchSize);
+ try {
+ Record record;
+ while ((record = recordReceiver.getFromReader()) != null) {
+ if (record.getColumnNumber() != columnNumber) {
+ // 源头读取字段列数与目的表字段写入列数不相等，直接报错
+ throw DataXException
+ .asDataXException(
+ DBUtilErrorCode.CONF_ERROR,
+ String.format(
+ "列配置信息有错误. 因为您配置的任务中，源头读取字段数:%s 与 目的表要写入的字段数:%s 不相等. 请检查您的配置并作出修改.",
+ record.getColumnNumber(),
+ columnNumber));
+ }
+
+ writeBuffer.add(record);
+
+ if (writeBuffer.size() >= batchSize) {
+ doOneInsert(connection, writeBuffer);
+ writeBuffer.clear();
+ }
+ }
+ if (!writeBuffer.isEmpty()) {
+ doOneInsert(connection, writeBuffer);
+ writeBuffer.clear();
+ }
+ } catch (Exception e) {
+ throw DataXException.asDataXException(
+ DBUtilErrorCode.WRITE_DATA_ERROR, e);
+ } finally {
+ writeBuffer.clear();
+ DBUtil.closeDBResources(null, null, connection);
+ }
+ }
+
+ protected void doBatchInsert(Connection connection, List buffer) throws SQLException {
+ Statement statement = null;
+ try {
+ connection.setAutoCommit(false);
+ statement = connection.createStatement();
+
+ for (Record record : buffer) {
+ String sql = generateInsertSql(record);
+ statement.addBatch(sql);
+ }
+ statement.executeBatch();
+ connection.commit();
+ } catch (SQLException e) {
+ LOG.warn("回滚此次写入, 采用每次写入一行方式提交. 因为:" + e.getMessage());
+ connection.rollback();
+ doOneInsert(connection, buffer);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(
+ DBUtilErrorCode.WRITE_DATA_ERROR, e);
+ } finally {
+ DBUtil.closeDBResources(statement, null);
+ }
+ }
+
+ protected void doOneInsert(Connection connection, List buffer) {
+ Statement statement = null;
+ String sql = null;
+ try {
+ connection.setAutoCommit(true);
+ statement = connection.createStatement();
+
+ for (Record record : buffer) {
+ try {
+ sql = generateInsertSql(record);
+ int status = statement.executeUpdate(sql);
+ sql = null;
+ } catch (SQLException e) {
+ LOG.error("sql: " + sql, e.getMessage());
+ this.taskPluginCollector.collectDirtyRecord(record, e);
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("插入异常, sql: " + sql);
+ throw DataXException.asDataXException(
+ DBUtilErrorCode.WRITE_DATA_ERROR, e);
+ } finally {
+ DBUtil.closeDBResources(statement, null);
+ }
+ }
+
+ private String generateInsertSql(Record record) throws SQLException {
+ StringBuilder sqlSb = new StringBuilder("insert into " + this.table + "(" +
+ StringUtils.join(columns, ",") + ") values(");
+ for (int i = 0; i < columns.size(); i++) {
+ int columnSqltype = this.resultSetMetaData.getMiddle().get(i);
+ checkColumnType(columnSqltype, sqlSb, record.getColumn(i), i);
+ if((i+1) != columns.size()) {
+ sqlSb.append(",");
+ }
+ }
+ sqlSb.append(")");
+ return sqlSb.toString();
+ }
+
+ private void checkColumnType(int columnSqltype, StringBuilder sqlSb, Column column, int columnIndex) throws SQLException {
+ java.util.Date utilDate;
+ switch (columnSqltype) {
+ case Types.CHAR:
+ case Types.NCHAR:
+ case Types.CLOB:
+ case Types.NCLOB:
+ case Types.VARCHAR:
+ case Types.LONGVARCHAR:
+ case Types.NVARCHAR:
+ case Types.LONGNVARCHAR:
+ String strValue = column.asString();
+ if(null == strValue) {
+ sqlSb.append("null");
+ } else {
+ String optStr = column.asString().replace("\\","\\\\");
+ sqlSb.append("'").append(optStr).append("'");
+ }
+ break;
+
+ case Types.SMALLINT:
+ case Types.INTEGER:
+ case Types.BIGINT:
+ case Types.NUMERIC:
+ case Types.DECIMAL:
+ case Types.FLOAT:
+ case Types.REAL:
+ case Types.DOUBLE:
+ String numValue = column.asString();
+ if(emptyAsNull && "".equals(numValue) || numValue == null){
+ sqlSb.append("null");
+ } else{
+ sqlSb.append(numValue);
+ }
+ break;
+
+ //tinyint is a little special in some database like mysql {boolean->tinyint(1)}
+ case Types.TINYINT:
+ Long longValue = column.asLong();
+ if (null == longValue) {
+ sqlSb.append("null");
+ } else {
+ sqlSb.append(longValue);
+ }
+ break;
+
+ case Types.DATE:
+ java.sql.Date sqlDate = null;
+ try {
+ if("".equals(column.getRawData())) {
+ utilDate = null;
+ } else {
+ utilDate = column.asDate();
+ }
+ } catch (DataXException e) {
+ throw new SQLException(String.format(
+ "Date 类型转换错误：[%s]", column));
+ }
+
+ if (null != utilDate) {
+ sqlDate = new java.sql.Date(utilDate.getTime());
+ sqlSb.append("'").append(sqlDate).append("'");
+ } else {
+ sqlSb.append("null");
+ }
+ break;
+
+ case Types.TIME:
+ java.sql.Time sqlTime = null;
+ try {
+ if("".equals(column.getRawData())) {
+ utilDate = null;
+ } else {
+ utilDate = column.asDate();
+ }
+ } catch (DataXException e) {
+ throw new SQLException(String.format(
+ "TIME 类型转换错误：[%s]", column));
+ }
+
+ if (null != utilDate) {
+ sqlTime = new java.sql.Time(utilDate.getTime());
+ sqlSb.append("'").append(sqlTime).append("'");
+ } else {
+ sqlSb.append("null");
+ }
+ break;
+
+ case Types.TIMESTAMP:
+ java.sql.Timestamp sqlTimestamp = null;
+ try {
+ if("".equals(column.getRawData())) {
+ utilDate = null;
+ } else {
+ utilDate = column.asDate();
+ }
+ } catch (DataXException e) {
+ throw new SQLException(String.format(
+ "TIMESTAMP 类型转换错误：[%s]", column));
+ }
+
+ if (null != utilDate) {
+ sqlTimestamp = new java.sql.Timestamp(
+ utilDate.getTime());
+ sqlSb.append("'").append(sqlTimestamp).append("'");
+ } else {
+ sqlSb.append("null");
+ }
+ break;
+
+ case Types.BOOLEAN:
+ case Types.BIT:
+ String bitValue = column.asString();
+ if(bitValue == null) {
+ sqlSb.append("null");
+ } else {
+ sqlSb.append(bitValue);
+ }
+ break;
+ default:
+ throw DataXException
+ .asDataXException(
+ DBUtilErrorCode.UNSUPPORTED_TYPE,
+ String.format(
+ "您的配置文件中的列配置信息有误. 因为DataX 不支持数据库写入这种字段类型. 字段名:[%s], 字段类型:[%d], 字段Java类型:[%s]. 请修改表中该字段的类型或者不同步该字段.",
+ this.resultSetMetaData.getLeft()
+ .get(columnIndex),
+ this.resultSetMetaData.getMiddle()
+ .get(columnIndex),
+ this.resultSetMetaData.getRight()
+ .get(columnIndex)));
+ }
+ }
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertUtil.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertUtil.java
new file mode 100644
index 000000000..11550b979
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertUtil.java
@@ -0,0 +1,134 @@
+package com.alibaba.datax.plugin.writer.adswriter.insert;
+
+import com.alibaba.datax.common.exception.DataXException;
+
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.common.util.ListUtil;
+import com.alibaba.datax.plugin.rdbms.util.DBUtil;
+import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
+import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
+import com.alibaba.datax.plugin.writer.adswriter.AdsException;
+import com.alibaba.datax.plugin.writer.adswriter.AdsWriterErrorCode;
+import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnInfo;
+import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
+import com.alibaba.datax.plugin.writer.adswriter.load.AdsHelper;
+import com.alibaba.datax.plugin.writer.adswriter.util.Key;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutableTriple;
+import org.apache.commons.lang3.tuple.Triple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class AdsInsertUtil {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AdsInsertUtil.class);
+
+ public Connection getAdsConnect(Configuration conf) {
+ String userName = conf.getString(Key.USERNAME);
+ String passWord = conf.getString(Key.PASSWORD);
+ String adsURL = conf.getString(Key.ADS_URL);
+ String schema = conf.getString(Key.SCHEMA);
+ String jdbcUrl = "jdbc:mysql://" + adsURL + "/" + schema + "?useUnicode=true&characterEncoding=UTF-8";
+
+ Connection connection = DBUtil.getConnection(DataBaseType.ADS, userName, passWord, jdbcUrl);
+ return connection;
+ }
+
+
+ public static List getAdsTableColumnNames(Configuration conf) {
+ List tableColumns = new ArrayList();
+ String userName = conf.getString(Key.USERNAME);
+ String passWord = conf.getString(Key.PASSWORD);
+ String adsUrl = conf.getString(Key.ADS_URL);
+ String schema = conf.getString(Key.SCHEMA);
+ String tableName = conf.getString(Key.ADS_TABLE);
+ AdsHelper adsHelper = new AdsHelper(adsUrl, userName, passWord, schema);
+ TableInfo tableInfo= null;
+ try {
+ tableInfo = adsHelper.getTableInfo(tableName);
+ } catch (AdsException e) {
+ throw DataXException.asDataXException(AdsWriterErrorCode.GET_ADS_TABLE_MEATA_FAILED, e);
+ }
+
+ List columnInfos = tableInfo.getColumns();
+ for(ColumnInfo columnInfo: columnInfos) {
+ tableColumns.add(columnInfo.getName());
+ }
+
+ LOG.info("table:[{}] all columns:[\n{}\n].", tableName,
+ StringUtils.join(tableColumns, ","));
+ return tableColumns;
+ }
+
+ public static Triple, List, List> getColumnMetaData
+ (Configuration configuration, List userColumns) {
+ Triple, List, List> columnMetaData = new ImmutableTriple, List, List>(
+ new ArrayList(), new ArrayList(),
+ new ArrayList());
+
+ List columnInfoList = getAdsTableColumns(configuration);
+ for(String column : userColumns) {
+ for (ColumnInfo columnInfo : columnInfoList) {
+ if(column.equals(columnInfo.getName())) {
+ columnMetaData.getLeft().add(columnInfo.getName());
+ columnMetaData.getMiddle().add(columnInfo.getDataType().sqlType);
+ columnMetaData.getRight().add(
+ columnInfo.getDataType().name);
+ }
+ }
+ }
+ return columnMetaData;
+ }
+
+ public static List getAdsTableColumns(Configuration conf) {
+ String userName = conf.getString(Key.USERNAME);
+ String passWord = conf.getString(Key.PASSWORD);
+ String adsUrl = conf.getString(Key.ADS_URL);
+ String schema = conf.getString(Key.SCHEMA);
+ String tableName = conf.getString(Key.ADS_TABLE);
+ AdsHelper adsHelper = new AdsHelper(adsUrl, userName, passWord, schema);
+ TableInfo tableInfo= null;
+ try {
+ tableInfo = adsHelper.getTableInfo(tableName);
+ } catch (AdsException e) {
+ throw DataXException.asDataXException(AdsWriterErrorCode.GET_ADS_TABLE_MEATA_FAILED, e);
+ }
+
+ List columnInfos = tableInfo.getColumns();
+
+ return columnInfos;
+ }
+
+ public static void dealColumnConf(Configuration originalConfig, List tableColumns) {
+ List userConfiguredColumns = originalConfig.getList(Key.COLUMN, String.class);
+ if (null == userConfiguredColumns || userConfiguredColumns.isEmpty()) {
+ throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_VALUE,
+ "您的配置文件中的列配置信息有误. 因为您未配置写入数据库表的列名称，DataX获取不到列信息. 请检查您的配置并作出修改.");
+ } else {
+ if (1 == userConfiguredColumns.size() && "*".equals(userConfiguredColumns.get(0))) {
+ LOG.warn("您的配置文件中的列配置信息存在风险. 因为您配置的写入数据库表的列为*，当您的表字段个数、类型有变动时，可能影响任务正确性甚至会运行出错。请检查您的配置并作出修改.");
+
+ // 回填其值，需要以 String 的方式转交后续处理
+ originalConfig.set(Key.COLUMN, tableColumns);
+ } else if (userConfiguredColumns.size() > tableColumns.size()) {
+ throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_VALUE,
+ String.format("您的配置文件中的列配置信息有误. 因为您所配置的写入数据库表的字段个数:%s 大于目的表的总字段总个数:%s. 请检查您的配置并作出修改.",
+ userConfiguredColumns.size(), tableColumns.size()));
+ } else {
+ // 确保用户配置的 column 不重复
+ ListUtil.makeSureNoValueDuplicate(userConfiguredColumns, false);
+
+ // 检查列是否都为数据库表中正确的列（通过执行一次 select column from table 进行判断）
+ ListUtil.makeSureBInA(tableColumns, userConfiguredColumns, true);
+ }
+ }
+ }
+
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/AdsHelper.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/AdsHelper.java
new file mode 100644
index 000000000..5f429294c
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/AdsHelper.java
@@ -0,0 +1,388 @@
+/**
+ *
+ */
+package com.alibaba.datax.plugin.writer.adswriter.load;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.plugin.rdbms.util.DBUtil;
+import com.alibaba.datax.plugin.writer.adswriter.AdsException;
+import com.alibaba.datax.plugin.writer.adswriter.AdsWriterErrorCode;
+import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnDataType;
+import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnInfo;
+import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+public class AdsHelper {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AdsHelper.class);
+
+ private String adsURL;
+ private String userName;
+ private String password;
+ private String schema;
+
+ public AdsHelper(String adsUrl, String userName, String password, String schema) {
+ this.adsURL = adsUrl;
+ this.userName = userName;
+ this.password = password;
+ this.schema = schema;
+ }
+
+ public String getAdsURL() {
+ return adsURL;
+ }
+
+ public void setAdsURL(String adsURL) {
+ this.adsURL = adsURL;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public String getSchema() {
+ return schema;
+ }
+
+ public void setSchema(String schema) {
+ this.schema = schema;
+ }
+
+ /**
+ * Obtain the table meta information.
+ *
+ * @param table The table
+ * @return The table meta information
+ * @throws com.alibaba.datax.plugin.writer.adswriter.AdsException
+ */
+ public TableInfo getTableInfo(String table) throws AdsException {
+
+ if (table == null) {
+ throw new AdsException(AdsException.ADS_TABLEMETA_TABLE_NULL, "Table is null.", null);
+ }
+
+ if (adsURL == null) {
+ throw new AdsException(AdsException.ADS_CONN_URL_NOT_SET, "ADS JDBC connection URL was not set.", null);
+ }
+
+ if (userName == null) {
+ throw new AdsException(AdsException.ADS_CONN_USERNAME_NOT_SET,
+ "ADS JDBC connection user name was not set.", null);
+ }
+
+ if (password == null) {
+ throw new AdsException(AdsException.ADS_CONN_PASSWORD_NOT_SET, "ADS JDBC connection password was not set.",
+ null);
+ }
+
+ if (schema == null) {
+ throw new AdsException(AdsException.ADS_CONN_SCHEMA_NOT_SET, "ADS JDBC connection schema was not set.",
+ null);
+ }
+
+ String sql = "select ordinal_position,column_name,data_type,type_name,column_comment from information_schema.columns where table_schema='"
+ + schema + "' and table_name='" + table + "' order by ordinal_position";
+
+ Connection connection = null;
+ Statement statement = null;
+ ResultSet rs = null;
+ try {
+ Class.forName("com.mysql.jdbc.Driver");
+ String url = "jdbc:mysql://" + adsURL + "/" + schema + "?useUnicode=true&characterEncoding=UTF-8&socketTimeout=3600000";
+
+ Properties connectionProps = new Properties();
+ connectionProps.put("user", userName);
+ connectionProps.put("password", password);
+ connection = DriverManager.getConnection(url, connectionProps);
+ statement = connection.createStatement();
+
+ rs = statement.executeQuery(sql);
+
+ TableInfo tableInfo = new TableInfo();
+ List columnInfoList = new ArrayList();
+ while (DBUtil.asyncResultSetNext(rs)) {
+ ColumnInfo columnInfo = new ColumnInfo();
+ columnInfo.setOrdinal(rs.getInt(1));
+ columnInfo.setName(rs.getString(2));
+ //columnInfo.setDataType(ColumnDataType.getDataType(rs.getInt(3))); //for ads version < 0.7
+ //columnInfo.setDataType(ColumnDataType.getTypeByName(rs.getString(3).toUpperCase())); //for ads version 0.8
+ columnInfo.setDataType(ColumnDataType.getTypeByName(rs.getString(4).toUpperCase())); //for ads version 0.8 & 0.7
+ columnInfo.setComment(rs.getString(5));
+ columnInfoList.add(columnInfo);
+ }
+ if (columnInfoList.isEmpty()) {
+ throw DataXException.asDataXException(AdsWriterErrorCode.NO_ADS_TABLE, table + "不存在或者查询不到列信息. ");
+ }
+
+ tableInfo.setColumns(columnInfoList);
+ tableInfo.setTableSchema(schema);
+ tableInfo.setTableName(table);
+
+ return tableInfo;
+
+ } catch (ClassNotFoundException e) {
+ throw new AdsException(AdsException.OTHER, e.getMessage(), e);
+ } catch (SQLException e) {
+ throw new AdsException(AdsException.OTHER, e.getMessage(), e);
+ } catch ( DataXException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AdsException(AdsException.OTHER, e.getMessage(), e);
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ if (connection != null) {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Submit LOAD DATA command.
+ *
+ * @param table The target ADS table
+ * @param partition The partition option in the form of "(partition_name,...)"
+ * @param sourcePath The source path
+ * @param overwrite
+ * @return
+ * @throws AdsException
+ */
+ public String loadData(String table, String partition, String sourcePath, boolean overwrite) throws AdsException {
+
+ if (table == null) {
+ throw new AdsException(AdsException.ADS_LOADDATA_TABLE_NULL, "ADS LOAD DATA table is null.", null);
+ }
+
+ if (sourcePath == null) {
+ throw new AdsException(AdsException.ADS_LOADDATA_SOURCEPATH_NULL, "ADS LOAD DATA source path is null.",
+ null);
+ }
+
+ if (adsURL == null) {
+ throw new AdsException(AdsException.ADS_CONN_URL_NOT_SET, "ADS JDBC connection URL was not set.", null);
+ }
+
+ if (userName == null) {
+ throw new AdsException(AdsException.ADS_CONN_USERNAME_NOT_SET,
+ "ADS JDBC connection user name was not set.", null);
+ }
+
+ if (password == null) {
+ throw new AdsException(AdsException.ADS_CONN_PASSWORD_NOT_SET, "ADS JDBC connection password was not set.",
+ null);
+ }
+
+ if (schema == null) {
+ throw new AdsException(AdsException.ADS_CONN_SCHEMA_NOT_SET, "ADS JDBC connection schema was not set.",
+ null);
+ }
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("LOAD DATA FROM ");
+ if (sourcePath.startsWith("'") && sourcePath.endsWith("'")) {
+ sb.append(sourcePath);
+ } else {
+ sb.append("'" + sourcePath + "'");
+ }
+ if (overwrite) {
+ sb.append(" OVERWRITE");
+ }
+ sb.append(" INTO TABLE ");
+ sb.append(schema + "." + table);
+ if (partition != null && !partition.trim().equals("")) {
+ String partitionTrim = partition.trim();
+ if(partitionTrim.startsWith("(") && partitionTrim.endsWith(")")) {
+ sb.append(" PARTITION " + partition);
+ } else {
+ sb.append(" PARTITION " + "(" + partition + ")");
+ }
+ }
+
+ Connection connection = null;
+ Statement statement = null;
+ ResultSet rs = null;
+ try {
+ Class.forName("com.mysql.jdbc.Driver");
+ String url = "jdbc:mysql://" + adsURL + "/" + schema + "?useUnicode=true&characterEncoding=UTF-8&socketTimeout=3600000";
+
+ Properties connectionProps = new Properties();
+ connectionProps.put("user", userName);
+ connectionProps.put("password", password);
+ connection = DriverManager.getConnection(url, connectionProps);
+ statement = connection.createStatement();
+ LOG.info("正在从ODPS数据库导数据到ADS中: "+sb.toString());
+ LOG.info("由于ADS的限制，ADS导数据最少需要20分钟，请耐心等待");
+ rs = statement.executeQuery(sb.toString());
+
+ String jobId = null;
+ while (DBUtil.asyncResultSetNext(rs)) {
+ jobId = rs.getString(1);
+ }
+
+ if (jobId == null) {
+ throw new AdsException(AdsException.ADS_LOADDATA_JOBID_NOT_AVAIL,
+ "Job id is not available for the submitted LOAD DATA." + jobId, null);
+ }
+
+ return jobId;
+
+ } catch (ClassNotFoundException e) {
+ throw new AdsException(AdsException.ADS_LOADDATA_FAILED, e.getMessage(), e);
+ } catch (SQLException e) {
+ throw new AdsException(AdsException.ADS_LOADDATA_FAILED, e.getMessage(), e);
+ } catch (Exception e) {
+ throw new AdsException(AdsException.ADS_LOADDATA_FAILED, e.getMessage(), e);
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ if (connection != null) {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Check the load data job status.
+ *
+ * @param jobId The job id to
+ * @return true if load data job succeeded, false if load data job failed.
+ * @throws AdsException
+ */
+ public boolean checkLoadDataJobStatus(String jobId) throws AdsException {
+
+ if (adsURL == null) {
+ throw new AdsException(AdsException.ADS_CONN_URL_NOT_SET, "ADS JDBC connection URL was not set.", null);
+ }
+
+ if (userName == null) {
+ throw new AdsException(AdsException.ADS_CONN_USERNAME_NOT_SET,
+ "ADS JDBC connection user name was not set.", null);
+ }
+
+ if (password == null) {
+ throw new AdsException(AdsException.ADS_CONN_PASSWORD_NOT_SET, "ADS JDBC connection password was not set.",
+ null);
+ }
+
+ if (schema == null) {
+ throw new AdsException(AdsException.ADS_CONN_SCHEMA_NOT_SET, "ADS JDBC connection schema was not set.",
+ null);
+ }
+
+ Connection connection = null;
+ Statement statement = null;
+ ResultSet rs = null;
+ try {
+ Class.forName("com.mysql.jdbc.Driver");
+ String url = "jdbc:mysql://" + adsURL + "/" + schema + "?useUnicode=true&characterEncoding=UTF-8&socketTimeout=3600000";
+
+ Properties connectionProps = new Properties();
+ connectionProps.put("user", userName);
+ connectionProps.put("password", password);
+ connection = DriverManager.getConnection(url, connectionProps);
+ statement = connection.createStatement();
+
+ String sql = "select state from information_schema.job_instances where job_id like '" + jobId + "'";
+ rs = statement.executeQuery(sql);
+
+ String state = null;
+ while (DBUtil.asyncResultSetNext(rs)) {
+ state = rs.getString(1);
+ }
+
+ if (state == null) {
+ throw new AdsException(AdsException.JOB_NOT_EXIST, "Target job does not exist for id: " + jobId, null);
+ }
+
+ if (state.equals("SUCCEEDED")) {
+ return true;
+ } else if (state.equals("FAILED")) {
+ throw new AdsException(AdsException.JOB_FAILED, "Target job failed for id: " + jobId, null);
+ } else {
+ return false;
+ }
+
+ } catch (ClassNotFoundException e) {
+ throw new AdsException(AdsException.OTHER, e.getMessage(), e);
+ } catch (SQLException e) {
+ throw new AdsException(AdsException.OTHER, e.getMessage(), e);
+ } catch (Exception e) {
+ throw new AdsException(AdsException.OTHER, e.getMessage(), e);
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ if (connection != null) {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ // Ignore exception
+ }
+ }
+ }
+
+ }
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TableMetaHelper.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TableMetaHelper.java
new file mode 100644
index 000000000..1ecad7561
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TableMetaHelper.java
@@ -0,0 +1,87 @@
+package com.alibaba.datax.plugin.writer.adswriter.load;
+
+import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnDataType;
+import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnInfo;
+import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
+import com.alibaba.datax.plugin.writer.adswriter.odps.DataType;
+import com.alibaba.datax.plugin.writer.adswriter.odps.FieldSchema;
+import com.alibaba.datax.plugin.writer.adswriter.odps.TableMeta;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Table meta helper for ADS writer.
+ *
+ * @since 0.0.1
+ */
+public class TableMetaHelper {
+
+ private TableMetaHelper() {
+ }
+
+ /**
+ * Create temporary ODPS table.
+ *
+ * @param tableMeta table meta
+ * @param lifeCycle for temporary table
+ * @return ODPS temporary table meta
+ */
+ public static TableMeta createTempODPSTable(TableInfo tableMeta, int lifeCycle) {
+ TableMeta tempTable = new TableMeta();
+ tempTable.setComment(tableMeta.getComments());
+ tempTable.setLifeCycle(lifeCycle);
+ String tableSchema = tableMeta.getTableSchema();
+ String tableName = tableMeta.getTableName();
+ tempTable.setTableName(generateTempTableName(tableSchema, tableName));
+ List tempColumns = new ArrayList();
+ List columns = tableMeta.getColumns();
+ for (ColumnInfo column : columns) {
+ FieldSchema tempColumn = new FieldSchema();
+ tempColumn.setName(column.getName());
+ tempColumn.setType(toODPSDataType(column.getDataType()));
+ tempColumn.setComment(column.getComment());
+ tempColumns.add(tempColumn);
+ }
+ tempTable.setCols(tempColumns);
+ tempTable.setPartitionKeys(null);
+ return tempTable;
+ }
+
+ private static String toODPSDataType(ColumnDataType columnDataType) {
+ int type;
+ switch (columnDataType.type) {
+ case ColumnDataType.BOOLEAN:
+ type = DataType.STRING;
+ break;
+ case ColumnDataType.BYTE:
+ case ColumnDataType.SHORT:
+ case ColumnDataType.INT:
+ case ColumnDataType.LONG:
+ type = DataType.INTEGER;
+ break;
+ case ColumnDataType.DECIMAL:
+ case ColumnDataType.DOUBLE:
+ case ColumnDataType.FLOAT:
+ type = DataType.DOUBLE;
+ break;
+ case ColumnDataType.DATE:
+ case ColumnDataType.TIME:
+ case ColumnDataType.TIMESTAMP:
+ case ColumnDataType.STRING:
+ case ColumnDataType.MULTI_VALUE:
+ type = DataType.STRING;
+ break;
+ default:
+ throw new IllegalArgumentException("columnDataType=" + columnDataType);
+ }
+ return DataType.toString(type);
+ }
+
+ private static String generateTempTableName(String tableSchema, String tableName) {
+ int randNum = 1000 + new Random(System.currentTimeMillis()).nextInt(1000);
+ return tableSchema + "__" + tableName + "_" + System.currentTimeMillis() + randNum;
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TransferProjectConf.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TransferProjectConf.java
new file mode 100644
index 000000000..bff4b7b90
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TransferProjectConf.java
@@ -0,0 +1,65 @@
+package com.alibaba.datax.plugin.writer.adswriter.load;
+
+import com.alibaba.datax.common.util.Configuration;
+
+/**
+ * Created by xiafei.qiuxf on 15/4/13.
+ */
+public class TransferProjectConf {
+
+ public final static String KEY_ACCESS_ID = "odps.accessId";
+ public final static String KEY_ACCESS_KEY = "odps.accessKey";
+ public final static String KEY_ACCOUNT = "odps.account";
+ public final static String KEY_ODPS_SERVER = "odps.odpsServer";
+ public final static String KEY_ODPS_TUNNEL = "odps.tunnelServer";
+ public final static String KEY_ACCOUNT_TYPE = "odps.accountType";
+ public final static String KEY_PROJECT = "odps.project";
+
+ private String accessId;
+ private String accessKey;
+ private String account;
+ private String odpsServer;
+ private String odpsTunnel;
+ private String accountType;
+ private String project;
+
+ public static TransferProjectConf create(Configuration adsWriterConf) {
+ TransferProjectConf res = new TransferProjectConf();
+ res.accessId = adsWriterConf.getString(KEY_ACCESS_ID);
+ res.accessKey = adsWriterConf.getString(KEY_ACCESS_KEY);
+ res.account = adsWriterConf.getString(KEY_ACCOUNT);
+ res.odpsServer = adsWriterConf.getString(KEY_ODPS_SERVER);
+ res.odpsTunnel = adsWriterConf.getString(KEY_ODPS_TUNNEL);
+ res.accountType = adsWriterConf.getString(KEY_ACCOUNT_TYPE, "aliyun");
+ res.project = adsWriterConf.getString(KEY_PROJECT);
+ return res;
+ }
+
+ public String getAccessId() {
+ return accessId;
+ }
+
+ public String getAccessKey() {
+ return accessKey;
+ }
+
+ public String getAccount() {
+ return account;
+ }
+
+ public String getOdpsServer() {
+ return odpsServer;
+ }
+
+ public String getOdpsTunnel() {
+ return odpsTunnel;
+ }
+
+ public String getAccountType() {
+ return accountType;
+ }
+
+ public String getProject() {
+ return project;
+ }
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/DataType.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/DataType.java
new file mode 100644
index 000000000..595b1dfd2
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/DataType.java
@@ -0,0 +1,77 @@
+package com.alibaba.datax.plugin.writer.adswriter.odps;
+
+/**
+ * ODPS 数据类型.
+ *
+ * 当前定义了如下类型：
+ *
+ * INTEGER
+ * DOUBLE
+ * BOOLEAN
+ * STRING
+ * DATETIME
+ *
+ *
+ *
+ * @since 0.0.1
+ */
+public class DataType {
+
+ public final static byte INTEGER = 0;
+ public final static byte DOUBLE = 1;
+ public final static byte BOOLEAN = 2;
+ public final static byte STRING = 3;
+ public final static byte DATETIME = 4;
+
+ public static String toString(int type) {
+ switch (type) {
+ case INTEGER:
+ return "bigint";
+ case DOUBLE:
+ return "double";
+ case BOOLEAN:
+ return "boolean";
+ case STRING:
+ return "string";
+ case DATETIME:
+ return "datetime";
+ default:
+ throw new IllegalArgumentException("type=" + type);
+ }
+ }
+
+ /**
+ * 字符串的数据类型转换为byte常量定义的数据类型.
+ *
+ * 转换规则：
+ *
+ * tinyint, int, bigint, long - {@link #INTEGER}
+ * double, float - {@link #DOUBLE}
+ * string - {@link #STRING}
+ * boolean, bool - {@link #BOOLEAN}
+ * datetime - {@link #DATETIME}
+ *
+ *
+ *
+ * @param type 字符串的数据类型
+ * @return byte常量定义的数据类型
+ * @throws IllegalArgumentException
+ */
+ public static byte convertToDataType(String type) throws IllegalArgumentException {
+ type = type.toLowerCase().trim();
+ if ("string".equals(type)) {
+ return STRING;
+ } else if ("bigint".equals(type) || "int".equals(type) || "tinyint".equals(type) || "long".equals(type)) {
+ return INTEGER;
+ } else if ("boolean".equals(type) || "bool".equals(type)) {
+ return BOOLEAN;
+ } else if ("double".equals(type) || "float".equals(type)) {
+ return DOUBLE;
+ } else if ("datetime".equals(type)) {
+ return DATETIME;
+ } else {
+ throw new IllegalArgumentException("unkown type: " + type);
+ }
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/FieldSchema.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/FieldSchema.java
new file mode 100644
index 000000000..701ee261c
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/FieldSchema.java
@@ -0,0 +1,63 @@
+package com.alibaba.datax.plugin.writer.adswriter.odps;
+
+/**
+ * ODPS列属性，包含列名和类型 列名和类型与SQL的DESC表或分区显示的列名和类型一致
+ *
+ * @since 0.0.1
+ */
+public class FieldSchema {
+
+ /** 列名 */
+ private String name;
+
+ /** 列类型，如：string, bigint, boolean, datetime等等 */
+ private String type;
+
+ private String comment;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("FieldSchema [name=").append(name).append(", type=").append(type).append(", comment=")
+ .append(comment).append("]");
+ return builder.toString();
+ }
+
+ /**
+ * @return "col_name data_type [COMMENT col_comment]"
+ */
+ public String toDDL() {
+ StringBuilder builder = new StringBuilder();
+ builder.append(name).append(" ").append(type);
+ String comment = this.comment;
+ if (comment != null && comment.length() > 0) {
+ builder.append(" ").append("COMMENT \"" + comment + "\"");
+ }
+ return builder.toString();
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/TableMeta.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/TableMeta.java
new file mode 100644
index 000000000..d0adc4eae
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/TableMeta.java
@@ -0,0 +1,114 @@
+package com.alibaba.datax.plugin.writer.adswriter.odps;
+
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * ODPS table meta.
+ *
+ * @since 0.0.1
+ */
+public class TableMeta {
+
+ private String tableName;
+
+ private List cols;
+
+ private List partitionKeys;
+
+ private int lifeCycle;
+
+ private String comment;
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public List getCols() {
+ return cols;
+ }
+
+ public void setCols(List cols) {
+ this.cols = cols;
+ }
+
+ public List getPartitionKeys() {
+ return partitionKeys;
+ }
+
+ public void setPartitionKeys(List partitionKeys) {
+ this.partitionKeys = partitionKeys;
+ }
+
+ public int getLifeCycle() {
+ return lifeCycle;
+ }
+
+ public void setLifeCycle(int lifeCycle) {
+ this.lifeCycle = lifeCycle;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("TableMeta [tableName=").append(tableName).append(", cols=").append(cols)
+ .append(", partitionKeys=").append(partitionKeys).append(", lifeCycle=").append(lifeCycle)
+ .append(", comment=").append(comment).append("]");
+ return builder.toString();
+ }
+
+ /**
+ * @return cols = this.cols;
+ if (cols != null && cols.size() > 0) {
+ builder.append("(").append(toDDL(cols)).append(")").append(" ");
+ }
+ String comment = this.comment;
+ if (comment != null && comment.length() > 0) {
+ builder.append("COMMENT \"" + comment + "\" ");
+ }
+ List partitionKeys = this.partitionKeys;
+ if (partitionKeys != null && partitionKeys.size() > 0) {
+ builder.append("PARTITIONED BY ");
+ builder.append("(").append(toDDL(partitionKeys)).append(")").append(" ");
+ }
+ if (lifeCycle > 0) {
+ builder.append("LIFECYCLE " + lifeCycle).append(" ");
+ }
+ builder.append(";");
+ return builder.toString();
+ }
+
+ private String toDDL(List cols) {
+ StringBuilder builder = new StringBuilder();
+ Iterator iter = cols.iterator();
+ builder.append(iter.next().toDDL());
+ while (iter.hasNext()) {
+ builder.append(", ").append(iter.next().toDDL());
+ }
+ return builder.toString();
+ }
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/package-info.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/package-info.java
new file mode 100644
index 000000000..92dfd09da
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/package-info.java
@@ -0,0 +1,6 @@
+/**
+ * ODPS meta.
+ *
+ * @since 0.0.1
+ */
+package com.alibaba.datax.plugin.writer.adswriter.odps;
\ No newline at end of file
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/package-info.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/package-info.java
new file mode 100644
index 000000000..139a39106
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/package-info.java
@@ -0,0 +1,6 @@
+/**
+ * ADS Writer.
+ *
+ * @since 0.0.1
+ */
+package com.alibaba.datax.plugin.writer.adswriter;
\ No newline at end of file
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/AdsUtil.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/AdsUtil.java
new file mode 100644
index 000000000..778ce5eb8
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/AdsUtil.java
@@ -0,0 +1,134 @@
+package com.alibaba.datax.plugin.writer.adswriter.util;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.writer.adswriter.load.AdsHelper;
+import com.alibaba.datax.plugin.writer.adswriter.AdsWriterErrorCode;
+import com.alibaba.datax.plugin.writer.adswriter.load.TransferProjectConf;
+import com.alibaba.datax.plugin.writer.adswriter.odps.FieldSchema;
+import com.alibaba.datax.plugin.writer.adswriter.odps.TableMeta;
+import com.alibaba.datax.plugin.writer.odpswriter.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by judy.lt on 2015/1/30.
+ */
+public class AdsUtil {
+ private static final Logger LOG = LoggerFactory.getLogger(AdsUtil.class);
+
+ /*检查配置文件中必填的配置项是否都已填
+ * */
+ public static void checkNecessaryConfig(Configuration originalConfig, String writeMode) {
+ //检查ADS必要参数
+ originalConfig.getNecessaryValue(Key.ADS_URL,
+ AdsWriterErrorCode.REQUIRED_VALUE);
+ originalConfig.getNecessaryValue(Key.USERNAME,
+ AdsWriterErrorCode.REQUIRED_VALUE);
+ originalConfig.getNecessaryValue(Key.PASSWORD,
+ AdsWriterErrorCode.REQUIRED_VALUE);
+ originalConfig.getNecessaryValue(Key.SCHEMA,
+ AdsWriterErrorCode.REQUIRED_VALUE);
+ if(Constant.LOADMODE.equals(writeMode)) {
+ originalConfig.getNecessaryValue(Key.Life_CYCLE,
+ AdsWriterErrorCode.REQUIRED_VALUE);
+ Integer lifeCycle = originalConfig.getInt(Key.Life_CYCLE);
+ if (lifeCycle <= 0) {
+ throw DataXException.asDataXException(AdsWriterErrorCode.INVALID_CONFIG_VALUE, "配置项[lifeCycle]的值必须大于零.");
+ }
+ originalConfig.getNecessaryValue(Key.ADS_TABLE,
+ AdsWriterErrorCode.REQUIRED_VALUE);
+ Boolean overwrite = originalConfig.getBool(Key.OVER_WRITE);
+ if (overwrite == null) {
+ throw DataXException.asDataXException(AdsWriterErrorCode.REQUIRED_VALUE, "配置项[overWrite]是必填项.");
+ }
+ }
+ }
+
+ /*生成AdsHelp实例
+ * */
+ public static AdsHelper createAdsHelper(Configuration originalConfig){
+ //Get adsUrl,userName,password,schema等参数,创建AdsHelp实例
+ String adsUrl = originalConfig.getString(Key.ADS_URL);
+ String userName = originalConfig.getString(Key.USERNAME);
+ String password = originalConfig.getString(Key.PASSWORD);
+ String schema = originalConfig.getString(Key.SCHEMA);
+ return new AdsHelper(adsUrl,userName,password,schema);
+ }
+
+ public static AdsHelper createAdsHelperWithOdpsAccount(Configuration originalConfig) {
+ String adsUrl = originalConfig.getString(Key.ADS_URL);
+ String userName = originalConfig.getString(TransferProjectConf.KEY_ACCESS_ID);
+ String password = originalConfig.getString(TransferProjectConf.KEY_ACCESS_KEY);
+ String schema = originalConfig.getString(Key.SCHEMA);
+ return new AdsHelper(adsUrl, userName, password, schema);
+ }
+
+ /*生成ODPSWriter Plugin所需要的配置文件
+ * */
+ public static Configuration generateConf(Configuration originalConfig, String odpsTableName, TableMeta tableMeta, TransferProjectConf transConf){
+ Configuration newConfig = originalConfig.clone();
+ newConfig.set(Key.ODPSTABLENAME, odpsTableName);
+ newConfig.set(Key.ODPS_SERVER, transConf.getOdpsServer());
+ newConfig.set(Key.TUNNEL_SERVER,transConf.getOdpsTunnel());
+ newConfig.set(Key.ACCESS_ID,transConf.getAccessId());
+ newConfig.set(Key.ACCESS_KEY,transConf.getAccessKey());
+ newConfig.set(Key.PROJECT,transConf.getProject());
+ newConfig.set(Key.TRUNCATE, true);
+ newConfig.set(Key.PARTITION,null);
+// newConfig.remove(Key.PARTITION);
+ List cols = tableMeta.getCols();
+ List allColumns = new ArrayList();
+ if(cols != null && !cols.isEmpty()){
+ for(FieldSchema col:cols){
+ allColumns.add(col.getName());
+ }
+ }
+ newConfig.set(Key.COLUMN,allColumns);
+ return newConfig;
+ }
+
+ /*生成ADS数据导入时的source_path
+ * */
+ public static String generateSourcePath(String project, String tmpOdpsTableName, String odpsPartition){
+ StringBuilder builder = new StringBuilder();
+ String partition = transferOdpsPartitionToAds(odpsPartition);
+ builder.append("odps://").append(project).append("/").append(tmpOdpsTableName);
+ if(odpsPartition != null && !odpsPartition.isEmpty()){
+ builder.append("/").append(partition);
+ }
+ return builder.toString();
+ }
+
+ public static String transferOdpsPartitionToAds(String odpsPartition){
+ if(odpsPartition == null || odpsPartition.isEmpty())
+ return null;
+ String adsPartition = formatPartition(odpsPartition);;
+ String[] partitions = adsPartition.split("/");
+ for(int last = partitions.length; last > 0; last--){
+
+ String partitionPart = partitions[last-1];
+ String newPart = partitionPart.replace(".*", "*").replace("*", ".*");
+ if(newPart.split("=")[1].equals(".*")){
+ adsPartition = adsPartition.substring(0,adsPartition.length()-partitionPart.length());
+ }else{
+ break;
+ }
+ if(adsPartition.endsWith("/")){
+ adsPartition = adsPartition.substring(0,adsPartition.length()-1);
+ }
+ }
+ if (adsPartition.contains("*"))
+ throw DataXException.asDataXException(AdsWriterErrorCode.ODPS_PARTITION_FAILED, "");
+ return adsPartition;
+ }
+
+ public static String formatPartition(String partition) {
+ return partition.trim().replaceAll(" *= *", "=")
+ .replaceAll(" */ *", ",").replaceAll(" *, *", ",")
+ .replaceAll("'", "").replaceAll(",", "/");
+ }
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Constant.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Constant.java
new file mode 100644
index 000000000..3842cd011
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Constant.java
@@ -0,0 +1,13 @@
+package com.alibaba.datax.plugin.writer.adswriter.util;
+
+public class Constant {
+
+ public static final String LOADMODE = "load";
+
+ public static final String INSERTMODE = "insert";
+
+ public static final String REPLACEMODE = "replace";
+
+ public static final int DEFAULT_BATCH_SIZE = 32;
+
+}
diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Key.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Key.java
new file mode 100644
index 000000000..e0822878b
--- /dev/null
+++ b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Key.java
@@ -0,0 +1,48 @@
+package com.alibaba.datax.plugin.writer.adswriter.util;
+
+
+public final class Key {
+
+ public final static String ADS_URL = "url";
+
+ public final static String USERNAME = "username";
+
+ public final static String PASSWORD = "password";
+
+ public final static String SCHEMA = "schema";
+
+ public final static String ADS_TABLE = "table";
+
+ public final static String Life_CYCLE = "lifeCycle";
+
+ public final static String OVER_WRITE = "overWrite";
+
+ public final static String WRITE_MODE = "writeMode";
+
+
+ public final static String COLUMN = "column";
+
+ public final static String EMPTY_AS_NULL = "emptyAsNull";
+
+ public final static String BATCH_SIZE = "batchSize";
+
+ /**
+ * 以下是odps writer的key
+ */
+ public final static String PARTITION = "partition";
+
+ public final static String ODPSTABLENAME = "table";
+
+ public final static String ODPS_SERVER = "odpsServer";
+
+ public final static String TUNNEL_SERVER = "tunnelServer";
+
+ public final static String ACCESS_ID = "accessId";
+
+ public final static String ACCESS_KEY = "accessKey";
+
+ public final static String PROJECT = "project";
+
+ public final static String TRUNCATE = "truncate";
+
+}
\ No newline at end of file
diff --git a/adswriter/src/main/resources/plugin.json b/adswriter/src/main/resources/plugin.json
new file mode 100644
index 000000000..a70fb3646
--- /dev/null
+++ b/adswriter/src/main/resources/plugin.json
@@ -0,0 +1,6 @@
+{
+ "name": "adswriter",
+ "class": "com.alibaba.datax.plugin.writer.adswriter.AdsWriter",
+ "description": "",
+ "developer": "alibaba"
+}
\ No newline at end of file
diff --git a/adswriter/src/main/resources/plugin_job_template.json b/adswriter/src/main/resources/plugin_job_template.json
new file mode 100644
index 000000000..0753a226e
--- /dev/null
+++ b/adswriter/src/main/resources/plugin_job_template.json
@@ -0,0 +1,13 @@
+{
+ "name": "adswriter",
+ "parameter": {
+ "url": "",
+ "username": "",
+ "password": "",
+ "schema": "",
+ "table": "",
+ "partition": "",
+ "overWrite": "",
+ "lifeCycle": 2
+ }
+}
\ No newline at end of file
diff --git a/common/datax-common.iml b/common/datax-common.iml
new file mode 100644
index 000000000..87f7dace6
--- /dev/null
+++ b/common/datax-common.iml
@@ -0,0 +1,30 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/common/pom.xml b/common/pom.xml
new file mode 100755
index 000000000..86c0a22dd
--- /dev/null
+++ b/common/pom.xml
@@ -0,0 +1,81 @@
+
+ 4.0.0
+
+ com.alibaba.datax
+ datax-all
+ 0.0.1-SNAPSHOT
+
+
+ datax-common
+ datax-common
+ jar
+
+
+
+ org.apache.commons
+ commons-lang3
+
+
+ com.google.guava
+ guava
+ 18.0
+
+
+ com.alibaba
+ fastjson
+
+
+ commons-io
+ commons-io
+
+
+
+ junit
+ junit
+ test
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+ ch.qos.logback
+ logback-classic
+
+
+
+ org.apache.httpcomponents
+ httpclient
+ 4.4
+ test
+
+
+ org.apache.httpcomponents
+ fluent-hc
+ 4.4
+ test
+
+
+ org.apache.commons
+ commons-math3
+ 3.1.1
+
+
+
+
+
+
+
+ maven-compiler-plugin
+
+ 1.6
+ 1.6
+ ${project-sourceEncoding}
+
+
+
+
+
\ No newline at end of file
diff --git a/common/src/main/java/com/alibaba/datax/common/base/BaseObject.java b/common/src/main/java/com/alibaba/datax/common/base/BaseObject.java
new file mode 100755
index 000000000..e7d06a950
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/base/BaseObject.java
@@ -0,0 +1,25 @@
+package com.alibaba.datax.common.base;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
+public class BaseObject {
+
+ @Override
+ public int hashCode() {
+ return HashCodeBuilder.reflectionHashCode(this, false);
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ return EqualsBuilder.reflectionEquals(this, object, false);
+ }
+
+ @Override
+ public String toString() {
+ return ToStringBuilder.reflectionToString(this,
+ ToStringStyle.MULTI_LINE_STYLE);
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/constant/CommonConstant.java b/common/src/main/java/com/alibaba/datax/common/constant/CommonConstant.java
new file mode 100755
index 000000000..423e16f92
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/constant/CommonConstant.java
@@ -0,0 +1,9 @@
+package com.alibaba.datax.common.constant;
+
+public final class CommonConstant {
+ /**
+ * 用于插件对自身 split 的每个 task 标识其使用的资源，以告知core 对 reader/writer split 之后的 task 进行拼接时需要根据资源标签进行更有意义的 shuffle 操作
+ */
+ public static String LOAD_BALANCE_RESOURCE_MARK = "loadBalanceResourceMark";
+
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/constant/PluginType.java b/common/src/main/java/com/alibaba/datax/common/constant/PluginType.java
new file mode 100755
index 000000000..ceee089e9
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/constant/PluginType.java
@@ -0,0 +1,20 @@
+package com.alibaba.datax.common.constant;
+
+/**
+ * Created by jingxing on 14-8-31.
+ */
+public enum PluginType {
+ //pluginType还代表了资源目录，很难扩展,或者说需要足够必要才扩展。先mark Handler（其实和transformer一样），再讨论
+ READER("reader"), TRANSFORMER("transformer"), WRITER("writer"), HANDLER("handler");
+
+ private String pluginType;
+
+ private PluginType(String pluginType) {
+ this.pluginType = pluginType;
+ }
+
+ @Override
+ public String toString() {
+ return this.pluginType;
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/element/BoolColumn.java b/common/src/main/java/com/alibaba/datax/common/element/BoolColumn.java
new file mode 100755
index 000000000..7699e152a
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/BoolColumn.java
@@ -0,0 +1,115 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+/**
+ * Created by jingxing on 14-8-24.
+ */
+public class BoolColumn extends Column {
+
+ public BoolColumn(Boolean bool) {
+ super(bool, Column.Type.BOOL, 1);
+ }
+
+ public BoolColumn(final String data) {
+ this(true);
+ this.validate(data);
+ if (null == data) {
+ this.setRawData(null);
+ this.setByteSize(0);
+ } else {
+ this.setRawData(Boolean.valueOf(data));
+ this.setByteSize(1);
+ }
+ return;
+ }
+
+ public BoolColumn() {
+ super(null, Column.Type.BOOL, 1);
+ }
+
+ @Override
+ public Boolean asBoolean() {
+ if (null == super.getRawData()) {
+ return null;
+ }
+
+ return (Boolean) super.getRawData();
+ }
+
+ @Override
+ public Long asLong() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return this.asBoolean() ? 1L : 0L;
+ }
+
+ @Override
+ public Double asDouble() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return this.asBoolean() ? 1.0d : 0.0d;
+ }
+
+ @Override
+ public String asString() {
+ if (null == super.getRawData()) {
+ return null;
+ }
+
+ return this.asBoolean() ? "true" : "false";
+ }
+
+ @Override
+ public BigInteger asBigInteger() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return BigInteger.valueOf(this.asLong());
+ }
+
+ @Override
+ public BigDecimal asBigDecimal() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return BigDecimal.valueOf(this.asLong());
+ }
+
+ @Override
+ public Date asDate() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bool类型不能转为Date .");
+ }
+
+ @Override
+ public byte[] asBytes() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Boolean类型不能转为Bytes .");
+ }
+
+ private void validate(final String data) {
+ if (null == data) {
+ return;
+ }
+
+ if ("true".equalsIgnoreCase(data) || "false".equalsIgnoreCase(data)) {
+ return;
+ }
+
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT,
+ String.format("String[%s]不能转为Bool .", data));
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/element/BytesColumn.java b/common/src/main/java/com/alibaba/datax/common/element/BytesColumn.java
new file mode 100755
index 000000000..d3cc59936
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/BytesColumn.java
@@ -0,0 +1,84 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+import org.apache.commons.lang3.ArrayUtils;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+/**
+ * Created by jingxing on 14-8-24.
+ */
+public class BytesColumn extends Column {
+
+ public BytesColumn() {
+ this(null);
+ }
+
+ public BytesColumn(byte[] bytes) {
+ super(ArrayUtils.clone(bytes), Column.Type.BYTES, null == bytes ? 0
+ : bytes.length);
+ }
+
+ @Override
+ public byte[] asBytes() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return (byte[]) this.getRawData();
+ }
+
+ @Override
+ public String asString() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ try {
+ return ColumnCast.bytes2String(this);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT,
+ String.format("Bytes[%s]不能转为String .", this.toString()));
+ }
+ }
+
+ @Override
+ public Long asLong() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为Long .");
+ }
+
+ @Override
+ public BigDecimal asBigDecimal() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为BigDecimal .");
+ }
+
+ @Override
+ public BigInteger asBigInteger() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为BigInteger .");
+ }
+
+ @Override
+ public Double asDouble() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为Long .");
+ }
+
+ @Override
+ public Date asDate() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为Date .");
+ }
+
+ @Override
+ public Boolean asBoolean() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为Boolean .");
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/element/Column.java b/common/src/main/java/com/alibaba/datax/common/element/Column.java
new file mode 100755
index 000000000..ed68e88d6
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/Column.java
@@ -0,0 +1,75 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.fastjson.JSON;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+/**
+ * Created by jingxing on 14-8-24.
+ *
+ */
+public abstract class Column {
+
+ private Type type;
+
+ private Object rawData;
+
+ private int byteSize;
+
+ public Column(final Object object, final Type type, int byteSize) {
+ this.rawData = object;
+ this.type = type;
+ this.byteSize = byteSize;
+ }
+
+ public Object getRawData() {
+ return this.rawData;
+ }
+
+ public Type getType() {
+ return this.type;
+ }
+
+ public int getByteSize() {
+ return this.byteSize;
+ }
+
+ protected void setType(Type type) {
+ this.type = type;
+ }
+
+ protected void setRawData(Object rawData) {
+ this.rawData = rawData;
+ }
+
+ protected void setByteSize(int byteSize) {
+ this.byteSize = byteSize;
+ }
+
+ public abstract Long asLong();
+
+ public abstract Double asDouble();
+
+ public abstract String asString();
+
+ public abstract Date asDate();
+
+ public abstract byte[] asBytes();
+
+ public abstract Boolean asBoolean();
+
+ public abstract BigDecimal asBigDecimal();
+
+ public abstract BigInteger asBigInteger();
+
+ @Override
+ public String toString() {
+ return JSON.toJSONString(this);
+ }
+
+ public enum Type {
+ BAD, NULL, INT, LONG, DOUBLE, STRING, BOOL, DATE, BYTES
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java b/common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java
new file mode 100755
index 000000000..89d0a7c62
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java
@@ -0,0 +1,199 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.util.Configuration;
+import org.apache.commons.lang3.time.DateFormatUtils;
+import org.apache.commons.lang3.time.FastDateFormat;
+
+import java.io.UnsupportedEncodingException;
+import java.text.ParseException;
+import java.util.*;
+
+public final class ColumnCast {
+
+ public static void bind(final Configuration configuration) {
+ StringCast.init(configuration);
+ DateCast.init(configuration);
+ BytesCast.init(configuration);
+ }
+
+ public static Date string2Date(final StringColumn column)
+ throws ParseException {
+ return StringCast.asDate(column);
+ }
+
+ public static byte[] string2Bytes(final StringColumn column)
+ throws UnsupportedEncodingException {
+ return StringCast.asBytes(column);
+ }
+
+ public static String date2String(final DateColumn column) {
+ return DateCast.asString(column);
+ }
+
+ public static String bytes2String(final BytesColumn column)
+ throws UnsupportedEncodingException {
+ return BytesCast.asString(column);
+ }
+}
+
+class StringCast {
+ static String datetimeFormat = "yyyy-MM-dd HH:mm:ss";
+
+ static String dateFormat = "yyyy-MM-dd";
+
+ static String timeFormat = "HH:mm:ss";
+
+ static List extraFormats = Collections.emptyList();
+
+ static String timeZone = "GMT+8";
+
+ static FastDateFormat dateFormatter;
+
+ static FastDateFormat timeFormatter;
+
+ static FastDateFormat datetimeFormatter;
+
+ static TimeZone timeZoner;
+
+ static String encoding = "UTF-8";
+
+ static void init(final Configuration configuration) {
+ StringCast.datetimeFormat = configuration.getString(
+ "common.column.datetimeFormat", StringCast.datetimeFormat);
+ StringCast.dateFormat = configuration.getString(
+ "common.column.dateFormat", StringCast.dateFormat);
+ StringCast.timeFormat = configuration.getString(
+ "common.column.timeFormat", StringCast.timeFormat);
+ StringCast.extraFormats = configuration.getList(
+ "common.column.extraFormats", Collections.emptyList(), String.class);
+
+ StringCast.timeZone = configuration.getString("common.column.timeZone",
+ StringCast.timeZone);
+ StringCast.timeZoner = TimeZone.getTimeZone(StringCast.timeZone);
+
+ StringCast.datetimeFormatter = FastDateFormat.getInstance(
+ StringCast.datetimeFormat, StringCast.timeZoner);
+ StringCast.dateFormatter = FastDateFormat.getInstance(
+ StringCast.dateFormat, StringCast.timeZoner);
+ StringCast.timeFormatter = FastDateFormat.getInstance(
+ StringCast.timeFormat, StringCast.timeZoner);
+
+ StringCast.encoding = configuration.getString("common.column.encoding",
+ StringCast.encoding);
+ }
+
+ static Date asDate(final StringColumn column) throws ParseException {
+ if (null == column.asString()) {
+ return null;
+ }
+
+ try {
+ return StringCast.datetimeFormatter.parse(column.asString());
+ } catch (ParseException ignored) {
+ }
+
+ try {
+ return StringCast.dateFormatter.parse(column.asString());
+ } catch (ParseException ignored) {
+ }
+
+ ParseException e;
+ try {
+ return StringCast.timeFormatter.parse(column.asString());
+ } catch (ParseException ignored) {
+ e = ignored;
+ }
+
+ for (String format : StringCast.extraFormats) {
+ try{
+ return FastDateFormat.getInstance(format, StringCast.timeZoner).parse(column.asString());
+ } catch (ParseException ignored){
+ e = ignored;
+ }
+ }
+ throw e;
+ }
+
+ static byte[] asBytes(final StringColumn column)
+ throws UnsupportedEncodingException {
+ if (null == column.asString()) {
+ return null;
+ }
+
+ return column.asString().getBytes(StringCast.encoding);
+ }
+}
+
+/**
+ * 后续为了可维护性，可以考虑直接使用 apache 的DateFormatUtils.
+ *
+ * 迟南已经修复了该问题，但是为了维护性，还是直接使用apache的内置函数
+ */
+class DateCast {
+
+ static String datetimeFormat = "yyyy-MM-dd HH:mm:ss";
+
+ static String dateFormat = "yyyy-MM-dd";
+
+ static String timeFormat = "HH:mm:ss";
+
+ static String timeZone = "GMT+8";
+
+ static TimeZone timeZoner = TimeZone.getTimeZone(DateCast.timeZone);
+
+ static void init(final Configuration configuration) {
+ DateCast.datetimeFormat = configuration.getString(
+ "common.column.datetimeFormat", datetimeFormat);
+ DateCast.timeFormat = configuration.getString(
+ "common.column.timeFormat", timeFormat);
+ DateCast.dateFormat = configuration.getString(
+ "common.column.dateFormat", dateFormat);
+ DateCast.timeZone = configuration.getString("common.column.timeZone",
+ DateCast.timeZone);
+ DateCast.timeZoner = TimeZone.getTimeZone(DateCast.timeZone);
+ return;
+ }
+
+ static String asString(final DateColumn column) {
+ if (null == column.asDate()) {
+ return null;
+ }
+
+ switch (column.getSubType()) {
+ case DATE:
+ return DateFormatUtils.format(column.asDate(), DateCast.dateFormat,
+ DateCast.timeZoner);
+ case TIME:
+ return DateFormatUtils.format(column.asDate(), DateCast.timeFormat,
+ DateCast.timeZoner);
+ case DATETIME:
+ return DateFormatUtils.format(column.asDate(),
+ DateCast.datetimeFormat, DateCast.timeZoner);
+ default:
+ throw DataXException
+ .asDataXException(CommonErrorCode.CONVERT_NOT_SUPPORT,
+ "时间类型出现不支持类型，目前仅支持DATE/TIME/DATETIME。该类型属于编程错误，请反馈给DataX开发团队 .");
+ }
+ }
+}
+
+class BytesCast {
+ static String encoding = "utf-8";
+
+ static void init(final Configuration configuration) {
+ BytesCast.encoding = configuration.getString("common.column.encoding",
+ BytesCast.encoding);
+ return;
+ }
+
+ static String asString(final BytesColumn column)
+ throws UnsupportedEncodingException {
+ if (null == column.asBytes()) {
+ return null;
+ }
+
+ return new String(column.asBytes(), encoding);
+ }
+}
\ No newline at end of file
diff --git a/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java b/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java
new file mode 100755
index 000000000..6626a6fbd
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java
@@ -0,0 +1,130 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+/**
+ * Created by jingxing on 14-8-24.
+ */
+public class DateColumn extends Column {
+
+ private DateType subType = DateType.DATETIME;
+
+ public static enum DateType {
+ DATE, TIME, DATETIME
+ }
+
+ /**
+ * 构建值为null的DateColumn，使用Date子类型为DATETIME
+ * */
+ public DateColumn() {
+ this((Long)null);
+ }
+
+ /**
+ * 构建值为stamp(Unix时间戳)的DateColumn，使用Date子类型为DATETIME
+ * 实际存储有date改为long的ms，节省存储
+ * */
+ public DateColumn(final Long stamp) {
+ super(stamp, Column.Type.DATE, (null == stamp ? 0 : 8));
+ }
+
+ /**
+ * 构建值为date(java.util.Date)的DateColumn，使用Date子类型为DATETIME
+ * */
+ public DateColumn(final Date date) {
+ this(date == null ? null : date.getTime());
+ }
+
+ /**
+ * 构建值为date(java.sql.Date)的DateColumn，使用Date子类型为DATE，只有日期，没有时间
+ * */
+ public DateColumn(final java.sql.Date date) {
+ this(date == null ? null : date.getTime());
+ this.setSubType(DateType.DATE);
+ }
+
+ /**
+ * 构建值为time(java.sql.Time)的DateColumn，使用Date子类型为TIME，只有时间，没有日期
+ * */
+ public DateColumn(final java.sql.Time time) {
+ this(time == null ? null : time.getTime());
+ this.setSubType(DateType.TIME);
+ }
+
+ /**
+ * 构建值为ts(java.sql.Timestamp)的DateColumn，使用Date子类型为DATETIME
+ * */
+ public DateColumn(final java.sql.Timestamp ts) {
+ this(ts == null ? null : ts.getTime());
+ this.setSubType(DateType.DATETIME);
+ }
+
+ @Override
+ public Long asLong() {
+
+ return (Long)this.getRawData();
+ }
+
+ @Override
+ public String asString() {
+ try {
+ return ColumnCast.date2String(this);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT,
+ String.format("Date[%s]类型不能转为String .", this.toString()));
+ }
+ }
+
+ @Override
+ public Date asDate() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return new Date((Long)this.getRawData());
+ }
+
+ @Override
+ public byte[] asBytes() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Date类型不能转为Bytes .");
+ }
+
+ @Override
+ public Boolean asBoolean() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Date类型不能转为Boolean .");
+ }
+
+ @Override
+ public Double asDouble() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Date类型不能转为Double .");
+ }
+
+ @Override
+ public BigInteger asBigInteger() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Date类型不能转为BigInteger .");
+ }
+
+ @Override
+ public BigDecimal asBigDecimal() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Date类型不能转为BigDecimal .");
+ }
+
+ public DateType getSubType() {
+ return subType;
+ }
+
+ public void setSubType(DateType subType) {
+ this.subType = subType;
+ }
+}
\ No newline at end of file
diff --git a/common/src/main/java/com/alibaba/datax/common/element/DoubleColumn.java b/common/src/main/java/com/alibaba/datax/common/element/DoubleColumn.java
new file mode 100755
index 000000000..17170ea6c
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/DoubleColumn.java
@@ -0,0 +1,161 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+public class DoubleColumn extends Column {
+
+ public DoubleColumn(final String data) {
+ this(data, null == data ? 0 : data.length());
+ this.validate(data);
+ }
+
+ public DoubleColumn(Long data) {
+ this(data == null ? (String) null : String.valueOf(data));
+ }
+
+ public DoubleColumn(Integer data) {
+ this(data == null ? (String) null : String.valueOf(data));
+ }
+
+ /**
+ * Double无法表示准确的小数数据，我们不推荐使用该方法保存Double数据，建议使用String作为构造入参
+ *
+ * */
+ public DoubleColumn(final Double data) {
+ this(data == null ? (String) null
+ : new BigDecimal(String.valueOf(data)).toPlainString());
+ }
+
+ /**
+ * Float无法表示准确的小数数据，我们不推荐使用该方法保存Float数据，建议使用String作为构造入参
+ *
+ * */
+ public DoubleColumn(final Float data) {
+ this(data == null ? (String) null
+ : new BigDecimal(String.valueOf(data)).toPlainString());
+ }
+
+ public DoubleColumn(final BigDecimal data) {
+ this(null == data ? (String) null : data.toPlainString());
+ }
+
+ public DoubleColumn(final BigInteger data) {
+ this(null == data ? (String) null : data.toString());
+ }
+
+ public DoubleColumn() {
+ this((String) null);
+ }
+
+ private DoubleColumn(final String data, int byteSize) {
+ super(data, Column.Type.DOUBLE, byteSize);
+ }
+
+ @Override
+ public BigDecimal asBigDecimal() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ try {
+ return new BigDecimal((String) this.getRawData());
+ } catch (NumberFormatException e) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT,
+ String.format("String[%s] 无法转换为Double类型 .",
+ (String) this.getRawData()));
+ }
+ }
+
+ @Override
+ public Double asDouble() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ String string = (String) this.getRawData();
+
+ boolean isDoubleSpecific = string.equals("NaN")
+ || string.equals("-Infinity") || string.equals("+Infinity");
+ if (isDoubleSpecific) {
+ return Double.valueOf(string);
+ }
+
+ BigDecimal result = this.asBigDecimal();
+ OverFlowUtil.validateDoubleNotOverFlow(result);
+
+ return result.doubleValue();
+ }
+
+ @Override
+ public Long asLong() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ BigDecimal result = this.asBigDecimal();
+ OverFlowUtil.validateLongNotOverFlow(result.toBigInteger());
+
+ return result.longValue();
+ }
+
+ @Override
+ public BigInteger asBigInteger() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+
+ return this.asBigDecimal().toBigInteger();
+ }
+
+ @Override
+ public String asString() {
+ if (null == this.getRawData()) {
+ return null;
+ }
+ return (String) this.getRawData();
+ }
+
+ @Override
+ public Boolean asBoolean() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Double类型无法转为Bool .");
+ }
+
+ @Override
+ public Date asDate() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Double类型无法转为Date类型 .");
+ }
+
+ @Override
+ public byte[] asBytes() {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT, "Double类型无法转为Bytes类型 .");
+ }
+
+ private void validate(final String data) {
+ if (null == data) {
+ return;
+ }
+
+ if (data.equalsIgnoreCase("NaN") || data.equalsIgnoreCase("-Infinity")
+ || data.equalsIgnoreCase("Infinity")) {
+ return;
+ }
+
+ try {
+ new BigDecimal(data);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONVERT_NOT_SUPPORT,
+ String.format("String[%s]无法转为Double类型 .", data));
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/common/src/main/java/com/alibaba/datax/common/element/LongColumn.java b/common/src/main/java/com/alibaba/datax/common/element/LongColumn.java
new file mode 100755
index 000000000..d8113f7c0
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/element/LongColumn.java
@@ -0,0 +1,135 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+import org.apache.commons.lang3.math.NumberUtils;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+public class LongColumn extends Column {
+
+ /**
+ * 从整形字符串表示转为LongColumn，支持Java科学计数法
+ *
+ * NOTE: > getMessage();
+
+ /**
+ * 从Task获取自定义收集信息
+ *
+ * */
+ List getMessage(String key);
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/plugin/PluginCollector.java b/common/src/main/java/com/alibaba/datax/common/plugin/PluginCollector.java
new file mode 100755
index 000000000..f2af398dd
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/plugin/PluginCollector.java
@@ -0,0 +1,9 @@
+package com.alibaba.datax.common.plugin;
+
+
+/**
+ * 这里只是一个标示类
+ * */
+public interface PluginCollector {
+
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/plugin/Pluginable.java b/common/src/main/java/com/alibaba/datax/common/plugin/Pluginable.java
new file mode 100755
index 000000000..ac28f6a29
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/plugin/Pluginable.java
@@ -0,0 +1,30 @@
+package com.alibaba.datax.common.plugin;
+
+import com.alibaba.datax.common.util.Configuration;
+
+public interface Pluginable {
+ String getDeveloper();
+
+ String getDescription();
+
+ void setPluginConf(Configuration pluginConf);
+
+ void init();
+
+ void destroy();
+
+ String getPluginName();
+
+ Configuration getPluginJobConf();
+
+ Configuration getPeerPluginJobConf();
+
+ public String getPeerPluginName();
+
+ void setPluginJobConf(Configuration jobConf);
+
+ void setPeerPluginJobConf(Configuration peerPluginJobConf);
+
+ public void setPeerPluginName(String peerPluginName);
+
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/plugin/RecordReceiver.java b/common/src/main/java/com/alibaba/datax/common/plugin/RecordReceiver.java
new file mode 100755
index 000000000..74f236f37
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/plugin/RecordReceiver.java
@@ -0,0 +1,26 @@
+/**
+ * (C) 2010-2013 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.datax.common.plugin;
+
+import com.alibaba.datax.common.element.Record;
+
+public interface RecordReceiver {
+
+ public Record getFromReader();
+
+ public void shutdown();
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/plugin/RecordSender.java b/common/src/main/java/com/alibaba/datax/common/plugin/RecordSender.java
new file mode 100755
index 000000000..0d6926098
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/plugin/RecordSender.java
@@ -0,0 +1,32 @@
+/**
+ * (C) 2010-2013 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.datax.common.plugin;
+
+import com.alibaba.datax.common.element.Record;
+
+public interface RecordSender {
+
+ public Record createRecord();
+
+ public void sendToWriter(Record record);
+
+ public void flush();
+
+ public void terminate();
+
+ public void shutdown();
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/plugin/TaskPluginCollector.java b/common/src/main/java/com/alibaba/datax/common/plugin/TaskPluginCollector.java
new file mode 100755
index 000000000..f0c85fe6c
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/plugin/TaskPluginCollector.java
@@ -0,0 +1,57 @@
+package com.alibaba.datax.common.plugin;
+
+import com.alibaba.datax.common.element.Record;
+
+/**
+ *
+ * 该接口提供给Task Plugin用来记录脏数据和自定义信息。
+ *
+ * @Override
+ * public String toString() {
+ * return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
+ * }
+ *
+ *
+ */
+public interface ErrorCode {
+ // 错误码编号
+ String getCode();
+
+ // 错误码描述
+ String getDescription();
+
+ /** 必须提供toString的实现
+ *
+ *
+ * @Override
+ * public String toString() {
+ * return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
+ * }
+ *
+ *
+ */
+ String toString();
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/spi/Hook.java b/common/src/main/java/com/alibaba/datax/common/spi/Hook.java
new file mode 100755
index 000000000..d510f57c1
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/spi/Hook.java
@@ -0,0 +1,27 @@
+package com.alibaba.datax.common.spi;
+
+import com.alibaba.datax.common.util.Configuration;
+
+import java.util.Map;
+
+/**
+ * Created by xiafei.qiuxf on 14/12/17.
+ */
+public interface Hook {
+
+ /**
+ * 返回名字
+ *
+ * @return
+ */
+ public String getName();
+
+ /**
+ * TODO 文档
+ *
+ * @param jobConf
+ * @param msg
+ */
+ public void invoke(Configuration jobConf, Map msg);
+
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/spi/Reader.java b/common/src/main/java/com/alibaba/datax/common/spi/Reader.java
new file mode 100755
index 000000000..fec41a9f0
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/spi/Reader.java
@@ -0,0 +1,52 @@
+package com.alibaba.datax.common.spi;
+
+import java.util.List;
+
+import com.alibaba.datax.common.base.BaseObject;
+import com.alibaba.datax.common.plugin.AbstractJobPlugin;
+import com.alibaba.datax.common.plugin.AbstractTaskPlugin;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.common.plugin.RecordSender;
+
+/**
+ * 每个Reader插件在其内部内部实现Job、Task两个内部类。
+ *
+ *
+ * */
+public abstract class Reader extends BaseObject {
+
+ /**
+ * 每个Reader插件必须实现Job内部类。
+ *
+ * */
+ public static abstract class Job extends AbstractJobPlugin {
+
+ /**
+ * 切分任务
+ *
+ * @param adviceNumber
+ *
+ * 着重说明下，adviceNumber是框架建议插件切分的任务数，插件开发人员最好切分出来的任务数>=
+ * adviceNumber。 split(int adviceNumber);
+ }
+
+ public static abstract class Task extends AbstractTaskPlugin {
+ public abstract void startRead(RecordSender recordSender);
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/spi/Writer.java b/common/src/main/java/com/alibaba/datax/common/spi/Writer.java
new file mode 100755
index 000000000..457eb6860
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/spi/Writer.java
@@ -0,0 +1,40 @@
+package com.alibaba.datax.common.spi;
+
+import com.alibaba.datax.common.base.BaseObject;
+import com.alibaba.datax.common.plugin.AbstractJobPlugin;
+import com.alibaba.datax.common.plugin.AbstractTaskPlugin;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.common.plugin.RecordReceiver;
+
+import java.util.List;
+
+/**
+ * 每个Writer插件需要实现Writer类，并在其内部实现Job、Task两个内部类。
+ *
+ *
+ * */
+public abstract class Writer extends BaseObject {
+ /**
+ * 每个Writer插件必须实现Job内部类
+ */
+ public abstract static class Job extends AbstractJobPlugin {
+ /**
+ * 切分任务。 split(int mandatoryNumber);
+ }
+
+ /**
+ * 每个Writer插件必须实现Task内部类
+ */
+ public abstract static class Task extends AbstractTaskPlugin {
+
+ public abstract void startWrite(RecordReceiver lineReceiver);
+
+ public boolean supportFailOver(){return false;}
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/statistics/PerfRecord.java b/common/src/main/java/com/alibaba/datax/common/statistics/PerfRecord.java
new file mode 100644
index 000000000..5174fcad2
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/statistics/PerfRecord.java
@@ -0,0 +1,246 @@
+package com.alibaba.datax.common.statistics;
+
+import com.alibaba.datax.common.util.HostUtils;
+import com.google.common.base.Objects;
+import org.apache.commons.lang3.time.DateFormatUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Date;
+
+/**
+ * Created by liqiang on 15/8/23.
+ */
+public class PerfRecord implements Comparable {
+ private static Logger perf = LoggerFactory.getLogger(PerfRecord.class);
+ private static String datetimeFormat = "yyyy-MM-dd HH:mm:ss";
+
+
+ public enum PHASE {
+ /**
+ * task total运行的时间，前10为框架统计，后面为部分插件的个性统计
+ */
+ TASK_TOTAL(0),
+
+ READ_TASK_INIT(1),
+ READ_TASK_PREPARE(2),
+ READ_TASK_DATA(3),
+ READ_TASK_POST(4),
+ READ_TASK_DESTROY(5),
+
+ WRITE_TASK_INIT(6),
+ WRITE_TASK_PREPARE(7),
+ WRITE_TASK_DATA(8),
+ WRITE_TASK_POST(9),
+ WRITE_TASK_DESTROY(10),
+
+ /**
+ * SQL_QUERY: sql query阶段, 部分reader的个性统计
+ */
+ SQL_QUERY(100),
+ /**
+ * 数据从sql全部读出来
+ */
+ RESULT_NEXT_ALL(101),
+
+ /**
+ * only odps block close
+ */
+ ODPS_BLOCK_CLOSE(102),
+
+ WAIT_READ_TIME(103),
+
+ WAIT_WRITE_TIME(104);
+
+ private int val;
+
+ PHASE(int val) {
+ this.val = val;
+ }
+
+ public int toInt(){
+ return val;
+ }
+ }
+
+ public enum ACTION{
+ start,
+ end
+ }
+
+ private final int taskGroupId;
+ private final int taskId;
+ private final PHASE phase;
+ private volatile ACTION action;
+ private volatile Date startTime;
+ private volatile long elapsedTimeInNs = -1;
+ private volatile long count = 0;
+ private volatile long size = 0;
+
+ private volatile long startTimeInNs;
+ private volatile boolean isReport = false;
+
+ public PerfRecord(int taskGroupId, int taskId, PHASE phase) {
+ this.taskGroupId = taskGroupId;
+ this.taskId = taskId;
+ this.phase = phase;
+ }
+
+ public static void addPerfRecord(int taskGroupId, int taskId, PHASE phase, long startTime,long elapsedTimeInNs) {
+ if(PerfTrace.getInstance().isEnable()) {
+ PerfRecord perfRecord = new PerfRecord(taskGroupId, taskId, phase);
+ perfRecord.elapsedTimeInNs = elapsedTimeInNs;
+ perfRecord.action = ACTION.end;
+ perfRecord.startTime = new Date(startTime);
+ //在PerfTrace里注册
+ PerfTrace.getInstance().tracePerfRecord(perfRecord);
+ perf.info(perfRecord.toString());
+ }
+ }
+
+ public void start() {
+ if(PerfTrace.getInstance().isEnable()) {
+ this.startTime = new Date();
+ this.startTimeInNs = System.nanoTime();
+ this.action = ACTION.start;
+ //在PerfTrace里注册
+ PerfTrace.getInstance().tracePerfRecord(this);
+ perf.info(toString());
+ }
+ }
+
+ public void addCount(long count) {
+ this.count += count;
+ }
+
+ public void addSize(long size) {
+ this.size += size;
+ }
+
+ public void end() {
+ if(PerfTrace.getInstance().isEnable()) {
+ this.elapsedTimeInNs = System.nanoTime() - startTimeInNs;
+ this.action = ACTION.end;
+ PerfTrace.getInstance().tracePerfRecord(this);
+ perf.info(toString());
+ }
+ }
+
+ public void end(long elapsedTimeInNs) {
+ if(PerfTrace.getInstance().isEnable()) {
+ this.elapsedTimeInNs = elapsedTimeInNs;
+ this.action = ACTION.end;
+ perf.info(toString());
+ }
+ }
+
+ public String toString() {
+ return String.format("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s"
+ , getJobId(), taskGroupId, taskId, phase, action,
+ DateFormatUtils.format(startTime, datetimeFormat), elapsedTimeInNs, count, size,getHostIP());
+ }
+
+
+ @Override
+ public int compareTo(PerfRecord o) {
+ if (o == null) {
+ return 1;
+ }
+ return this.elapsedTimeInNs > o.elapsedTimeInNs ? 1 : this.elapsedTimeInNs == o.elapsedTimeInNs ? 0 : -1;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(getJobId(),taskGroupId,taskId,phase,startTime);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if(!(o instanceof PerfRecord)){
+ return false;
+ }
+
+ PerfRecord dst = (PerfRecord)o;
+
+ if(!Objects.equal(this.getJobId(),dst.getJobId())) return false;
+ if(!Objects.equal(this.taskGroupId,dst.taskGroupId)) return false;
+ if(!Objects.equal(this.taskId,dst.taskId)) return false;
+ if(!Objects.equal(this.phase,dst.phase)) return false;
+ if(!Objects.equal(this.startTime,dst.startTime)) return false;
+
+ return true;
+ }
+
+ public PerfRecord copy() {
+ PerfRecord copy = new PerfRecord(this.taskGroupId, this.getTaskId(), this.phase);
+ copy.action = this.action;
+ copy.startTime = this.startTime;
+ copy.elapsedTimeInNs = this.elapsedTimeInNs;
+ copy.count = this.count;
+ copy.size = this.size;
+ return copy;
+ }
+ public int getTaskGroupId() {
+ return taskGroupId;
+ }
+
+ public int getTaskId() {
+ return taskId;
+ }
+
+ public PHASE getPhase() {
+ return phase;
+ }
+
+ public ACTION getAction() {
+ return action;
+ }
+
+ public long getElapsedTimeInNs() {
+ return elapsedTimeInNs;
+ }
+
+ public long getCount() {
+ return count;
+ }
+
+ public long getSize() {
+ return size;
+ }
+
+ public long getJobId(){
+ return PerfTrace.getInstance().getJobId();
+ }
+
+ public String getHostIP(){
+ return HostUtils.IP;
+ }
+
+ public String getHostName(){
+ return HostUtils.HOSTNAME;
+ }
+
+ public Date getStartTime() {
+ return startTime;
+ }
+
+ public long getStartTimeInNs() {
+ return startTimeInNs;
+ }
+
+ public String getDatetime(){
+ if(startTime == null){
+ return "null time";
+ }
+ return DateFormatUtils.format(startTime, datetimeFormat);
+ }
+
+ public boolean isReport() {
+ return isReport;
+ }
+
+ public void setIsReport(boolean isReport) {
+ this.isReport = isReport;
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/statistics/PerfTrace.java b/common/src/main/java/com/alibaba/datax/common/statistics/PerfTrace.java
new file mode 100644
index 000000000..eb0706079
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/statistics/PerfTrace.java
@@ -0,0 +1,422 @@
+package com.alibaba.datax.common.statistics;
+
+import com.alibaba.datax.common.util.Configuration;
+import com.google.common.base.Optional;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * PerfTrace 记录 job（local模式），taskGroup（distribute模式），因为这2种都是jvm，即一个jvm里只需要有1个PerfTrace。
+ */
+
+public class PerfTrace {
+
+ private static Logger LOG = LoggerFactory.getLogger(PerfTrace.class);
+ private static PerfTrace instance;
+ private static final Object lock = new Object();
+ private String perfTraceId;
+ private volatile boolean enable;
+ private volatile boolean isJob;
+ private long jobId;
+ private int priority;
+ private int batchSize = 500;
+ private volatile boolean perfReportEnalbe = true;
+
+ //jobid_jobversion,instanceid,taskid, src_mark, dst_mark,
+ private Map taskDetails = new ConcurrentHashMap();
+ //PHASE => PerfRecord
+ private ConcurrentHashMap perfRecordMaps = new ConcurrentHashMap();
+ private Configuration jobInfo;
+ private final List startReportPool = new ArrayList();
+ private final List endReportPool = new ArrayList();
+ private final List totalEndReport = new ArrayList();
+ private final Set waitingReportSet = new HashSet();
+
+
+ /**
+ * 单实例
+ *
+ * @param isJob
+ * @param jobId
+ * @param taskGroupId
+ * @return
+ */
+ public static PerfTrace getInstance(boolean isJob, long jobId, int taskGroupId,int priority, boolean enable) {
+
+ if (instance == null) {
+ synchronized (lock) {
+ if (instance == null) {
+ instance = new PerfTrace(isJob, jobId, taskGroupId,priority, enable);
+ }
+ }
+ }
+ return instance;
+ }
+
+ /**
+ * 因为一个JVM只有一个，因此在getInstance(isJob,jobId,taskGroupId)调用完成实例化后，方便后续调用，直接返回该实例
+ *
+ * @return
+ */
+ public static PerfTrace getInstance() {
+ if (instance == null) {
+ LOG.error("PerfTrace instance not be init! must have some error! ");
+ synchronized (lock) {
+ if (instance == null) {
+ instance = new PerfTrace(false, -1111, -1111, 0, false);
+ }
+ }
+ }
+ return instance;
+ }
+
+ private PerfTrace(boolean isJob, long jobId, int taskGroupId, int priority, boolean enable) {
+ this.perfTraceId = isJob ? "job_" + jobId : String.format("taskGroup_%s_%s", jobId, taskGroupId);
+ this.enable = enable;
+ this.isJob = isJob;
+ this.jobId = jobId;
+ this.priority = priority;
+ LOG.info(String.format("PerfTrace traceId=%s, isEnable=%s, priority=%s", this.perfTraceId, this.enable, this.priority));
+
+ }
+
+ public void addTaskDetails(int taskId, String detail) {
+ if (enable) {
+ String before = "";
+ int index = detail.indexOf("?");
+ String current = detail.substring(0, index == -1 ? detail.length() : index);
+ if(current.indexOf("[")>=0){
+ current+="]";
+ }
+ if (taskDetails.containsKey(taskId)) {
+ before = taskDetails.get(taskId).trim();
+ }
+ if (StringUtils.isEmpty(before)) {
+ before = "";
+ } else {
+ before += ",";
+ }
+ this.taskDetails.put(taskId, before + current);
+ }
+ }
+
+ public void tracePerfRecord(PerfRecord perfRecord) {
+ if (enable) {
+ //ArrayList非线程安全
+ switch (perfRecord.getAction()) {
+ case start:
+ synchronized (startReportPool) {
+ startReportPool.add(perfRecord);
+ }
+ break;
+ case end:
+ synchronized (endReportPool) {
+ endReportPool.add(perfRecord);
+ }
+ break;
+ }
+ }
+ }
+
+ public String summarizeNoException(){
+ String res;
+ try {
+ res = summarize();
+ } catch (Exception e) {
+ res = "PerfTrace summarize has Exception "+e.getMessage();
+ }
+ return res;
+ }
+
+ //任务结束时，对当前的perf总汇总统计
+ private synchronized String summarize() {
+ if (!enable) {
+ return "PerfTrace not enable!";
+ }
+
+ if (totalEndReport.size() > 0) {
+ sumEndPerfRecords(totalEndReport);
+ }
+
+ StringBuilder info = new StringBuilder();
+ info.append("\n === total summarize info === \n");
+ info.append("\n 1. all phase average time info and max time task info: \n\n");
+ info.append(String.format("%-20s | %18s | %18s | %18s | %18s | %-100s\n", "PHASE", "AVERAGE USED TIME", "ALL TASK NUM", "MAX USED TIME", "MAX TASK ID", "MAX TASK INFO"));
+
+ List keys = new ArrayList(perfRecordMaps.keySet());
+ Collections.sort(keys, new Comparator() {
+ @Override
+ public int compare(PerfRecord.PHASE o1, PerfRecord.PHASE o2) {
+ return o1.toInt() - o2.toInt();
+ }
+ });
+ for (PerfRecord.PHASE phase : keys) {
+ SumPerfRecord sumPerfRecord = perfRecordMaps.get(phase);
+ if (sumPerfRecord == null) {
+ continue;
+ }
+ long averageTime = sumPerfRecord.getAverageTime();
+ long maxTime = sumPerfRecord.getMaxTime();
+ int maxTaskId = sumPerfRecord.maxTaskId;
+ int maxTaskGroupId = sumPerfRecord.getMaxTaskGroupId();
+ info.append(String.format("%-20s | %18s | %18s | %18s | %18s | %-100s\n",
+ phase, unitTime(averageTime), sumPerfRecord.totalCount, unitTime(maxTime), jobId + "-" + maxTaskGroupId + "-" + maxTaskId, taskDetails.get(maxTaskId)));
+ }
+
+ SumPerfRecord countSumPerf = Optional.fromNullable(perfRecordMaps.get(PerfRecord.PHASE.READ_TASK_DATA)).or(new SumPerfRecord());
+
+ long averageRecords = countSumPerf.getAverageRecords();
+ long averageBytes = countSumPerf.getAverageBytes();
+ long maxRecord = countSumPerf.getMaxRecord();
+ long maxByte = countSumPerf.getMaxByte();
+ int maxTaskId4Records = countSumPerf.getMaxTaskId4Records();
+ int maxTGID4Records = countSumPerf.getMaxTGID4Records();
+
+ info.append("\n\n 2. record average count and max count task info :\n\n");
+ info.append(String.format("%-20s | %18s | %18s | %18s | %18s | %18s | %-100s\n", "PHASE", "AVERAGE RECORDS", "AVERAGE BYTES", "MAX RECORDS", "MAX RECORD`S BYTES", "MAX TASK ID", "MAX TASK INFO"));
+ if (maxTaskId4Records > -1) {
+ info.append(String.format("%-20s | %18s | %18s | %18s | %18s | %18s | %-100s\n"
+ , PerfRecord.PHASE.READ_TASK_DATA, averageRecords, unitSize(averageBytes), maxRecord, unitSize(maxByte), jobId + "-" + maxTGID4Records + "-" + maxTaskId4Records, taskDetails.get(maxTaskId4Records)));
+
+ }
+ return info.toString();
+ }
+
+ //缺省传入的时间是nano
+ public static String unitTime(long time) {
+ return unitTime(time, TimeUnit.NANOSECONDS);
+ }
+
+ public static String unitTime(long time, TimeUnit timeUnit) {
+ return String.format("%,.3fs", ((float) timeUnit.toNanos(time)) / 1000000000);
+ }
+
+ public static String unitSize(long size) {
+ if (size > 1000000000) {
+ return String.format("%,.2fG", (float) size / 1000000000);
+ } else if (size > 1000000) {
+ return String.format("%,.2fM", (float) size / 1000000);
+ } else if (size > 1000) {
+ return String.format("%,.2fK", (float) size / 1000);
+ } else {
+ return size + "B";
+ }
+ }
+
+
+ public synchronized ConcurrentHashMap getPerfRecordMaps() {
+ synchronized (endReportPool) {
+ // perfRecordMaps.get(perfRecord.getPhase()).add(perfRecord);
+ waitingReportSet.addAll(endReportPool);
+ totalEndReport.addAll(endReportPool);
+ endReportPool.clear();
+ }
+ if(totalEndReport.size() > 0 ){
+ sumEndPerfRecords(totalEndReport);
+ }
+ return perfRecordMaps;
+ }
+
+ public List getWaitingReportList() {
+ return new ArrayList(waitingReportSet);
+ }
+
+ public List getStartReportPool() {
+ return startReportPool;
+ }
+
+ public List getEndReportPool() {
+ return endReportPool;
+ }
+
+ public List getTotalEndReport() {
+ return totalEndReport;
+ }
+
+ public Map getTaskDetails() {
+ return taskDetails;
+ }
+
+ public boolean isEnable() {
+ return enable;
+ }
+
+ public boolean isJob() {
+ return isJob;
+ }
+
+ public long getJobId() {
+ return jobId;
+ }
+
+ private String cluster;
+ private String jobDomain;
+ private String srcType;
+ private String dstType;
+ private String srcGuid;
+ private String dstGuid;
+ private String dataxType;
+
+ public void setJobInfo(Configuration jobInfo) {
+ this.jobInfo = jobInfo;
+ if (jobInfo != null) {
+ cluster = jobInfo.getString("cluster");
+
+ String srcDomain = jobInfo.getString("srcDomain", "null");
+ String dstDomain = jobInfo.getString("dstDomain", "null");
+ jobDomain = srcDomain + "|" + dstDomain;
+ srcType = jobInfo.getString("srcType");
+ dstType = jobInfo.getString("dstType");
+ srcGuid = jobInfo.getString("srcGuid");
+ dstGuid = jobInfo.getString("dstGuid");
+ long jobId = jobInfo.getLong("jobId");
+ if (jobId > 0) {
+ //同步中心任务
+ dataxType = "dsc";
+ } else {
+ dataxType = "datax3";
+ }
+ } else {
+ dataxType = "datax3";
+ }
+ }
+
+ public Configuration getJobInfo() {
+ return jobInfo;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ public void setPerfReportEnalbe(boolean perfReportEnalbe) {
+ this.perfReportEnalbe = perfReportEnalbe;
+ }
+
+
+ private void sumEndPerfRecords(List totalEndReport) {
+ if (!enable || totalEndReport == null) {
+ return;
+ }
+
+ for (PerfRecord perfRecord : totalEndReport) {
+ perfRecordMaps.putIfAbsent(perfRecord.getPhase(), new SumPerfRecord());
+ perfRecordMaps.get(perfRecord.getPhase()).add(perfRecord);
+ }
+
+ totalEndReport.clear();
+ }
+
+
+
+ public static class SumPerfRecord {
+ private long perfTimeTotal = 0;
+ private long averageTime = 0;
+ private long maxTime = 0;
+ private int maxTaskId = -1;
+ private int maxTaskGroupId = -1;
+ private int totalCount = 0;
+
+ private long recordsTotal = 0;
+ private long sizesTotal = 0;
+ private long averageRecords = 0;
+ private long averageBytes = 0;
+ private long maxRecord = 0;
+ private long maxByte = 0;
+ private int maxTaskId4Records = -1;
+ private int maxTGID4Records = -1;
+
+ synchronized void add(PerfRecord perfRecord) {
+ if (perfRecord == null) {
+ return;
+ }
+ perfTimeTotal += perfRecord.getElapsedTimeInNs();
+ if (perfRecord.getElapsedTimeInNs() > maxTime) {
+ maxTime = perfRecord.getElapsedTimeInNs();
+ maxTaskId = perfRecord.getTaskId();
+ maxTaskGroupId = perfRecord.getTaskGroupId();
+ }
+
+ recordsTotal += perfRecord.getCount();
+ sizesTotal += perfRecord.getSize();
+ if (perfRecord.getCount() > maxRecord) {
+ maxRecord = perfRecord.getCount();
+ maxByte = perfRecord.getSize();
+ maxTaskId4Records = perfRecord.getTaskId();
+ maxTGID4Records = perfRecord.getTaskGroupId();
+ }
+
+ totalCount++;
+ }
+
+ public long getPerfTimeTotal() {
+ return perfTimeTotal;
+ }
+
+ public long getAverageTime() {
+ if (totalCount > 0) {
+ averageTime = perfTimeTotal / totalCount;
+ }
+ return averageTime;
+ }
+
+ public long getMaxTime() {
+ return maxTime;
+ }
+
+ public int getMaxTaskId() {
+ return maxTaskId;
+ }
+
+ public int getMaxTaskGroupId() {
+ return maxTaskGroupId;
+ }
+
+ public long getRecordsTotal() {
+ return recordsTotal;
+ }
+
+ public long getSizesTotal() {
+ return sizesTotal;
+ }
+
+ public long getAverageRecords() {
+ if (totalCount > 0) {
+ averageRecords = recordsTotal / totalCount;
+ }
+ return averageRecords;
+ }
+
+ public long getAverageBytes() {
+ if (totalCount > 0) {
+ averageBytes = sizesTotal / totalCount;
+ }
+ return averageBytes;
+ }
+
+ public long getMaxRecord() {
+ return maxRecord;
+ }
+
+ public long getMaxByte() {
+ return maxByte;
+ }
+
+ public int getMaxTaskId4Records() {
+ return maxTaskId4Records;
+ }
+
+ public int getMaxTGID4Records() {
+ return maxTGID4Records;
+ }
+
+ public int getTotalCount() {
+ return totalCount;
+ }
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/statistics/VMInfo.java b/common/src/main/java/com/alibaba/datax/common/statistics/VMInfo.java
new file mode 100644
index 000000000..85535fddd
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/statistics/VMInfo.java
@@ -0,0 +1,412 @@
+package com.alibaba.datax.common.statistics;
+
+import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.MemoryPoolMXBean;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.management.RuntimeMXBean;
+import java.lang.reflect.Method;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by liqiang on 15/11/12.
+ */
+public class VMInfo {
+ private static final Logger LOG = LoggerFactory.getLogger(VMInfo.class);
+ static final long MB = 1024 * 1024;
+ static final long GB = 1024 * 1024 * 1024;
+ public static Object lock = new Object();
+ private static VMInfo vmInfo;
+
+ /**
+ * @return null or vmInfo. null is something error, job no care it.
+ */
+ public static VMInfo getVmInfo() {
+ if (vmInfo == null) {
+ synchronized (lock) {
+ if (vmInfo == null) {
+ try {
+ vmInfo = new VMInfo();
+ } catch (Exception e) {
+ LOG.warn("no need care, the fail is ignored : vmInfo init failed " + e.getMessage(), e);
+ }
+ }
+ }
+
+ }
+ return vmInfo;
+ }
+
+ // 数据的MxBean
+ private final OperatingSystemMXBean osMXBean;
+ private final RuntimeMXBean runtimeMXBean;
+ private final List garbageCollectorMXBeanList;
+ private final List memoryPoolMXBeanList;
+ /**
+ * 静态信息
+ */
+ private final String osInfo;
+ private final String jvmInfo;
+
+ /**
+ * cpu个数
+ */
+ private final int totalProcessorCount;
+
+ /**
+ * 机器的各个状态，用于中间打印和统计上报
+ */
+ private final PhyOSStatus startPhyOSStatus;
+ private final ProcessCpuStatus processCpuStatus = new ProcessCpuStatus();
+ private final ProcessGCStatus processGCStatus = new ProcessGCStatus();
+ private final ProcessMemoryStatus processMomoryStatus = new ProcessMemoryStatus();
+ //ms
+ private long lastUpTime = 0;
+ //nano
+ private long lastProcessCpuTime = 0;
+
+
+ private VMInfo() {
+ //初始化静态信息
+ osMXBean = java.lang.management.ManagementFactory.getOperatingSystemMXBean();
+ runtimeMXBean = java.lang.management.ManagementFactory.getRuntimeMXBean();
+ garbageCollectorMXBeanList = java.lang.management.ManagementFactory.getGarbageCollectorMXBeans();
+ memoryPoolMXBeanList = java.lang.management.ManagementFactory.getMemoryPoolMXBeans();
+
+ osInfo = runtimeMXBean.getVmVendor() + " " + runtimeMXBean.getSpecVersion() + " " + runtimeMXBean.getVmVersion();
+ jvmInfo = osMXBean.getName() + " " + osMXBean.getArch() + " " + osMXBean.getVersion();
+ totalProcessorCount = osMXBean.getAvailableProcessors();
+
+ //构建startPhyOSStatus
+ startPhyOSStatus = new PhyOSStatus();
+ LOG.info("VMInfo# operatingSystem class => " + osMXBean.getClass().getName());
+ if (VMInfo.isSunOsMBean(osMXBean)) {
+ {
+ startPhyOSStatus.totalPhysicalMemory = VMInfo.getLongFromOperatingSystem(osMXBean, "getTotalPhysicalMemorySize");
+ startPhyOSStatus.freePhysicalMemory = VMInfo.getLongFromOperatingSystem(osMXBean, "getFreePhysicalMemorySize");
+ startPhyOSStatus.maxFileDescriptorCount = VMInfo.getLongFromOperatingSystem(osMXBean, "getMaxFileDescriptorCount");
+ startPhyOSStatus.currentOpenFileDescriptorCount = VMInfo.getLongFromOperatingSystem(osMXBean, "getOpenFileDescriptorCount");
+ }
+ }
+
+ //初始化processGCStatus;
+ for (GarbageCollectorMXBean garbage : garbageCollectorMXBeanList) {
+ GCStatus gcStatus = new GCStatus();
+ gcStatus.name = garbage.getName();
+ processGCStatus.gcStatusMap.put(garbage.getName(), gcStatus);
+ }
+
+ //初始化processMemoryStatus
+ if (memoryPoolMXBeanList != null && !memoryPoolMXBeanList.isEmpty()) {
+ for (MemoryPoolMXBean pool : memoryPoolMXBeanList) {
+ MemoryStatus memoryStatus = new MemoryStatus();
+ memoryStatus.name = pool.getName();
+ memoryStatus.initSize = pool.getUsage().getInit();
+ memoryStatus.maxSize = pool.getUsage().getMax();
+ processMomoryStatus.memoryStatusMap.put(pool.getName(), memoryStatus);
+ }
+ }
+ }
+
+ public String toString() {
+ return "the machine info => \n\n"
+ + "\tosInfo:\t" + osInfo + "\n"
+ + "\tjvmInfo:\t" + jvmInfo + "\n"
+ + "\tcpu num:\t" + totalProcessorCount + "\n\n"
+ + startPhyOSStatus.toString() + "\n"
+ + processGCStatus.toString() + "\n"
+ + processMomoryStatus.toString() + "\n";
+ }
+
+ public String totalString() {
+ return (processCpuStatus.getTotalString() + processGCStatus.getTotalString());
+ }
+
+ public void getDelta() {
+ getDelta(true);
+ }
+
+ public synchronized void getDelta(boolean print) {
+
+ try {
+ if (VMInfo.isSunOsMBean(osMXBean)) {
+ long curUptime = runtimeMXBean.getUptime();
+ long curProcessTime = getLongFromOperatingSystem(osMXBean, "getProcessCpuTime");
+ //百分比， uptime是ms，processTime是nano
+ if ((curUptime > lastUpTime) && (curProcessTime >= lastProcessCpuTime)) {
+ float curDeltaCpu = (float) (curProcessTime - lastProcessCpuTime) / ((curUptime - lastUpTime) * totalProcessorCount * 10000);
+ processCpuStatus.setMaxMinCpu(curDeltaCpu);
+ processCpuStatus.averageCpu = (float) curProcessTime / (curUptime * totalProcessorCount * 10000);
+
+ lastUpTime = curUptime;
+ lastProcessCpuTime = curProcessTime;
+ }
+ }
+
+ for (GarbageCollectorMXBean garbage : garbageCollectorMXBeanList) {
+
+ GCStatus gcStatus = processGCStatus.gcStatusMap.get(garbage.getName());
+ if (gcStatus == null) {
+ gcStatus = new GCStatus();
+ gcStatus.name = garbage.getName();
+ processGCStatus.gcStatusMap.put(garbage.getName(), gcStatus);
+ }
+
+ long curTotalGcCount = garbage.getCollectionCount();
+ gcStatus.setCurTotalGcCount(curTotalGcCount);
+
+ long curtotalGcTime = garbage.getCollectionTime();
+ gcStatus.setCurTotalGcTime(curtotalGcTime);
+ }
+
+ if (memoryPoolMXBeanList != null && !memoryPoolMXBeanList.isEmpty()) {
+ for (MemoryPoolMXBean pool : memoryPoolMXBeanList) {
+
+ MemoryStatus memoryStatus = processMomoryStatus.memoryStatusMap.get(pool.getName());
+ if (memoryStatus == null) {
+ memoryStatus = new MemoryStatus();
+ memoryStatus.name = pool.getName();
+ processMomoryStatus.memoryStatusMap.put(pool.getName(), memoryStatus);
+ }
+ memoryStatus.commitedSize = pool.getUsage().getCommitted();
+ memoryStatus.setMaxMinUsedSize(pool.getUsage().getUsed());
+ long maxMemory = memoryStatus.commitedSize > 0 ? memoryStatus.commitedSize : memoryStatus.maxSize;
+ memoryStatus.setMaxMinPercent(maxMemory > 0 ? (float) 100 * memoryStatus.usedSize / maxMemory : -1);
+ }
+ }
+
+ if (print) {
+ LOG.info(processCpuStatus.getDeltaString() + processMomoryStatus.getDeltaString() + processGCStatus.getDeltaString());
+ }
+
+ } catch (Exception e) {
+ LOG.warn("no need care, the fail is ignored : vmInfo getDelta failed " + e.getMessage(), e);
+ }
+ }
+
+ public static boolean isSunOsMBean(OperatingSystemMXBean operatingSystem) {
+ final String className = operatingSystem.getClass().getName();
+
+ return "com.sun.management.UnixOperatingSystem".equals(className);
+ }
+
+ public static long getLongFromOperatingSystem(OperatingSystemMXBean operatingSystem, String methodName) {
+ try {
+ final Method method = operatingSystem.getClass().getMethod(methodName, (Class[]) null);
+ method.setAccessible(true);
+ return (Long) method.invoke(operatingSystem, (Object[]) null);
+ } catch (final Exception e) {
+ LOG.info(String.format("OperatingSystemMXBean %s failed, Exception = %s ", methodName, e.getMessage()));
+ }
+
+ return -1;
+ }
+
+ private class PhyOSStatus {
+ long totalPhysicalMemory = -1;
+ long freePhysicalMemory = -1;
+ long maxFileDescriptorCount = -1;
+ long currentOpenFileDescriptorCount = -1;
+
+ public String toString() {
+ return String.format("\ttotalPhysicalMemory:\t%,.2fG\n"
+ + "\tfreePhysicalMemory:\t%,.2fG\n"
+ + "\tmaxFileDescriptorCount:\t%s\n"
+ + "\tcurrentOpenFileDescriptorCount:\t%s\n",
+ (float) totalPhysicalMemory / GB, (float) freePhysicalMemory / GB, maxFileDescriptorCount, currentOpenFileDescriptorCount);
+ }
+ }
+
+ private class ProcessCpuStatus {
+ // 百分比的值 比如30.0 表示30.0%
+ float maxDeltaCpu = -1;
+ float minDeltaCpu = -1;
+ float curDeltaCpu = -1;
+ float averageCpu = -1;
+
+ public void setMaxMinCpu(float curCpu) {
+ this.curDeltaCpu = curCpu;
+ if (maxDeltaCpu < curCpu) {
+ maxDeltaCpu = curCpu;
+ }
+
+ if (minDeltaCpu == -1 || minDeltaCpu > curCpu) {
+ minDeltaCpu = curCpu;
+ }
+ }
+
+ public String getDeltaString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n\t [delta cpu info] => \n");
+ sb.append("\t\t");
+ sb.append(String.format("%-30s | %-30s | %-30s | %-30s \n", "curDeltaCpu", "averageCpu", "maxDeltaCpu", "minDeltaCpu"));
+ sb.append("\t\t");
+ sb.append(String.format("%-30s | %-30s | %-30s | %-30s \n",
+ String.format("%,.2f%%", processCpuStatus.curDeltaCpu),
+ String.format("%,.2f%%", processCpuStatus.averageCpu),
+ String.format("%,.2f%%", processCpuStatus.maxDeltaCpu),
+ String.format("%,.2f%%\n", processCpuStatus.minDeltaCpu)));
+
+ return sb.toString();
+ }
+
+ public String getTotalString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n\t [total cpu info] => \n");
+ sb.append("\t\t");
+ sb.append(String.format("%-30s | %-30s | %-30s \n", "averageCpu", "maxDeltaCpu", "minDeltaCpu"));
+ sb.append("\t\t");
+ sb.append(String.format("%-30s | %-30s | %-30s \n",
+ String.format("%,.2f%%", processCpuStatus.averageCpu),
+ String.format("%,.2f%%", processCpuStatus.maxDeltaCpu),
+ String.format("%,.2f%%\n", processCpuStatus.minDeltaCpu)));
+
+ return sb.toString();
+ }
+
+ }
+
+ private class ProcessGCStatus {
+ final Map gcStatusMap = Maps.newHashMap();
+
+ public String toString() {
+ return "\tGC Names\t" + gcStatusMap.keySet() + "\n";
+ }
+
+ public String getDeltaString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n\t [delta gc info] => \n");
+ sb.append("\t\t ");
+ sb.append(String.format("%-20s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s \n", "NAME", "curDeltaGCCount", "totalGCCount", "maxDeltaGCCount", "minDeltaGCCount", "curDeltaGCTime", "totalGCTime", "maxDeltaGCTime", "minDeltaGCTime"));
+ for (GCStatus gc : gcStatusMap.values()) {
+ sb.append("\t\t ");
+ sb.append(String.format("%-20s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s \n",
+ gc.name, gc.curDeltaGCCount, gc.totalGCCount, gc.maxDeltaGCCount, gc.minDeltaGCCount,
+ String.format("%,.3fs",(float)gc.curDeltaGCTime/1000),
+ String.format("%,.3fs",(float)gc.totalGCTime/1000),
+ String.format("%,.3fs",(float)gc.maxDeltaGCTime/1000),
+ String.format("%,.3fs",(float)gc.minDeltaGCTime/1000)));
+
+ }
+ return sb.toString();
+ }
+
+ public String getTotalString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n\t [total gc info] => \n");
+ sb.append("\t\t ");
+ sb.append(String.format("%-20s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s \n", "NAME", "totalGCCount", "maxDeltaGCCount", "minDeltaGCCount", "totalGCTime", "maxDeltaGCTime", "minDeltaGCTime"));
+ for (GCStatus gc : gcStatusMap.values()) {
+ sb.append("\t\t ");
+ sb.append(String.format("%-20s | %-18s | %-18s | %-18s | %-18s | %-18s | %-18s \n",
+ gc.name, gc.totalGCCount, gc.maxDeltaGCCount, gc.minDeltaGCCount,
+ String.format("%,.3fs",(float)gc.totalGCTime/1000),
+ String.format("%,.3fs",(float)gc.maxDeltaGCTime/1000),
+ String.format("%,.3fs",(float)gc.minDeltaGCTime/1000)));
+
+ }
+ return sb.toString();
+ }
+ }
+
+ private class ProcessMemoryStatus {
+ final Map memoryStatusMap = Maps.newHashMap();
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\t");
+ sb.append(String.format("%-30s | %-30s | %-30s \n", "MEMORY_NAME", "allocation_size", "init_size"));
+ for (MemoryStatus ms : memoryStatusMap.values()) {
+ sb.append("\t");
+ sb.append(String.format("%-30s | %-30s | %-30s \n",
+ ms.name, String.format("%,.2fMB", (float) ms.maxSize / MB), String.format("%,.2fMB", (float) ms.initSize / MB)));
+ }
+ return sb.toString();
+ }
+
+ public String getDeltaString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n\t [delta memory info] => \n");
+ sb.append("\t\t ");
+ sb.append(String.format("%-30s | %-30s | %-30s | %-30s | %-30s \n", "NAME", "used_size", "used_percent", "max_used_size", "max_percent"));
+ for (MemoryStatus ms : memoryStatusMap.values()) {
+ sb.append("\t\t ");
+ sb.append(String.format("%-30s | %-30s | %-30s | %-30s | %-30s \n",
+ ms.name, String.format("%,.2f", (float) ms.usedSize / MB) + "MB",
+ String.format("%,.2f", (float) ms.percent) + "%",
+ String.format("%,.2f", (float) ms.maxUsedSize / MB) + "MB",
+ String.format("%,.2f", (float) ms.maxpercent) + "%"));
+
+ }
+ return sb.toString();
+ }
+ }
+
+ private class GCStatus {
+ String name;
+ long maxDeltaGCCount = -1;
+ long minDeltaGCCount = -1;
+ long curDeltaGCCount;
+ long totalGCCount = 0;
+ long maxDeltaGCTime = -1;
+ long minDeltaGCTime = -1;
+ long curDeltaGCTime;
+ long totalGCTime = 0;
+
+ public void setCurTotalGcCount(long curTotalGcCount) {
+ this.curDeltaGCCount = curTotalGcCount - totalGCCount;
+ this.totalGCCount = curTotalGcCount;
+
+ if (maxDeltaGCCount < curDeltaGCCount) {
+ maxDeltaGCCount = curDeltaGCCount;
+ }
+
+ if (minDeltaGCCount == -1 || minDeltaGCCount > curDeltaGCCount) {
+ minDeltaGCCount = curDeltaGCCount;
+ }
+ }
+
+ public void setCurTotalGcTime(long curTotalGcTime) {
+ this.curDeltaGCTime = curTotalGcTime - totalGCTime;
+ this.totalGCTime = curTotalGcTime;
+
+ if (maxDeltaGCTime < curDeltaGCTime) {
+ maxDeltaGCTime = curDeltaGCTime;
+ }
+
+ if (minDeltaGCTime == -1 || minDeltaGCTime > curDeltaGCTime) {
+ minDeltaGCTime = curDeltaGCTime;
+ }
+ }
+ }
+
+ private class MemoryStatus {
+ String name;
+ long initSize;
+ long maxSize;
+ long commitedSize;
+ long usedSize;
+ float percent;
+ long maxUsedSize = -1;
+ float maxpercent = 0;
+
+ void setMaxMinUsedSize(long curUsedSize) {
+ if (maxUsedSize < curUsedSize) {
+ maxUsedSize = curUsedSize;
+ }
+ this.usedSize = curUsedSize;
+ }
+
+ void setMaxMinPercent(float curPercent) {
+ if (maxpercent < curPercent) {
+ maxpercent = curPercent;
+ }
+ this.percent = curPercent;
+ }
+ }
+
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/util/Configuration.java b/common/src/main/java/com/alibaba/datax/common/util/Configuration.java
new file mode 100755
index 000000000..456920805
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/util/Configuration.java
@@ -0,0 +1,1078 @@
+package com.alibaba.datax.common.util;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.spi.ErrorCode;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.CharUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Configuration 提供多级JSON配置信息无损存储
+ * 实例代码:
+ * 获取job的配置信息
+ *
+ *
+ *
+ *
+ *
+ * Configuration 存在两种较好地实现方式
+ * 目前使用的第二种实现方式，使用第一种的问题在于: secretKeyPathSet =
+ new HashSet();
+
+ private Object root = null;
+
+ /**
+ * 初始化空白的Configuration
+ */
+ public static Configuration newDefault() {
+ return Configuration.from("{}");
+ }
+
+ /**
+ * 从JSON字符串加载Configuration
+ */
+ public static Configuration from(String json) {
+ json = StrUtil.replaceVariable(json);
+ checkJSON(json);
+
+ try {
+ return new Configuration(json);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
+ e);
+ }
+
+ }
+
+ /**
+ * 从包括json的File对象加载Configuration
+ */
+ public static Configuration from(File file) {
+ try {
+ return Configuration.from(IOUtils
+ .toString(new FileInputStream(file)));
+ } catch (FileNotFoundException e) {
+ throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
+ String.format("配置信息错误，您提供的配置文件[%s]不存在. 请检查您的配置文件.", file.getAbsolutePath()));
+ } catch (IOException e) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.CONFIG_ERROR,
+ String.format("配置信息错误. 您提供配置文件[%s]读取失败，错误原因: %s. 请检查您的配置文件的权限设置.",
+ file.getAbsolutePath(), e));
+ }
+ }
+
+ /**
+ * 从包括json的InputStream对象加载Configuration
+ */
+ public static Configuration from(InputStream is) {
+ try {
+ return Configuration.from(IOUtils.toString(is));
+ } catch (IOException e) {
+ throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
+ String.format("请检查您的配置文件. 您提供的配置文件读取失败，错误原因: %s. 请检查您的配置文件的权限设置.", e));
+ }
+ }
+
+ /**
+ * 从Map对象加载Configuration
+ */
+ public static Configuration from(final Map object) {
+ return Configuration.from(Configuration.toJSONString(object));
+ }
+
+ /**
+ * 从List对象加载Configuration
+ */
+ public static Configuration from(final List object) {
+ return Configuration.from(Configuration.toJSONString(object));
+ }
+
+ public String getNecessaryValue(String key, ErrorCode errorCode) {
+ String value = this.getString(key, null);
+ if (StringUtils.isBlank(value)) {
+ throw DataXException.asDataXException(errorCode,
+ String.format("您提供配置文件有误，[%s]是必填参数，不允许为空或者留白 .", key));
+ }
+
+ return value;
+ }
+
+ public String getUnnecessaryValue(String key,String defaultValue,ErrorCode errorCode) {
+ String value = this.getString(key, defaultValue);
+ if (StringUtils.isBlank(value)) {
+ value = defaultValue;
+ }
+ return value;
+ }
+
+ public Boolean getNecessaryBool(String key, ErrorCode errorCode) {
+ Boolean value = this.getBool(key);
+ if (value == null) {
+ throw DataXException.asDataXException(errorCode,
+ String.format("您提供配置文件有误，[%s]是必填参数，不允许为空或者留白 .", key));
+ }
+
+ return value;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址具体的对象。
+ *
+ *
+ * NOTE: 目前仅支持Map以及List下标寻址, 例如:
+ *
+ *
+ * 对于如下JSON
+ *
+ * {"a": {"b": {"c": [0,1,2,3]}}}
+ *
+ * config.get("") 返回整个Map
+ * getList(final String path) {
+ List list = this.get(path, List.class);
+ if (null == list) {
+ return null;
+ }
+ return list;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址List对象，如果对象不存在，返回null
+ */
+ @SuppressWarnings("unchecked")
+ public List getList(final String path, Class t) {
+ Object object = this.get(path, List.class);
+ if (null == object) {
+ return null;
+ }
+
+ List result = new ArrayList();
+
+ List origin = (List) object;
+ for (final Object each : origin) {
+ result.add((T) each);
+ }
+
+ return result;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址List对象，如果对象不存在，返回默认List
+ */
+ @SuppressWarnings("unchecked")
+ public List getList(final String path,
+ final List defaultList) {
+ Object object = this.getList(path);
+ if (null == object) {
+ return defaultList;
+ }
+ return (List) object;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址List对象，如果对象不存在，返回默认List
+ */
+ public List getList(final String path, final List defaultList,
+ Class t) {
+ List list = this.getList(path, t);
+ if (null == list) {
+ return defaultList;
+ }
+ return list;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址包含Configuration的List，如果对象不存在，返回默认null
+ */
+ public List getListConfiguration(final String path) {
+ List lists = getList(path);
+ if (lists == null) {
+ return null;
+ }
+
+ List result = new ArrayList();
+ for (final Object object : lists) {
+ result.add(Configuration.from(Configuration.toJSONString(object)));
+ }
+ return result;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址Map对象，如果对象不存在，返回null
+ */
+ @SuppressWarnings("unchecked")
+ public Map getMap(final String path) {
+ Map result = this.get(path, Map.class);
+ if (null == result) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址Map对象，如果对象不存在，返回null;
+ */
+ @SuppressWarnings("unchecked")
+ public Map getMap(final String path, Class t) {
+ Map map = this.get(path, Map.class);
+ if (null == map) {
+ return null;
+ }
+
+ Map result = new HashMap();
+ for (final String key : map.keySet()) {
+ result.put(key, (T) map.get(key));
+ }
+
+ return result;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址Map对象，如果对象不存在，返回默认map
+ */
+ @SuppressWarnings("unchecked")
+ public Map getMap(final String path,
+ final Map defaultMap) {
+ Object object = this.getMap(path);
+ if (null == object) {
+ return defaultMap;
+ }
+ return (Map) object;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址Map对象，如果对象不存在，返回默认map
+ */
+ public Map getMap(final String path,
+ final Map defaultMap, Class t) {
+ Map result = getMap(path, t);
+ if (null == result) {
+ return defaultMap;
+ }
+ return result;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址包含Configuration的Map，如果对象不存在，返回默认null
+ */
+ @SuppressWarnings("unchecked")
+ public Map getMapConfiguration(final String path) {
+ Map map = this.get(path, Map.class);
+ if (null == map) {
+ return null;
+ }
+
+ Map result = new HashMap();
+ for (final String key : map.keySet()) {
+ result.put(key, Configuration.from(Configuration.toJSONString(map
+ .get(key))));
+ }
+
+ return result;
+ }
+
+ /**
+ * 根据用户提供的json path，寻址具体的对象，并转为用户提供的类型
+ *
+ *
+ * NOTE: 目前仅支持Map以及List下标寻址, 例如:
+ *
+ *
+ * 对于如下JSON
+ *
+ * {"a": {"b": {"c": [0,1,2,3]}}}
+ *
+ * config.get("") 返回整个Map T get(final String path, Class clazz) {
+ this.checkPath(path);
+ return (T) this.get(path);
+ }
+
+ /**
+ * 格式化Configuration输出
+ */
+ public String beautify() {
+ return JSON.toJSONString(this.getInternal(),
+ SerializerFeature.PrettyFormat);
+ }
+
+ /**
+ * 根据用户提供的json path，插入指定对象，并返回之前存在的对象(如果存在)
+ *
+ *
+ * 目前仅支持.以及数组下标寻址, 例如:
+ *
+ *
+ * config.set("a.b.c[3]", object);
+ *
+ * 、List)，不要使用自定义对象，否则后续对于JSON序列化等情况会出现未定义行为。
+ *
+ * @param path
+ * JSON path对象
+ * @param object
+ * 需要插入的对象
+ * @return Java表示的JSON对象
+ */
+ public Object set(final String path, final Object object) {
+ checkPath(path);
+
+ Object result = this.get(path);
+
+ setObject(path, extractConfiguration(object));
+
+ return result;
+ }
+
+ /**
+ * 获取Configuration下所有叶子节点的key
+ *
+ *
+ * 对于
+ * {"a": {"b": {"c": [0,1,2,3]}}, "x": "y"}
+ *
+ * 下属的key包括: a.b.c[0],a.b.c[1],a.b.c[2],a.b.c[3],x
+ */
+ public Set getKeys() {
+ Set collect = new HashSet();
+ this.getKeysRecursive(this.getInternal(), "", collect);
+ return collect;
+ }
+
+ /**
+ * 删除path对应的值，如果path不存在，将抛出异常。
+ */
+ public Object remove(final String path) {
+ final Object result = this.get(path);
+ if (null == result) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.RUNTIME_ERROR,
+ String.format("配置文件对应Key[%s]并不存在，该情况是代码编程错误. 请联系DataX团队的同学.", path));
+ }
+
+ this.set(path, null);
+ return result;
+ }
+
+ /**
+ * 合并其他Configuration，并修改两者冲突的KV配置
+ *
+ * @param another
+ * 合并加入的第三方Configuration
+ * @param updateWhenConflict
+ * 当合并双方出现KV冲突时候，选择更新当前KV，或者忽略该KV
+ * @return 返回合并后对象
+ */
+ public Configuration merge(final Configuration another,
+ boolean updateWhenConflict) {
+ Set keys = another.getKeys();
+
+ for (final String key : keys) {
+ // 如果使用更新策略，凡是another存在的key，均需要更新
+ if (updateWhenConflict) {
+ this.set(key, another.get(key));
+ continue;
+ }
+
+ // 使用忽略策略，只有another Configuration存在但是当前Configuration不存在的key，才需要更新
+ boolean isCurrentExists = this.get(key) != null;
+ if (isCurrentExists) {
+ continue;
+ }
+
+ this.set(key, another.get(key));
+ }
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return this.toJSON();
+ }
+
+ /**
+ * 将Configuration作为JSON输出
+ */
+ public String toJSON() {
+ return Configuration.toJSONString(this.getInternal());
+ }
+
+ /**
+ * 拷贝当前Configuration，注意，这里使用了深拷贝，避免冲突
+ */
+ public Configuration clone() {
+ Configuration config = Configuration
+ .from(Configuration.toJSONString(this.getInternal()));
+ config.addSecretKeyPath(this.secretKeyPathSet);
+ return config;
+ }
+
+ /**
+ * 按照configuration要求格式的path
+ * 比如：
+ * a.b.c
+ * a.b[2].c
+ * @param path
+ */
+ public void addSecretKeyPath(String path) {
+ if(StringUtils.isNotBlank(path)) {
+ this.secretKeyPathSet.add(path);
+ }
+ }
+
+ public void addSecretKeyPath(Set pathSet) {
+ if(pathSet != null) {
+ this.secretKeyPathSet.addAll(pathSet);
+ }
+ }
+
+ public void setSecretKeyPathSet(Set keyPathSet) {
+ if(keyPathSet != null) {
+ this.secretKeyPathSet = keyPathSet;
+ }
+ }
+
+ public boolean isSecretPath(String path) {
+ return this.secretKeyPathSet.contains(path);
+ }
+
+ @SuppressWarnings("unchecked")
+ void getKeysRecursive(final Object current, String path, Set collect) {
+ boolean isRegularElement = !(current instanceof Map || current instanceof List);
+ if (isRegularElement) {
+ collect.add(path);
+ return;
+ }
+
+ boolean isMap = current instanceof Map;
+ if (isMap) {
+ Map mapping = ((Map) current);
+ for (final String key : mapping.keySet()) {
+ if (StringUtils.isBlank(path)) {
+ getKeysRecursive(mapping.get(key), key.trim(), collect);
+ } else {
+ getKeysRecursive(mapping.get(key), path + "." + key.trim(),
+ collect);
+ }
+ }
+ return;
+ }
+
+ boolean isList = current instanceof List;
+ if (isList) {
+ List lists = (List) current;
+ for (int i = 0; i < lists.size(); i++) {
+ getKeysRecursive(lists.get(i), path + String.format("[%d]", i),
+ collect);
+ }
+ return;
+ }
+
+ return;
+ }
+
+ public Object getInternal() {
+ return this.root;
+ }
+
+ private void setObject(final String path, final Object object) {
+ Object newRoot = setObjectRecursive(this.root, split2List(path), 0,
+ object);
+
+ if (isSuitForRoot(newRoot)) {
+ this.root = newRoot;
+ return;
+ }
+
+ throw DataXException.asDataXException(CommonErrorCode.RUNTIME_ERROR,
+ String.format("值[%s]无法适配您提供[%s]， 该异常代表系统编程错误, 请联系DataX开发团队!",
+ ToStringBuilder.reflectionToString(object), path));
+ }
+
+ @SuppressWarnings("unchecked")
+ private Object extractConfiguration(final Object object) {
+ if (object instanceof Configuration) {
+ return extractFromConfiguration(object);
+ }
+
+ if (object instanceof List) {
+ List result = new ArrayList();
+ for (final Object each : (List) object) {
+ result.add(extractFromConfiguration(each));
+ }
+ return result;
+ }
+
+ if (object instanceof Map) {
+ Map result = new HashMap();
+ for (final String key : ((Map) object).keySet()) {
+ result.put(key,
+ extractFromConfiguration(((Map) object)
+ .get(key)));
+ }
+ return result;
+ }
+
+ return object;
+ }
+
+ private Object extractFromConfiguration(final Object object) {
+ if (object instanceof Configuration) {
+ return ((Configuration) object).getInternal();
+ }
+
+ return object;
+ }
+
+ Object buildObject(final List paths, final Object object) {
+ if (null == paths) {
+ throw DataXException.asDataXException(
+ CommonErrorCode.RUNTIME_ERROR,
+ "Path不能为null，该异常代表系统编程错误, 请联系DataX开发团队 !");
+ }
+
+ if (1 == paths.size() && StringUtils.isBlank(paths.get(0))) {
+ return object;
+ }
+
+ Object child = object;
+ for (int i = paths.size() - 1; i >= 0; i--) {
+ String path = paths.get(i);
+
+ if (isPathMap(path)) {
+ Map mapping = new HashMap();
+ mapping.put(path, child);
+ child = mapping;
+ continue;
+ }
+
+ if (isPathList(path)) {
+ List lists = new ArrayList(
+ this.getIndex(path) + 1);
+ expand(lists, this.getIndex(path) + 1);
+ lists.set(this.getIndex(path), child);
+ child = lists;
+ continue;
+ }
+
+ throw DataXException.asDataXException(
+ CommonErrorCode.RUNTIME_ERROR, String.format(
+ "路径[%s]出现非法值类型[%s]，该异常代表系统编程错误, 请联系DataX开发团队! .",
+ StringUtils.join(paths, "."), path));
+ }
+
+ return child;
+ }
+
+ @SuppressWarnings("unchecked")
+ Object setObjectRecursive(Object current, final List paths,
+ int index, final Object value) {
+
+ // 如果是已经超出path，我们就返回value即可，作为最底层叶子节点
+ boolean isLastIndex = index == paths.size();
+ if (isLastIndex) {
+ return value;
+ }
+
+ String path = paths.get(index).trim();
+ boolean isNeedMap = isPathMap(path);
+ if (isNeedMap) {
+ Map mapping;
+
+ // 当前不是map，因此全部替换为map，并返回新建的map对象
+ boolean isCurrentMap = current instanceof Map;
+ if (!isCurrentMap) {
+ mapping = new HashMap();
+ mapping.put(
+ path,
+ buildObject(paths.subList(index + 1, paths.size()),
+ value));
+ return mapping;
+ }
+
+ // 当前是map，但是没有对应的key，也就是我们需要新建对象插入该map，并返回该map
+ mapping = ((Map) current);
+ boolean hasSameKey = mapping.containsKey(path);
+ if (!hasSameKey) {
+ mapping.put(
+ path,
+ buildObject(paths.subList(index + 1, paths.size()),
+ value));
+ return mapping;
+ }
+
+ // 当前是map，而且还竟然存在这个值，好吧，继续递归遍历
+ current = mapping.get(path);
+ mapping.put(path,
+ setObjectRecursive(current, paths, index + 1, value));
+ return mapping;
+ }
+
+ boolean isNeedList = isPathList(path);
+ if (isNeedList) {
+ List lists;
+ int listIndexer = getIndex(path);
+
+ // 当前是list，直接新建并返回即可
+ boolean isCurrentList = current instanceof List;
+ if (!isCurrentList) {
+ lists = expand(new ArrayList(), listIndexer + 1);
+ lists.set(
+ listIndexer,
+ buildObject(paths.subList(index + 1, paths.size()),
+ value));
+ return lists;
+ }
+
+ // 当前是list，但是对应的indexer是没有具体的值，也就是我们新建对象然后插入到该list，并返回该List
+ lists = (List) current;
+ lists = expand(lists, listIndexer + 1);
+
+ boolean hasSameIndex = lists.get(listIndexer) != null;
+ if (!hasSameIndex) {
+ lists.set(
+ listIndexer,
+ buildObject(paths.subList(index + 1, paths.size()),
+ value));
+ return lists;
+ }
+
+ // 当前是list，并且存在对应的index，没有办法继续递归寻找
+ current = lists.get(listIndexer);
+ lists.set(listIndexer,
+ setObjectRecursive(current, paths, index + 1, value));
+ return lists;
+ }
+
+ throw DataXException.asDataXException(CommonErrorCode.RUNTIME_ERROR,
+ "该异常代表系统编程错误, 请联系DataX开发团队 !");
+ }
+
+ private Object findObject(final String path) {
+ boolean isRootQuery = StringUtils.isBlank(path);
+ if (isRootQuery) {
+ return this.root;
+ }
+
+ Object target = this.root;
+
+ for (final String each : split2List(path)) {
+ if (isPathMap(each)) {
+ target = findObjectInMap(target, each);
+ continue;
+ } else {
+ target = findObjectInList(target, each);
+ continue;
+ }
+ }
+
+ return target;
+ }
+
+ @SuppressWarnings("unchecked")
+ private Object findObjectInMap(final Object target, final String index) {
+ boolean isMap = (target instanceof Map);
+ if (!isMap) {
+ throw new IllegalArgumentException(String.format(
+ "您提供的配置文件有误. 路径[%s]需要配置Json格式的Map对象，但该节点发现实际类型是[%s]. 请检查您的配置并作出修改.",
+ index, target.getClass().toString()));
+ }
+
+ Object result = ((Map) target).get(index);
+ if (null == result) {
+ throw new IllegalArgumentException(String.format(
+ "您提供的配置文件有误. 路径[%s]值为null，datax无法识别该配置. 请检查您的配置并作出修改.", index));
+ }
+
+ return result;
+ }
+
+ @SuppressWarnings({ "unchecked" })
+ private Object findObjectInList(final Object target, final String each) {
+ boolean isList = (target instanceof List);
+ if (!isList) {
+ throw new IllegalArgumentException(String.format(
+ "您提供的配置文件有误. 路径[%s]需要配置Json格式的Map对象，但该节点发现实际类型是[%s]. 请检查您的配置并作出修改.",
+ each, target.getClass().toString()));
+ }
+
+ String index = each.replace("[", "").replace("]", "");
+ if (!StringUtils.isNumeric(index)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "系统编程错误，列表下标必须为数字类型，但该节点发现实际类型是[%s] ，该异常代表系统编程错误, 请联系DataX开发团队 !",
+ index));
+ }
+
+ return ((List) target).get(Integer.valueOf(index));
+ }
+
+ private List expand(List list, int size) {
+ int expand = size - list.size();
+ while (expand-- > 0) {
+ list.add(null);
+ }
+ return list;
+ }
+
+ private boolean isPathList(final String path) {
+ return path.contains("[") && path.contains("]");
+ }
+
+ private boolean isPathMap(final String path) {
+ return StringUtils.isNotBlank(path) && !isPathList(path);
+ }
+
+ private int getIndex(final String index) {
+ return Integer.valueOf(index.replace("[", "").replace("]", ""));
+ }
+
+ private boolean isSuitForRoot(final Object object) {
+ if (null != object && (object instanceof List || object instanceof Map)) {
+ return true;
+ }
+
+ return false;
+ }
+
+ private String split(final String path) {
+ return StringUtils.replace(path, "[", ".[");
+ }
+
+ private List split2List(final String path) {
+ return Arrays.asList(StringUtils.split(split(path), "."));
+ }
+
+ private void checkPath(final String path) {
+ if (null == path) {
+ throw new IllegalArgumentException(
+ "系统编程错误, 该异常代表系统编程错误, 请联系DataX开发团队!.");
+ }
+
+ for (final String each : StringUtils.split(".")) {
+ if (StringUtils.isBlank(each)) {
+ throw new IllegalArgumentException(String.format(
+ "系统编程错误, 路径[%s]不合法, 路径层次之间不能出现空白字符 .", path));
+ }
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private String toJSONPath(final String path) {
+ return (StringUtils.isBlank(path) ? "$" : "$." + path).replace("$.[",
+ "$[");
+ }
+
+ private static void checkJSON(final String json) {
+ if (StringUtils.isBlank(json)) {
+ throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
+ "配置信息错误. 因为您提供的配置信息不是合法的JSON格式, JSON不能为空白. 请按照标准json格式提供配置信息. ");
+ }
+ }
+
+ private Configuration(final String json) {
+ try {
+ this.root = JSON.parse(json);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
+ String.format("配置信息错误. 您提供的配置信息不是合法的JSON格式: %s . 请按照标准json格式提供配置信息. ", e.getMessage()));
+ }
+ }
+
+ private static String toJSONString(final Object object) {
+ return JSON.toJSONString(object);
+ }
+
+ public Set getSecretKeyPathSet() {
+ return secretKeyPathSet;
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/util/FilterUtil.java b/common/src/main/java/com/alibaba/datax/common/util/FilterUtil.java
new file mode 100755
index 000000000..37b319a19
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/util/FilterUtil.java
@@ -0,0 +1,52 @@
+package com.alibaba.datax.common.util;
+
+import java.util.*;
+import java.util.regex.Pattern;
+
+/**
+ * 提供从 List 中根据 regular 过滤的通用工具(返回值已经去重). 使用场景，比如：odpsreader
+ * 的分区筛选，hdfsreader/txtfilereader的路径筛选等
+ */
+public final class FilterUtil {
+
+ //已经去重
+ public static List filterByRegular(List allStrs,
+ String regular) {
+ List matchedValues = new ArrayList();
+
+ // 语法习惯上的兼容处理(pt=* 实际正则应该是：pt=.*)
+ String newReqular = regular.replace(".*", "*").replace("*", ".*");
+
+ Pattern p = Pattern.compile(newReqular);
+
+ for (String partition : allStrs) {
+ if (p.matcher(partition).matches()) {
+ if (!matchedValues.contains(partition)) {
+ matchedValues.add(partition);
+ }
+ }
+ }
+
+ return matchedValues;
+ }
+
+ //已经去重
+ public static List filterByRegulars(List allStrs,
+ List regulars) {
+ List matchedValues = new ArrayList();
+
+ List tempMatched = null;
+ for (String regular : regulars) {
+ tempMatched = filterByRegular(allStrs, regular);
+ if (null != tempMatched && !tempMatched.isEmpty()) {
+ for (String temp : tempMatched) {
+ if (!matchedValues.contains(temp)) {
+ matchedValues.add(temp);
+ }
+ }
+ }
+ }
+
+ return matchedValues;
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/util/HostUtils.java b/common/src/main/java/com/alibaba/datax/common/util/HostUtils.java
new file mode 100644
index 000000000..3980076c6
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/util/HostUtils.java
@@ -0,0 +1,50 @@
+package com.alibaba.datax.common.util;
+
+import com.google.common.base.CharMatcher;
+import com.google.common.io.ByteStreams;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * Created by liqiang on 15/8/25.
+ */
+public class HostUtils {
+
+ public static final String IP;
+ public static final String HOSTNAME;
+ private static final Logger log = LoggerFactory.getLogger(HostUtils.class);
+
+ static {
+ String ip;
+ String hostname;
+ try {
+ InetAddress addr = InetAddress.getLocalHost();
+ ip = addr.getHostAddress();
+ hostname = addr.getHostName();
+ } catch (UnknownHostException e) {
+ log.error("Can't find out address: " + e.getMessage(), e);
+ ip = "UNKNOWN";
+ hostname = "UNKNOWN";
+ }
+ if (ip.equals("127.0.0.1") || ip.equals("::1") || ip.equals("UNKNOWN")) {
+ try {
+ Process process = Runtime.getRuntime().exec("hostname -i");
+ if (process.waitFor() == 0) {
+ ip = new String(ByteStreams.toByteArray(process.getInputStream()), "UTF8");
+ }
+ process = Runtime.getRuntime().exec("hostname");
+ if (process.waitFor() == 0) {
+ hostname = CharMatcher.BREAKING_WHITESPACE.trimFrom(new String(ByteStreams.toByteArray(process.getInputStream()), "UTF8"));
+ }
+ } catch (Exception e) {
+ log.warn("get hostname failed {}", e.getMessage());
+ }
+ }
+ IP = ip;
+ HOSTNAME = hostname;
+ log.info("IP {} HOSTNAME {}", IP, HOSTNAME);
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/util/ListUtil.java b/common/src/main/java/com/alibaba/datax/common/util/ListUtil.java
new file mode 100755
index 000000000..d7a5b7646
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/util/ListUtil.java
@@ -0,0 +1,139 @@
+package com.alibaba.datax.common.util;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * 提供针对 DataX中使用的 List 较为常见的一些封装。 比如：checkIfValueDuplicate 可以用于检查用户配置的 writer
+ * 的列不能重复。makeSureNoValueDuplicate亦然，只是会严格报错。
+ */
+public final class ListUtil {
+
+ public static boolean checkIfValueDuplicate(List aList,
+ boolean caseSensitive) {
+ if (null == aList || aList.isEmpty()) {
+ throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
+ "您提供的作业配置有误，List不能为空.");
+ }
+
+ try {
+ makeSureNoValueDuplicate(aList, caseSensitive);
+ } catch (Exception e) {
+ return true;
+ }
+ return false;
+ }
+
+ public static void makeSureNoValueDuplicate(List aList,
+ boolean caseSensitive) {
+ if (null == aList || aList.isEmpty()) {
+ throw new IllegalArgumentException("您提供的作业配置有误, List不能为空.");
+ }
+
+ if (1 == aList.size()) {
+ return;
+ } else {
+ List list = null;
+ if (!caseSensitive) {
+ list = valueToLowerCase(aList);
+ } else {
+ list = new ArrayList(aList);
+ }
+
+ Collections.sort(list);
+
+ for (int i = 0, len = list.size() - 1; i < len; i++) {
+ if (list.get(i).equals(list.get(i + 1))) {
+ throw DataXException
+ .asDataXException(
+ CommonErrorCode.CONFIG_ERROR,
+ String.format(
+ "您提供的作业配置信息有误, String:[%s] 不允许重复出现在列表中: [%s].",
+ list.get(i),
+ StringUtils.join(aList, ",")));
+ }
+ }
+ }
+ }
+
+ public static boolean checkIfBInA(List aList, List bList,
+ boolean caseSensitive) {
+ if (null == aList || aList.isEmpty() || null == bList
+ || bList.isEmpty()) {
+ throw new IllegalArgumentException("您提供的作业配置有误, List不能为空.");
+ }
+
+ try {
+ makeSureBInA(aList, bList, caseSensitive);
+ } catch (Exception e) {
+ return false;
+ }
+ return true;
+ }
+
+ public static void makeSureBInA(List aList, List bList,
+ boolean caseSensitive) {
+ if (null == aList || aList.isEmpty() || null == bList
+ || bList.isEmpty()) {
+ throw new IllegalArgumentException("您提供的作业配置有误, List不能为空.");
+ }
+
+ List all = null;
+ List part = null;
+
+ if (!caseSensitive) {
+ all = valueToLowerCase(aList);
+ part = valueToLowerCase(bList);
+ } else {
+ all = new ArrayList(aList);
+ part = new ArrayList(bList);
+ }
+
+ for (String oneValue : part) {
+ if (!all.contains(oneValue)) {
+ throw DataXException
+ .asDataXException(
+ CommonErrorCode.CONFIG_ERROR,
+ String.format(
+ "您提供的作业配置信息有误, String:[%s] 不存在于列表中:[%s].",
+ oneValue, StringUtils.join(aList, ",")));
+ }
+ }
+
+ }
+
+ public static boolean checkIfValueSame(List aList) {
+ if (null == aList || aList.isEmpty()) {
+ throw new IllegalArgumentException("您提供的作业配置有误, List不能为空.");
+ }
+
+ if (1 == aList.size()) {
+ return true;
+ } else {
+ Boolean firstValue = aList.get(0);
+ for (int i = 1, len = aList.size(); i < len; i++) {
+ if (firstValue.booleanValue() != aList.get(i).booleanValue()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+
+ public static List valueToLowerCase(List aList) {
+ if (null == aList || aList.isEmpty()) {
+ throw new IllegalArgumentException("您提供的作业配置有误, List不能为空.");
+ }
+ List result = new ArrayList(aList.size());
+ for (String oneValue : aList) {
+ result.add(null != oneValue ? oneValue.toLowerCase() : null);
+ }
+
+ return result;
+ }
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/util/RangeSplitUtil.java b/common/src/main/java/com/alibaba/datax/common/util/RangeSplitUtil.java
new file mode 100755
index 000000000..791f9ea12
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/util/RangeSplitUtil.java
@@ -0,0 +1,209 @@
+package com.alibaba.datax.common.util;
+
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.math.BigInteger;
+import java.util.*;
+
+/**
+ * 提供通用的根据数字范围、字符串范围等进行切分的通用功能.
+ */
+public final class RangeSplitUtil {
+
+ public static String[] doAsciiStringSplit(String left, String right, int expectSliceNumber) {
+ int radix = 128;
+
+ BigInteger[] tempResult = doBigIntegerSplit(stringToBigInteger(left, radix),
+ stringToBigInteger(right, radix), expectSliceNumber);
+ String[] result = new String[tempResult.length];
+
+ //处理第一个字符串（因为：在转换为数字，再还原的时候，如果首字符刚好是 basic,则不知道应该添加多少个 basic）
+ result[0] = left;
+ result[tempResult.length - 1] = right;
+
+ for (int i = 1, len = tempResult.length - 1; i < len; i++) {
+ result[i] = bigIntegerToString(tempResult[i], radix);
+ }
+
+ return result;
+ }
+
+
+ public static long[] doLongSplit(long left, long right, int expectSliceNumber) {
+ BigInteger[] result = doBigIntegerSplit(BigInteger.valueOf(left),
+ BigInteger.valueOf(right), expectSliceNumber);
+ long[] returnResult = new long[result.length];
+ for (int i = 0, len = result.length; i < len; i++) {
+ returnResult[i] = result[i].longValue();
+ }
+ return returnResult;
+ }
+
+ public static BigInteger[] doBigIntegerSplit(BigInteger left, BigInteger right, int expectSliceNumber) {
+ if (expectSliceNumber < 1) {
+ throw new IllegalArgumentException(String.format(
+ "切分份数不能小于1. 此处:expectSliceNumber=[%s].", expectSliceNumber));
+ }
+
+ if (null == left || null == right) {
+ throw new IllegalArgumentException(String.format(
+ "对 BigInteger 进行切分时，其左右区间不能为 null. 此处:left=[%s],right=[%s].", left, right));
+ }
+
+ if (left.compareTo(right) == 0) {
+ return new BigInteger[]{left, right};
+ } else {
+ // 调整大小顺序，确保 left < right
+ if (left.compareTo(right) > 0) {
+ BigInteger temp = left;
+ left = right;
+ right = temp;
+ }
+
+ //left < right
+ BigInteger endAndStartGap = right.subtract(left);
+
+ BigInteger step = endAndStartGap.divide(BigInteger.valueOf(expectSliceNumber));
+ BigInteger remainder = endAndStartGap.remainder(BigInteger.valueOf(expectSliceNumber));
+
+ //remainder 不可能超过expectSliceNumber,所以不需要检查remainder的 Integer 的范围
+
+ // 这里不能 step.intValue()==0，因为可能溢出
+ if (step.compareTo(BigInteger.ZERO) == 0) {
+ expectSliceNumber = remainder.intValue();
+ }
+
+ BigInteger[] result = new BigInteger[expectSliceNumber + 1];
+ result[0] = left;
+ result[expectSliceNumber] = right;
+
+ BigInteger lowerBound;
+ BigInteger upperBound = left;
+ for (int i = 1; i < expectSliceNumber; i++) {
+ lowerBound = upperBound;
+ upperBound = lowerBound.add(step);
+ upperBound = upperBound.add((remainder.compareTo(BigInteger.valueOf(i)) >= 0)
+ ? BigInteger.ONE : BigInteger.ZERO);
+ result[i] = upperBound;
+ }
+
+ return result;
+ }
+ }
+
+ private static void checkIfBetweenRange(int value, int left, int right) {
+ if (value < left || value > right) {
+ throw new IllegalArgumentException(String.format("parameter can not <[%s] or >[%s].",
+ left, right));
+ }
+ }
+
+ /**
+ * 由于只支持 ascii 码对应字符，所以radix 范围为[1,128]
+ */
+ public static BigInteger stringToBigInteger(String aString, int radix) {
+ if (null == aString) {
+ throw new IllegalArgumentException("参数 bigInteger 不能为空.");
+ }
+
+ checkIfBetweenRange(radix, 1, 128);
+
+ BigInteger result = BigInteger.ZERO;
+ BigInteger radixBigInteger = BigInteger.valueOf(radix);
+
+ int tempChar;
+ int k = 0;
+
+ for (int i = aString.length() - 1; i >= 0; i--) {
+ tempChar = aString.charAt(i);
+ if (tempChar >= 128) {
+ throw new IllegalArgumentException(String.format("根据字符串进行切分时仅支持 ASCII 字符串，而字符串:[%s]非 ASCII 字符串.", aString));
+ }
+ result = result.add(BigInteger.valueOf(tempChar).multiply(radixBigInteger.pow(k)));
+ k++;
+ }
+
+ return result;
+ }
+
+ /**
+ * 把BigInteger 转换为 String.注意：radix 和 basic 范围都为[1,128], radix + basic 的范围也必须在[1,128].
+ */
+ private static String bigIntegerToString(BigInteger bigInteger, int radix) {
+ if (null == bigInteger) {
+ throw new IllegalArgumentException("参数 bigInteger 不能为空.");
+ }
+
+ checkIfBetweenRange(radix, 1, 128);
+
+ StringBuilder resultStringBuilder = new StringBuilder();
+
+ List list = new ArrayList();
+ BigInteger radixBigInteger = BigInteger.valueOf(radix);
+ BigInteger currentValue = bigInteger;
+
+ BigInteger quotient = currentValue.divide(radixBigInteger);
+ while (quotient.compareTo(BigInteger.ZERO) > 0) {
+ list.add(currentValue.remainder(radixBigInteger).intValue());
+ currentValue = currentValue.divide(radixBigInteger);
+ quotient = currentValue;
+ }
+ Collections.reverse(list);
+
+ if (list.isEmpty()) {
+ list.add(0, bigInteger.remainder(radixBigInteger).intValue());
+ }
+
+ Map map = new HashMap();
+ for (int i = 0; i < radix; i++) {
+ map.put(i, (char) (i));
+ }
+
+// String msg = String.format("%s 转为 %s 进制，结果为：%s", bigInteger.longValue(), radix, list);
+// System.out.println(msg);
+
+ for (Integer aList : list) {
+ resultStringBuilder.append(map.get(aList));
+ }
+
+ return resultStringBuilder.toString();
+ }
+
+ /**
+ * 获取字符串中的最小字符和最大字符（依据 ascii 进行判断）.要求字符串必须非空，并且为 ascii 字符串.
+ * 返回的Pair，left=最小字符，right=最大字符.
+ */
+ public static Pair getMinAndMaxCharacter(String aString) {
+ if (!isPureAscii(aString)) {
+ throw new IllegalArgumentException(String.format("根据字符串进行切分时仅支持 ASCII 字符串，而字符串:[%s]非 ASCII 字符串.", aString));
+ }
+
+ char min = aString.charAt(0);
+ char max = min;
+
+ char temp;
+ for (int i = 1, len = aString.length(); i < len; i++) {
+ temp = aString.charAt(i);
+ min = min < temp ? min : temp;
+ max = max > temp ? max : temp;
+ }
+
+ return new ImmutablePair(min, max);
+ }
+
+ private static boolean isPureAscii(String aString) {
+ if (null == aString) {
+ return false;
+ }
+
+ for (int i = 0, len = aString.length(); i < len; i++) {
+ char ch = aString.charAt(i);
+ if (ch >= 127 || ch < 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/util/RetryUtil.java b/common/src/main/java/com/alibaba/datax/common/util/RetryUtil.java
new file mode 100755
index 000000000..51f3f277c
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/util/RetryUtil.java
@@ -0,0 +1,171 @@
+package com.alibaba.datax.common.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.*;
+
+public final class RetryUtil {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RetryUtil.class);
+
+ private static final long MAX_SLEEP_MILLISECOND = 256 * 1000;
+
+ /**
+ * 重试次数工具方法.
+ *
+ * @param callable 实际逻辑
+ * @param retryTimes 最大重试次数（>1）
+ * @param sleepTimeInMilliSecond 运行失败后休眠对应时间再重试
+ * @param exponential 休眠时间是否指数递增
+ * @param 返回值类型
+ * @return 经过重试的callable的执行结果
+ */
+ public static T executeWithRetry(Callable callable,
+ int retryTimes,
+ long sleepTimeInMilliSecond,
+ boolean exponential) throws Exception {
+ Retry retry = new Retry();
+ return retry.doRetry(callable, retryTimes, sleepTimeInMilliSecond, exponential);
+ }
+
+ /**
+ * 在外部线程执行并且重试。每次执行需要在timeoutMs内执行完，不然视为失败。
+ * 执行异步操作的线程池从外部传入，线程池的共享粒度由外部控制。比如，HttpClientUtil共享一个线程池。
+ *
+ * 限制条件：仅仅能够在阻塞的时候interrupt线程
+ *
+ * @param callable 实际逻辑
+ * @param retryTimes 最大重试次数（>1）
+ * @param sleepTimeInMilliSecond 运行失败后休眠对应时间再重试
+ * @param exponential 休眠时间是否指数递增
+ * @param timeoutMs callable执行超时时间，毫秒
+ * @param executor 执行异步操作的线程池
+ * @param 返回值类型
+ * @return 经过重试的callable的执行结果
+ */
+ public static T asyncExecuteWithRetry(Callable callable,
+ int retryTimes,
+ long sleepTimeInMilliSecond,
+ boolean exponential,
+ long timeoutMs,
+ ThreadPoolExecutor executor) throws Exception {
+ Retry retry = new AsyncRetry(timeoutMs, executor);
+ return retry.doRetry(callable, retryTimes, sleepTimeInMilliSecond, exponential);
+ }
+
+ /**
+ * 创建异步执行的线程池。特性如下：
+ * core大小为0，初始状态下无线程，无初始消耗。
+ * max大小为5，最多五个线程。
+ * 60秒超时时间，闲置超过60秒线程会被回收。
+ * 使用SynchronousQueue，任务不会排队，必须要有可用线程才能提交成功，否则会RejectedExecutionException。
+ *
+ * @return 线程池
+ */
+ public static ThreadPoolExecutor createThreadPoolExecutor() {
+ return new ThreadPoolExecutor(0, 5,
+ 60L, TimeUnit.SECONDS,
+ new SynchronousQueue());
+ }
+
+
+ private static class Retry {
+
+ public T doRetry(Callable callable, int retryTimes, long sleepTimeInMilliSecond, boolean exponential)
+ throws Exception {
+
+ if (null == callable) {
+ throw new IllegalArgumentException("系统编程错误, 入参callable不能为空 ! ");
+ }
+
+ if (retryTimes < 1) {
+ throw new IllegalArgumentException(String.format(
+ "系统编程错误, 入参retrytime[%d]不能小于1 !", retryTimes));
+ }
+
+ Exception saveException = null;
+ for (int i = 0; i < retryTimes; i++) {
+ try {
+ return call(callable);
+ } catch (Exception e) {
+ saveException = e;
+
+ if (i + 1 < retryTimes && sleepTimeInMilliSecond > 0) {
+ long startTime = System.currentTimeMillis();
+
+ long timeToSleep;
+ if (exponential) {
+ timeToSleep = sleepTimeInMilliSecond * (long) Math.pow(2, i);
+ if(timeToSleep >= MAX_SLEEP_MILLISECOND) {
+ timeToSleep = MAX_SLEEP_MILLISECOND;
+ }
+ } else {
+ timeToSleep = sleepTimeInMilliSecond;
+ if(timeToSleep >= MAX_SLEEP_MILLISECOND) {
+ timeToSleep = MAX_SLEEP_MILLISECOND;
+ }
+ }
+
+ try {
+ Thread.sleep(timeToSleep);
+ } catch (InterruptedException ignored) {
+ }
+
+ long realTimeSleep = System.currentTimeMillis()-startTime;
+
+ LOG.error(String.format("Exception when calling callable, 即将尝试执行第%s次重试.本次重试计划等待[%s]ms,实际等待[%s]ms, 异常Msg:[%s]",
+ i+1, timeToSleep,realTimeSleep, e.getMessage()));
+
+ }
+ }
+ }
+ throw saveException;
+ }
+
+ protected T call(Callable callable) throws Exception {
+ return callable.call();
+ }
+ }
+
+ private static class AsyncRetry extends Retry {
+
+ private long timeoutMs;
+ private ThreadPoolExecutor executor;
+
+ public AsyncRetry(long timeoutMs, ThreadPoolExecutor executor) {
+ this.timeoutMs = timeoutMs;
+ this.executor = executor;
+ }
+
+ /**
+ * 使用传入的线程池异步执行任务，并且等待。
+ *
+ * future.get()方法，等待指定的毫秒数。如果任务在超时时间内结束，则正常返回。
+ * 如果抛异常（可能是执行超时、执行异常、被其他线程cancel或interrupt），都记录日志并且网上抛异常。
+ * 正常和非正常的情况都会判断任务是否结束，如果没有结束，则cancel任务。cancel参数为true，表示即使
+ * 任务正在执行，也会interrupt线程。
+ *
+ * @param callable
+ * @param
+ * @return
+ * @throws Exception
+ */
+ @Override
+ protected T call(Callable callable) throws Exception {
+ Future future = executor.submit(callable);
+ try {
+ return future.get(timeoutMs, TimeUnit.MILLISECONDS);
+ } catch (Exception e) {
+ LOG.warn("Try once failed", e);
+ throw e;
+ } finally {
+ if (!future.isDone()) {
+ future.cancel(true);
+ LOG.warn("Try once task not done, cancel it, active count: " + executor.getActiveCount());
+ }
+ }
+ }
+ }
+
+}
diff --git a/common/src/main/java/com/alibaba/datax/common/util/StrUtil.java b/common/src/main/java/com/alibaba/datax/common/util/StrUtil.java
new file mode 100755
index 000000000..82222b0d4
--- /dev/null
+++ b/common/src/main/java/com/alibaba/datax/common/util/StrUtil.java
@@ -0,0 +1,85 @@
+package com.alibaba.datax.common.util;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Validate;
+
+import java.text.DecimalFormat;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class StrUtil {
+
+ private final static long KB_IN_BYTES = 1024;
+
+ private final static long MB_IN_BYTES = 1024 * KB_IN_BYTES;
+
+ private final static long GB_IN_BYTES = 1024 * MB_IN_BYTES;
+
+ private final static long TB_IN_BYTES = 1024 * GB_IN_BYTES;
+
+ private final static DecimalFormat df = new DecimalFormat("0.00");
+
+ private static final Pattern VARIABLE_PATTERN = Pattern
+ .compile("(\\$)\\{?(\\w+)\\}?");
+
+ private static String SYSTEM_ENCODING = System.getProperty("file.encoding");
+
+ static {
+ if (SYSTEM_ENCODING == null) {
+ SYSTEM_ENCODING = "UTF-8";
+ }
+ }
+
+ private StrUtil() {
+ }
+
+ public static String stringify(long byteNumber) {
+ if (byteNumber / TB_IN_BYTES > 0) {
+ return df.format((double) byteNumber / (double) TB_IN_BYTES) + "TB";
+ } else if (byteNumber / GB_IN_BYTES > 0) {
+ return df.format((double) byteNumber / (double) GB_IN_BYTES) + "GB";
+ } else if (byteNumber / MB_IN_BYTES > 0) {
+ return df.format((double) byteNumber / (double) MB_IN_BYTES) + "MB";
+ } else if (byteNumber / KB_IN_BYTES > 0) {
+ return df.format((double) byteNumber / (double) KB_IN_BYTES) + "KB";
+ } else {
+ return String.valueOf(byteNumber) + "B";
+ }
+ }
+
+
+ public static String replaceVariable(final String param) {
+ Map mapping = new HashMap();
+
+ Matcher matcher = VARIABLE_PATTERN.matcher(param);
+ while (matcher.find()) {
+ String variable = matcher.group(2);
+ String value = System.getProperty(variable);
+ if (StringUtils.isBlank(value)) {
+ value = matcher.group();
+ }
+ mapping.put(matcher.group(), value);
+ }
+
+ String retString = param;
+ for (final String key : mapping.keySet()) {
+ retString = retString.replace(key, mapping.get(key));
+ }
+
+ return retString;
+ }
+
+ public static String compressMiddle(String s, int headLength, int tailLength) {
+ Validate.notNull(s, "Input string must not be null");
+ Validate.isTrue(headLength > 0, "Head length must be larger than 0");
+ Validate.isTrue(tailLength > 0, "Tail length must be larger than 0");
+
+ if(headLength + tailLength >= s.length()) {
+ return s;
+ }
+ return s.substring(0, headLength) + "..." + s.substring(s.length() - tailLength);
+ }
+
+}
diff --git a/common/src/test/java/com/alibaba/datax/common/base/BaseTest.java b/common/src/test/java/com/alibaba/datax/common/base/BaseTest.java
new file mode 100755
index 000000000..bbc88d9bb
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/base/BaseTest.java
@@ -0,0 +1,30 @@
+package com.alibaba.datax.common.base;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.BeforeClass;
+
+import com.alibaba.datax.common.element.ColumnCast;
+import com.alibaba.datax.common.element.ColumnCastTest;
+import com.alibaba.datax.common.util.Configuration;
+import org.junit.Test;
+
+public class BaseTest {
+
+ @BeforeClass
+ public static void beforeClass() throws IOException {
+ String path = ColumnCastTest.class.getClassLoader().getResource(".")
+ .getFile();
+ ColumnCast.bind(Configuration.from(FileUtils.readFileToString(new File(
+ StringUtils.join(new String[] { path, "all.json" },
+ File.separator)))));
+ }
+
+ @Test
+ public void emptyTest() {
+
+ }
+}
diff --git a/common/src/test/java/com/alibaba/datax/common/element/BoolColumnTest.java b/common/src/test/java/com/alibaba/datax/common/element/BoolColumnTest.java
new file mode 100755
index 000000000..d1db4d0c6
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/element/BoolColumnTest.java
@@ -0,0 +1,79 @@
+package com.alibaba.datax.common.element;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.datax.common.base.BaseTest;
+import com.alibaba.datax.common.exception.DataXException;
+
+public class BoolColumnTest extends BaseTest {
+ @Test
+ public void test_true() {
+ BoolColumn bool = new BoolColumn(true);
+ Assert.assertTrue(bool.asBoolean().equals(true));
+ Assert.assertTrue(bool.asString().equals("true"));
+ Assert.assertTrue(bool.asDouble().equals(1.0d));
+ Assert.assertTrue(bool.asLong().equals(1L));
+
+ try {
+ bool.asDate();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+
+ try {
+ bool.asBytes();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+ }
+
+ @Test
+ public void test_false() {
+ BoolColumn bool = new BoolColumn(false);
+ Assert.assertTrue(bool.asBoolean().equals(false));
+ Assert.assertTrue(bool.asString().equals("false"));
+ Assert.assertTrue(bool.asDouble().equals(0.0d));
+ Assert.assertTrue(bool.asLong().equals(0L));
+
+ try {
+ bool.asDate();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+
+ try {
+ bool.asBytes();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+ }
+
+ @Test
+ public void test_null() {
+ BoolColumn bool = new BoolColumn();
+ Assert.assertTrue(bool.asBoolean() == null);
+ Assert.assertTrue(bool.asString() == null);
+ Assert.assertTrue(bool.asDouble() == null);
+ Assert.assertTrue(bool.asLong() == null);
+
+ try {
+ bool.asDate();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+
+ try {
+ bool.asBytes();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+ }
+
+ @Test
+ public void test_nullReference() {
+ Boolean b = null;
+ BoolColumn boolColumn = new BoolColumn(b);
+ Assert.assertTrue(boolColumn.asBoolean() == null);
+ }
+}
diff --git a/common/src/test/java/com/alibaba/datax/common/element/ColumnCastTest.java b/common/src/test/java/com/alibaba/datax/common/element/ColumnCastTest.java
new file mode 100755
index 000000000..ff800b5cb
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/element/ColumnCastTest.java
@@ -0,0 +1,95 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.util.Configuration;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateFormatUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.Date;
+import java.sql.Time;
+import java.text.ParseException;
+
+public class ColumnCastTest {
+ private Configuration produce() throws IOException {
+ String path = ColumnCastTest.class.getClassLoader().getResource(".")
+ .getFile();
+ String content = FileUtils.readFileToString(new File(StringUtils.join(
+ new String[] { path, "all.json" }, File.separator)));
+ return Configuration.from(content);
+ }
+
+ @Test
+ public void test_string() throws IOException, ParseException {
+ Configuration configuration = this.produce();
+ StringCast.init(configuration);
+
+ System.out.println(StringCast.asDate(new StringColumn("2014-09-18")));
+ Assert.assertTrue(StringCast.asDate(new StringColumn("2014-09-18"))
+ .getTime() == 1410969600000L);
+
+ Assert.assertTrue(StringCast.asDate(new StringColumn("20140918"))
+ .getTime() == 1410969600000L);
+
+ Assert.assertTrue(StringCast.asDate(new StringColumn("08:00:00"))
+ .getTime() == 0L);
+
+ Assert.assertTrue(StringCast.asDate(
+ new StringColumn("2014-09-18 16:00:00")).getTime() == 1411027200000L);
+ configuration
+ .set("common.column.datetimeFormat", "yyyy/MM/dd HH:mm:ss");
+ StringCast.init(configuration);
+ Assert.assertTrue(StringCast.asDate(
+ new StringColumn("2014/09/18 16:00:00")).getTime() == 1411027200000L);
+
+ configuration.set("common.column.timeZone", "GMT");
+ StringCast.init(configuration);
+
+ java.util.Date date = StringCast.asDate(new StringColumn(
+ "2014/09/18 16:00:00"));
+ System.out.println(DateFormatUtils.format(date, "yyyy/MM/dd HH:mm:ss"));
+ Assert.assertTrue("2014/09/19 00:00:00".equals(DateFormatUtils.format(
+ date, "yyyy/MM/dd HH:mm:ss")));
+
+ }
+
+ @Test
+ public void test_date() throws IOException {
+ Assert.assertTrue(DateCast.asString(
+ new DateColumn(System.currentTimeMillis())).startsWith("201"));
+
+ Configuration configuration = this.produce();
+ configuration
+ .set("common.column.datetimeFormat", "MM/dd/yyyy HH:mm:ss");
+ DateCast.init(configuration);
+ System.out.println(DateCast.asString(new DateColumn(System
+ .currentTimeMillis())));
+ Assert.assertTrue(!DateCast.asString(
+ new DateColumn(System.currentTimeMillis())).startsWith("2014"));
+
+ DateColumn dateColumn = new DateColumn(new Time(0L));
+ System.out.println(dateColumn.asString());
+ Assert.assertTrue(dateColumn.asString().equals("08:00:00"));
+
+ configuration.set("common.column.timeZone", "GMT");
+ DateCast.init(configuration);
+ System.err.println(DateCast.asString(dateColumn));
+ Assert.assertTrue(dateColumn.asString().equals("00:00:00"));
+
+ configuration.set("common.column.timeZone", "GMT+8");
+ DateCast.init(configuration);
+ System.out.println(dateColumn.asString());
+ Assert.assertTrue(dateColumn.asString().equals("08:00:00"));
+
+ dateColumn = new DateColumn(new Date(0L));
+ System.out.println(dateColumn.asString());
+ Assert.assertTrue(dateColumn.asString().equals("1970-01-01"));
+
+ dateColumn = new DateColumn(new java.util.Date(0L));
+ System.out.println(dateColumn.asString());
+ Assert.assertTrue(dateColumn.asString().equals("01/01/1970 08:00:00"));
+ }
+}
\ No newline at end of file
diff --git a/common/src/test/java/com/alibaba/datax/common/element/DateColumnTest.java b/common/src/test/java/com/alibaba/datax/common/element/DateColumnTest.java
new file mode 100755
index 000000000..2c063db12
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/element/DateColumnTest.java
@@ -0,0 +1,109 @@
+package com.alibaba.datax.common.element;
+
+import com.alibaba.datax.common.base.BaseTest;
+import com.alibaba.datax.common.exception.DataXException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Date;
+
+public class DateColumnTest extends BaseTest {
+ @Test
+ public void test() {
+ long time = System.currentTimeMillis();
+ DateColumn date = new DateColumn(time);
+ Assert.assertTrue(date.getType().equals(Column.Type.DATE));
+ Assert.assertTrue(date.asDate().getTime() == time);
+ Assert.assertTrue(date.asLong().equals(time));
+ System.out.println(date.asString());
+ Assert.assertTrue(date.asString().startsWith("201"));
+
+ try {
+ date.asBytes();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+
+ try {
+ date.asDouble();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+ }
+
+ @Test
+ public void test_null() {
+ DateColumn date = new DateColumn();
+ DateColumn nul1 = new DateColumn((Long)null);
+ DateColumn nul2 = new DateColumn((Date)null);
+ DateColumn nul3 = new DateColumn((java.sql.Date)null);
+ DateColumn nul4 = new DateColumn((java.sql.Time)null);
+ DateColumn nul5 = new DateColumn((java.sql.Timestamp)null);
+ Assert.assertTrue(date.getType().equals(Column.Type.DATE));
+
+ Assert.assertTrue(date.asDate() == null);
+ Assert.assertTrue(date.asLong() == null);
+ Assert.assertTrue(date.asString() == null);
+
+ Assert.assertTrue(nul1.asDate() == null);
+ Assert.assertTrue(nul1.asLong() == null);
+ Assert.assertTrue(nul1.asString() == null);
+
+ Assert.assertTrue(nul2.asDate() == null);
+ Assert.assertTrue(nul2.asLong() == null);
+ Assert.assertTrue(nul2.asString() == null);
+
+ Assert.assertTrue(nul3.asDate() == null);
+ Assert.assertTrue(nul3.asLong() == null);
+ Assert.assertTrue(nul3.asString() == null);
+
+ Assert.assertTrue(nul4.asDate() == null);
+ Assert.assertTrue(nul4.asLong() == null);
+ Assert.assertTrue(nul4.asString() == null);
+
+ Assert.assertTrue(nul5.asDate() == null);
+ Assert.assertTrue(nul5.asLong() == null);
+ Assert.assertTrue(nul5.asString() == null);
+
+ try {
+ date.asBytes();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+
+ try {
+ date.asDouble();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+
+ try {
+ date.asBoolean();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+
+ }
+
+ @Test
+ public void testDataColumn() throws Exception {
+ DateColumn date = new DateColumn(1449925250000L);
+ Assert.assertEquals(date.asString(),"2015-12-12 21:00:50");
+ Assert.assertEquals(date.asDate(),new Date(1449925250000L));
+
+ java.sql.Date dat2 = new java.sql.Date(1449925251001L);
+ date = new DateColumn(dat2);
+ Assert.assertEquals(date.asString(),"2015-12-12");
+ Assert.assertEquals(date.asDate(),new Date(1449925251001L));
+
+ java.sql.Time dat3 = new java.sql.Time(1449925252002L);
+ date = new DateColumn(dat3);
+ Assert.assertEquals(date.asString(),"21:00:52");
+ Assert.assertEquals(date.asDate(),new Date(1449925252002L));
+
+ java.sql.Timestamp ts = new java.sql.Timestamp(1449925253003L);
+ date = new DateColumn(ts);
+ Assert.assertEquals(date.asString(),"2015-12-12 21:00:53");
+ Assert.assertEquals(date.asDate(),new Date(1449925253003L));
+ }
+}
diff --git a/common/src/test/java/com/alibaba/datax/common/element/DoubleColumnTest.java b/common/src/test/java/com/alibaba/datax/common/element/DoubleColumnTest.java
new file mode 100755
index 000000000..ab8d12ed7
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/element/DoubleColumnTest.java
@@ -0,0 +1,250 @@
+package com.alibaba.datax.common.element;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+public class DoubleColumnTest {
+ @Test
+ public void test_null() {
+ DoubleColumn column = new DoubleColumn();
+
+ System.out.println(column.asString());
+ Assert.assertTrue(column.asString() == null);
+ System.out.println(column.toString());
+ Assert.assertTrue(column.toString().equals(
+ "{\"byteSize\":0,\"type\":\"DOUBLE\"}"));
+ Assert.assertTrue(column.asDouble() == null);
+ Assert.assertTrue(column.asString() == null);
+
+ try {
+ Assert.assertTrue(column.asBoolean() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+
+ try {
+ Assert.assertTrue(column.asDate() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+
+ try {
+ Assert.assertTrue(column.asBytes() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_double() {
+ DoubleColumn column = new DoubleColumn(1.0d);
+
+ System.out.println(column.asString());
+ Assert.assertTrue(column.asString().equals("1.0"));
+ System.out.println(column.toString());
+ Assert.assertTrue(column.toString().equals(
+ "{\"byteSize\":3,\"rawData\":\"1.0\",\"type\":\"DOUBLE\"}"));
+
+ System.out.println(column.asDouble());
+ Assert.assertTrue(column.asDouble().equals(1.0d));
+
+ try {
+ Assert.assertTrue(column.asBoolean() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+
+ try {
+ Assert.assertTrue(column.asBytes() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_float() {
+ DoubleColumn column = new DoubleColumn(1.0f);
+
+ System.out.println(column.asString());
+ Assert.assertTrue(column.asString().equals("1.0"));
+ System.out.println(column.toString());
+ Assert.assertTrue(column.toString().equals(
+ "{\"byteSize\":3,\"rawData\":\"1.0\",\"type\":\"DOUBLE\"}"));
+
+ System.out.println(column.asDouble());
+ Assert.assertTrue(column.asDouble().equals(1.0d));
+
+ try {
+ Assert.assertTrue(column.asBoolean() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+
+ try {
+ Assert.assertTrue(column.asBytes() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_string() {
+ DoubleColumn column = new DoubleColumn("1.0");
+
+ System.out.println(column.asString());
+ Assert.assertTrue(column.asString().equals("1.0"));
+ System.out.println(column.toString());
+ Assert.assertTrue(column.toString().equals(
+ "{\"byteSize\":3,\"rawData\":\"1.0\",\"type\":\"DOUBLE\"}"));
+
+ System.out.println(column.asDouble());
+ Assert.assertTrue(column.asDouble().equals(1.0d));
+
+ try {
+ Assert.assertTrue(column.asBoolean() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+
+ try {
+ Assert.assertTrue(column.asBytes() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_BigDecimal() {
+ DoubleColumn column = new DoubleColumn(new BigDecimal("1E-100"));
+
+ System.out.println(column.asString());
+ System.out.println(column.asString().length());
+ Assert.assertTrue(column.asString().length() == 102);
+
+ Assert.assertTrue(column.asString().equals(
+ new BigDecimal("1E-100").toPlainString()));
+
+ Assert.assertTrue(column
+ .toString()
+ .equals("{\"byteSize\":102,\"rawData\":\"0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001\",\"type\":\"DOUBLE\"}"));
+
+ System.out.println(column.asDouble());
+ Assert.assertTrue(column.asDouble().equals(1.0E-100));
+
+ try {
+ Assert.assertTrue(column.asBoolean() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+
+ try {
+ Assert.assertTrue(column.asBytes() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_overflow() {
+ DoubleColumn column = new DoubleColumn(new BigDecimal("1E-1000"));
+
+ System.out.println(column.asString());
+
+ Assert.assertTrue(column.asBigDecimal().equals(
+ new BigDecimal("1E-1000")));
+
+ Assert.assertTrue(column.asBigInteger().compareTo(BigInteger.ZERO) == 0);
+ Assert.assertTrue(column.asLong().equals(0L));
+
+ try {
+ column.asDouble();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.assertTrue(true);
+ }
+
+ column = new DoubleColumn(new BigDecimal("1E1000"));
+ Assert.assertTrue(column.asBigDecimal().compareTo(
+ new BigDecimal("1E1000")) == 0);
+ Assert.assertTrue(column.asBigInteger().compareTo(
+ new BigDecimal("1E1000").toBigInteger()) == 0);
+ try {
+ column.asDouble();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.assertTrue(true);
+ }
+
+ try {
+ column.asLong();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_NaN() {
+ DoubleColumn column = new DoubleColumn(String.valueOf(Double.NaN));
+ Assert.assertTrue(column.asString().equals("NaN"));
+ try {
+ column.asBigDecimal();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+
+ column = new DoubleColumn(String.valueOf(Double.POSITIVE_INFINITY));
+ Assert.assertTrue(column.asString().equals("Infinity"));
+ try {
+ column.asBigDecimal();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+
+ column = new DoubleColumn(String.valueOf(Double.NEGATIVE_INFINITY));
+ Assert.assertTrue(column.asString().equals("-Infinity"));
+ try {
+ column.asBigDecimal();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_doubleFormat() {
+ System.out.println(new BigDecimal("9801523474.1234567890987654321")
+ .toPlainString());
+
+ System.out.println("double: " + 9801523474.399621d);
+ System.out.println("bigdecimal: " + new BigDecimal(9801523474.399621d).toPlainString());
+ System.out.println("bigdecimal: " + new BigDecimal(String.valueOf(9801523474.399621d)).toPlainString());
+ System.out.println(new DoubleColumn(9801523474.399621d).asString());
+ Assert.assertTrue("9801523474.39962".equals(new DoubleColumn(
+ 9801523474.399621d).asString()));
+
+ Assert.assertTrue(!new DoubleColumn(Double.MAX_VALUE).asString()
+ .contains("E"));
+ Assert.assertTrue(!new DoubleColumn(Float.MAX_VALUE).asString()
+ .contains("E"));
+ }
+}
diff --git a/common/src/test/java/com/alibaba/datax/common/element/LongColumnTest.java b/common/src/test/java/com/alibaba/datax/common/element/LongColumnTest.java
new file mode 100755
index 000000000..aa9f242f9
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/element/LongColumnTest.java
@@ -0,0 +1,217 @@
+package com.alibaba.datax.common.element;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.Date;
+
+public class LongColumnTest {
+
+ @Test
+ public void test_null() {
+ LongColumn column = new LongColumn();
+ System.out.println(column.asString());
+ Assert.assertTrue(column.asString() == null);
+ System.out.println(column.toString());
+ Assert.assertTrue(column.toString().equals(
+ "{\"byteSize\":0,\"type\":\"LONG\"}"));
+ Assert.assertTrue(column.asBoolean() == null);
+ Assert.assertTrue(column.asDouble() == null);
+ Assert.assertTrue(column.asString() == null);
+ Assert.assertTrue(column.asDate() == null);
+
+ try {
+ Assert.assertTrue(column.asBytes() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_normal() {
+ LongColumn column = new LongColumn(1);
+ System.out.println(column.asString());
+ Assert.assertTrue(column.asString().equals("1"));
+ System.out.println(column.toString());
+ Assert.assertEquals(column.toString(),
+ "{\"byteSize\":8,\"rawData\":1,\"type\":\"LONG\"}");
+ Assert.assertTrue(column.asBoolean().equals(true));
+
+ System.out.println(column.asDouble());
+ Assert.assertTrue(column.asDouble().equals(1.0d));
+ Assert.assertTrue(column.asDate().equals(new Date(1L)));
+
+ try {
+ Assert.assertTrue(column.asBytes() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_max() {
+ LongColumn column = new LongColumn(Long.MAX_VALUE);
+ System.out.println(column.asString());
+ Assert.assertTrue(column.asString().equals(
+ String.valueOf(Long.MAX_VALUE)));
+ System.out.println(column.toString());
+ Assert.assertTrue(column
+ .toString()
+ .equals(String
+ .format("{\"byteSize\":8,\"rawData\":9223372036854775807,\"type\":\"LONG\"}",
+ Long.MAX_VALUE)));
+ Assert.assertTrue(column.asBoolean().equals(true));
+
+ System.out.println(column.asDouble());
+ Assert.assertTrue(column.asDouble().equals((double) Long.MAX_VALUE));
+ Assert.assertTrue(column.asDate().equals(new Date(Long.MAX_VALUE)));
+
+ try {
+ Assert.assertTrue(column.asBytes() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_min() {
+ LongColumn column = new LongColumn(Long.MIN_VALUE);
+ System.out.println(column.asString());
+ Assert.assertTrue(column.asString().equals(
+ String.valueOf(Long.MIN_VALUE)));
+ System.out.println(column.toString());
+ Assert.assertTrue(column
+ .toString()
+ .equals(String
+ .format("{\"byteSize\":8,\"rawData\":-9223372036854775808,\"type\":\"LONG\"}",
+ Long.MIN_VALUE)));
+ Assert.assertTrue(column.asBoolean().equals(true));
+
+ System.out.println(column.asDouble());
+ Assert.assertTrue(column.asDouble().equals((double) Long.MIN_VALUE));
+ Assert.assertTrue(column.asDate().equals(new Date(Long.MIN_VALUE)));
+
+ try {
+ Assert.assertTrue(column.asBytes() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_string() {
+ LongColumn column = new LongColumn(String.valueOf(Long.MIN_VALUE));
+ System.out.println(column.asString());
+ Assert.assertTrue(column.asString().equals(
+ String.valueOf(Long.MIN_VALUE)));
+ System.out.println(column.toString());
+ Assert.assertTrue(column
+ .toString()
+ .equals("{\"byteSize\":20,\"rawData\":-9223372036854775808,\"type\":\"LONG\"}"));
+ Assert.assertTrue(column.asBoolean().equals(true));
+
+ System.out.println(column.asDouble());
+ Assert.assertTrue(column.asDouble().equals((double) Long.MIN_VALUE));
+ Assert.assertTrue(column.asDate().equals(new Date(Long.MIN_VALUE)));
+
+ try {
+ Assert.assertTrue(column.asBytes() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_science() {
+ LongColumn column = new LongColumn(String.valueOf("4.7E+38"));
+ System.out.println(column.asString());
+ Assert.assertTrue(column.asString().equals(
+ "470000000000000000000000000000000000000"));
+ System.out.println(column.toString());
+ Assert.assertTrue(column.asBoolean().equals(true));
+
+ System.out.println(">>" + column.asBigDecimal());
+ System.out.println(">>" + new BigDecimal("4.7E+38").toPlainString());
+ Assert.assertTrue(column.asBigDecimal().toPlainString()
+ .equals(new BigDecimal("4.7E+38").toPlainString()));
+
+ try {
+ Assert.assertTrue(column.asBytes() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_bigInteger() {
+ LongColumn column = new LongColumn(BigInteger.valueOf(Long.MIN_VALUE));
+ System.out.println(column.asString());
+ Assert.assertTrue(column.asString().equals(
+ String.valueOf(Long.MIN_VALUE)));
+ System.out.println(column.toString());
+ Assert.assertEquals(column.toString()
+ ,String.format("{\"byteSize\":8,\"rawData\":-9223372036854775808,\"type\":\"LONG\"}",
+ Long.MIN_VALUE));
+ Assert.assertTrue(column.asBoolean().equals(true));
+
+ System.out.println(column.asDouble());
+ Assert.assertTrue(column.asDouble().equals((double) Long.MIN_VALUE));
+ Assert.assertTrue(column.asDate().equals(new Date(Long.MIN_VALUE)));
+
+ try {
+ Assert.assertTrue(column.asBytes() == null);
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_overflow() {
+ LongColumn column = new LongColumn(String.valueOf(Long.MAX_VALUE)
+ + "000");
+
+ Assert.assertTrue(column.asBoolean().equals(true));
+ Assert.assertTrue(column.asBigDecimal().equals(
+ new BigDecimal(String.valueOf(Long.MAX_VALUE) + "000")));
+ Assert.assertTrue(column.asString().equals(
+ String.valueOf(Long.MAX_VALUE) + "000"));
+ Assert.assertTrue(column.asBigInteger().equals(
+ new BigInteger(String.valueOf(Long.MAX_VALUE) + "000")));
+
+ try {
+ column.asLong();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.assertTrue(true);
+ }
+
+ column = new LongColumn(String.valueOf(Long.MIN_VALUE) + "000");
+
+ Assert.assertTrue(column.asBoolean().equals(true));
+ Assert.assertTrue(column.asBigDecimal().equals(
+ new BigDecimal(String.valueOf(Long.MIN_VALUE) + "000")));
+ Assert.assertTrue(column.asString().equals(
+ String.valueOf(Long.MIN_VALUE) + "000"));
+ Assert.assertTrue(column.asBigInteger().equals(
+ new BigInteger(String.valueOf(Long.MIN_VALUE) + "000")));
+
+ try {
+ column.asLong();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.assertTrue(true);
+ }
+
+ }
+}
diff --git a/common/src/test/java/com/alibaba/datax/common/element/ScientificTester.java b/common/src/test/java/com/alibaba/datax/common/element/ScientificTester.java
new file mode 100755
index 000000000..e6a3a1e19
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/element/ScientificTester.java
@@ -0,0 +1,12 @@
+package com.alibaba.datax.common.element;
+
+import org.apache.commons.lang3.math.NumberUtils;
+import org.junit.Test;
+
+public class ScientificTester {
+ @Test
+ public void test() {
+ System.out.println(NumberUtils.createBigDecimal("10E+6").toBigInteger().toString());
+ System.err.println((String) null);
+ }
+}
diff --git a/common/src/test/java/com/alibaba/datax/common/element/StringColumnTest.java b/common/src/test/java/com/alibaba/datax/common/element/StringColumnTest.java
new file mode 100755
index 000000000..ce32ef5a5
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/element/StringColumnTest.java
@@ -0,0 +1,216 @@
+package com.alibaba.datax.common.element;
+
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.datax.common.base.BaseTest;
+import com.alibaba.datax.common.exception.DataXException;
+
+public class StringColumnTest extends BaseTest {
+
+ @Test
+ public void test_double() {
+ DoubleColumn real = new DoubleColumn("3.14");
+ Assert.assertTrue(real.asString().equals("3.14"));
+ Assert.assertTrue(real.asDouble().equals(3.14d));
+ Assert.assertTrue(real.asLong().equals(3L));
+
+ try {
+ real.asBoolean();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+
+ try {
+ real.asDate();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+ }
+
+ @Test
+ public void test_int() {
+ LongColumn integer = new LongColumn("3");
+ Assert.assertTrue(integer.asString().equals("3"));
+ Assert.assertTrue(integer.asDouble().equals(3.0d));
+ Assert.assertTrue(integer.asBoolean().equals(true));
+ Assert.assertTrue(integer.asLong().equals(3L));
+ System.out.println(integer.asDate());
+ }
+
+ @Test
+ public void test_string() {
+ StringColumn string = new StringColumn("bazhen");
+ Assert.assertTrue(string.asString().equals("bazhen"));
+ try {
+ string.asLong();
+ Assert.assertTrue(false);
+
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+ try {
+ string.asDouble();
+ Assert.assertTrue(false);
+
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+ try {
+ string.asDate();
+ Assert.assertTrue(false);
+
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+
+ Assert.assertTrue(new String(string.asString().getBytes())
+ .equals("bazhen"));
+ }
+
+ @Test
+ public void test_bool() {
+ StringColumn string = new StringColumn("true");
+ Assert.assertTrue(string.asString().equals("true"));
+ Assert.assertTrue(string.asBoolean().equals(true));
+
+ try {
+ string.asDate();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+
+ try {
+ string.asDouble();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+
+ try {
+ string.asLong();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof DataXException);
+ }
+ }
+
+ @Test
+ public void test_null() throws UnsupportedEncodingException {
+ StringColumn string = new StringColumn();
+ Assert.assertTrue(string.asString() == null);
+ Assert.assertTrue(string.asLong() == null);
+ Assert.assertTrue(string.asDouble() == null);
+ Assert.assertTrue(string.asDate() == null);
+ Assert.assertTrue(string.asBytes() == null);
+ }
+
+ @Test
+ public void test_overflow() {
+ StringColumn column = new StringColumn(
+ new BigDecimal("1E-1000").toPlainString());
+
+ System.out.println(column.asString());
+
+ Assert.assertTrue(column.asBigDecimal().equals(
+ new BigDecimal("1E-1000")));
+
+ Assert.assertTrue(column.asBigInteger().compareTo(BigInteger.ZERO) == 0);
+ Assert.assertTrue(column.asLong().equals(0L));
+
+ try {
+ column.asDouble();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.assertTrue(true);
+ }
+
+ column = new StringColumn(new BigDecimal("1E1000").toPlainString());
+ Assert.assertTrue(column.asBigDecimal().compareTo(
+ new BigDecimal("1E1000")) == 0);
+ Assert.assertTrue(column.asBigInteger().compareTo(
+ new BigDecimal("1E1000").toBigInteger()) == 0);
+ try {
+ column.asDouble();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.assertTrue(true);
+ }
+
+ try {
+ column.asLong();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.assertTrue(true);
+ }
+
+ column = new StringColumn(new BigDecimal("-1E1000").toPlainString());
+ Assert.assertTrue(column.asBigDecimal().compareTo(
+ new BigDecimal("-1E1000")) == 0);
+ Assert.assertTrue(column.asBigInteger().compareTo(
+ new BigDecimal("-1E1000").toBigInteger()) == 0);
+ try {
+ column.asDouble();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.assertTrue(true);
+ }
+
+ try {
+ column.asLong();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void test_NaN() {
+ StringColumn column = new StringColumn(String.valueOf(Double.NaN));
+ Assert.assertTrue(column.asDouble().equals(Double.NaN));
+ try {
+ column.asBigDecimal();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+
+ column = new StringColumn(String.valueOf(Double.POSITIVE_INFINITY));
+ Assert.assertTrue(column.asDouble().equals(Double.POSITIVE_INFINITY));
+ try {
+ column.asBigDecimal();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+
+ column = new StringColumn(String.valueOf(Double.NEGATIVE_INFINITY));
+ Assert.assertTrue(column.asDouble().equals(Double.NEGATIVE_INFINITY));
+ try {
+ column.asBigDecimal();
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void testEmptyString() {
+ StringColumn column = new StringColumn("");
+ try {
+ BigDecimal num = column.asBigDecimal();
+ } catch(Exception e) {
+ Assert.assertTrue(e.getMessage().contains("String [\"\"] 不能转为BigDecimal"));
+ }
+
+ }
+}
diff --git a/common/src/test/java/com/alibaba/datax/common/exception/DataXExceptionTest.java b/common/src/test/java/com/alibaba/datax/common/exception/DataXExceptionTest.java
new file mode 100755
index 000000000..96cebb966
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/exception/DataXExceptionTest.java
@@ -0,0 +1,29 @@
+package com.alibaba.datax.common.exception;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.datax.common.spi.ErrorCode;
+
+public class DataXExceptionTest {
+
+ private DataXException dataXException;
+
+ @Test
+ public void basicTest() {
+ ErrorCode errorCode = FakeErrorCode.FAKE_ERROR_CODE_ONLY_FOR_TEST_00;
+ String errorMsg = "basicTest";
+ dataXException = DataXException.asDataXException(errorCode, errorMsg);
+ Assert.assertEquals(errorCode.toString() + " - " + errorMsg,
+ dataXException.getMessage());
+ }
+
+ @Test
+ public void basicTest_中文() {
+ ErrorCode errorCode = FakeErrorCode.FAKE_ERROR_CODE_ONLY_FOR_TEST_01;
+ String errorMsg = "basicTest中文";
+ dataXException = DataXException.asDataXException(errorCode, errorMsg);
+ Assert.assertEquals(errorCode.toString() + " - " + errorMsg,
+ dataXException.getMessage());
+ }
+}
diff --git a/common/src/test/java/com/alibaba/datax/common/exception/FakeErrorCode.java b/common/src/test/java/com/alibaba/datax/common/exception/FakeErrorCode.java
new file mode 100755
index 000000000..27e2bdafa
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/exception/FakeErrorCode.java
@@ -0,0 +1,37 @@
+package com.alibaba.datax.common.exception;
+
+import com.alibaba.datax.common.spi.ErrorCode;
+
+public enum FakeErrorCode implements ErrorCode {
+
+ FAKE_ERROR_CODE_ONLY_FOR_TEST_00("FakeErrorCode-00",
+ "only a test, FakeErrorCode."), FAKE_ERROR_CODE_ONLY_FOR_TEST_01(
+ "FakeErrorCode-01",
+ "only a test, FakeErrorCode，测试中文."),
+
+ ;
+
+ private final String code;
+ private final String description;
+
+ private FakeErrorCode(String code, String description) {
+ this.code = code;
+ this.description = description;
+ }
+
+ @Override
+ public String getCode() {
+ return this.code;
+ }
+
+ @Override
+ public String getDescription() {
+ return this.description;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Code:[%s], Describe:[%s]", this.code,
+ this.description);
+ }
+}
diff --git a/common/src/test/java/com/alibaba/datax/common/statistics/PerfRecordTest.java b/common/src/test/java/com/alibaba/datax/common/statistics/PerfRecordTest.java
new file mode 100644
index 000000000..66c07570e
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/statistics/PerfRecordTest.java
@@ -0,0 +1,471 @@
+package com.alibaba.datax.common.statistics;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Field;
+import java.util.List;
+
+/**
+ * Created by liqiang on 15/8/26.
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class PerfRecordTest {
+ private static Logger LOG = LoggerFactory.getLogger(PerfRecordTest.class);
+ private final int TGID = 1;
+
+
+ @Before
+ public void setUp() throws Exception {
+ Field instance=PerfTrace.class.getDeclaredField("instance");
+ instance.setAccessible(true);
+ instance.set(null,null);
+ }
+
+ public boolean hasRecordInList(List perfRecordList,PerfRecord perfRecord){
+ if(perfRecordList==null || perfRecordList.size()==0){
+ return false;
+ }
+
+ for(PerfRecord perfRecord1:perfRecordList){
+ if(perfRecord.equals(perfRecord1)){
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ @Test
+ public void test001PerfRecordEquals() throws Exception {
+ PerfTrace.getInstance(true, 1001, 1, 0, true);
+
+ PerfRecord initPerfRecord = new PerfRecord(TGID, 1, PerfRecord.PHASE.WRITE_TASK_INIT);
+ initPerfRecord.start();
+ Thread.sleep(50);
+ initPerfRecord.end();
+
+ PerfRecord initPerfRecord2 = initPerfRecord.copy();
+
+ Assert.assertTrue(initPerfRecord.equals(initPerfRecord2));
+
+ PerfRecord initPerfRecord3 = new PerfRecord(TGID, 1, PerfRecord.PHASE.READ_TASK_DESTROY);
+ initPerfRecord3.start();
+ Thread.sleep(1050);
+ initPerfRecord3.end();
+
+ Assert.assertTrue(!initPerfRecord.equals(initPerfRecord3));
+
+ PerfRecord initPerfRecord4 = new PerfRecord(TGID, 1, PerfRecord.PHASE.WRITE_TASK_INIT);
+ initPerfRecord4.start();
+ Thread.sleep(2050);
+ initPerfRecord4.end();
+
+ System.out.println(initPerfRecord4.toString());
+ System.out.println(initPerfRecord.toString());
+
+ Assert.assertTrue(!initPerfRecord.equals(initPerfRecord4));
+
+ PerfRecord initPerfRecord5 = new PerfRecord(TGID, 1, PerfRecord.PHASE.WRITE_TASK_INIT);
+ initPerfRecord5.start();
+ Thread.sleep(50);
+ initPerfRecord5.end();
+
+ initPerfRecord5.addCount(100);
+ initPerfRecord5.addSize(200);
+
+ Assert.assertTrue(!initPerfRecord.equals(initPerfRecord5));
+
+ PerfRecord initPerfRecord6 = initPerfRecord.copy();
+ initPerfRecord6.addCount(1001);
+ initPerfRecord6.addSize(1001);
+
+ Assert.assertTrue(initPerfRecord.equals(initPerfRecord6));
+
+ }
+
+ @Test
+ public void test002Normal() throws Exception {
+
+ PerfTrace.getInstance(true, 1001, 1, 0, true);
+
+ PerfRecord initPerfRecord = new PerfRecord(TGID, 1, PerfRecord.PHASE.WRITE_TASK_INIT);
+ initPerfRecord.start();
+ Thread.sleep(1050);
+ initPerfRecord.end();
+
+ Assert.assertTrue(initPerfRecord.getAction().name().equals("end"));
+ Assert.assertTrue(initPerfRecord.getElapsedTimeInNs() >= 1050000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WRITE_TASK_INIT).getTotalCount() == 1);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), initPerfRecord));
+
+
+ LOG.debug("task writer starts to do prepare ...");
+ PerfRecord preparePerfRecord = new PerfRecord(TGID, 1, PerfRecord.PHASE.WRITE_TASK_PREPARE);
+ preparePerfRecord.start();
+ Thread.sleep(1020);
+ preparePerfRecord.end();
+
+ Assert.assertTrue(preparePerfRecord.getAction().name().equals("end"));
+ Assert.assertTrue(preparePerfRecord.getElapsedTimeInNs() >= 1020000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WRITE_TASK_PREPARE).getTotalCount() == 1);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), preparePerfRecord));
+
+
+ LOG.debug("task writer starts to write ...");
+ PerfRecord dataPerfRecord = new PerfRecord(TGID, 1, PerfRecord.PHASE.READ_TASK_DATA);
+ dataPerfRecord.start();
+
+ Thread.sleep(1200);
+ dataPerfRecord.addCount(1001);
+ dataPerfRecord.addSize(1002);
+ dataPerfRecord.end();
+
+ Assert.assertTrue(dataPerfRecord.getAction().name().equals("end"));
+ Assert.assertTrue(dataPerfRecord.getElapsedTimeInNs() >= 1020000000);
+ Assert.assertTrue(dataPerfRecord.getCount() == 1001);
+ Assert.assertTrue(dataPerfRecord.getSize() == 1002);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.READ_TASK_DATA).getTotalCount() == 1);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), dataPerfRecord));
+
+
+ PerfRecord destoryPerfRecord = new PerfRecord(TGID, 1, PerfRecord.PHASE.READ_TASK_DESTROY);
+ destoryPerfRecord.start();
+
+ Thread.sleep(250);
+ destoryPerfRecord.end();
+
+ Assert.assertTrue(destoryPerfRecord.getAction().name().equals("end"));
+ Assert.assertTrue(destoryPerfRecord.getElapsedTimeInNs() >= 250000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.READ_TASK_DESTROY).getTotalCount() == 1);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), destoryPerfRecord));
+
+ PerfRecord waitTimePerfRecord = new PerfRecord(TGID, 1, PerfRecord.PHASE.WAIT_READ_TIME);
+ waitTimePerfRecord.start();
+
+ Thread.sleep(250);
+ waitTimePerfRecord.end();
+
+ Assert.assertTrue(waitTimePerfRecord.getAction().name().equals("end"));
+ Assert.assertTrue(waitTimePerfRecord.getElapsedTimeInNs() >= 250000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WAIT_READ_TIME).getTotalCount() == 1);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), waitTimePerfRecord));
+
+
+
+ PerfRecord initPerfRecord2 = new PerfRecord(TGID, 2, PerfRecord.PHASE.WRITE_TASK_INIT);
+ initPerfRecord2.start();
+ Thread.sleep(50);
+ initPerfRecord2.end();
+
+ Assert.assertTrue(initPerfRecord2.getAction().name().equals("end"));
+ Assert.assertTrue(initPerfRecord2.getElapsedTimeInNs() >= 50000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WRITE_TASK_INIT).getTotalCount() == 2);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), initPerfRecord2));
+
+ LOG.debug("task writer starts to do prepare ...");
+ PerfRecord preparePerfRecord2 = new PerfRecord(TGID, 2, PerfRecord.PHASE.WRITE_TASK_PREPARE);
+ preparePerfRecord2.start();
+ Thread.sleep(20);
+ preparePerfRecord2.end();
+ LOG.debug("task writer starts to write ...");
+
+ Assert.assertTrue(preparePerfRecord2.getAction().name().equals("end"));
+ Assert.assertTrue(preparePerfRecord2.getElapsedTimeInNs() >= 20000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WRITE_TASK_PREPARE).getTotalCount() == 2);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), preparePerfRecord2));
+
+
+ PerfRecord dataPerfRecor2 = new PerfRecord(TGID, 2, PerfRecord.PHASE.READ_TASK_DATA);
+ dataPerfRecor2.start();
+
+ Thread.sleep(2200);
+ dataPerfRecor2.addCount(2001);
+ dataPerfRecor2.addSize(2002);
+ dataPerfRecor2.end();
+
+ Assert.assertTrue(dataPerfRecor2.getAction().name().equals("end"));
+ Assert.assertTrue(dataPerfRecor2.getElapsedTimeInNs() >= 2200000000L);
+ Assert.assertTrue(dataPerfRecor2.getCount() == 2001);
+ Assert.assertTrue(dataPerfRecor2.getSize() == 2002);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.READ_TASK_DATA).getTotalCount() == 2);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), dataPerfRecor2));
+
+
+ PerfRecord destoryPerfRecord2 = new PerfRecord(TGID, 2, PerfRecord.PHASE.READ_TASK_DESTROY);
+ destoryPerfRecord2.start();
+
+ Thread.sleep(1250);
+ destoryPerfRecord2.end();
+
+ Assert.assertTrue(destoryPerfRecord2.getAction().name().equals("end"));
+ Assert.assertTrue(destoryPerfRecord2.getElapsedTimeInNs() >= 1250000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.READ_TASK_DESTROY).getTotalCount() == 2);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), destoryPerfRecord2));
+
+ PerfRecord waitPerfRecord2 = new PerfRecord(TGID, 2, PerfRecord.PHASE.WAIT_READ_TIME);
+ waitPerfRecord2.start();
+
+ Thread.sleep(1250);
+ waitPerfRecord2.end();
+
+ Assert.assertTrue(waitPerfRecord2.getAction().name().equals("end"));
+ Assert.assertTrue(waitPerfRecord2.getElapsedTimeInNs() >= 1250000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WAIT_READ_TIME).getTotalCount() == 2);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), waitPerfRecord2));
+
+
+ PerfTrace.getInstance().addTaskDetails(1, " ");
+ PerfTrace.getInstance().addTaskDetails(1, "task 1 some thing abcdf");
+ PerfTrace.getInstance().addTaskDetails(2,"before char");
+ PerfTrace.getInstance().addTaskDetails(2,"task 2 some thing abcdf");
+
+ Assert.assertTrue(PerfTrace.getInstance().getTaskDetails().get(1).equals("task 1 some thing abcdf"));
+ Assert.assertTrue(PerfTrace.getInstance().getTaskDetails().get(2).equals("before char,task 2 some thing abcdf"));
+ System.out.println(PerfTrace.getInstance().summarizeNoException());
+ }
+ @Test
+ public void test003Disable() throws Exception {
+
+ PerfTrace.getInstance(true, 1001, 1, 0, false);
+
+ PerfRecord initPerfRecord = new PerfRecord(TGID, 1, PerfRecord.PHASE.WRITE_TASK_INIT);
+ initPerfRecord.start();
+ Thread.sleep(1050);
+ initPerfRecord.end();
+
+ Assert.assertTrue(initPerfRecord.getDatetime().equals("null time"));
+ Assert.assertTrue(initPerfRecord.getElapsedTimeInNs() == -1);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WRITE_TASK_INIT) == null);
+
+
+ LOG.debug("task writer starts to do prepare ...");
+ PerfRecord preparePerfRecord = new PerfRecord(TGID, 1, PerfRecord.PHASE.WRITE_TASK_PREPARE);
+ preparePerfRecord.start();
+ Thread.sleep(1020);
+ preparePerfRecord.end();
+ LOG.debug("task writer starts to write ...");
+
+ Assert.assertTrue(preparePerfRecord.getDatetime().equals("null time"));
+ Assert.assertTrue(preparePerfRecord.getElapsedTimeInNs() == -1);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WRITE_TASK_PREPARE) == null);
+
+
+ PerfRecord dataPerfRecord = new PerfRecord(TGID, 1, PerfRecord.PHASE.READ_TASK_DATA);
+ dataPerfRecord.start();
+
+ Thread.sleep(1200);
+ dataPerfRecord.addCount(1001);
+ dataPerfRecord.addSize(1001);
+ dataPerfRecord.end();
+
+ Assert.assertTrue(dataPerfRecord.getDatetime().equals("null time"));
+ Assert.assertTrue(dataPerfRecord.getElapsedTimeInNs() == -1);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.READ_TASK_DATA) == null);
+
+ PerfRecord waitPerfRecor1 = new PerfRecord(TGID, 1, PerfRecord.PHASE.WAIT_WRITE_TIME);
+ waitPerfRecor1.start();
+
+ Thread.sleep(2200);
+ waitPerfRecor1.end();
+
+ Assert.assertTrue(waitPerfRecor1.getDatetime().equals("null time"));
+ Assert.assertTrue(waitPerfRecor1.getElapsedTimeInNs() == -1);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WAIT_WRITE_TIME) == null);
+
+
+ PerfRecord initPerfRecord2 = new PerfRecord(TGID, 2, PerfRecord.PHASE.WRITE_TASK_INIT);
+ initPerfRecord2.start();
+ Thread.sleep(50);
+ initPerfRecord2.end();
+
+ Assert.assertTrue(initPerfRecord2.getDatetime().equals("null time"));
+ Assert.assertTrue(initPerfRecord2.getElapsedTimeInNs() == -1);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WRITE_TASK_INIT) == null);
+
+ LOG.debug("task writer starts to do prepare ...");
+ PerfRecord preparePerfRecord2 = new PerfRecord(TGID, 2, PerfRecord.PHASE.WRITE_TASK_PREPARE);
+ preparePerfRecord2.start();
+ Thread.sleep(20);
+ preparePerfRecord2.end();
+ LOG.debug("task writer starts to write ...");
+
+ Assert.assertTrue(preparePerfRecord2.getDatetime().equals("null time"));
+ Assert.assertTrue(preparePerfRecord2.getElapsedTimeInNs() == -1);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WRITE_TASK_PREPARE) == null);
+
+
+
+ PerfRecord dataPerfRecor2 = new PerfRecord(TGID, 2, PerfRecord.PHASE.READ_TASK_DATA);
+ dataPerfRecor2.start();
+
+ Thread.sleep(2200);
+ dataPerfRecor2.addCount(2001);
+ dataPerfRecor2.addSize(2001);
+ dataPerfRecor2.end();
+
+ Assert.assertTrue(dataPerfRecor2.getDatetime().equals("null time"));
+ Assert.assertTrue(dataPerfRecor2.getElapsedTimeInNs() == -1);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.READ_TASK_DATA) == null);
+
+ PerfRecord waitPerfRecor2 = new PerfRecord(TGID, 2, PerfRecord.PHASE.WAIT_WRITE_TIME);
+ waitPerfRecor2.start();
+
+ Thread.sleep(2200);
+ waitPerfRecor2.end();
+
+ Assert.assertTrue(waitPerfRecor2.getDatetime().equals("null time"));
+ Assert.assertTrue(waitPerfRecor2.getElapsedTimeInNs() == -1);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WAIT_WRITE_TIME) == null);
+
+ PerfTrace.getInstance().addTaskDetails(1, "task 1 some thing abcdf");
+ PerfTrace.getInstance().addTaskDetails(2, "task 2 some thing abcdf");
+
+ Assert.assertTrue(PerfTrace.getInstance().getTaskDetails().size()==0);
+ System.out.println(PerfTrace.getInstance().summarizeNoException());
+ }
+
+ @Test
+ public void test004Normal2() throws Exception {
+ int priority = 0;
+ try {
+ priority = Integer.parseInt(System.getenv("SKYNET_PRIORITY"));
+ }catch (NumberFormatException e){
+ LOG.warn("prioriy set to 0, because NumberFormatException, the value is: "+System.getProperty("PROIORY"));
+ }
+
+ System.out.println("priority====" + priority);
+
+ PerfTrace.getInstance(false, 1001001001001L, 1, 0, true);
+
+ PerfRecord initPerfRecord = new PerfRecord(TGID, 10000001, PerfRecord.PHASE.WRITE_TASK_INIT);
+ initPerfRecord.start();
+ Thread.sleep(1050);
+ initPerfRecord.end();
+
+ Assert.assertTrue(initPerfRecord.getAction().name().equals("end"));
+ Assert.assertTrue(initPerfRecord.getElapsedTimeInNs() >= 1050000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WRITE_TASK_INIT).getTotalCount() == 1);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), initPerfRecord));
+
+
+ LOG.debug("task writer starts to do prepare ...");
+ PerfRecord preparePerfRecord = new PerfRecord(TGID, 10000001, PerfRecord.PHASE.WRITE_TASK_PREPARE);
+ preparePerfRecord.start();
+ Thread.sleep(1020);
+ preparePerfRecord.end();
+
+ Assert.assertTrue(preparePerfRecord.getAction().name().equals("end"));
+ Assert.assertTrue(preparePerfRecord.getElapsedTimeInNs() >= 1020000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WRITE_TASK_PREPARE).getTotalCount() == 1);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), preparePerfRecord));
+
+ LOG.debug("task wait time ...");
+ PerfRecord waitPerfRecord = new PerfRecord(TGID, 10000001, PerfRecord.PHASE.WAIT_WRITE_TIME);
+ waitPerfRecord.start();
+ Thread.sleep(1030);
+ waitPerfRecord.end();
+
+ Assert.assertTrue(waitPerfRecord.getAction().name().equals("end"));
+ Assert.assertTrue(waitPerfRecord.getElapsedTimeInNs() >= 1030000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WAIT_WRITE_TIME).getTotalCount() == 1);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), waitPerfRecord));
+
+
+
+ LOG.debug("task writer starts to write ...");
+
+ PerfRecord dataPerfRecord = new PerfRecord(TGID, 10000001, PerfRecord.PHASE.READ_TASK_DATA);
+ dataPerfRecord.start();
+
+ Thread.sleep(1200);
+ dataPerfRecord.addCount(1001);
+ dataPerfRecord.addSize(1002);
+ dataPerfRecord.end();
+
+ Assert.assertTrue(dataPerfRecord.getAction().name().equals("end"));
+ Assert.assertTrue(dataPerfRecord.getElapsedTimeInNs() >= 1020000000);
+ Assert.assertTrue(dataPerfRecord.getCount() == 1001);
+ Assert.assertTrue(dataPerfRecord.getSize() == 1002);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.READ_TASK_DATA).getTotalCount() == 1);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), dataPerfRecord));
+
+
+ PerfRecord initPerfRecord2 = new PerfRecord(TGID, 10000002, PerfRecord.PHASE.WRITE_TASK_INIT);
+ initPerfRecord2.start();
+ Thread.sleep(50);
+ initPerfRecord2.end();
+
+ Assert.assertTrue(initPerfRecord2.getAction().name().equals("end"));
+ Assert.assertTrue(initPerfRecord2.getElapsedTimeInNs() >= 50000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WRITE_TASK_INIT).getTotalCount() == 2);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), initPerfRecord2));
+
+ LOG.debug("task wait time ...");
+ PerfRecord waitPerfRecord2 = new PerfRecord(TGID, 10000002, PerfRecord.PHASE.WAIT_WRITE_TIME);
+ waitPerfRecord2.start();
+ Thread.sleep(2030);
+ waitPerfRecord2.end();
+
+ Assert.assertTrue(waitPerfRecord2.getAction().name().equals("end"));
+ Assert.assertTrue(waitPerfRecord2.getElapsedTimeInNs() >= 2030000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WAIT_WRITE_TIME).getTotalCount() == 2);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), waitPerfRecord2));
+
+
+ LOG.debug("task writer starts to do prepare ...");
+ PerfRecord preparePerfRecord2 = new PerfRecord(TGID, 10000002, PerfRecord.PHASE.WRITE_TASK_PREPARE);
+ preparePerfRecord2.start();
+ Thread.sleep(20);
+ preparePerfRecord2.end();
+
+ Assert.assertTrue(preparePerfRecord2.getAction().name().equals("end"));
+ Assert.assertTrue(preparePerfRecord2.getElapsedTimeInNs() >= 20000000);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.WRITE_TASK_PREPARE).getTotalCount() == 2);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), preparePerfRecord2));
+
+
+ LOG.debug("task writer starts to write ...");
+
+ PerfRecord dataPerfRecor2 = new PerfRecord(TGID, 10000002, PerfRecord.PHASE.READ_TASK_DATA);
+ dataPerfRecor2.start();
+
+ Thread.sleep(2200);
+ dataPerfRecor2.addCount(2001);
+ dataPerfRecor2.addSize(2002);
+ dataPerfRecor2.end();
+
+ Assert.assertTrue(dataPerfRecor2.getAction().name().equals("end"));
+ Assert.assertTrue(dataPerfRecor2.getElapsedTimeInNs() >= 2200000000L);
+ Assert.assertTrue(dataPerfRecor2.getCount() == 2001);
+ Assert.assertTrue(dataPerfRecor2.getSize() == 2002);
+ Assert.assertTrue(PerfTrace.getInstance().getPerfRecordMaps().get(PerfRecord.PHASE.READ_TASK_DATA).getTotalCount() == 2);
+ Assert.assertTrue(hasRecordInList(PerfTrace.getInstance().getWaitingReportList(), dataPerfRecor2));
+
+
+ PerfTrace.getInstance().addTaskDetails(10000001, "task 100000011 some thing abcdf");
+ PerfTrace.getInstance().addTaskDetails(10000002, "task 100000012 some thing abcdf");
+ PerfTrace.getInstance().addTaskDetails(10000004, "task 100000012 some thing abcdf?123?345");
+ PerfTrace.getInstance().addTaskDetails(10000005, "task 100000012 some thing abcdf?456");
+ PerfTrace.getInstance().addTaskDetails(10000006, "[task 100000012? some thing abcdf?456");
+
+ Assert.assertTrue(PerfTrace.getInstance().getTaskDetails().get(10000001).equals("task 100000011 some thing abcdf"));
+ Assert.assertTrue(PerfTrace.getInstance().getTaskDetails().get(10000002).equals("task 100000012 some thing abcdf"));
+
+ PerfRecord.addPerfRecord(TGID, 10000003, PerfRecord.PHASE.TASK_TOTAL, System.currentTimeMillis(), 12300123L * 1000L * 1000L);
+ PerfRecord.addPerfRecord(TGID, 10000004, PerfRecord.PHASE.TASK_TOTAL, System.currentTimeMillis(), 22300123L * 1000L * 1000L);
+ PerfRecord.addPerfRecord(TGID, 10000005, PerfRecord.PHASE.SQL_QUERY, System.currentTimeMillis(), 4L);
+ PerfRecord.addPerfRecord(TGID, 10000006, PerfRecord.PHASE.RESULT_NEXT_ALL, System.currentTimeMillis(), 3000L);
+ PerfRecord.addPerfRecord(TGID, 10000006, PerfRecord.PHASE.ODPS_BLOCK_CLOSE, System.currentTimeMillis(), 2000000L);
+
+ System.out.println(PerfTrace.getInstance().summarizeNoException());
+
+
+ }
+
+}
\ No newline at end of file
diff --git a/common/src/test/java/com/alibaba/datax/common/statistics/VMInfoTest.java b/common/src/test/java/com/alibaba/datax/common/statistics/VMInfoTest.java
new file mode 100644
index 000000000..ce97007c5
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/statistics/VMInfoTest.java
@@ -0,0 +1,109 @@
+package com.alibaba.datax.common.statistics;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.management.*;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by liqiang on 15/11/12.
+ */
+public class VMInfoTest {
+ static final long MB = 1024 * 1024;
+
+ @Test
+ public void testOs() throws Exception {
+
+ RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
+ System.out.println(runtimeMXBean.getName());
+ System.out.println("jvm运营商:" + runtimeMXBean.getVmVendor());
+ System.out.println("jvm规范版本:" + runtimeMXBean.getSpecVersion());
+ System.out.println("jvm实现版本:" + runtimeMXBean.getVmVersion());
+
+
+ OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean();
+ System.out.println(osMXBean.getName());
+ System.out.println(osMXBean.getArch());
+ System.out.println(osMXBean.getVersion());
+ System.out.println(osMXBean.getAvailableProcessors());
+
+
+ if (VMInfo.isSunOsMBean(osMXBean)) {
+ long totalPhysicalMemory = VMInfo.getLongFromOperatingSystem(osMXBean, "getTotalPhysicalMemorySize");
+ long freePhysicalMemory = VMInfo.getLongFromOperatingSystem(osMXBean, "getFreePhysicalMemorySize");
+ System.out.println("总物理内存(M):" + totalPhysicalMemory / MB);
+ System.out.println("剩余物理内存(M):" + freePhysicalMemory / MB);
+
+ long maxFileDescriptorCount = VMInfo.getLongFromOperatingSystem(osMXBean, "getMaxFileDescriptorCount");
+ long currentOpenFileDescriptorCount = VMInfo.getLongFromOperatingSystem(osMXBean, "getOpenFileDescriptorCount");
+ long getProcessCpuTime = VMInfo.getLongFromOperatingSystem(osMXBean, "getProcessCpuTime");
+ System.out.println(osMXBean.getSystemLoadAverage());
+ System.out.println("maxFileDescriptorCount=>" + maxFileDescriptorCount);
+ System.out.println("currentOpenFileDescriptorCount=>" + currentOpenFileDescriptorCount);
+ System.out.println("jvm运行时间（毫秒）:" + runtimeMXBean.getUptime());
+ System.out.println("getProcessCpuTime=>" + getProcessCpuTime);
+
+ long startTime = System.currentTimeMillis();
+ while (true) {
+ if (System.currentTimeMillis() > startTime + 1000) {
+ break;
+ }
+ }
+// system = ManagementFactory.getOperatingSystemMXBean();
+// runtime = ManagementFactory.getRuntimeMXBean();
+ System.out.println("test!!" + 2 * 2 * 2 * 123456789);
+ System.out.println("test!!" + 123456789 * 987654321);
+ System.out.println("test!!" + 2 * 2 * 2 * 2);
+ System.out.println("test!!" + 3 * 2 * 4);
+ System.out.println("test123!!");
+ long upTime = runtimeMXBean.getUptime();
+ long processTime = VMInfo.getLongFromOperatingSystem(osMXBean, "getProcessCpuTime");
+ System.out.println("jvm运行时间（毫秒）:" + upTime);
+ System.out.println("getProcessCpuTime=>" + processTime);
+
+ System.out.println(String.format("%,.1f", (float) processTime / (upTime * osMXBean.getAvailableProcessors() * 10000)));
+
+
+ List garbages = ManagementFactory.getGarbageCollectorMXBeans();
+ for (GarbageCollectorMXBean garbage : garbages) {
+ System.out.println("垃圾收集器：名称=" + garbage.getName() + ",收集=" + garbage.getCollectionCount() + ",总花费时间=" + garbage.getCollectionTime() + ",内存区名称=" + Arrays.deepToString(garbage.getMemoryPoolNames()));
+ }
+
+ List pools = ManagementFactory.getMemoryPoolMXBeans();
+ if (pools != null && !pools.isEmpty()) {
+ for (MemoryPoolMXBean pool : pools) {
+ //只打印一些各个内存区都有的属性，一些区的特殊属性，可看文档或百度
+ // 最大值，初始值，如果没有定义的话，返回-1，所以真正使用时，要注意
+ System.out.println("vm内存区:\n\t名称=" + pool.getName() + "\n\t所属内存管理者=" + Arrays.deepToString(pool.getMemoryManagerNames()) + "\n\t ObjectName=" + "\n\t初始大小(M)=" + pool.getUsage().getInit() / MB + "\n\t最大(上限)(M)=" + pool.getUsage().getMax() / MB + "\n\t已用大小(M)=" + pool.getUsage().getUsed() / MB + "\n\t已提交(已申请)(M)=" + pool.getUsage().getCommitted() / MB + "\n\t使用率=" + (pool.getUsage().getUsed() * 100 / pool.getUsage().getCommitted()) + "%");
+ }
+ }
+
+ }
+ }
+
+ @Test
+ public void testVMInfo() throws Exception {
+ VMInfo vmInfo = VMInfo.getVmInfo();
+ Assert.assertTrue(vmInfo != null);
+ System.out.println(vmInfo.toString());
+ vmInfo.getDelta();
+ int count = 0;
+
+ while(count < 10) {
+ long startTime = System.currentTimeMillis();
+ while (true) {
+ if (System.currentTimeMillis() > startTime + 1000) {
+ break;
+ }
+ }
+ vmInfo.getDelta();
+ count++;
+ Thread.sleep(1000);
+ }
+
+ vmInfo.getDelta(false);
+ System.out.println(vmInfo.totalString());
+ }
+}
\ No newline at end of file
diff --git a/common/src/test/java/com/alibaba/datax/common/util/ConfigurationTest.java b/common/src/test/java/com/alibaba/datax/common/util/ConfigurationTest.java
new file mode 100755
index 000000000..1b8a56858
--- /dev/null
+++ b/common/src/test/java/com/alibaba/datax/common/util/ConfigurationTest.java
@@ -0,0 +1,699 @@
+package com.alibaba.datax.common.util;
+
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.fastjson.JSON;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.*;
+
+public class ConfigurationTest {
+
+ @Test
+ public void test_get() {
+ Configuration configuration = Configuration
+ .from("{\"a\":[{\"b\":[{\"c\":[\"bazhen\"]}]}]}");
+
+ String path = "";
+ Assert.assertTrue(JSON.toJSONString(configuration.get(path)).equals(
+ "{\"a\":[{\"b\":[{\"c\":[\"bazhen\"]}]}]}"));
+
+ path = "a[0].b[0].c[0]";
+ Assert.assertTrue(JSON.toJSONString(configuration.get(path)).equals(
+ "\"bazhen\""));
+
+ configuration = Configuration.from("{\"a\": [[[0]]]}");
+ path = "a[0][0][0]";
+ System.out.println(JSON.toJSONString(configuration.get(path)));
+ Assert.assertTrue(JSON.toJSONString(configuration.get(path))
+ .equals("0"));
+
+ path = "a[0]";
+ System.out.println(JSON.toJSONString(configuration.get(path)));
+ Assert.assertTrue(JSON.toJSONString(configuration.get(path)).equals(
+ "[[0]]"));
+
+ path = "c[0]";
+ System.out.println(JSON.toJSONString(configuration.get(path)));
+ Assert.assertTrue(JSON.toJSONString(configuration.get(path)).equals(
+ "null"));
+
+ configuration = Configuration.from("[1,2]");
+ System.out.println(configuration.get("[0]"));
+ Assert.assertTrue(configuration.getString("[0]").equals("1"));
+ Assert.assertTrue(configuration.getString("[1]").equals("2"));
+
+ }
+
+ @Test
+ public void test_buildObject() {
+
+ // 非法参数
+ try {
+ Configuration.from("{}").buildObject(null, "bazhen");
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+
+ // 测试单元素
+ Assert.assertTrue(Configuration.from("{}")
+ .buildObject(new ArrayList(), "bazhen")
+ .equals("bazhen"));
+ Assert.assertTrue(Configuration.from("{}").buildObject(
+ new ArrayList(), new HashMap()) instanceof Map);
+ Assert.assertTrue(Configuration.from("{}").buildObject(
+ new ArrayList(), null) == null);
+
+ // 测试多级元素
+ String path = null;
+ String json = null;
+
+ path = "";
+ json = JSON.toJSONString(Configuration.from("{}").buildObject(
+ Arrays.asList(StringUtils.split(path, ".")), "bazhen"));
+ System.out.println(json);
+ Assert.assertTrue("\"bazhen\"".equals(json));
+
+ path = "a";
+ json = JSON.toJSONString(Configuration.from("{}").buildObject(
+ Arrays.asList(StringUtils.split(path, ".")), "bazhen"));
+ System.out.println(json);
+ Assert.assertTrue("{\"a\":\"bazhen\"}".equals(json));
+
+ path = "a";
+ json = JSON.toJSONString(Configuration.from("{}").buildObject(
+ Arrays.asList(StringUtils.split(path, ".")),
+ new HashMap()));
+ System.out.println(json);
+ Assert.assertTrue("{\"a\":{}}".equals(json));
+
+ path = "a";
+ json = JSON.toJSONString(Configuration.from("{}").buildObject(
+ Arrays.asList(StringUtils.split(path, ".")),
+ new ArrayList()));
+ System.out.println(json);
+ Assert.assertTrue("{\"a\":[]}".equals(json));
+
+ path = "a";
+ json = JSON.toJSONString(Configuration.from("{}").buildObject(
+ Arrays.asList(StringUtils.split(path, ".")), 1L));
+ System.out.println(json);
+ Assert.assertTrue("{\"a\":1}".equals(json));
+
+ path = "a";
+ json = JSON.toJSONString(Configuration.from("{}").buildObject(
+ Arrays.asList(StringUtils.split(path, ".")), 1.1));
+ System.out.println(json);
+ Assert.assertTrue("{\"a\":1.1}".equals(json));
+
+ path = "[0]";
+ json = JSON.toJSONString(Configuration.from("{}").buildObject(
+ Arrays.asList(StringUtils.split(path, ".")), "bazhen"));
+ System.out.println(json);
+ Assert.assertTrue("[\"bazhen\"]".equals(json));
+
+ path = "[1]";
+ json = JSON.toJSONString(Configuration.from("{}").buildObject(
+ Arrays.asList(StringUtils.split(path, ".")), "bazhen"));
+ System.out.println(json);
+ Assert.assertTrue("[null,\"bazhen\"]".equals(json));
+
+ path = "a.b.c.d.e.f";
+ json = JSON.toJSONString(Configuration.from("{}").buildObject(
+ Arrays.asList(StringUtils.split(path, ".")), "bazhen"));
+ System.out.println(json);
+ Assert.assertTrue("{\"a\":{\"b\":{\"c\":{\"d\":{\"e\":{\"f\":\"bazhen\"}}}}}}"
+ .equals(json));
+
+ path = "[1].[1]";
+ json = JSON.toJSONString(Configuration.from("{}").buildObject(
+ Arrays.asList(StringUtils.split(path, ".")), "bazhen"));
+ System.out.println(json);
+ Assert.assertTrue("[null,[null,\"bazhen\"]]".equals(json));
+
+ path = "a.[10].b.[0].c.[1]";
+ json = JSON.toJSONString(Configuration.from("{}").buildObject(
+ Arrays.asList(StringUtils.split(path, ".")), "bazhen"));
+ System.out.println(json);
+ Assert.assertTrue("{\"a\":[null,null,null,null,null,null,null,null,null,null,{\"b\":[{\"c\":[null,\"bazhen\"]}]}]}"
+ .equals(json));
+ }
+
+ @Test
+ public void test_setObjectRecursive() {
+ // 当current完全为空，类似新插入对象
+
+ String path = "";
+ Object root = null;
+
+ root = Configuration.from("{}").setObjectRecursive(null,
+ Arrays.asList(StringUtils.split(path, ".")), 0, "bazhen");
+ System.out.println(root);
+ Assert.assertTrue(JSON.toJSONString(root).equals("\"bazhen\""));
+
+ root = JSON.toJSONString(Configuration.from("{}").setObjectRecursive(
+ null, Arrays.asList(StringUtils.split(path, ".")), 0,
+ new ArrayList()));
+ System.out.println(root);
+ Assert.assertTrue(root.equals("[]"));
+
+ root = JSON.toJSONString(Configuration.from("{}").setObjectRecursive(
+ null, Arrays.asList(StringUtils.split(path, ".")), 0,
+ new HashMap()));
+ System.out.println(root);
+ Assert.assertTrue(root.equals("{}"));
+
+ root = JSON.toJSONString(Configuration.from("{}").setObjectRecursive(
+ null, Arrays.asList(StringUtils.split(path, ".")), 0, 0L));
+ System.out.println(root);
+ Assert.assertTrue(root.equals("0"));
+
+ // 当current当前为空，但是path存在路径，类似新插入对象
+ path = "a";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(null,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue("{\"a\":\"bazhen\"}".equals(root));
+
+ path = "a.b";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(null,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue("{\"a\":{\"b\":\"bazhen\"}}".equals(root));
+
+ path = "a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(null,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue("{\"a\":{\"b\":{\"c\":{\"d\":{\"e\":{\"f\":{\"g\":{\"h\":{\"i\":{\"j\":{\"k\":{\"l\":{\"m\":{\"n\":{\"o\":{\"p\":{\"q\":{\"r\":{\"s\":{\"t\":{\"u\":{\"v\":{\"w\":{\"x\":{\"y\":{\"z\":\"bazhen\"}}}}}}}}}}}}}}}}}}}}}}}}}}"
+ .equals(root));
+
+ path = "1.1";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(null,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue("{\"1\":{\"1\":\"bazhen\"}}".equals(root));
+
+ path = "-.-";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(null,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue("{\"-\":{\"-\":\"bazhen\"}}".equals(root));
+
+ path = "[0]";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(null,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue(root.equals("[\"bazhen\"]"));
+
+ path = "[0].[0]";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(null,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue(root.equals("[[\"bazhen\"]]"));
+
+ path = "[0].[0].[0].[0].[0].[0].[0].[0].[0]";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(null,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue(root.equals("[[[[[[[[[\"bazhen\"]]]]]]]]]"));
+
+ path = "[0].[1].[2].[3].[4].[5].[6].[7].[8]";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(null,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue(root
+ .equals("[[null,[null,null,[null,null,null,[null,null,null,null,[null,null,null,null,null,[null,null,null,null,null,null,[null,null,null,null,null,null,null,[null,null,null,null,null,null,null,null,\"bazhen\"]]]]]]]]]"));
+
+ path = "a.[0].b.[0].c.[0]";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(null,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue(root
+ .equals("{\"a\":[{\"b\":[{\"c\":[\"bazhen\"]}]}]}"));
+
+ // 初始化为list，测试插入对象
+
+ root = JSON.parse("[]");
+ path = "a";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(root,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue("{\"a\":\"bazhen\"}".equals(root));
+
+ root = JSON.parse("[]");
+ path = "a.b";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(root,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue("{\"a\":{\"b\":\"bazhen\"}}".equals(root));
+
+ root = JSON.parse("[]");
+ path = "[0]";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(root,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue(root.equals("[\"bazhen\"]"));
+
+ root = JSON.parse("[]");
+ path = "[0].[0]";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(root,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue(root.equals("[[\"bazhen\"]]"));
+
+ // 初始化为map，测试插入对象
+ root = JSON.parse("{}");
+ path = "a";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(root,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue("{\"a\":\"bazhen\"}".equals(root));
+
+ root = JSON.parse("{}");
+ path = "a.b";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(root,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue("{\"a\":{\"b\":\"bazhen\"}}".equals(root));
+
+ root = JSON.parse("{}");
+ path = "[0]";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(root,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue(root.equals("[\"bazhen\"]"));
+
+ root = JSON.parse("{}");
+ path = "[0].[0]";
+ root = JSON
+ .toJSONString(Configuration.from("{}").setObjectRecursive(root,
+ Arrays.asList(StringUtils.split(path, ".")), 0,
+ "bazhen"));
+ System.out.println(root);
+ Assert.assertTrue(root.equals("[[\"bazhen\"]]"));
+
+ root = JSON.parse("{\"a\": \"a\", \"b\":\"b\"}");
+ path = "a.[0]";
+ root = Configuration.from("{}").setObjectRecursive(root,
+ Arrays.asList(StringUtils.split(path, ".")), 0, "bazhen");
+ System.out.println(root);
+ System.out.println(JSON.toJSONString(root).equals(
+ "{\"a\":[\"bazhen\"],\"b\":\"b\"}"));
+
+ root = JSON
+ .parse("{\"a\":{\"b\":{\"c\":[0],\"B\": \"B\"},\"A\": \"A\"}}");
+ path = "a.b.c.[0]";
+ root = Configuration.from("{}").setObjectRecursive(root,
+ Arrays.asList(StringUtils.split(path, ".")), 0, "bazhen");
+ System.out.println(root);
+ Assert.assertTrue(JSON.toJSONString(root).equals(
+ "{\"a\":{\"A\":\"A\",\"b\":{\"B\":\"B\",\"c\":[\"bazhen\"]}}}"));
+ }
+
+ @Test
+ public void test_setConfiguration() {
+ Configuration configuration = Configuration.from("{}");
+ configuration.set("b", Configuration.from("{}"));
+ System.out.println(configuration.toJSON());
+ Assert.assertTrue(configuration.toJSON().equals("{\"b\":{}}"));
+
+ configuration = Configuration.newDefault();
+ List list = new ArrayList();
+ for (int i = 0; i < 3; i++) {
+ list.add(Configuration.newDefault());
+ }
+ configuration.set("a", list);
+ System.out.println(configuration.toJSON());
+ Assert.assertTrue("{\"a\":[{},{},{}]}".equals(configuration.toJSON()));
+
+ Map map = new HashMap