diff --git a/develop/reader/hdfsreader/index.html b/develop/reader/hdfsreader/index.html index 53635fca0..1c83c14ea 100644 --- a/develop/reader/hdfsreader/index.html +++ b/develop/reader/hdfsreader/index.html @@ -870,6 +870,15 @@ + + +
  • + + + hdfsSitePath + + +
    • @@ -2369,6 +2378,15 @@ + + +
  • + + + hdfsSitePath + + +
    • @@ -2576,6 +2594,13 @@

    配置项说明hdfsSitePath +否 +string +无 +hdfs-site.xml 的路径，详细解释见下 +

    path

    @@ -2673,6 +2698,14 @@

    csvReaderConfigboolean skipEmptyRecords = true;//是否跳过空行 boolean captureRawRecord = true; +

    hdfsSitePath

    +

    这是 4.2.4 引入的新配置想，用于指定 hdfs-site.xml 文件的路径，比如对 HDP/CDH 而言，可以这样配置：

    +
    {
+  "hdfsSitePath": "/etc/hadoop/conf/hdfs-site.xml"
+}
+
    +

    如果配置了 hdfsSitePath , 则插件会从该文件中获得访问 HDFS 文件系统必要的配置，从而在大部分情况下不在需要配置 hadoopConfig，减少配置量。

    +

    对于把 Addax 部署在 Hadoop 集群上的场景，推荐使用这种方式。

    类型转换

    diff --git a/develop/search/search_index.json b/develop/search/search_index.json index f26678ae4..2586bcb53 100644 --- a/develop/search/search_index.json +++ b/develop/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\u200b\\u3000\\-\u3001\u3002\uff0c\uff0e\uff1f\uff01\uff1b]+","pipeline":["stemmer"]},"docs":[{"location":"","title":"Addax \u4ecb\u7ecd","text":""},{"location":"#_1","title":"\u6982\u89c8","text":"

    Addax \u662f\u4e00\u4e2a\u5f02\u6784\u6570\u636e\u6e90\u79bb\u7ebf\u540c\u6b65\u5de5\u5177\uff0c\u6700\u521d\u6765\u6e90\u4e8e\u963f\u91cc\u7684 DataX \uff0c\u81f4\u529b\u4e8e\u5b9e\u73b0\u5305\u62ec\u5173\u7cfb\u578b\u6570\u636e\u5e93(MySQL\u3001Oracle \u7b49)\u3001HDFS\u3001Hive\u3001HBase\u3001FTP \u7b49\u5404\u79cd\u5f02\u6784\u6570\u636e\u6e90\u4e4b\u95f4\u7a33\u5b9a\u9ad8\u6548\u7684\u6570\u636e\u540c\u6b65\u529f\u80fd\u3002

    \u4e3a\u4e86\u89e3\u51b3\u5f02\u6784\u6570\u636e\u6e90\u540c\u6b65\u95ee\u9898\uff0cAddax \u5c06\u590d\u6742\u7684\u7f51\u72b6\u7684\u540c\u6b65\u94fe\u8def\u53d8\u6210\u4e86\u661f\u578b\u6570\u636e\u94fe\u8def\uff0cAddax \u4f5c\u4e3a\u4e2d\u95f4\u4f20\u8f93\u8f7d\u4f53\u8d1f\u8d23\u8fde\u63a5\u5404\u79cd\u6570\u636e\u6e90\u3002\u5f53\u9700\u8981\u63a5\u5165\u4e00\u4e2a\u65b0\u7684\u6570\u636e\u6e90\u7684\u65f6\u5019\uff0c\u53ea\u9700\u8981\u5c06\u6b64\u6570\u636e\u6e90\u5bf9\u63a5\u5230 Addax\uff0c\u4fbf\u80fd\u8ddf\u5df2\u6709\u7684\u6570\u636e\u6e90\u505a\u5230\u65e0\u7f1d\u6570\u636e\u540c\u6b65\u3002

    "},{"location":"#_2","title":"\u6846\u67b6\u8bbe\u8ba1","text":"
    graph LR\nMySQL\nsubgraph Addax\n    direction LR\n    subgraph reader[\"Reader Plugin\"]\n        mr[\"MySQLReader\"]\n    end\n    subgraph writer[\"Writer Plugin\"]\n    hw[\"HDFSWriter\"]\n    end\n    Framework\n    mr --> Framework --> writer\nend\n\nMySQL ==> Addax ==> HDFS\n

    Addax \u672c\u8eab\u4f5c\u4e3a\u79bb\u7ebf\u6570\u636e\u540c\u6b65\u6846\u67b6\uff0c\u91c7\u7528 Framework + plugin \u67b6\u6784\u6784\u5efa\u3002\u5c06\u6570\u636e\u6e90\u8bfb\u53d6\u548c\u5199\u5165\u62bd\u8c61\u6210\u4e3a Reader/Writer \u63d2\u4ef6\uff0c\u7eb3\u5165\u5230\u6574\u4e2a\u540c\u6b65\u6846\u67b6\u4e2d\u3002

    Addax Framework \u63d0\u4f9b\u4e86\u7b80\u5355\u7684\u63a5\u53e3\u4e0e\u63d2\u4ef6\u4ea4\u4e92\uff0c\u63d0\u4f9b\u7b80\u5355\u7684\u63d2\u4ef6\u63a5\u5165\u673a\u5236\uff0c\u53ea\u9700\u8981\u4efb\u610f\u52a0\u4e0a\u4e00\u79cd\u63d2\u4ef6\uff0c\u5c31\u80fd\u65e0\u7f1d\u5bf9\u63a5\u5176\u4ed6\u6570\u636e\u6e90\u3002

    "},{"location":"#_3","title":"\u6838\u5fc3\u67b6\u6784","text":"

    \u672c\u5c0f\u8282\u6309\u4e00\u4e2a Addax \u4f5c\u4e1a\u751f\u547d\u5468\u671f\u7684\u65f6\u5e8f\u56fe\uff0c\u4ece\u6574\u4f53\u67b6\u6784\u8bbe\u8ba1\u975e\u5e38\u7b80\u8981\u8bf4\u660e\u5404\u4e2a\u6a21\u5757\u76f8\u4e92\u5173\u7cfb\u3002

    graph TB\nsubgraph Job\nend\nsubgraph task\n  direction TB\n  t1[\"Task\"]\n  t2[\"Task\"]\n  t3[\"Task\"]\n  t4[\"Task\"]\n  t5[\"Task\"]\n  t6[\"Task\"]\nend\nsubgraph taskgroup[\" \"]\n    direction TB\n  subgraph tg1[\"TaskGroup\"]\n    subgraph tg1_Task[\"Task\"]\n      tg1_r[\"Reader\"]\n      tg1_c[\"Channel\"]\n      tg1_w[\"Writer\"]\n    end\n    t7[\"Task\"]\n    t8[\"Task\"]\n  end\n\n  subgraph tg2[\"TaskGroup\"]\n    subgraph tg2_Task[\"Task\"]\n      direction LR\n      tg2_r[\"Reader\"]\n      tg2_c[\"Channel\"]\n      tg2_w[\"Writer\"]\n    end\n    t9[\"Task\"]\n    t10[\"Task\"]\n  end\n\n  subgraph tg3[\"TaskGroup\"]\n    direction LR\n    subgraph tg3_Task[\"Task\"]\n      tg3_r[\"Reader\"]\n      tg3_c[\"Channel\"]\n      tg3_w[\"Writer\"]\n    end\n    t11[\"Task\"]\n    t12[\"Task\"]\n  end\nend\n\nJob == split ==> task\ntask == Schedule ==> taskgroup
    "},{"location":"#_4","title":"\u6838\u5fc3\u6a21\u5757\u4ecb\u7ecd","text":"
    1. Addax \u5b8c\u6210\u5355\u4e2a\u6570\u636e\u540c\u6b65\u7684\u4f5c\u4e1a\uff0c\u6211\u4eec\u79f0\u4e4b\u4e3a Job\uff0cAddax \u63a5\u6536\u5230\u4e00\u4e2a Job \u4e4b\u540e\uff0c\u5c06\u542f\u52a8\u4e00\u4e2a\u8fdb\u7a0b\u6765\u5b8c\u6210\u6574\u4e2a\u4f5c\u4e1a\u540c\u6b65\u8fc7\u7a0b\u3002Addax Job \u6a21\u5757\u662f\u5355\u4e2a\u4f5c\u4e1a\u7684\u4e2d\u67a2\u7ba1\u7406\u8282\u70b9\uff0c\u627f\u62c5\u4e86\u6570\u636e\u6e05\u7406\u3001\u5b50\u4efb\u52a1\u5207\u5206(\u5c06\u5355\u4e00\u4f5c\u4e1a\u8ba1\u7b97\u8f6c\u5316\u4e3a\u591a\u4e2a\u5b50 Task)\u3001TaskGroup \u7ba1\u7406\u7b49\u529f\u80fd\u3002
    2. Addax Job \u542f\u52a8\u540e\uff0c\u4f1a\u6839\u636e\u4e0d\u540c\u7684\u6e90\u7aef\u5207\u5206\u7b56\u7565\uff0c\u5c06 Job \u5207\u5206\u6210\u591a\u4e2a\u5c0f\u7684 Task (\u5b50\u4efb\u52a1)\uff0c\u4ee5\u4fbf\u4e8e\u5e76\u53d1\u6267\u884c\u3002Task \u4fbf\u662f Addax \u4f5c\u4e1a\u7684\u6700\u5c0f\u5355\u5143\uff0c\u6bcf\u4e00\u4e2a Task \u90fd\u4f1a\u8d1f\u8d23\u4e00\u90e8\u5206\u6570\u636e\u7684\u540c\u6b65\u5de5\u4f5c\u3002
    3. \u5207\u5206\u591a\u4e2a Task \u4e4b\u540e\uff0cAddax Job \u4f1a\u8c03\u7528 Scheduler \u6a21\u5757\uff0c\u6839\u636e\u914d\u7f6e\u7684\u5e76\u53d1\u6570\u636e\u91cf\uff0c\u5c06\u62c6\u5206\u6210\u7684 Task \u91cd\u65b0\u7ec4\u5408\uff0c\u7ec4\u88c5\u6210 TaskGroup(\u4efb\u52a1\u7ec4)\u3002\u6bcf\u4e00\u4e2a TaskGroup \u8d1f\u8d23\u4ee5\u4e00\u5b9a\u7684\u5e76\u53d1\u8fd0\u884c\u5b8c\u6bd5\u5206\u914d\u597d\u7684\u6240\u6709 Task\uff0c\u9ed8\u8ba4\u5355\u4e2a\u4efb\u52a1\u7ec4\u7684\u5e76\u53d1\u6570\u91cf\u4e3a 5\u3002
    4. \u6bcf\u4e00\u4e2a Task \u90fd\u7531 TaskGroup \u8d1f\u8d23\u542f\u52a8\uff0cTask \u542f\u52a8\u540e\uff0c\u4f1a\u56fa\u5b9a\u542f\u52a8 Reader\u2014>Channel\u2014>Writer \u7684\u7ebf\u7a0b\u6765\u5b8c\u6210\u4efb\u52a1\u540c\u6b65\u5de5\u4f5c\u3002
    5. Addax \u4f5c\u4e1a\u8fd0\u884c\u8d77\u6765\u4e4b\u540e\uff0c Job \u76d1\u63a7\u5e76\u7b49\u5f85\u591a\u4e2a TaskGroup \u6a21\u5757\u4efb\u52a1\u5b8c\u6210\uff0c\u7b49\u5f85\u6240\u6709 TaskGroup \u4efb\u52a1\u5b8c\u6210\u540e Job \u6210\u529f\u9000\u51fa\u3002\u5426\u5219\uff0c\u5f02\u5e38\u9000\u51fa\uff0c\u8fdb\u7a0b\u9000\u51fa\u503c\u975e 0
    "},{"location":"#_5","title":"\u8c03\u5ea6\u6d41\u7a0b","text":"

    \u4e3e\u4f8b\u6765\u8bf4\uff0c\u7528\u6237\u63d0\u4ea4\u4e86\u4e00\u4e2a\u4f5c\u4e1a\uff0c\u5e76\u4e14\u914d\u7f6e\u4e86 20 \u4e2a\u5e76\u53d1\uff0c\u76ee\u7684\u662f\u5c06\u4e00\u4e2a 100 \u5f20\u5206\u8868\u7684 MySQL \u6570\u636e\u540c\u6b65\u5230 Oracle \u91cc\u9762\u3002 \u8c03\u5ea6\u51b3\u7b56\u601d\u8def\u662f\uff1a

    1. Addax Job \u6839\u636e\u5206\u5e93\u5206\u8868\u5207\u5206\u6210\u4e86 100 \u4e2a Task\u3002
    2. \u6839\u636e 20 \u4e2a\u5e76\u53d1\uff0c\u8ba1\u7b97\u5171\u9700\u8981\u5206\u914d 20/5 = 4 \u4e2a TaskGroup\u3002
    3. 4 \u4e2a TaskGroup \u5e73\u5206\u5207\u5206\u597d\u7684 100 \u4e2a Task\uff0c\u6bcf\u4e00\u4e2a TaskGroup \u8d1f\u8d23\u4ee5 5 \u4e2a\u5e76\u53d1\u5171\u8ba1\u8fd0\u884c 25 \u4e2a Task\u3002
    "},{"location":"#_6","title":"\u6838\u5fc3\u4f18\u52bf","text":""},{"location":"#_7","title":"\u53ef\u9760\u7684\u6570\u636e\u8d28\u91cf\u76d1\u63a7","text":"

    \u652f\u6301\u6240\u6709\u7684\u5f3a\u6570\u636e\u7c7b\u578b\uff0c\u6bcf\u4e00\u79cd\u63d2\u4ef6\u90fd\u6709\u81ea\u5df1\u7684\u6570\u636e\u7c7b\u578b\u8f6c\u6362\u7b56\u7565\uff0c\u8ba9\u6570\u636e\u53ef\u4ee5\u5b8c\u6574\u65e0\u635f\u7684\u4f20\u8f93\u5230\u76ee\u7684\u7aef\u3002

    \u8fd0\u884c\u8fc7\u7a0b\u4e2d\u53ef\u4ee5\u5c06\u4f5c\u4e1a\u672c\u8eab\u72b6\u6001\u3001\u6570\u636e\u6d41\u91cf\u3001\u6570\u636e\u901f\u5ea6\u3001\u6267\u884c\u8fdb\u5ea6\u7b49\u4fe1\u606f\u8fdb\u884c\u5168\u9762\u7684\u5c55\u793a\uff0c\u8ba9\u7528\u6237\u53ef\u4ee5\u5b9e\u65f6\u4e86\u89e3\u4f5c\u4e1a\u72b6\u6001\u3002\u5e76\u53ef\u5728\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u667a\u80fd\u5224\u65ad\u6e90\u7aef\u548c\u76ee\u7684\u7aef\u7684\u901f\u5ea6\u5bf9\u6bd4\u60c5\u51b5\uff0c\u7ed9\u4e88\u7528\u6237\u66f4\u591a\u6027\u80fd\u6392\u67e5\u4fe1\u606f\u3002

    \u5728\u5927\u91cf\u6570\u636e\u7684\u4f20\u8f93\u8fc7\u7a0b\u4e2d\uff0c\u5fc5\u5b9a\u4f1a\u7531\u4e8e\u5404\u79cd\u539f\u56e0\u5bfc\u81f4\u5f88\u591a\u6570\u636e\u4f20\u8f93\u62a5\u9519(\u6bd4\u5982\u7c7b\u578b\u8f6c\u6362\u9519\u8bef)\uff0c\u8fd9\u79cd\u6570\u636e Addax \u8ba4\u4e3a\u5c31\u662f\u810f\u6570\u636e\u3002Addax \u76ee\u524d\u53ef\u4ee5\u5b9e\u73b0\u810f\u6570\u636e\u7cbe\u786e\u8fc7\u6ee4\u3001\u8bc6\u522b\u3001\u91c7\u96c6\u3001\u5c55\u793a\uff0c\u4e3a\u7528\u6237\u63d0\u4f9b\u591a\u79cd\u7684\u810f\u6570\u636e\u5904\u7406\u6a21\u5f0f\uff0c\u8ba9\u7528\u6237\u51c6\u786e\u628a\u63a7\u6570\u636e\u8d28\u91cf\u5173\uff01

    "},{"location":"#_8","title":"\u4e30\u5bcc\u7684\u6570\u636e\u8f6c\u6362\u529f\u80fd","text":"

    \u4f5c\u4e3a\u4e00\u4e2a\u670d\u52a1\u4e8e\u5927\u6570\u636e\u7684 ETL \u5de5\u5177\uff0c\u9664\u4e86\u63d0\u4f9b\u6570\u636e\u5feb\u7167\u642c\u8fc1\u529f\u80fd\u4e4b\u5916\uff0c\u8fd8\u63d0\u4f9b\u4e86\u4e30\u5bcc\u6570\u636e\u8f6c\u6362\u7684\u529f\u80fd\uff0c\u8ba9\u6570\u636e\u5728\u4f20\u8f93\u8fc7\u7a0b\u4e2d\u53ef\u4ee5\u8f7b\u677e\u5b8c\u6210\u6570\u636e\u8131\u654f\uff0c\u8865\u5168\uff0c\u8fc7\u6ee4\u7b49\u6570\u636e\u8f6c\u6362\u529f\u80fd\uff0c\u53e6\u5916\u8fd8\u63d0\u4f9b\u4e86\u81ea\u52a8 groovy \u51fd\u6570\uff0c\u8ba9\u7528\u6237\u81ea\u5b9a\u4e49\u8f6c\u6362\u51fd\u6570\u3002\u8be6\u60c5\u8bf7\u770b transformer \u8be6\u7ec6\u4ecb\u7ecd\u3002

    "},{"location":"#_9","title":"\u7cbe\u51c6\u7684\u901f\u5ea6\u63a7\u5236","text":"

    \u63d0\u4f9b\u4e86\u5305\u62ec\u901a\u9053(\u5e76\u53d1)\u3001\u8bb0\u5f55\u6d41\u3001\u5b57\u8282\u6d41\u4e09\u79cd\u6d41\u63a7\u6a21\u5f0f\uff0c\u53ef\u4ee5\u968f\u610f\u63a7\u5236\u4f60\u7684\u4f5c\u4e1a\u901f\u5ea6\uff0c\u8ba9\u4f60\u7684\u4f5c\u4e1a\u5728\u5e93\u53ef\u4ee5\u627f\u53d7\u7684\u8303\u56f4\u5185\u8fbe\u5230\u6700\u4f73\u7684\u540c\u6b65\u901f\u5ea6\u3002

    {\n  \"speed\": {\n    \"channel\": 5,\n    \"byte\": 1048576,\n    \"record\": 10000\n  }\n}\n
    "},{"location":"#_10","title":"\u5f3a\u52b2\u5730\u540c\u6b65\u6027\u80fd","text":"

    \u6bcf\u4e00\u79cd\u8bfb\u63d2\u4ef6\u90fd\u6709\u4e00\u79cd\u6216\u591a\u79cd\u5207\u5206\u7b56\u7565\uff0c\u90fd\u80fd\u5c06\u4f5c\u4e1a\u5408\u7406\u5207\u5206\u6210\u591a\u4e2a Task \u5e76\u884c\u6267\u884c\uff0c\u5355\u673a\u591a\u7ebf\u7a0b\u6267\u884c\u6a21\u578b\u53ef\u4ee5\u8ba9\u901f\u5ea6\u968f\u5e76\u53d1\u6210\u7ebf\u6027\u589e\u957f\u3002 \u5728\u6e90\u7aef\u548c\u76ee\u7684\u7aef\u6027\u80fd\u90fd\u8db3\u591f\u7684\u60c5\u51b5\u4e0b\uff0c\u5355\u4e2a\u4f5c\u4e1a\u4e00\u5b9a\u53ef\u4ee5\u6253\u6ee1\u7f51\u5361\u3002

    "},{"location":"#_11","title":"\u5065\u58ee\u7684\u5bb9\u9519\u673a\u5236","text":"

    \u4f5c\u4e1a\u662f\u6781\u6613\u53d7\u5916\u90e8\u56e0\u7d20\u7684\u5e72\u6270\uff0c\u7f51\u7edc\u95ea\u65ad\u3001\u6570\u636e\u6e90\u4e0d\u7a33\u5b9a\u7b49\u56e0\u7d20\u5f88\u5bb9\u6613\u8ba9\u540c\u6b65\u5230\u4e00\u534a\u7684\u4f5c\u4e1a\u62a5\u9519\u505c\u6b62\u3002\u56e0\u6b64\u7a33\u5b9a\u6027\u662f Addax \u7684\u57fa\u672c\u8981\u6c42\uff0c\u5728 Addax \u7684\u8bbe\u8ba1\u4e2d\uff0c\u91cd\u70b9\u5b8c\u5584\u4e86\u6846\u67b6\u548c\u63d2\u4ef6\u7684\u7a33\u5b9a\u6027\u3002 \u76ee\u524d Addax \u53ef\u4ee5\u505a\u5230\u7ebf\u7a0b\u7ea7\u522b\u3001\u4f5c\u4e1a\u7ea7\u522b\u591a\u5c42\u6b21\u5c40\u90e8/\u5168\u5c40\u7684\u91cd\u8bd5\uff0c\u4fdd\u8bc1\u7528\u6237\u7684\u4f5c\u4e1a\u7a33\u5b9a\u8fd0\u884c\u3002

    "},{"location":"debug/","title":"\u5982\u4f55\u8c03\u8bd5\u9879\u76ee","text":"

    \u5982\u679c\u4f60\u60f3\u5728\u672c\u5730\u4fee\u6539\u6216\u8005\u8c03\u8bd5\u4ee3\u7801\uff0c\u53ef\u4ee5\u53c2\u8003\u4ee5\u4e0b\u65b9\u5f0f\uff0c\u4ee5\u4e0b\u7684\u64cd\u4f5c\u5747\u4ee5 Intellij IDEA \u5f00\u53d1\u5de5\u5177\u4e3a\u4f8b\u3002

    \u8c03\u8bd5\u53ef\u4ee5\u5206\u6210\u4e24\u65b9\u65b9\u5f0f\uff0c\u4e00\u79cd\u662f\u4ee3\u7801\u548c\u4e8c\u8fdb\u5236\u5305\u90fd\u5728\u672c\u5730\uff0c\u6211\u4eec\u79f0\u4e4b\u4e3a\u672c\u5730\u8c03\u8bd5\uff1b\u53e6\u5916\u4e00\u79cd\u6e90\u4ee3\u7801\u5728\u672c\u5730\uff0c\u4f46\u4e8c\u8fdb\u5236\u7a0b\u5e8f\u5df2\u7ecf\u90e8\u7f72\u5728\u8fdc\u7a0b\uff0c\u8fd9\u79cd\u60c5\u51b5\u4e0b\u6211\u4eec\u79f0\u4e4b\u4e3a\u8fdc\u7a0b\u8c03\u8bd5\u3002

    \u4e0b\u9762\u5206\u522b\u63cf\u8ff0

    "},{"location":"debug/#_2","title":"\u672c\u5730\u8c03\u8bd5","text":""},{"location":"debug/#_3","title":"\u4e00\u4e9b\u8bbe\u5b9a","text":"

    \u6211\u4eec\u5047\u5b9a\u672c\u5730\u90e8\u7f72\u7684 Addax \u5728 /opt/app/addax/4.0.3 \u6587\u4ef6\u5939\u4e0b\u3002\u5176 job \u76ee\u5f55\u4e0b\u6709\u8fd9\u6837\u7684\u4e00\u4e2a job.json \u914d\u7f6e\u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    job/job.json 
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 00:00:00\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u4e0a\u8ff0 job \u6587\u4ef6\u8fd0\u884c\u6ca1\u6709\u7b26\u5408\u6211\u4eec\u7684\u9884\u671f\uff0c\u731c\u6d4b\u662f streamreader \u8fd9\u4e2a\u63d2\u4ef6\u7684 parseMixupFunctions \u51fd\u6570\u6709\u95ee\u9898\uff0c\u6211\u60f3\u8c03\u8bd5\u770b\u5177\u4f53\u95ee\u9898\u5728\u54ea\u91cc\u3002

    "},{"location":"debug/#idea","title":"\u914d\u7f6e IDEA","text":"

    \u6253\u5f00 IDEA\u5de5\u5177\uff0c\u5e76\u6253\u5f00 addax \u9879\u76ee\u6e90\u4ee3\u7801\uff0c\u6253\u5f00 plugin/reader/streamreader/StreamReader.java \u6587\u4ef6\uff0c\u627e\u5230 parseMixupFunctions \u51fd\u6570\uff0c\u5e76\u5728\u51fd\u6570\u7533\u660e\u5904\u7684\u70b9\u51fb\u5de6\u4fa7\u8fb9\u7f18\u5904\u589e\u52a0\u65ad\u70b9\u3002\u5982\u4e0b\u56fe\u6240\u793a\uff1a

    \u70b9\u51fb IDEA \u7684 Run->Edit Configurations... \u83dc\u5355\uff0c\u5728\u5f39\u51fa\u7684 Run/Debug Configurations \u7a97\u53e3\uff0c\u70b9\u51fb\u5de6\u4e0a\u89d2\u7684 + \u6309\u94ae\uff0c\u7136\u540e\u9009\u62e9 \u9009\u62e9 Application \uff0c\u5728\u53f3\u4fa7\u914d\u7f6e\u6846\u4e2d\uff0c\u586b\u5199\u76f8\u5173\u4fe1\u606f\u5982\u4e0b\uff1a

    \u5176\u4ed6\u4fdd\u6301\u4e0d\u53d8\uff0c\u70b9\u51fb Apply \u6309\u94ae\u3002\u5f97\u5230\u7c7b\u4f3c\u4e0b\u56fe\u914d\u7f6e\u5185\u5bb9

    \u70b9\u51fb OK \u6309\u94ae\u4fdd\u5b58\u4e0a\u8ff0\u914d\u7f6e\uff0c\u56de\u5230 IDEA \u4e3b\u7a97\u53e3\uff0c\u5728\u7a97\u53e3\u83dc\u5355\u680f\u6709\u7eff\u8272\ud83d\udd28\u7684\u53f3\u4fa7\uff0c\u5e94\u8be5\u53ef\u4ee5\u770b\u5230\u521a\u624d\u914d\u7f6e\u7684\u63cf\u8ff0\u6587\u4ef6\uff0c\u7c7b\u4f3c\u4e0b\u56fe\uff1a

    \u70b9\u51fb\u4e0a\u8ff0\u622a\u56fe\u4e2d\u7684\u7eff\u8272 DEBUG \u5c0f\u866b\u6309\u94ae\uff0c\u8fdb\u5165\u8c03\u8bd5\uff0c\u5f97\u5230\u7c7b\u4f3c\u4e0b\u56fe\u7684\u8c03\u8bd5\u7a97\u53e3\uff1a

    "},{"location":"debug/#_4","title":"\u8fdc\u7a0b\u8c03\u8bd5","text":""},{"location":"debug/#_5","title":"\u4e00\u4e9b\u5047\u5b9a","text":"

    \u5047\u5b9a\u7a0b\u5e8f\u90e8\u7f72\u5728\u8fdc\u7a0b\u670d\u52a1\u5668\u4e0a\uff0c\u9700\u8981\u76f4\u63a5\u8c03\u8bd5\u8fdc\u7a0b\u670d\u52a1\u5668\u4e0a\u8fd0\u884c\u7684\u7a0b\u5e8f\uff0c\u5047\u5b9a\u8fdc\u7a0b\u670d\u52a1\u5668IP\u5730\u5740\u4e3a 192.168.1.100\uff0c\bAddax \u90e8\u7f72\u5728 /opt/addax/4.0.3 \u76ee\u5f55\u4e0b\uff0c\u5176 job \u6587\u4ef6\u5939\u4e0b\uff0c\u4e5f\u6709\u4e00\u4e2a\u548c\u672c\u5730\u8c03\u8bd5\u4e2d\u63cf\u8ff0\u7684 job.json \u6587\u4ef6\u3002 \u540c\u6837\u7684\uff0c\u4e0a\u8ff0 job \u6587\u4ef6\u8fd0\u884c\u6ca1\u6709\u7b26\u5408\u6211\u4eec\u7684\u9884\u671f\uff0c\u731c\u6d4b\u662f streamreader \u8fd9\u4e2a\u63d2\u4ef6\u7684 parseMixupFunctions \u51fd\u6570\u6709\u95ee\u9898\uff0c\u6211\u60f3\u8c03\u8bd5\u770b\u5177\u4f53\u95ee\u9898\u5728\u54ea\u91cc\u3002

    \u6ce8\u610f\uff1a\u8fdc\u7a0b\u8c03\u8bd5\u9700\u8981\u5728\u670d\u52a1\u5668\u6253\u5f00 9999 \u7aef\u53e3\uff0c\u56e0\u6b64\u8981\u4fdd\u8bc1\u670d\u52a1\u5668\u4e0a\u7684 9999 \u7aef\u53e3\u6ca1\u6709\u88ab\u5360\u7528\uff0c\u5982\u679c\u88ab\u5360\u7528\uff0c\u5219\u9700\u8981\u4fee\u6539\u6b64\u7aef\u53e3\u3002

    \u4fee\u6539\u65b9\u5f0f\u5982\u4e0b\uff1a

    1. \u6253\u5f00 bin/addax.sh \u811a\u672c
    2. \u5b9a\u4f4d\u5230\u5927\u7ea624\u884c\u5904\uff0c\u627e\u5230 address=0.0.0.0:9999 \u5b57\u7b26\u4e32
    3. \u5c06 9999 \u4fee\u6539\u6210\u5176\u4ed6\u672a\u88ab\u5360\u7528\u7684\u7aef\u53e3
    4. \u4fdd\u5b58\u9000\u51fa
    "},{"location":"debug/#idea_1","title":"\u914d\u7f6e IDEA","text":"

    \u6253\u5f00 IDEA\u5de5\u5177\uff0c\u5e76\u6253\u5f00 addax \u9879\u76ee\u6e90\u4ee3\u7801\uff0c\u6253\u5f00 plugin/reader/streamreader/StreamReader.java \u6587\u4ef6\uff0c\u627e\u5230 parseMixupFunctions \u51fd\u6570\uff0c\u5e76\u5728\u51fd\u6570\u7533\u660e\u5904\u7684\u70b9\u51fb\u5de6\u4fa7\u8fb9\u7f18\u5904\u589e\u52a0\u65ad\u70b9\u3002\u5982\u4e0b\u56fe\u6240\u793a\uff1a

    \u70b9\u51fb IDEA \u7684 Run->Edit Configurations... \u83dc\u5355\uff0c\u5728\u5f39\u51fa\u7684 Run/Debug Configurations \u7a97\u53e3\uff0c\u70b9\u51fb\u5de6\u4e0a\u89d2\u7684 + \u6309\u94ae\uff0c\u7136\u540e\u9009\u62e9 \u9009\u62e9 Remove JVM Debug \uff0c\u5728\u53f3\u4fa7\u914d\u7f6e\u6846\u4e2d\uff0c\u586b\u5199\u76f8\u5173\u4fe1\u606f\u5982\u4e0b\uff1a

    \u5176\u4ed6\u4fdd\u6301\u4e0d\u53d8\uff0c\u70b9\u51fb Apply \u6309\u94ae\uff0c\u5f97\u5230\u5982\u4e0b\u914d\u7f6e\u4fe1\u606f\uff1a

    \u70b9\u51fb OK \u6309\u94ae\u4fdd\u5b58\u5e76\u8fd4\u56de\u5230 IDEA \u4e3b\u7a97\u53e3

    \u786e\u4fdd\u5728\u7a97\u53e3\u5de5\u5177\u680f\u6709\u7eff\u8272\ud83d\udd28\u7684\u53f3\u4fa7\u9009\u62e9\u7684\u662f\u4e0a\u8ff0\u586b\u5199 Name \u7684\u63cf\u8ff0\u914d\u7f6e\uff0c\u5426\u5219\u5728\u4e0b\u62c9\u6846\u4e2d\u9009\u62e9\u521a\u624d\u7684\u914d\u7f6e\u3002

    "},{"location":"debug/#_6","title":"\u8fd0\u884c\u8c03\u8bd5","text":"

    \u8fd0\u884c\u8fdc\u7a0b\u8c03\u8bd5\u5206\u6210\u4e24\u4e2a\u6b65\u9aa4\uff0c\u4e00\u662f\u542f\u52a8\u7a0b\u5e8f\uff0c\u4e8c\u662f\u8c03\u8bd5\u5de5\u5177\u8054\u63a5\u5230\u8fd0\u884c\u7684\u7a0b\u5e8f\u4e0a\u3002

    \u5728\u8fdc\u7a0b\u670d\u52a1\u5668\u4e0a\u8fd0\u884c\u5982\u4e0b\u547d\u4ee4\uff1a

    bin/addax.sh -d job/job.json

    \u5982\u679c\u8fd0\u884c\u6b63\u5e38\uff0c\u4f1a\u5f97\u5230\u5982\u4e0b\u4fe1\u606f\uff1a

    bin/addax.sh -d job/job.json\nListening for transport dt_socket at address: 9999\n

    \u8868\u793a\u7a0b\u5e8f\u76d1\u542c\u5728 9999 \u7aef\u53e3\u4e0a\uff0c\u7b49\u5f85\u8054\u63a5\u3002

    \u8fd4\u56de IDEA \u7a97\u53e3\uff0c\u70b9\u51fb\u5de5\u5177\u680f\u4e0a\u7eff\u8272 DEBUG \u5c0f\u866b\u6309\u94ae\uff0c\u5f00\u59cb\u8c03\u8bd5\uff0c\u5982\u679c\u8fd0\u884c\u6b63\u5e38\uff0c\u4f1a\u5f97\u5230\u7c7b\u4f3c\u4e0b\u56fe\u7684\u8c03\u8bd5\u7a97\u53e3\uff1a

    "},{"location":"encrypt_password/","title":"\u52a0\u5bc6\u914d\u7f6e\u6587\u4ef6\u7684\u4e2d\u5bc6\u7801","text":"

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u5bc6\u7801\u662f\u660e\u6587\u7684\uff0c\u8fd9\u5e26\u6765\u4e86\u4e00\u5b9a\u7684\u5b89\u5168\u9690\u60a3\uff0c\u4ece 4.0.9 \u7248\u672c\u8d77\uff0c\u6211\u4eec\u589e\u52a0\u4e86\u5bf9\u914d\u7f6e\u6587\u4ef6\u7684\u7684\u5bc6\u7801\u52a0\u5bc6\u529f\u80fd\u3002 \u540c\u65f6\u6211\u4eec\u63d0\u4f9b\u4e86\u4e00\u4e2a shell \u811a\u672c encrypt_password.sh \u6765\u5e2e\u52a9\u4f60\u52a0\u5bc6\u914d\u7f6e\u6587\u4ef6\u7684\u5bc6\u7801\u3002

    \u5047\u5b9a\u4f60\u7684\u539f\u59cb\u5bc6\u7801\u662f 123456\uff0c\u4f60\u5e0c\u671b\u5728\u914d\u7f6e\u6587\u4ef6\u4e2d\u4f7f\u7528\u52a0\u5bc6\u5bc6\u7801\u914d\u7f6e\u3002 \u9996\u5148\u6267\u884c\u4e0b\u9762\u7684\u6307\u4ee4\uff1a

    $ bin/encrypt_password.sh 123456\nThe encrypt string is : '${enc:tFd05jnm1mSq+PEK9t/Rgg==}', you can paste it to json file.\n

    \u4e0a\u8ff0\u7ed3\u679c\u4e2d\u7684 tFd05jnm1mSq+PEK9t/Rgg== \u4e3a 123456 \u7684\u5bc6\u6587\u3002 ${enc: \u5f00\u5934\u662f\u4e3a\u4e86\u8ba9 addax \u77e5\u9053\u8fd9\u662f\u4e00\u4e2a\u52a0\u5bc6\u5bc6\u6587\u3002

    \u4f60\u5c06\u4e0a\u8ff0\u5b57\u7b26\u4e32 ${enc:tFd05jnm1mSq+PEK9t/Rgg==} \u66ff\u6362\u4f60\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u8bbe\u7f6e\u5bc6\u7801\u4e3a 123456 \u7684\u5730\u65b9\u5373\u53ef\u3002

    "},{"location":"plugin_development/","title":"\u63d2\u4ef6\u5f00\u53d1","text":"

    \u672c\u6307\u5357\u4e3b\u8981\u9762\u5411\u90a3\u4e9b\u9700\u8981\u5f00\u53d1\u7b26\u5408\u81ea\u5df1\u9700\u6c42\u7684 Addax \u63d2\u4ef6\u5f00\u53d1\u4eba\u5458\u3002

    "},{"location":"plugin_development/#addax","title":"Addax \u6d41\u7a0b","text":"

    Addax \u8fd0\u884c\u4e00\u4e2a\u4efb\u52a1\u7684\u5927\u81f4\u6d41\u7a0b\u5982\u4e0b\uff1a

    \u542f\u52a8\u6b65\u9aa4\u4e3a\uff1a

    1. \u89e3\u6790\u914d\u7f6e\uff0c\u5305\u62ec job.json\u3001core.json\u3001plugin.json \u4e09\u4e2a\u914d\u7f6e
    2. \u8bbe\u7f6e jobId \u5230 configuration \u5f53\u4e2d
    3. \u542f\u52a8 Engine\uff0c\u901a\u8fc7 Engine.start() \u8fdb\u5165\u542f\u52a8\u7a0b\u5e8f
    4. \u8bbe\u7f6e RUNTIME_MODE \u5230 configuration \u5f53\u4e2d
    5. \u901a\u8fc7 JobContainer \u7684 start() \u65b9\u6cd5\u542f\u52a8
    6. \u4f9d\u6b21\u6267\u884c job \u7684 preHandler()\u3001init()\u3001prepare()\u3001split()\u3001schedule()\u3001post()\u3001postHandle() \u7b49\u65b9\u6cd5\u3002
    7. init() \u65b9\u6cd5\u6d89\u53ca\u5230\u6839\u636e configuration \u6765\u521d\u59cb\u5316 reader \u548c writer \u63d2\u4ef6\uff0c\u8fd9\u91cc\u6d89\u53ca\u5230 jar \u5305\u70ed\u52a0\u8f7d\u4ee5\u53ca\u8c03\u7528\u63d2\u4ef6 init() \u64cd\u4f5c\u65b9\u6cd5\uff0c\u540c\u65f6\u8bbe\u7f6e reader \u548c writer \u7684 configuration \u4fe1\u606f
    8. prepare() \u65b9\u6cd5\u6d89\u53ca\u5230\u521d\u59cb\u5316 reader \u548c writer \u63d2\u4ef6\u7684\u521d\u59cb\u5316\uff0c\u901a\u8fc7\u8c03\u7528\u63d2\u4ef6\u7684 prepare() \u65b9\u6cd5\u5b9e\u73b0\uff0c\u6bcf\u4e2a\u63d2\u4ef6\u90fd\u6709\u81ea\u5df1\u7684 jarLoader\uff0c\u901a\u8fc7\u96c6\u6210 URLClassloader \u5b9e\u73b0\u800c\u6765
    9. split() \u65b9\u6cd5\u901a\u8fc7 adjustChannelNumber() \u65b9\u6cd5\u8c03\u6574 channel \u4e2a\u6570\uff0c\u540c\u65f6\u6267\u884c reader \u548c writer \u6700\u7ec6\u7c92\u5ea6\u7684\u5207\u5206\uff0c\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0cwriter \u7684\u5207\u5206\u7ed3\u679c\u8981\u53c2\u7167 reader \u7684\u5207\u5206\u7ed3\u679c\uff0c\u8fbe\u5230\u5207\u5206\u540e\u6570\u76ee\u76f8\u7b49\uff0c\u624d\u80fd\u6ee1\u8db3 1\uff1a1 \u7684\u901a\u9053\u6a21\u578b
    10. channel \u7684\u8ba1\u6570\u4e3b\u8981\u662f\u6839\u636e byte \u548c record \u7684\u9650\u901f\u6765\u5b9e\u73b0\u7684\uff0c\u5728 split() \u7684\u51fd\u6570\u4e2d\u7b2c\u4e00\u6b65\u5c31\u662f\u8ba1\u7b97 channel \u7684\u5927\u5c0f
    11. split() \u65b9\u6cd5 reader \u63d2\u4ef6\u4f1a\u6839\u636e channel \u7684\u503c\u8fdb\u884c\u62c6\u5206\uff0c\u4f46\u662f\u6709\u4e9b reader \u63d2\u4ef6\u53ef\u80fd\u4e0d\u4f1a\u53c2\u8003 channel \u7684\u503c\uff0cwriter \u63d2\u4ef6\u4f1a\u5b8c\u5168\u6839\u636e reader \u7684\u63d2\u4ef6 1:1 \u8fdb\u884c\u8fd4\u56de
    12. split() \u65b9\u6cd5\u5185\u90e8\u7684 mergeReaderAndWriterTaskConfigs() \u8d1f\u8d23\u5408\u5e76 reader\u3001writer\u3001\u4ee5\u53ca transformer \u4e09\u8005\u5173\u7cfb\uff0c\u751f\u6210 task \u7684\u914d\u7f6e\uff0c\u5e76\u4e14\u91cd\u5199 job.content \u7684\u914d\u7f6e
    13. schedule() \u65b9\u6cd5\u6839\u636e split() \u62c6\u5206\u751f\u6210\u7684 task \u914d\u7f6e\u5206\u914d\u751f\u6210 taskGroup \u5bf9\u8c61\uff0c\u6839\u636e task \u7684\u6570\u91cf\u548c\u5355\u4e2a taskGroup \u652f\u6301\u7684 task \u6570\u91cf\u8fdb\u884c\u914d\u7f6e\uff0c\u4e24\u8005\u76f8\u9664\u5c31\u53ef\u4ee5\u5f97\u51fa taskGroup \u7684\u6570\u91cf
    14. schedule() \u5185\u90e8\u901a\u8fc7 AbstractScheduler \u7684 schedule() \u6267\u884c\uff0c\u7ee7\u7eed\u6267\u884cstartAllTaskGroup() \u65b9\u6cd5\u521b\u5efa\u6240\u6709\u7684 TaskGroupContainer \u7ec4\u7ec7\u76f8\u5173\u7684 task\uff0cTaskGroupContainerRunner \u8d1f\u8d23\u8fd0\u884c TaskGroupContainer \u6267\u884c\u5206\u914d\u7684 task\u3002scheduler \u7684\u5177\u4f53\u5b9e\u73b0\u7c7b\u4e3a ProcessInnerScheduler\u3002
    15. taskGroupContainerExecutorService \u542f\u52a8\u56fa\u5b9a\u7684\u7ebf\u7a0b\u6c60\u7528\u4ee5\u6267\u884c TaskGroupContainerRunner \u5bf9\u8c61\uff0cTaskGroupContainerRunner \u7684 run() \u65b9\u6cd5\u8c03\u7528 taskGroupContainer. start() \u65b9\u6cd5\uff0c\u9488\u5bf9\u6bcf\u4e2a channel \u521b\u5efa\u4e00\u4e2a TaskExecutor\uff0c\u901a\u8fc7 taskExecutor.doStart() \u542f\u52a8\u4efb\u52a1\u3002
    "},{"location":"plugin_development/#_2","title":"\u63d2\u4ef6\u673a\u5236","text":"

    Addax \u4e3a\u4e86\u5e94\u5bf9\u4e0d\u540c\u6570\u636e\u6e90\u7684\u5dee\u5f02\u3001\u540c\u65f6\u63d0\u4f9b\u4e00\u81f4\u5730\u540c\u6b65\u539f\u8bed\u548c\u6269\u5c55\u80fd\u529b\uff0c\u91c7\u7528\u4e86 \u6846\u67b6 + \u63d2\u4ef6 \u7684\u6a21\u5f0f\uff1a

    \u4f5c\u4e3a\u63d2\u4ef6\u5f00\u53d1\u4eba\u5458\uff0c\u5219\u9700\u8981\u5173\u6ce8\u4e24\u4e2a\u95ee\u9898\uff1a

    1. \u6570\u636e\u6e90\u672c\u8eab\u7684\u8bfb\u5199\u6570\u636e\u6b63\u786e\u6027\u3002
    2. \u5982\u4f55\u4e0e\u6846\u67b6\u6c9f\u901a\u3001\u5408\u7406\u6b63\u786e\u5730\u4f7f\u7528\u6846\u67b6\u3002
    "},{"location":"plugin_development/#_3","title":"\u63d2\u4ef6\u89c6\u89d2\u770b\u6846\u67b6","text":""},{"location":"plugin_development/#_4","title":"\u903b\u8f91\u6267\u884c\u6a21\u578b","text":"

    \u63d2\u4ef6\u5f00\u53d1\u8005\u4e0d\u7528\u5173\u5fc3\u592a\u591a\uff0c\u57fa\u672c\u53ea\u9700\u8981\u5173\u6ce8\u7279\u5b9a\u7cfb\u7edf\u8bfb\u548c\u5199\uff0c\u4ee5\u53ca\u81ea\u5df1\u7684\u4ee3\u7801\u5728\u903b\u8f91\u4e0a\u662f\u600e\u6837\u88ab\u6267\u884c\u7684\uff0c\u54ea\u4e00\u4e2a\u65b9\u6cd5\u662f\u5728\u4ec0\u4e48\u65f6\u5019\u88ab\u8c03\u7528\u7684\u3002\u5728\u6b64\u4e4b\u524d\uff0c\u9700\u8981\u660e\u786e\u4ee5\u4e0b\u6982\u5ff5\uff1a

    \u7b80\u800c\u8a00\u4e4b\uff0c Job\u62c6\u5206\u6210Task\uff0c\u5206\u522b\u5728\u6846\u67b6\u63d0\u4f9b\u7684\u5bb9\u5668\u4e2d\u6267\u884c\uff0c\u63d2\u4ef6\u53ea\u9700\u8981\u5b9e\u73b0 Job \u548c Task \u4e24\u90e8\u5206\u903b\u8f91\u3002

    "},{"location":"plugin_development/#_5","title":"\u7f16\u7a0b\u63a5\u53e3","text":"

    \u90a3\u4e48\uff0cJob \u548c Task \u7684\u903b\u8f91\u5e94\u662f\u600e\u4e48\u5bf9\u5e94\u5230\u5177\u4f53\u7684\u4ee3\u7801\u4e2d\u7684\uff1f

    \u9996\u5148\uff0c\u63d2\u4ef6\u7684\u5165\u53e3\u7c7b\u5fc5\u987b\u6269\u5c55 Reader \u6216 Writer \u62bd\u8c61\u7c7b\uff0c\u5e76\u4e14\u5b9e\u73b0\u5206\u522b\u5b9e\u73b0 Job \u548c Task \u4e24\u4e2a\u5185\u90e8\u62bd\u8c61\u7c7b\uff0cJob \u548c Task \u7684\u5b9e\u73b0\u5fc5\u987b\u662f \u5185\u90e8\u7c7b \u7684\u5f62\u5f0f\uff0c\u539f\u56e0\u89c1 \u52a0\u8f7d\u539f\u7406 \u4e00\u8282\u3002\u4ee5 Reader \u4e3a\u4f8b\uff1a

    public class SomeReader\n        extends Reader\n{\n    public static class Job\n            extends Reader.Job\n    {\n        @Override\n        public void init()\n        {\n        }\n\n        @Override\n        public void prepare()\n        {\n        }\n\n        @Override\n        public List<Configuration> split(int adviceNumber)\n        {\n            return null;\n        }\n\n        @Override\n        public void post()\n        {\n        }\n\n        @Override\n        public void destroy()\n        {\n        }\n    }\n\n    public static class Task\n            extends Reader.Task\n    {\n\n        @Override\n        public void init()\n        {\n        }\n\n        @Override\n        public void prepare()\n        {\n        }\n\n        @Override\n        public void startRead(RecordSender recordSender)\n        {\n        }\n\n        @Override\n        public void post()\n        {\n        }\n\n        @Override\n        public void destroy()\n        {\n        }\n    }\n}\n

    Job \u63a5\u53e3\u529f\u80fd\u5982\u4e0b\uff1a

    Task \u63a5\u53e3\u529f\u80fd\u5982\u4e0b\uff1a

    \u9700\u8981\u6ce8\u610f\u7684\u662f\uff1a

    \u6846\u67b6\u6309\u7167\u5982\u4e0b\u7684\u987a\u5e8f\u6267\u884c Job \u548c Task \u7684\u63a5\u53e3\uff1a

    stateDiagram-v2\ndirection TB\nInit:::job --> Prepare:::job\nPrepare --> Split:::job\nSplit --> Schedule:::fw\nstate Schedule {\n    direction LR\n    init\\nprepare\\nstartRead\\npost\\ndestroy1 --> init\\nprepare\\nstartRead\\npost\\ndestroy : Channel\n}\nSchedule --> Post:::job\n\nclassDef job fill:yellow\nclassDef fw fill:#c6fac4\nclassDef ctask fill:blue

    \u4e0a\u56fe\u4e2d\uff0c\u9ec4\u8272\u8868\u793a Job \u90e8\u5206\u7684\u6267\u884c\u9636\u6bb5\uff0c\u7070\u8272\u8868\u793a Task \u90e8\u5206\u7684\u6267\u884c\u9636\u6bb5\uff0c\u7eff\u8272\u8868\u793a\u6846\u67b6\u6267\u884c\u9636\u6bb5\u3002

    \u76f8\u5173\u7c7b\u5173\u7cfb\u5982\u4e0b\uff1a

    %%{init: {\"theme\": \"neutral\"}}%%\nclassDiagram\n    class Pluginable {\n    + init()\n    + destroy()\n    + others()\n    }\n    class AbstractPlugin {\n        + prepare()\n        + post()\n        + others()\n    }\n    class AbstractJobPlugin {\n        + getJobPluginCollector(): JobPluginCollector\n        + setJobPluginCollector(JobPluginCollector)\n    }\n\n    class AbstractTaskPlugin {\n        + getTaskPluginCollector(): TaskPluginCollector\n        + setTaskPluginCollector(TaskPluginCollector)\n    }\n    class Reader_Job {\n        + split(init): List<<Configuration>>\n    }\n\n    class Writer_Job {\n        + split(init): List<<Configuration>>\n    }\n\n    class Reader_Task {\n        + startRead(RecordSender)\n    }\n\n    class Writer_Task {\n        + startWrite(RecordReceiver)\n    }\n\n    AbstractJobPlugin <|-- Reader_Job\n    AbstractJobPlugin <|-- Writer_Job\n\n    AbstractTaskPlugin <|-- Reader_Task\n    AbstractTaskPlugin <|-- Writer_Task\n\n    AbstractPlugin <|-- AbstractJobPlugin\n    AbstractPlugin <|-- AbstractTaskPlugin\n\n    Pluginable <|-- AbstractPlugin\n
    "},{"location":"plugin_development/#_6","title":"\u63d2\u4ef6\u5b9a\u4e49","text":"

    \u5728\u6bcf\u4e2a\u63d2\u4ef6\u7684\u9879\u76ee\u4e2d\uff0c\u90fd\u6709\u4e00\u4e2aplugin.json\u6587\u4ef6\uff0c\u8fd9\u4e2a\u6587\u4ef6\u5b9a\u4e49\u4e86\u63d2\u4ef6\u7684\u76f8\u5173\u4fe1\u606f\uff0c\u5305\u62ec\u5165\u53e3\u7c7b\u3002\u4f8b\u5982\uff1a

    {\n  \"name\": \"mysqlwriter\",\n  \"class\": \"com.wgzhao.addax.plugin.writer.mysqlwriter.MysqlWriter\",\n  \"description\": \"Use Jdbc connect to database, execute insert sql.\",\n  \"developer\": \"wgzhao\"\n}\n
    "},{"location":"plugin_development/#_7","title":"\u6253\u5305\u53d1\u5e03","text":"

    Addax \u4f7f\u7528 assembly \u6253\u5305\uff0c\u6253\u5305\u547d\u4ee4\u5982\u4e0b\uff1a

    mvn clean package\nmvn package assembly:single\n

    Addax \u63d2\u4ef6\u9700\u8981\u9075\u5faa\u7edf\u4e00\u7684\u76ee\u5f55\u7ed3\u6784\uff1a

    ${ADDAX_HOME}\n\u251c\u2500\u2500 bin\n\u2502     \u251c\u2500\u2500 addax.sh\n\u251c\u2500\u2500 conf\n\u2502     \u251c\u2500\u2500 core.json\n\u2502     \u2514\u2500\u2500 logback.xml\n\u251c\u2500\u2500 job\n\u251c\u2500\u2500 lib\n\u2502     \u251c\u2500\u2500 addax-common-<version>.jar\n\u2502     \u251c\u2500\u2500 addax-core-<version>.jar\n\u2502     \u251c\u2500\u2500 addax-rdbms-<version>.jar\n\u2502     \u251c\u2500\u2500 addax-storage-<version>.jar\n\u251c\u2500\u2500 log\n\u251c\u2500\u2500 plugin\n\u2502     \u251c\u2500\u2500 reader\n\u2502     \u2502     \u251c\u2500\u2500 cassandrareader\n\u2502     \u2502     \u2502     \u251c\u2500\u2500 cassandrareader-<version>.jar\n\u2502     \u2502     \u2502     \u251c\u2500\u2500 libs\n\u2502     \u2502     \u2502     \u2502     \u251c\u2500\u2500 <symbol link to shared folder>\n\u2502     \u2502     \u2502     \u251c\u2500\u2500 plugin.json\n\u2502     \u2502     \u2502     \u2514\u2500\u2500 plugin_job_template.json\n\u2502     \u2514\u2500\u2500 writer\n\u2502         \u251c\u2500\u2500 cassandrawriter\n\u2502         \u2502     \u251c\u2500\u2500 cassandrawriter-<version>.jar\n\u2502         \u2502     \u251c\u2500\u2500 libs\n\u2502         \u2502     \u2502     \u251c\u2500\u2500 <symbol link to shared folder>\n\u2502         \u2502     \u251c\u2500\u2500 plugin.json\n\u2502         \u2502     \u2514\u2500\u2500 plugin_job_template.json\n\u251c\u2500\u2500 shared\n

    \u63d2\u4ef6\u76ee\u5f55\u5206\u4e3a reader \u548c writer \u5b50\u76ee\u5f55\uff0c\u8bfb\u5199\u63d2\u4ef6\u5206\u522b\u5b58\u653e\u3002\u63d2\u4ef6\u76ee\u5f55\u89c4\u8303\u5982\u4e0b\uff1a

    \u5c3d\u7ba1\u6846\u67b6\u52a0\u8f7d\u63d2\u4ef6\u65f6\uff0c\u4f1a\u628a ${PLUGIN_HOME} \u4e0b\u6240\u6709\u7684 jar \u5305\u6dfb\u52a0\u5230 classpath \u73af\u5883\u53d8\u91cf\u4e2d\uff0c\u4f46\u8fd8\u662f\u63a8\u8350\u4f9d\u8d56\u5e93\u7684 jar \u548c\u63d2\u4ef6\u672c\u8eab\u7684 jar \u5206\u5f00\u5b58\u653e\u3002

    \u7279\u522b\u63d0\u9192

    \u63d2\u4ef6\u7684\u76ee\u5f55\u540d\u5b57\u5fc5\u987b\u548c plugin.json \u4e2d\u5b9a\u4e49\u7684\u63d2\u4ef6\u540d\u79f0\u4e00\u81f4\u3002

    "},{"location":"plugin_development/#_8","title":"\u914d\u7f6e\u6587\u4ef6","text":"

    Addax \u4f7f\u7528 json \u4f5c\u4e3a\u914d\u7f6e\u6587\u4ef6\u7684\u683c\u5f0f\u3002\u4e00\u4e2a\u5178\u578b\u7684 Addax \u4efb\u52a1\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"postgresqlreader\",\n        \"parameter\": {\n          \"username\": \"pgtest\",\n          \"password\": \"pgtest\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"addax_tbl\"\n            ],\n            \"jdbcUrl\": \"jdbc:postgresql://localhost:5432/pgtest\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"postgresqlwriter\",\n        \"parameter\": {\n          \"column\": [\n            \"*\"\n          ],\n          \"preSql\": [\n            \"truncate table @table\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:postgresql://127.0.0.1:5432/pgtest\",\n            \"table\": [\n              \"addax_tbl1\"\n            ]\n          },\n          \"username\": \"pgtest\",\n          \"password\": \"pgtest\",\n          \"writeMode\": \"insert\"\n        }\n      }\n    }\n  }\n}\n

    Addax \u6846\u67b6\u6709 core.json \u914d\u7f6e\u6587\u4ef6\uff0c\u6307\u5b9a\u4e86\u6846\u67b6\u7684\u9ed8\u8ba4\u884c\u4e3a\u3002\u4efb\u52a1\u7684\u914d\u7f6e\u91cc\u5934\u53ef\u4ee5\u6307\u5b9a\u6846\u67b6\u4e2d\u5df2\u7ecf\u5b58\u5728\u7684\u914d\u7f6e\u9879\uff0c\u800c\u4e14\u5177\u6709\u66f4\u9ad8\u7684\u4f18\u5148\u7ea7\uff0c\u4f1a\u8986\u76d6 core.json \u4e2d\u7684\u9ed8\u8ba4\u503c\u3002

    \u914d\u7f6e\u4e2djob.content.reader.parameter \u7684 value \u90e8\u5206\u4f1a\u4f20\u7ed9 Reader.Job\uff1bjob.content.writer.parameter \u7684 value \u90e8\u5206\u4f1a\u4f20\u7ed9Writer.Job \uff0c Reader.Job \u548c Writer.Job \u53ef\u4ee5\u901a\u8fc7 super.getPluginJobConf() \u6765\u83b7\u53d6\u3002

    "},{"location":"plugin_development/#_9","title":"\u5982\u4f55\u8bbe\u8ba1\u914d\u7f6e\u53c2\u6570","text":"

    \u914d\u7f6e\u6587\u4ef6\u7684\u8bbe\u8ba1\u662f\u63d2\u4ef6\u5f00\u53d1\u7684\u7b2c\u4e00\u6b65\uff01

    \u4efb\u52a1\u914d\u7f6e\u4e2d reader \u548c writer \u4e0b parameter \u90e8\u5206\u662f\u63d2\u4ef6\u7684\u914d\u7f6e\u53c2\u6570\uff0c\u63d2\u4ef6\u7684\u914d\u7f6e\u53c2\u6570\u5e94\u5f53\u9075\u5faa\u4ee5\u4e0b\u539f\u5219\uff1a

    {\n  \"connection\": [\n    {\n      \"table\": [\"table_1\", \"table_2\"],\n      \"jdbcUrl\": [\n        \"jdbc:mysql://127.0.0.1:3306/database_1\",\n        \"jdbc:mysql://127.0.0.2:3306/database_1_slave\"\n      ]\n    },\n    {\n      \"table\": [\"table_3\", \"table_4\"],\n      \"jdbcUrl\": [\n        \"jdbc:mysql://127.0.0.3:3306/database_2\",\n        \"jdbc:mysql://127.0.0.4:3306/database_2_slave\"\n      ]\n    }\n  ]\n}\n
    "},{"location":"plugin_development/#configuration","title":"\u5982\u4f55\u4f7f\u7528 Configuration \u7c7b","text":"

    \u4e3a\u4e86\u7b80\u5316\u5bf9 json \u7684\u64cd\u4f5c\uff0cAddax \u63d0\u4f9b\u4e86\u7b80\u5355\u7684 DSL \u914d\u5408 Configuration \u7c7b\u4f7f\u7528\u3002

    Configuration \u63d0\u4f9b\u4e86\u5e38\u89c1\u7684 get, \u5e26\u7c7b\u578bget\uff0c\u5e26\u9ed8\u8ba4\u503cget\uff0cset \u7b49\u8bfb\u5199\u914d\u7f6e\u9879\u7684\u64cd\u4f5c\uff0c\u4ee5\u53ca clone, toJSON \u7b49\u65b9\u6cd5\u3002\u914d\u7f6e\u9879\u8bfb\u5199\u64cd\u4f5c\u90fd\u9700\u8981\u4f20\u5165\u4e00\u4e2a path \u505a\u4e3a\u53c2\u6570\uff0c \u8fd9\u4e2a path \u5c31\u662f Addax \u5b9a\u4e49\u7684 DSL\u3002\u8bed\u6cd5\u6709\u4e24\u6761\uff1a

    1. \u5b50 map \u7528 .key \u8868\u793a\uff0cpath \u7684\u7b2c\u4e00\u4e2a\u70b9\u7701\u7565\u3002
    2. \u6570\u7ec4\u5143\u7d20\u7528 [index] \u8868\u793a\u3002

    \u6bd4\u5982\u64cd\u4f5c\u5982\u4e0b json\uff1a

    {\n  \"a\": {\n    \"b\": {\n      \"c\": 2\n    },\n    \"f\": [\n      1,\n      2,\n      {\n        \"g\": true,\n        \"h\": false\n      },\n      4\n    ]\n  },\n  \"x\": 4\n}\n

    \u6bd4\u5982\u8c03\u7528 configuration.get(path) \u65b9\u6cd5\uff0c\u5f53 path \u4e3a\u5982\u4e0b\u503c\u7684\u65f6\u5019\u5f97\u5230\u7684\u7ed3\u679c\u4e3a\uff1a

    \u6ce8\u610f\uff0c\u56e0\u4e3a\u63d2\u4ef6\u770b\u5230\u7684\u914d\u7f6e\u53ea\u662f\u6574\u4e2a\u914d\u7f6e\u7684\u4e00\u90e8\u5206\u3002\u4f7f\u7528 Configuration \u5bf9\u8c61\u65f6\uff0c\u9700\u8981\u6ce8\u610f\u5f53\u524d\u7684\u6839\u8def\u5f84\u662f\u4ec0\u4e48\u3002

    \u66f4\u591a Configuration \u7684\u64cd\u4f5c\u8bf7\u53c2\u8003 Configuration.java \u3002

    "},{"location":"plugin_development/#_10","title":"\u63d2\u4ef6\u6570\u636e\u4f20\u8f93","text":"

    \u8ddf\u4e00\u822c\u7684 \u751f\u4ea7\u8005-\u6d88\u8d39\u8005 \u6a21\u5f0f\u4e00\u6837\uff0cReader \u63d2\u4ef6\u548c Writer \u63d2\u4ef6\u4e4b\u95f4\u4e5f\u662f\u901a\u8fc7 channel \u6765\u5b9e\u73b0\u6570\u636e\u7684\u4f20\u8f93\u7684\u3002channel \u53ef\u4ee5\u662f\u5185\u5b58\u7684\uff0c\u4e5f\u53ef\u80fd\u662f\u6301\u4e45\u5316\u7684\uff0c\u63d2\u4ef6\u4e0d\u5fc5\u5173\u5fc3\u3002 \u63d2\u4ef6\u901a\u8fc7 RecordSender \u5f80 channel \u5199\u5165\u6570\u636e\uff0c\u901a\u8fc7 RecordReceiver \u4ece channel \u8bfb\u53d6\u6570\u636e\u3002

    channel \u4e2d\u7684\u4e00\u6761\u6570\u636e\u4e3a\u4e00\u4e2a Record \u7684\u5bf9\u8c61\uff0cRecord \u4e2d\u53ef\u4ee5\u653e\u591a\u4e2a Column \u5bf9\u8c61\uff0c\u8fd9\u53ef\u4ee5\u7b80\u5355\u7406\u89e3\u4e3a\u6570\u636e\u5e93\u4e2d\u7684\u8bb0\u5f55\u548c\u5217\u3002

    Record \u6709\u5982\u4e0b\u65b9\u6cd5\uff1a

    public interface Record\n{\n    // \u52a0\u5165\u4e00\u4e2a\u5217\uff0c\u653e\u5728\u6700\u540e\u7684\u4f4d\u7f6e\n    void addColumn(Column column);\n\n    // \u5728\u6307\u5b9a\u4e0b\u6807\u5904\u653e\u7f6e\u4e00\u4e2a\u5217\n    void setColumn(int i, final Column column);\n\n    // \u83b7\u53d6\u4e00\u4e2a\u5217\n    Column getColumn(int i);\n\n    // \u8f6c\u6362\u4e3ajson String\n    String toString();\n\n    // \u83b7\u53d6\u603b\u5217\u6570\n    int getColumnNumber();\n\n    // \u8ba1\u7b97\u6574\u6761\u8bb0\u5f55\u5728\u5185\u5b58\u4e2d\u5360\u7528\u7684\u5b57\u8282\u6570\n    int getByteSize();\n}\n

    \u56e0\u4e3a Record \u662f\u4e00\u4e2a\u63a5\u53e3\uff0cReader \u63d2\u4ef6\u9996\u5148\u8c03\u7528 RecordSender.createRecord() \u521b\u5efa\u4e00\u4e2a Record \u5b9e\u4f8b\uff0c\u7136\u540e\u628a Column \u4e00\u4e2a\u4e2a\u6dfb\u52a0\u5230 Record \u4e2d\u3002

    Writer \u63d2\u4ef6\u8c03\u7528 RecordReceiver.getFromReader() \u65b9\u6cd5\u83b7\u53d6 Record\uff0c\u7136\u540e\u628a Column \u904d\u5386\u51fa\u6765\uff0c\u5199\u5165\u76ee\u6807\u5b58\u50a8\u4e2d\u3002\u5f53 Reader \u5c1a\u672a\u9000\u51fa\uff0c\u4f20\u8f93\u8fd8\u5728\u8fdb\u884c\u65f6\uff0c\u5982\u679c\u6682\u65f6\u6ca1\u6709\u6570\u636e RecordReceiver.getFromReader() \u65b9\u6cd5\u4f1a\u963b\u585e\u76f4\u5230\u6709\u6570\u636e\u3002 \u5982\u679c\u4f20\u8f93\u5df2\u7ecf\u7ed3\u675f\uff0c\u4f1a\u8fd4\u56denull\uff0cWriter \u63d2\u4ef6\u53ef\u4ee5\u636e\u6b64\u5224\u65ad\u662f\u5426\u7ed3\u675f startWrite \u65b9\u6cd5\u3002

    "},{"location":"plugin_development/#_11","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u4e3a\u4e86\u89c4\u8303\u6e90\u7aef\u548c\u76ee\u7684\u7aef\u7c7b\u578b\u8f6c\u6362\u64cd\u4f5c\uff0c\u4fdd\u8bc1\u6570\u636e\u4e0d\u5931\u771f\uff0cAddax \u652f\u6301\u516d\u79cd\u5185\u90e8\u6570\u636e\u7c7b\u578b\uff1a

    \u5bf9\u5e94\u5730\uff0c\u6709 DateColumn\u3001LongColumn\u3001DoubleColumn\u3001BytesColumn\u3001StringColumn \u3001BoolColumn \u548c TimestampColumn \u4e03\u79cd Column \u7684\u5b9e\u73b0\u3002

    Column \u9664\u4e86\u63d0\u4f9b\u6570\u636e\u76f8\u5173\u7684\u65b9\u6cd5\u5916\uff0c\u8fd8\u63d0\u4f9b\u4e00\u7cfb\u5217\u4ee5 as \u5f00\u5934\u7684\u6570\u636e\u7c7b\u578b\u8f6c\u6362\u8f6c\u6362\u65b9\u6cd5\u3002

    %%{init: {\"theme\": \"neutral\"}}%%\nclassDiagram\ndirection TB\nclass Column {\n    <<interface>>\n    - rawData: Object\n    - type: Type\n    + getRawData(): Object\n    + getType(): Type\n    + getByteSize(): init\n    + asLong(): Long\n    + asDouble(): Doule\n    + asString(): String\n    + asDate(): Date\n    + asBytes(): Bytes\n    + asBigDecimal(): BigDecimal\n    + asBoolean(): Boolean\n}\nColumn <|-- Stringcolumn\nColumn <|-- Doublecolumn\nColumn <|-- Longcolumn\nColumn <|-- Datecolumn\nColumn <|-- Boolcolumn\nColumn <|-- Bytescolumn

    Addax \u7684\u5185\u90e8\u7c7b\u578b\u5728\u5b9e\u73b0\u4e0a\u4f1a\u9009\u7528\u4e0d\u540c\u7684 java \u7c7b\u578b\uff1a

    \u5185\u90e8\u7c7b\u578b \u5b9e\u73b0\u7c7b\u578b \u5907\u6ce8 Date java.util.Date Timestamp java.sql.Timestamp \u53ef\u4ee5\u7cbe\u786e\u5230\u7eb3\u79d2 Long java.math.BigInteger \u4f7f\u7528\u65e0\u9650\u7cbe\u5ea6\u7684\u5927\u6574\u6570\uff0c\u4fdd\u8bc1\u4e0d\u5931\u771f Double java.lang.String \u7528 String \u8868\u793a\uff0c\u4fdd\u8bc1\u4e0d\u5931\u771f Bytes byte[] String java.lang.String Bool java.lang.Boolean

    \u7c7b\u578b\u4e4b\u95f4\u76f8\u4e92\u8f6c\u6362\u7684\u5173\u7cfb\u5982\u4e0b\uff1a

    from/to Date Long Double Bytes String Bool Date - \u4f7f\u7528\u6beb\u79d2\u65f6\u95f4\u6233 \u4e0d\u652f\u6301 \u4e0d\u652f\u6301 \u6309\u914d\u7f6e\u683c\u5f0f\u8f6c\u6362 \u4e0d\u652f\u6301 Long \u4f5c\u4e3a\u6beb\u79d2\u65f6\u95f4\u6233\u6784\u9020 Date - BigDecimal.doubleValue() \u4e0d\u652f\u6301 BigInteger.toString() 0 \u4e3a false\uff0c\u5176\u4ed6\u4e3a true Double \u4e0d\u652f\u6301 BigDecimal.longValue() - \u4e0d\u652f\u6301 \u76f4\u63a5\u8fd4\u56de\u5185\u90e8 String \u4e0d\u652f\u6301 Bytes \u4e0d\u652f\u6301 \u4e0d\u652f\u6301 \u4e0d\u652f\u6301 - \u6309utf-8\u7f16\u7801\u8f6c\u6362\u4e3a byte[] \u4e0d\u652f\u6301 String \u6309\u914d\u7f6e\u7684 \u683c\u5f0f\u89e3\u6790 BigDecimal.longValueBigDecimal.doubleValue1 \u6309 utf-8 \u7f16\u78012\u8f6c\u6362\u4e3a byte[] - \"true\"\u4e3atrue, \"false\"\u4e3afalse\uff0c\u4e0d\u533a\u5206\u5927\u5c0f\u5199\u3002\u5176\u4ed6\u4e0d\u652f\u6301 Bool \u4e0d\u652f\u6301 true\u4e3a1L\uff0c\u5426\u52190L \u4e0d\u652f\u6301 true\u4e3a1.0\uff0c\u5426\u52190.0 \u4e0d\u652f\u6301 -"},{"location":"plugin_development/#_12","title":"\u810f\u6570\u636e\u5904\u7406","text":""},{"location":"plugin_development/#_13","title":"\u4ec0\u4e48\u662f\u810f\u6570\u636e","text":"

    \u76ee\u524d\u4e3b\u8981\u6709\u4e09\u7c7b\u810f\u6570\u636e\uff1a

    1. Reader \u8bfb\u5230\u4e0d\u652f\u6301\u7684\u7c7b\u578b\u3001\u4e0d\u5408\u6cd5\u7684\u503c\u3002
    2. \u4e0d\u652f\u6301\u7684\u7c7b\u578b\u8f6c\u6362\uff0c\u6bd4\u5982\uff1aBytes \u8f6c\u6362\u4e3a Date\u3002
    3. \u5199\u5165\u76ee\u6807\u7aef\u5931\u8d25\uff0c\u6bd4\u5982\uff1a\u5199 MySQL \u6574\u578b\u957f\u5ea6\u8d85\u957f\u3002
    "},{"location":"plugin_development/#_14","title":"\u5982\u4f55\u5904\u7406\u810f\u6570\u636e","text":"

    \u5728 Reader.Task \u548c Writer.Task \u4e2d\uff0c\u901a\u8fc7 AbstractTaskPlugin.getPluginCollector() \u53ef\u4ee5\u62ff\u5230\u4e00\u4e2a TaskPluginCollector\uff0c\u5b83\u63d0\u4f9b\u4e86\u4e00\u7cfb\u5217 collectDirtyRecord \u7684\u65b9\u6cd5\u3002 \u5f53\u810f\u6570\u636e\u51fa\u73b0\u65f6\uff0c\u53ea\u9700\u8981\u8c03\u7528\u5408\u9002\u7684 collectDirtyRecord \u65b9\u6cd5\uff0c\u628a\u88ab\u8ba4\u4e3a\u662f\u810f\u6570\u636e\u7684 Record \u4f20\u5165\u5373\u53ef\u3002

    \u7528\u6237\u53ef\u4ee5\u5728\u4efb\u52a1\u7684\u914d\u7f6e\u4e2d\u6307\u5b9a\u810f\u6570\u636e\u9650\u5236\u6761\u6570\u6216\u8005\u767e\u5206\u6bd4\u9650\u5236\uff0c\u5f53\u810f\u6570\u636e\u8d85\u51fa\u9650\u5236\u65f6\uff0c\u6846\u67b6\u4f1a\u7ed3\u675f\u540c\u6b65\u4efb\u52a1\uff0c\u9000\u51fa\u3002\u63d2\u4ef6\u9700\u8981\u4fdd\u8bc1\u810f\u6570\u636e\u90fd\u88ab\u6536\u96c6\u5230\uff0c\u5176\u4ed6\u5de5\u4f5c\u4ea4\u7ed9\u6846\u67b6\u5c31\u597d\u3002

    "},{"location":"plugin_development/#_15","title":"\u52a0\u8f7d\u539f\u7406","text":"
    1. \u6846\u67b6\u626b\u63cf plugin/reader \u548c plugin/writer\u76ee\u5f55\uff0c\u52a0\u8f7d\u6bcf\u4e2a\u63d2\u4ef6\u7684 plugin.json \u6587\u4ef6\u3002
    2. \u4ee5 plugin.json \u6587\u4ef6\u4e2d name \u4e3a key\uff0c\u7d22\u5f15\u6240\u6709\u7684\u63d2\u4ef6\u914d\u7f6e\u3002\u5982\u679c\u53d1\u73b0\u91cd\u540d\u7684\u63d2\u4ef6\uff0c\u6846\u67b6\u4f1a\u5f02\u5e38\u9000\u51fa\u3002
    3. \u7528\u6237\u5728\u63d2\u4ef6\u4e2d\u5728 reader/writer \u914d\u7f6e\u7684 name \u5b57\u6bb5\u6307\u5b9a\u63d2\u4ef6\u540d\u5b57\u3002\u6846\u67b6\u6839\u636e\u63d2\u4ef6\u7684\u7c7b\u578b\uff08reader/writer\uff09\u548c\u63d2\u4ef6\u540d\u79f0\u53bb\u63d2\u4ef6\u7684\u8def\u5f84\u4e0b\u626b\u63cf\u6240\u6709\u7684 jar\uff0c\u52a0\u5165 classpath\u3002
    4. \u6839\u636e\u63d2\u4ef6\u914d\u7f6e\u4e2d\u5b9a\u4e49\u7684\u5165\u53e3\u7c7b\uff0c\u6846\u67b6\u901a\u8fc7\u53cd\u5c04\u5b9e\u4f8b\u5316\u5bf9\u5e94\u7684 Job \u548c Task \u5bf9\u8c61\u3002

    1. \u5904\u7406 NaN, Infinity, -Infinity \u7b49\u6570\u503c\u00a0\u21a9

    2. \u9664\u975e\u53e6\u6709\u6307\u5b9a\u7f16\u7801\u683c\u5f0f\u00a0\u21a9

    "},{"location":"quickstart/","title":"\u5feb\u901f\u4f7f\u7528","text":""},{"location":"quickstart/#addax","title":"\u5b89\u88c5 Addax","text":""},{"location":"quickstart/#docker","title":"\u4f7f\u7528 Docker \u955c\u50cf","text":"

    \u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528 Docker \u955c\u50cf\uff0c\u53ea\u9700\u8981\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u5373\u53ef

    docker run -it --rm quay.io/wgzhao/addax:latest /opt/addax/bin/addax.sh /opt/addax/job/job.json\n
    "},{"location":"quickstart/#_2","title":"\u4e00\u952e\u5b89\u88c5","text":"

    \u5982\u679c\u4f60\u4e0d\u60f3\u7f16\u8bd1\uff0c\u4f60\u53ef\u4ee5\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\uff0c\u4e00\u952e\u5b89\u88c5\uff08\u5f53\u524d\u4ec5\u652f\u6301 Linux \u548c macOS \uff09

    /bin/bash -c \"$(curl -fsSL https://raw.githubusercontent.com/wgzhao/Addax/master/install.sh)\"\n

    \u5982\u679c\u662f macOS \uff0c\u9ed8\u8ba4\u5b89\u88c5\u5728 /usr/local/addax \u76ee\u5f55\u4e0b\uff0c \u5982\u679c\u662f Linux\uff0c \u5219\u5b89\u88c5\u5728 /opt/addax \u76ee\u5f55\u4e0b

    "},{"location":"quickstart/#_3","title":"\u6e90\u4ee3\u7801\u7f16\u8bd1\u5b89\u88c5","text":"

    \u4f60\u53ef\u4ee5\u9009\u62e9\u4ece\u6e90\u4ee3\u7801\u7f16\u8bd1\u5b89\u88c5\uff0c\u57fa\u672c\u64cd\u4f5c\u5982\u4e0b\uff1a

    git clone https://github.com/wgzhao/addax.git\ncd addax\nmvn clean package\nmvn package assembly:single\ncd target/addax/addax-<version>\n
    "},{"location":"quickstart/#_4","title":"\u5f00\u59cb\u7b2c\u4e00\u4e2a\u91c7\u96c6\u4efb\u52a1","text":"

    \u8981\u4f7f\u7528 Addax \u8fdb\u884c\u6570\u636e\u91c7\u96c6\uff0c\u53ea\u9700\u8981\u7f16\u5199\u4e00\u4e2a\u4efb\u52a1\u91c7\u96c6\u6587\u4ef6\uff0c\u8be5\u6587\u4ef6\u4e3a JSON \u683c\u5f0f\uff0c\u4ee5\u4e0b\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u914d\u7f6e\u6587\u4ef6\uff0c\u8be5\u4efb\u52a1\u7684\u76ee\u7684\u662f\u4ece\u5185\u5b58\u8bfb\u53d6\u8bfb\u53d6\u6307\u5b9a\u5185\u5bb9\u7684\u6570\u636e\uff0c\u5e76\u5c06\u5176\u6253\u5370\u51fa\u6765\uff0c\u6587\u4ef6\u4fdd\u5b58\u5728 job/test.json \u4e2d

    job/test.json 
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 00:00:00\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u6587\u4ef6\u4fdd\u5b58\u4e3a job/test.json

    \u7136\u540e\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\uff1a

    bin/addax.sh job/test.json\n

    \u5982\u679c\u6ca1\u6709\u62a5\u9519\uff0c\u5e94\u8be5\u4f1a\u6709\u7c7b\u4f3c\u8fd9\u6837\u7684\u8f93\u51fa

      ___      _     _\n / _ \\    | |   | |\n/ /_\\ \\ __| | __| | __ ___  __\n|  _  |/ _` |/ _` |/ _` \\ \\/ /\n| | | | (_| | (_| | (_| |>  <\n\\_| |_/\\__,_|\\__,_|\\__,_/_/\\_\\\n\n:: Addax version ::    (v4.0.3-SNAPSHOT)\n\n2021-08-23 13:45:17.199 [        main] INFO  VMInfo               - VMInfo# operatingSystem class => com.sun.management.internal.OperatingSystemImpl\n2021-08-23 13:45:17.223 [        main] INFO  Engine               -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"column\":[\n                        {\n                            \"type\":\"string\",\n                            \"value\":\"addax\"\n                        },\n                        {\n                            \"type\":\"long\",\n                            \"value\":19890604\n                        },\n                        {\n                            \"type\":\"date\",\n                            \"value\":\"1989-06-04 00:00:00\"\n                        },\n                        {\n                            \"type\":\"bool\",\n                            \"value\":true\n                        }\n                    ],\n                    \"sliceRecordCount\":10\n                },\n                \"name\":\"streamreader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":true\n                },\n                \"name\":\"streamwriter\"\n            }\n        }\n}\n\n2021-08-23 13:45:17.238 [        main] INFO  PerfTrace            - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-08-23 13:45:17.239 [        main] INFO  JobContainer         - Addax jobContainer starts job.\n2021-08-23 13:45:17.240 [        main] INFO  JobContainer         - Set jobId = 0\n2021-08-23 13:45:17.250 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do prepare work .\n2021-08-23 13:45:17.250 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do prepare work .\n2021-08-23 13:45:17.251 [       job-0] INFO  JobContainer         - Job set Channel-Number to 1 channels.\n2021-08-23 13:45:17.251 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] splits to [1] tasks.\n2021-08-23 13:45:17.252 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] splits to [1] tasks.\n2021-08-23 13:45:17.276 [       job-0] INFO  JobContainer         - Scheduler starts [1] taskGroups.\n2021-08-23 13:45:17.282 [ taskGroup-0] INFO  TaskGroupContainer   - taskGroupId=[0] start [1] channels for [1] tasks.\n2021-08-23 13:45:17.287 [ taskGroup-0] INFO  Channel              - Channel set byte_speed_limit to -1, No bps activated.\n2021-08-23 13:45:17.288 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\n2021-08-23 13:45:20.295 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2021-08-23 13:45:20.296 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do post work.\n2021-08-23 13:45:20.297 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do post work.\n2021-08-23 13:45:20.302 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2021-08-23 13:45:20.305 [       job-0] INFO  StandAloneJobContainerCommunicator - Total 10 records, 220 bytes | Speed 73B/s, 3 records/s | Error 0 records, 0 bytes |  All Task WaitWriterTime 0.000s |  All Task WaitReaderTime 0.011s | Percentage 100.00%\n2021-08-23 13:45:20.307 [       job-0] INFO  JobContainer         -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-08-23 13:45:17\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-08-23 13:45:20\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :               73B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :              3rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                  10\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"setupJob/","title":"\u4efb\u52a1\u914d\u7f6e","text":"

    \u4e00\u4e2a\u91c7\u96c6\u4efb\u52a1\u5c31\u662f\u4e00\u4e2a JSON \u683c\u5f0f\u914d\u7f6e\u6587\u4ef6\uff0c\u8be5\u914d\u7f6e\u6587\u4ef6\u7684\u6a21\u677f\u5982\u4e0b\uff1a

    {\n  \"job\": {\n    \"settings\": {},\n    \"content\": {\n      \"reader\": {},\n      \"writer\": {},\n      \"transformer\": []\n    }\n  }\n}\n

    \u4efb\u52a1\u914d\u7f6e\u7531 key \u4e3a job \u7684\u5b57\u5178\u7ec4\u6210\uff0c\u5176\u5b57\u5178\u5143\u7d20\u7531\u4e09\u90e8\u5206\u7ec4\u6210\uff1a

    "},{"location":"setupJob/#reader","title":"reader \u914d\u7f6e\u9879","text":"

    reader \u914d\u7f6e\u9879\u4f9d\u636e\u4e0d\u540c\u7684 reader \u63d2\u4ef6\u800c\u6709\u4e9b\u5fae\u4e0d\u540c\uff0c\u4f46\u5927\u90e8\u5206\u7684\u914d\u7f6e\u57fa\u672c\u76f8\u540c\uff0c\u7279\u522b\u662f\u9488\u5bf9\u5173\u7cfb\u578b\u6570\u636e\u5e93\u800c\u8a00\uff0c\u5176\u57fa\u672c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"name\": \"mysqlreader\",\n  \"parameter\": {\n    \"username\": \"\",\n    \"password\": \"\",\n    \"column\": [],\n    \"autoPk\": false,\n    \"splitPk\": \"\",\n    \"connection\": [\n      {\n        \"jdbcUrl\": [],\n        \"table\": []\n      }\n    ],\n    \"where\": \"\"\n  }\n}\n

    \u5176\u4e2d name \u662f\u63d2\u4ef6\u7684\u540d\u79f0\uff0c\u6bcf\u4e2a\u63d2\u4ef6\u7684\u540d\u79f0\u90fd\u662f\u552f\u4e00\u7684\uff0c\u6bcf\u4e2a\u63d2\u4ef6\u66f4\u8be6\u7ec6\u7684\u914d\u7f6e\u53ef\u4ee5\u53c2\u8003\u8bfb\u53d6\u63d2\u4ef6\u7ae0\u8282\u7684\u5404\u63d2\u4ef6\u5185\u5bb9

    "},{"location":"setupJob/#writer","title":"writer \u914d\u7f6e\u9879","text":"

    writer \u914d\u7f6e\u9879\u548c reader \u914d\u7f6e\u9879\u5dee\u4e0d\u591a\uff0c\u5176\u57fa\u672c\u6a21\u677f\u5982\u4e0b\uff1a

    {\n  \"name\": \"mysqlwriter\",\n  \"parameter\": {\n    \"username\": \"\",\n    \"password\": \"\",\n    \"writeMode\": \"\",\n    \"column\": [],\n    \"session\": [],\n    \"preSql\": [],\n    \"postSql\": [],\n    \"connection\": [\n      {\n        \"jdbcUrl\": \"\",\n        \"table\": []\n      }\n    ]\n  }\n}\n

    \u540c\u6837\u7684\uff0c\u8fd9\u91cc\u7684 name \u4e5f\u662f\u552f\u4e00\u7684\uff0c\u6bcf\u4e2a\u63d2\u4ef6\u66f4\u8be6\u7ec6\u7684\u914d\u7f6e\u53ef\u4ee5\u53c2\u8003\u5199\u5165\u63d2\u4ef6\u7ae0\u8282\u7684\u5404\u63d2\u4ef6\u5185\u5bb9

    "},{"location":"setupJob/#settings","title":"settings \u914d\u7f6e\u9879","text":"

    settings \u53ef\u914d\u7f6e\u7684\u5185\u5bb9\u5982\u4e0b\uff1a

    {\n  \"speed\": {\n    \"byte\": -1,\n    \"record\": 100,\n    \"channel\": 1\n  },\n  \"errorLimit\": {\n    \"record\": 0,\n    \"percentage\": 0.02\n  }\n}\n

    \u89e3\u91ca\u5982\u4e0b\uff1a

    "},{"location":"setupJob/#speed","title":"speed","text":"

    \u987e\u540d\u601d\u4e49\uff0c\u8fd9\u91cc\u662f\u7528\u6765\u505a\u6d41\u63a7\u7684\u914d\u7f6e\u9879\uff0c\u53ef\u4ee5\u4ece\u7f51\u7edc\u4f20\u8f93\u901f\u5ea6\uff0c\u6bcf\u79d2\u7684\u8bb0\u5f55\u6570\u4ee5\u53ca\u7ebf\u7a0b\u6570\u4e0a\u505a\u63a7\u5236\uff0c\u5206\u522b\u63cf\u8ff0\u5982\u4e0b\uff1a

    "},{"location":"setupJob/#speedbyte","title":"speed.byte","text":"

    \u8bbe\u7f6e\u6bcf\u79d2\u53ef\u83b7\u53d6\u7684\u5b57\u8282\u6570(Bps)\uff0c\u4e00\u822c\u662f\u4e3a\u4e86\u9632\u6b62\u6267\u884c\u4efb\u52a1\u65f6\u5c06\u6574\u4e2a\u5e26\u5bbd\u8dd1\u6ee1\uff0c\u4ece\u800c\u5f71\u54cd\u5230\u5176\u4ed6\u670d\u52a1\u3002\u5982\u679c\u4e0d\u505a\u9650\u5236\uff0c\u53ef\u8bbe\u7f6e\u4e3a -1

    "},{"location":"setupJob/#speedrecord","title":"speed.record","text":"

    \u8bbe\u7f6e\u8bb0\u5f55\u6bcf\u79d2\u53ef\u83b7\u53d6\u7684\u6700\u5927\u8bb0\u5f55\u6761\u6570\uff0c\u8be5\u53c2\u6570\u9700\u8981\u548c speed.byte \u914d\u5408\u4f7f\u7528

    "},{"location":"setupJob/#speedchannel","title":"speed.channel","text":"

    \u8bbe\u7f6e\u901a\u9053\u6570\uff0c\u8be5\u901a\u9053\u8def\u786e\u5b9a\u4e86\u603b\u7684 Task \u7ebf\u7a0b\u6570\uff0c\u5047\u5b9a\u8bbe\u5b9a speed.channel \u4e3a 13\uff0c \u5219\u4e00\u5171\u6709 13 \u4e2a Task. \u7136\u540e\u6839\u636e conf/core.json \u914d\u7f6e\u4e2d\u7684 taskGroup.channel \u914d\u7f6e\u6765\u786e\u5b9a\u8981\u521b\u5efa\u7684 taskGroup \u6570\u91cf\u3002\u5373 taskGroup = speed.channel / taskGroup.channel.

    "},{"location":"setupJob/#errorlimit","title":"errorLimit","text":"

    errorLimit \u7528\u6765\u914d\u7f6e\u5728\u6570\u636e\u5199\u5165\u62a5\u9519\u65f6\u7684\u884c\u4e3a\uff0c\u5177\u4f53\u5982\u4e0b

    "},{"location":"setupJob/#errorlimitrecord","title":"errorLimit.record","text":"

    \u5141\u8bb8\u9519\u8bef\u7684\u8bb0\u5f55\u6761\u6570\uff0c\u5982\u679c\u8d85\u8fc7\u8fd9\u4e2a\u6570\uff0c\u5219\u8ba4\u4e3a\u672c\u6b21\u4efb\u52a1\u5931\u8d25\uff0c\u5426\u5219\u8ba4\u4e3a\u6210\u529f

    "},{"location":"setupJob/#errorlimitpercentage","title":"errorLimit.percentage","text":"

    \u5141\u8bb8\u9519\u8bef\u8bb0\u5f55\u7684\u6bd4\u7387\uff0c\u8d85\u8fc7\u8fd9\u4e2a\u6bd4\u7387\uff0c\u5219\u8ba4\u4e3a\u672c\u6b21\u4efb\u52a1\u5931\u8d25\uff0c\u5426\u5219\u8ba4\u4e3a\u6210\u529f

    \u6ce8\u610f\uff0c\u4e0a\u8ff0\u53c2\u6570\u5728 conf/core.json \u914d\u7f6e\u6587\u4ef6\u5747\u6709\u9ed8\u8ba4\u914d\u7f6e\uff0c\u7528\u6765\u63a7\u5236\u5168\u5c40\u7684\u8bbe\u7f6e\u3002

    "},{"location":"statsreport/","title":"\u4efb\u52a1\u7ed3\u679c\u4e0a\u62a5","text":""},{"location":"statsreport/#_2","title":"\u5feb\u901f\u4ecb\u7ecd","text":"

    \u4e3b\u8981\u7528\u4e8e\u5c06\u5b9a\u65f6\u4efb\u52a1\u7684\u7ed3\u679c\u4e0a\u62a5\u7ed9\u6307\u5b9a\u670d\u52a1\u5668\uff0cAddax \u4f7f\u7528 HTTP \u534f\u8bae\u4ee5 POST \u65b9\u5f0f\u628a\u4e0a\u62a5\u6570\u636e\u4ee5 JSON \u683c\u5f0f\u53d1\u9001\u7ed9\u6307\u5b9a\u7684\u670d\u52a1\u5668\u3002

    \u53d1\u9001\u7684\u6570\u636e\u6837\u4f8b\u5982\u4e0b\uff1a

    {\n  \"jobName\": \"test\",\n  \"startTimeStamp\": 1587971621,\n  \"endTimeStamp\": 1587971621,\n  \"totalCosts\": 10,\n  \"byteSpeedPerSecond\": 33,\n  \"recordSpeedPerSecond\": 1,\n  \"totalReadRecords\": 6,\n  \"totalErrorRecords\": 0,\n  \"jobContent\": {\n    ...\n  }\n}\n
    \u4e0a\u8ff0\u53c2\u6570\u8bf4\u660e\u5982\u4e0b\uff1a

    \u53c2\u6570 \u63cf\u8ff0 \u5fc5\u9009 \u9ed8\u8ba4\u503c jobName \u4efb\u52a1\u540d \u662f jobName startTimeStamp \u4efb\u52a1\u6267\u884c\u7684\u5f00\u59cb\u65f6\u95f4 \u662f \u65e0 endTimeStamp \u4efb\u52a1\u6267\u884c\u7684\u7ed3\u675f\u65f6\u95f4 \u662f \u65e0 totalCosts \u4efb\u52a1\u603b\u8ba1\u8017\u65f6 \u662f \u65e0 byteSpeedPerSecond \u4efb\u52a1\u5e73\u5747\u6d41\u91cf \u662f \u65e0 recordSpeedPerSecond \u8bb0\u5f55\u5199\u5165\u901f\u5ea6 \u662f \u65e0 totalReadRecords \u8bfb\u51fa\u8bb0\u5f55\u603b\u6570 \u662f 0 totalErrorRecords \u8bfb\u5199\u5931\u8d25\u603b\u6570 \u662f 0 jobContent \u672c\u6b21\u4efb\u52a1\u7684json\u6587\u4ef6 \u662f \u65e0

    jobName \u7684\u8bbe\u7f6e\u89c4\u5219\u5982\u4e0b:

    1. \u5728\u547d\u4ee4\u884c\u901a\u8fc7\u4f20\u9012 -P\"-DjobName=xxxx\" \u65b9\u5f0f\u6307\u5b9a,\u5426\u5219
    2. \u914d\u7f6e\u6587\u4ef6\u7684 writer.parameters.path \u503c\u6309 / \u5206\u5272\u540e\u53d6\u7b2c2\uff0c3\u5217\u7528\u70b9(.)\u62fc\u63a5\u800c\u6210\uff0c\u5176\u542b\u4e49\u662f\u4e3a\u5e93\u540d\u53ca\u8868\u540d,\u5426\u5219
    3. \u5426\u5219\u8bbe\u7f6e\u4e3a jobName
    "},{"location":"transformer/","title":"\u6570\u636e\u8f6c\u6362","text":""},{"location":"transformer/#transformer","title":"Transformer \u5b9a\u4e49","text":"

    \u5728\u6570\u636e\u540c\u6b65\u3001\u4f20\u8f93\u8fc7\u7a0b\u4e2d\uff0c\u5b58\u5728\u7528\u6237\u5bf9\u4e8e\u6570\u636e\u4f20\u8f93\u8fdb\u884c\u7279\u6b8a\u5b9a\u5236\u5316\u7684\u9700\u6c42\u573a\u666f\uff0c\u5305\u62ec\u88c1\u526a\u5217\u3001\u8f6c\u6362\u5217\u7b49\u5de5\u4f5c\uff0c\u53ef\u4ee5\u501f\u52a9ETL\u7684T\u8fc7\u7a0b\u5b9e\u73b0(Transformer)\u3002Addax\u5305\u542b\u4e86\u5b8c\u6210\u7684E(Extract)\u3001T(Transformer)\u3001L(Load)\u652f\u6301\u3002

    "},{"location":"transformer/#_2","title":"\u8fd0\u884c\u6a21\u578b","text":"
    graph LR\nsource((\"source\"))\nsubgraph fr[\"Addax Framework\"]\n    direction LR\n    Reader ==> Transformer ==>Writer\nend\ntarget((\"target\"))\nsource ==> fr ==> target
    "},{"location":"transformer/#udf","title":"UDF \u51fd\u6570","text":""},{"location":"transformer/#dx_substr","title":"dx_substr","text":"

    dx_substr(idx, pos, length) -> str

    \u53c2\u6570

    \u8fd4\u56de\uff1a \u4ece\u5b57\u7b26\u4e32\u7684\u6307\u5b9a\u4f4d\u7f6e\uff08\u5305\u542b\uff09\u622a\u53d6\u6307\u5b9a\u957f\u5ea6\u7684\u5b57\u7b26\u4e32\u3002\u5982\u679c\u5f00\u59cb\u4f4d\u7f6e\u975e\u6cd5\u629b\u51fa\u5f02\u5e38\u3002\u5982\u679c\u5b57\u6bb5\u4e3a\u7a7a\u503c\uff0c\u76f4\u63a5\u8fd4\u56de\uff08\u5373\u4e0d\u53c2\u4e0e\u672ctransformer\uff09

    "},{"location":"transformer/#dx_pad","title":"dx_pad","text":"

    dx_pad(idx, flag, length, chr)

    \u53c2\u6570

    \u8fd4\u56de\uff1a \u5982\u679c\u6e90\u5b57\u7b26\u4e32\u957f\u5ea6\u5c0f\u4e8e\u76ee\u6807\u5b57\u6bb5\u957f\u5ea6\uff0c\u6309\u7167\u4f4d\u7f6e\u6dfb\u52a0pad\u5b57\u7b26\u540e\u8fd4\u56de\u3002\u5982\u679c\u957f\u4e8e\uff0c\u76f4\u63a5\u622a\u65ad\uff08\u90fd\u622a\u53f3\u8fb9\uff09\u3002\u5982\u679c\u5b57\u6bb5\u4e3a\u7a7a\u503c\uff0c\u8f6c\u6362\u4e3a\u7a7a\u5b57\u7b26\u4e32\u8fdb\u884cpad\uff0c\u5373\u6700\u540e\u7684\u5b57\u7b26\u4e32\u5168\u662f\u9700\u8981pad\u7684\u5b57\u7b26

    \u4e3e\u4f8b\uff1a

    "},{"location":"transformer/#dx_replace","title":"dx_replace","text":"

    dx_replace(idx, pos, length, str) -> str

    \u53c2\u6570

    \u8fd4\u56de\uff1a \u4ece\u5b57\u7b26\u4e32\u7684\u6307\u5b9a\u4f4d\u7f6e\uff08\u5305\u542b\uff09\u66ff\u6362\u6307\u5b9a\u957f\u5ea6\u7684\u5b57\u7b26\u4e32\u3002\u5982\u679c\u5f00\u59cb\u4f4d\u7f6e\u975e\u6cd5\u629b\u51fa\u5f02\u5e38\u3002\u5982\u679c\u5b57\u6bb5\u4e3a\u7a7a\u503c\uff0c\u76f4\u63a5\u8fd4\u56de\uff08\u5373\u4e0d\u53c2\u4e0e\u672ctransformer\uff09

    \u4e3e\u4f8b\uff1a

    "},{"location":"transformer/#dx_filter","title":"dx_filter","text":"

    dx_filter(idx, operator, expr) -> str

    \u53c2\u6570\uff1a

    \u8fd4\u56de\uff1a

    \u4e3e\u4f8b

    \u5173\u8054filter\u6682\u4e0d\u652f\u6301\uff0c\u5373\u591a\u4e2a\u5b57\u6bb5\u7684\u8054\u5408\u5224\u65ad\uff0c\u51fd\u53c2\u592a\u8fc7\u590d\u6742\uff0c\u7528\u6237\u96be\u4ee5\u4f7f\u7528\u3002

    "},{"location":"transformer/#dx_groovy","title":"dx_groovy","text":"

    dx_groovy(code, package) -> record

    \u53c2\u6570

    \u8fd4\u56de

    Record \u6570\u636e\u7c7b\u578b

    \u6ce8\u610f\uff1a

    \u4e3e\u4f8b:

    groovy \u5b9e\u73b0\u7684 subStr

    String code=\"Column column = record.getColumn(1);\\n\"+\n        \" String oriValue = column.asString();\\n\"+\n        \" String newValue = oriValue.substring(0, 3);\\n\"+\n        \" record.setColumn(1, new StringColumn(newValue));\\n\"+\n        \" return record;\";\n        dx_groovy(record);\n

    groovy \u5b9e\u73b0\u7684Replace

    String code2=\"Column column = record.getColumn(1);\\n\"+\n        \" String oriValue = column.asString();\\n\"+\n        \" String newValue = \\\"****\\\" + oriValue.substring(3, oriValue.length());\\n\"+\n        \" record.setColumn(1, new StringColumn(newValue));\\n\"+\n        \" return record;\";\n

    groovy \u5b9e\u73b0\u7684Pad

    String code3=\"Column column = record.getColumn(1);\\n\"+\n        \" String oriValue = column.asString();\\n\"+\n        \" String padString = \\\"12345\\\";\\n\"+\n        \" String finalPad = \\\"\\\";\\n\"+\n        \" int NeedLength = 8 - oriValue.length();\\n\"+\n        \"        while (NeedLength > 0) {\\n\"+\n        \"\\n\"+\n        \"            if (NeedLength >= padString.length()) {\\n\"+\n        \"                finalPad += padString;\\n\"+\n        \"                NeedLength -= padString.length();\\n\"+\n        \"            } else {\\n\"+\n        \"                finalPad += padString.substring(0, NeedLength);\\n\"+\n        \"                NeedLength = 0;\\n\"+\n        \"            }\\n\"+\n        \"        }\\n\"+\n        \" String newValue= finalPad + oriValue;\\n\"+\n        \" record.setColumn(1, new StringColumn(newValue));\\n\"+\n        \" return record;\";\n

    \u4ece 4.1.2 \u7248\u672c\u5f00\u59cb\uff0c dx_groovy \u652f\u6301\u4ece\u5916\u90e8\u6587\u4ef6\u52a0\u8f7d groovy \u4ee3\u7801\uff0c\u8bfb\u53d6\u6587\u4ef6\u7684\u76f8\u5bf9\u8def\u5f84\u4e3a $ADDAX_HOME \u53d8\u91cf\u6240\u5728\u7684\u76ee\u5f55\uff0c\u4e5f\u5c31\u662f Addax \u7684\u5b89\u88c5\u76ee\u5f55\u3002

    \u4ee5\u5b9e\u73b0 subStr \u4e3a\u4f8b\uff0c\u6211\u4eec\u53ef\u4ee5\u521b\u5efa job/substr.groovy \u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    Column column = record.getColumn(1)\nString oriValue = column.asString()\nString newValue = oriValue.substring(0, 3)\nrecord.setColumn(1, new StringColumn(newValue))\nreturn record\n

    \u7136\u540e\u5728 job \u6587\u4ef6\u4e2d\u8fd9\u6837\u53bb\u5b9a\u4e49\uff1a

    {\n  \"transformer\": [\n    {\n      \"name\": \"dx_groovy\",\n      \"parameter\": {\n        \"codeFile\": \"job/substr.groovy\"\n      }\n    }\n  ]\n}\n

    \u6587\u4ef6\u4e5f\u53ef\u4ee5\u4f7f\u7528\u7edd\u5bf9\u8def\u5f84\u6765\u6307\u5b9a\u3002

    "},{"location":"transformer/#job","title":"Job\u5b9a\u4e49","text":"

    \u672c\u4f8b\u4e2d\uff0c\u914d\u7f6e4\u4e2aUDF\u3002

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"My name is xxxx\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"password is Passw0rd\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 00:00:00\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            },\n            {\n              \"random\": \"0,10\",\n              \"type\": \"long\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true,\n          \"encoding\": \"UTF-8\"\n        }\n      },\n      \"transformer\": [\n        {\n          \"name\": \"dx_replace\",\n          \"parameter\": {\n            \"columnIndex\": 0,\n            \"paras\": [\n              \"11\",\n              \"6\",\n              \"wgzhao\"\n            ]\n          }\n        },\n        {\n          \"name\": \"dx_substr\",\n          \"parameter\": {\n            \"columnIndex\": 1,\n            \"paras\": [\n              \"0\",\n              \"12\"\n            ]\n          }\n        },\n        {\n          \"name\": \"dx_map\",\n          \"parameter\": {\n            \"columnIndex\": 2,\n            \"paras\": [\n              \"^\",\n              \"2\"\n            ]\n          }\n        },\n        {\n          \"name\": \"dx_filter\",\n          \"parameter\": {\n            \"columnIndex\": 6,\n            \"paras\": [\n              \"<\",\n              \"5\"\n            ]\n          }\n        }\n      ]\n    }\n  }\n}\n
    "},{"location":"transformer/#_3","title":"\u81ea\u5b9a\u4e49\u51fd\u6570","text":"

    \u5982\u679c\u81ea\u5e26\u7684\u51fd\u6570\u4e0d\u6ee1\u8db3\u6570\u636e\u8f6c\u6362\u8981\u6c42\uff0c\u6211\u4eec\u53ef\u4ee5\u5728 transformer \u7f16\u5199\u6ee1\u8db3 groovy \u89c4\u8303\u8981\u6c42\u7684\u4ee3\u7801\uff0c\u4e0b\u9762\u7ed9\u51fa\u4e00\u4e2a\u5b8c\u6574\u7684\u4f8b\u5b50

    \n

    \u4e0a\u8ff0 transformer \u4ee3\u7801\u9488\u5bf9\u6bcf\u6761\u8bb0\u5f55\u7684\u524d\u9762\u4e24\u4e2a\u5b57\u6bb5\u505a\u4e86\u4fee\u6539\uff0c\u5bf9\u7b2c\u4e00\u4e2a\u5b57\u6bb5\u7684\u5b57\u7b26\u4e32\uff0c\u5728\u5b57\u7b26\u4e32\u524d\u9762\u589e\u52a0 Header_ \u5b57\u7b26\uff1b \u7b2c\u4e8c\u4e2a\u6574\u6570\u5b57\u6bb5\u503c\u8fdb\u884c\u500d\u589e\u5904\u7406\u3002\u6700\u540e\u6267\u884c\u7684\u7ed3\u679c\u5982\u4e0b\uff1a

    $ bin/addax.sh job/transformer_demo.json \n\n  ___      _     _            \n / _ \\    | |   | |           \n/ /_\\ \\ __| | __| | __ ___  __\n|  _  |/ _` |/ _` |/ _` \\ \\/ /\n| | | | (_| | (_| | (_| |>  < \n\\_| |_/\\__,_|\\__,_|\\__,_/_/\\_\\\n\n:: Addax version ::    (v4.0.2-SNAPSHOT)\n\n2021-08-04 15:45:56.421 [        main] INFO  VMInfo               - VMInfo# operatingSystem class => com.sun.management.internal.OperatingSystemImpl\n2021-08-04 15:45:56.443 [        main] INFO  Engine               - \n\n.....\n\n2021-08-04 15:45:56.458 [        main] INFO  PerfTrace            - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-08-04 15:45:56.459 [        main] INFO  JobContainer         - Addax jobContainer starts job.\n2021-08-04 15:45:56.460 [        main] INFO  JobContainer         - Set jobId = 0\n2021-08-04 15:45:56.470 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do prepare work .\n2021-08-04 15:45:56.471 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do prepare work .\n2021-08-04 15:45:56.471 [       job-0] INFO  JobContainer         - Job set Channel-Number to 1 channels.\n2021-08-04 15:45:56.472 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] splits to [1] tasks.\n2021-08-04 15:45:56.472 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] splits to [1] tasks.\n2021-08-04 15:45:56.498 [       job-0] INFO  JobContainer         - Scheduler starts [1] taskGroups.\n2021-08-04 15:45:56.505 [ taskGroup-0] INFO  TaskGroupContainer   - taskGroupId=[0] start [1] channels for [1] tasks.\n2021-08-04 15:45:56.517 [ taskGroup-0] INFO  Channel              - Channel set byte_speed_limit to -1, No bps activated.\n2021-08-04 15:45:56.517 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\n2021-08-04 15:45:56.520 [ taskGroup-0] INFO  TransformerUtil      -  user config transformers [[dx_groovy]], loading...\n2021-08-04 15:45:56.531 [ taskGroup-0] INFO  TransformerUtil      -  1 of transformer init success. name=dx_groovy, isNative=true parameter = \n  {\"code\":\"record.setColumn(0, new StringColumn('Header_' + record.getColumn(0).asString()));record.setColumn(1, new LongColumn(record.getColumn(1).asLong() * 2));return record;\"}\n\nHeader_Addax    2       1989-06-04 00:00:01     true    test\nHeader_Addax    4       1989-06-03 00:00:01     true    test\nHeader_Addax    6       1989-06-02 00:00:01     true    test\nHeader_Addax    8       1989-06-01 00:00:01     true    test\nHeader_Addax    10      1989-05-31 00:00:01     true    test\nHeader_Addax    12      1989-05-30 00:00:01     true    test\nHeader_Addax    14      1989-05-29 00:00:01     true    test\nHeader_Addax    16      1989-05-28 00:00:01     true    test\nHeader_Addax    18      1989-05-27 00:00:01     true    test\nHeader_Addax    20      1989-05-26 00:00:01     true    test\n\n2021-08-04 15:45:59.515 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2021-08-04 15:45:59.517 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do post work.\n2021-08-04 15:45:59.518 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do post work.\n2021-08-04 15:45:59.521 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2021-08-04 15:45:59.524 [       job-0] INFO  StandAloneJobContainerCommunicator - Total 10 records, 330 bytes | Speed 110B/s, 3 records/s | Error 0 records, 0 bytes |  \n  All Task WaitWriterTime 0.000s |  All Task WaitReaderTime 0.000s | Transformer Success 10 records | Transformer Error 0 records | Transformer Filter 0 records \n  | Transformer usedTime 0.383s | Percentage 100.00%\n2021-08-04 15:45:59.527 [       job-0] INFO  JobContainer         - \n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-08-04 15:45:56\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-08-04 15:45:59\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :              110B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :              3rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                  10\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n\n2021-08-04 15:45:59.528 [       job-0] INFO  JobContainer         - \nTransformer\u6210\u529f\u8bb0\u5f55\u603b\u6570         :                  10\nTransformer\u5931\u8d25\u8bb0\u5f55\u603b\u6570         :                   0\nTransformer\u8fc7\u6ee4\u8bb0\u5f55\u603b\u6570         :                   0\n
    "},{"location":"transformer/#_4","title":"\u8ba1\u91cf\u548c\u810f\u6570\u636e","text":"

    Transform\u8fc7\u7a0b\u6d89\u53ca\u5230\u6570\u636e\u7684\u8f6c\u6362\uff0c\u53ef\u80fd\u9020\u6210\u6570\u636e\u7684\u589e\u52a0\u6216\u51cf\u5c11\uff0c\u56e0\u6b64\u66f4\u52a0\u9700\u8981\u7cbe\u786e\u5ea6\u91cf\uff0c\u5305\u62ec\uff1a

    \u6d89\u53ca\u5230\u8fd0\u884c\u8fc7\u7a0b\u7684\u8ba1\u91cf\u6570\u636e\u5c55\u73b0\u5b9a\u4e49\u5982\u4e0b\uff1a

    Total 1000000 records, 22000000 bytes | Transform 100000 records(in), 10000 records(out) | Speed 2.10MB/s, 100000 records/s | Error 0 records, 0 bytes | Percentage 100.00%\n

    \u6ce8\u610f\uff0c\u8fd9\u91cc\u4e3b\u8981\u8bb0\u5f55\u8f6c\u6362\u7684\u8f93\u5165\u8f93\u51fa\uff0c\u9700\u8981\u68c0\u6d4b\u6570\u636e\u8f93\u5165\u8f93\u51fa\u7684\u8bb0\u5f55\u6570\u91cf\u53d8\u5316\u3002

    \u6d89\u53ca\u5230\u6700\u7ec8\u4f5c\u4e1a\u7684\u8ba1\u91cf\u6570\u636e\u5c55\u73b0\u5b9a\u4e49\u5982\u4e0b\uff1a

    \u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2015-03-10 17:34:21\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2015-03-10 17:34:31\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                 10s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :            2.10MB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :         100000rec/s\n\u8f6c\u6362\u8f93\u5165\u603b\u6570                    :             1000000\n\u8f6c\u6362\u8f93\u51fa\u603b\u6570                    :             1000000\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :             1000000\n\u540c\u6b65\u5931\u8d25\u603b\u6570                    :                   0\n

    \u6ce8\u610f\uff0c\u8fd9\u91cc\u4e3b\u8981\u8bb0\u5f55\u8f6c\u6362\u7684\u8f93\u5165\u8f93\u51fa\uff0c\u9700\u8981\u68c0\u6d4b\u6570\u636e\u8f93\u5165\u8f93\u51fa\u7684\u8bb0\u5f55\u6570\u91cf\u53d8\u5316\u3002

    "},{"location":"reader/accessreader/","title":"Access Reader","text":"

    AccessReader \u5b9e\u73b0\u4e86\u4ece Access \u6570\u636e\u5e93\u4e0a\u8bfb\u53d6\u6570\u636e\u7684\u80fd\u529b\uff0c\u4ed6\u57fa\u4e8e Addax RDBMS Reader \u5b9e\u73b0\u3002

    "},{"location":"reader/accessreader/#_1","title":"\u793a\u4f8b","text":"

    \u6211\u4eec\u4e0b\u8f7d\u7528\u4e8e\u6d4b\u8bd5\u7528\u7684 AcessThemeDemo.zip \u6587\u4ef6\uff0c\u89e3\u538b\u540e\u5f97\u5230 AccessThemeDemo.mdb \u6587\u4ef6\uff0c\u8be5\u6587\u4ef6\u4e2d\u5305\u542b\u4e86\u4e00\u4e2a tbl_Users \u8868\uff0c\u6211\u4eec\u5c06\u8be5\u8868\u7684\u6570\u636e\u540c\u6b65\u5230\u7ec8\u7aef\u4e0a\u3002

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/access2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\":\n      {\n        \"reader\": {\n          \"name\": \"accessreader\",\n          \"parameter\": {\n            \"username\": \"root\",\n            \"password\": \"\",\n            \"column\": [\n              \"*\"\n            ],\n            \"connection\": {\n              \"table\": [\n                \"tbl_Users\"\n              ],\n              \"jdbcUrl\": \"jdbc:ucanaccess:///Users/wgzhao/Downloads/AccessThemeDemo.mdb\"\n            },\n            \"where\": \"\"\n          }\n        },\n        \"writer\": {\n          \"name\": \"streamwriter\",\n          \"parameter\": {\n            \"encoding\": \"utf-8\",\n            \"print\": true\n          }\n        }\n      }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/access2stream.json

    "},{"location":"reader/accessreader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/access2stream.json\n
    "},{"location":"reader/accessreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    AccessReader \u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/cassandrareader/","title":"Cassandra Reader","text":"

    CassandraReader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Cassandra \u8bfb\u53d6\u6570\u636e\u7684\u80fd\u529b\u3002

    "},{"location":"reader/cassandrareader/#_1","title":"\u914d\u7f6e","text":"

    \u4e0b\u9762\u662f\u914d\u7f6e\u4e00\u4e2a\u4ece Cassandra \u8bfb\u53d6\u6570\u636e\u5230\u7ec8\u7aef\u7684\u4f8b\u5b50

    job/cassandra2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"cassandrareader\",\n        \"parameter\": {\n          \"host\": \"localhost\",\n          \"port\": 9042,\n          \"useSSL\": false,\n          \"keyspace\": \"test\",\n          \"table\": \"addax_src\",\n          \"column\": [\n            \"textCol\",\n            \"blobCol\",\n            \"writetime(blobCol)\",\n            \"boolCol\",\n            \"smallintCol\",\n            \"tinyintCol\",\n            \"intCol\",\n            \"bigintCol\",\n            \"varintCol\",\n            \"floatCol\",\n            \"doubleCol\",\n            \"decimalCol\",\n            \"dateCol\",\n            \"timeCol\",\n            \"timeStampCol\",\n            \"uuidCol\",\n            \"inetCol\",\n            \"durationCol\",\n            \"listCol\",\n            \"mapCol\",\n            \"setCol\",\n            \"tupleCol\",\n            \"udtCol\"\n          ]\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/cassandrareader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 host \u662f list \u65e0 \u8fde\u63a5\u7684\u57df\u540d\u6216 IP\uff0c\u591a\u4e2a\u8282\u70b9\u4e4b\u95f4\u7528\u9017\u53f7\u5206\u9694 port \u662f int 9042 \u7aef\u53e3 username \u5426 string \u65e0 \u7528\u6237\u540d password \u5426 string \u65e0 \u5bc6\u7801 useSSL \u5426 boolean false \u662f\u5426\u4f7f\u7528SSL\u8fde\u63a5 keyspace \u662f string \u65e0 \u9700\u8981\u540c\u6b65\u7684\u8868\u6240\u5728\u7684 keyspace table \u662f string \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u96c6\u5408,\u5176\u4e2d\u7684\u5143\u7d20\u53ef\u4ee5\u6307\u5b9a\u5217\u7684\u540d\u79f0\u6216 writetime(column_name)\uff0c\u540e\u4e00\u79cd\u5f62\u5f0f\u4f1a\u8bfb\u53d6column_name\u5217\u7684\u65f6\u95f4\u6233\u800c\u4e0d\u662f\u6570\u636e where \u5426 string \u65e0 \u6570\u636e\u7b5b\u9009\u6761\u4ef6\u7684 cql \u8868\u8fbe\u5f0f allowFiltering \u5426 boolean \u65e0 \u662f\u5426\u5728\u670d\u52a1\u7aef\u8fc7\u6ee4\u6570\u636e\uff0c\u8be6\u7ec6\u63cf\u8ff0\u53c2\u8003\u5b98\u65b9\u6587\u6863\u7684\u76f8\u5173\u63cf\u8ff0 consistencyLevel \u5426 string LOCAL_QUORUM \u6570\u636e\u4e00\u81f4\u6027\u7ea7\u522b, \u53ef\u9009 ONE, QUORUM, LOCAL_QUORUM, EACH_QUORUM, ALL, ANY, TWO, THREE, LOCAL_ONE"},{"location":"reader/cassandrareader/#_3","title":"\u652f\u6301\u7684\u6570\u636e\u7c7b\u578b","text":"

    \u76ee\u524d\u652f\u6301\u9664 counter \u548c Custom \u7c7b\u578b\u4e4b\u5916\u7684\u6240\u6709\u7c7b\u578b\u3002

    \u4e0b\u9762\u5217\u51fa\u7c7b\u578b\u8f6c\u6362\u5217\u8868:

    Addax \u5185\u90e8\u7c7b\u578b Cassandra \u6570\u636e\u7c7b\u578b Long int, tinyint, smallint,varint,bigint,time,counter Double float, double, decimal String ascii,varchar, text,uuid,timeuuid,duration,list,map,set,tuple,udt,inet Date date, timestamp Boolean bool Bytes blob"},{"location":"reader/clickhousereader/","title":"ClickHouse Reader","text":"

    ClickHouseReader \u63d2\u4ef6\u652f\u6301\u4ece ClickHouse\u6570\u636e\u5e93\u8bfb\u53d6\u6570\u636e\u3002

    "},{"location":"reader/clickhousereader/#_1","title":"\u793a\u4f8b","text":""},{"location":"reader/clickhousereader/#_2","title":"\u8868\u7ed3\u6784\u53ca\u6570\u636e\u4fe1\u606f","text":"

    \u5047\u5b9a\u9700\u8981\u7684\u8bfb\u53d6\u7684\u8868\u7684\u7ed3\u6784\u4ee5\u53ca\u6570\u636e\u5982\u4e0b\uff1a

    CREATE TABLE ck_addax (\n    c_int8 Int8,\n    c_int16 Int16,\n    c_int32 Int32,\n    c_int64 Int64,\n    c_uint8 UInt8,\n    c_uint16 UInt16,\n    c_uint32 UInt32,\n    c_uint64 UInt64,\n    c_float32 Float32,\n    c_float64 Float64,\n    c_decimal Decimal(38,10),\n    c_string String,\n    c_fixstr FixedString(36),\n    c_uuid UUID,\n    c_date Date,\n    c_datetime DateTime('Asia/Chongqing'),\n    c_datetime64 DateTime64(3, 'Asia/Chongqing'),\n    c_enum Enum('hello' = 1, 'world'=2)\n) ENGINE = MergeTree() ORDER BY (c_int8, c_int16) SETTINGS index_granularity = 8192;\n\ninsert into ck_addax values(\n    127,\n    -32768,\n    2147483647,\n    -9223372036854775808,\n    255,\n    65535,\n    4294967295,\n    18446744073709551615,\n    0.999999999999,\n    0.99999999999999999,\n    1234567891234567891234567891.1234567891,\n    'Hello String',\n    '2c:16:db:a3:3a:4f',\n    '5F042A36-5B0C-4F71-ADFD-4DF4FCA1B863',\n    '2021-01-01',\n    '2021-01-01 11:22:33',\n    '2021-01-01 10:33:23.123',\n    'hello'\n);\n
    "},{"location":"reader/clickhousereader/#json","title":"\u914d\u7f6e json \u6587\u4ef6","text":"

    \u4e0b\u9762\u7684\u914d\u7f6e\u6587\u4ef6\u8868\u793a\u4ece ClickHouse \u6570\u636e\u5e93\u8bfb\u53d6\u6307\u5b9a\u7684\u8868\u6570\u636e\u5e76\u6253\u5370\u5230\u7ec8\u7aef

    job/clickhouse2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"clickhousereader\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"root\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"ck_addax\"\n            ],\n            \"jdbcUrl\": \"jdbc:clickhouse://127.0.0.1:8123/default\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/clickhouse2stream.json

    "},{"location":"reader/clickhousereader/#_3","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/clickhouse2stream.json\n

    \u5176\u8f93\u51fa\u4fe1\u606f\u5982\u4e0b\uff08\u5220\u9664\u4e86\u975e\u5173\u952e\u4fe1\u606f)

    021-01-06 14:39:35.742 [main] INFO  VMInfo - VMInfo# operatingSystem class => com.sun.management.internal.OperatingSystemImpl\n\n2021-01-06 14:39:35.767 [main] INFO  Engine -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"column\":[\n                        \"*\"\n                    ],\n                    \"connection\":[\n                        {\n                            \"jdbcUrl\":[\n                                \"jdbc:clickhouse://127.0.0.1:8123/\"\n                            ],\n                            \"table\":[\n                                \"ck_addax\"\n                            ]\n                        }\n                    ],\n                    \"username\":\"default\"\n                },\n                \"name\":\"clickhousereader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":true\n                },\n                \"name\":\"streamwriter\"\n            }\n    },\n    \"setting\":{\n        \"errorLimit\":{\n            \"record\":0,\n            \"percentage\":0.02\n        },\n        \"speed\":{\n            \"channel\":3\n        }\n    }\n}\n\n127 -32768  2147483647  -9223372036854775808    255 65535   4294967295  18446744073709551615    1   1   1234567891234567891234567891.1234567891Hello String 2c:16:db:a3:3a:4f   \n5f042a36-5b0c-4f71-adfd-4df4fca1b863    2021-01-01  2021-01-01 00:00:00 2021-01-01 00:00:00 hello\n\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-01-06 14:39:35\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-01-06 14:39:39\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :               77B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :              0rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                   1\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"reader/clickhousereader/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u53c2\u6570\u3002

    "},{"location":"reader/clickhousereader/#_5","title":"\u652f\u6301\u7684\u6570\u636e\u7c7b\u578b","text":"Addax \u5185\u90e8\u7c7b\u578b ClickHouse \u6570\u636e\u7c7b\u578b Long Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, Enum8, Enum16 Double Float32, Float64, Decimal String String, FixedString(N) Date Date, DateTime, DateTime64 Boolean UInt8 Bytes String"},{"location":"reader/clickhousereader/#_6","title":"\u9650\u5236","text":"

    \u9664\u4e0a\u8ff0\u7f57\u5217\u5b57\u6bb5\u7c7b\u578b\u5916\uff0c\u5176\u4ed6\u7c7b\u578b\u5747\u4e0d\u652f\u6301\uff0c\u5982 Array\u3001Nested \u7b49

    "},{"location":"reader/databendreader/","title":"Databend Reader","text":"

    DatabendReader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Databend \u8bfb\u53d6\u6570\u636e

    \u6ce8\u610f\uff0cdatabender \u6709\u517c\u5bb9 MySQL \u5ba2\u6237\u7aef\u7684\u534f\u8bae\u5b9e\u73b0\uff0c\u56e0\u6b64\u4f60\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528 MySQL Reader \u6765\u8bfb\u53d6 Databend \u6570\u636e\u3002

    "},{"location":"reader/databendreader/#_1","title":"\u793a\u4f8b","text":"

    \u6211\u4eec\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u65b9\u5f0f\u542f\u52a8 Databend \u6570\u636e\u5e93

    docker run  -tid  --rm  -p 8000:8000 \\\n   -e QUERY_DEFAULT_USER=databend \\\n   -e QUERY_DEFAULT_PASSWORD=databend \\\n   datafuselabs/databend\n

    \u7136\u540e\u521b\u5efa\u4e00\u5f20\u9700\u8981\u8bfb\u53d6\u7684\u8868

    (\n    id int,\n    name varchar(255),\n    salary float,\n    created_at datetime,\n    updated_at datetime\n);\n

    \u5e76\u586b\u5145\u5fc5\u8981\u7684\u6570\u636e

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/databend2stream.json 
    {\n  \"job\": {\n    \"content\": [\n      {\n        \"reader\": {\n          \"name\": \"databendreader\",\n          \"parameter\": {\n            \"column\": [\n              \"*\"\n            ],\n            \"connection\": {\n              \"jdbcUrl\": \"jdbc:databend://127.0.0.1:8000/default\",\n              \"table\": [\n                \"addax_reader\"\n              ]\n            },\n            \"username\": \"databend\",\n            \"password\": \"databend\"\n          }\n        },\n        \"writer\": {\n          \"name\": \"streamwriter\",\n          \"parameter\": {\n            \"print\": \"true\"\n          }\n        }\n      }\n    ],\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/databend2stream.json

    "},{"location":"reader/databendreader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/databend2stream.json\n
    "},{"location":"reader/databendreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u53c2\u6570\u3002

    "},{"location":"reader/databendreader/#_4","title":"\u9650\u5236","text":"

    \u6682\u65e0

    "},{"location":"reader/datareader/","title":"Data Reader","text":"

    DataReader \u63d2\u4ef6\u662f\u4e13\u95e8\u63d0\u4f9b\u7528\u4e8e\u5f00\u53d1\u548c\u6d4b\u8bd5\u73af\u5883\u4e2d\uff0c\u751f\u4ea7\u6ee1\u8db3\u4e00\u5b9a\u89c4\u5219\u8981\u6c42\u7684\u6570\u636e\u7684\u63d2\u4ef6\u3002

    \u5728\u5b9e\u9645\u5f00\u53d1\u548c\u6d4b\u8bd5\u4e2d\uff0c\u6211\u4eec\u9700\u8981\u6309\u7167\u4e00\u5b9a\u7684\u4e1a\u52a1\u89c4\u5219\u6765\u751f\u4ea7\u6d4b\u8bd5\u6570\u636e\uff0c\u800c\u4e0d\u4ec5\u4ec5\u662f\u968f\u673a\u5185\u5bb9\uff0c\u6bd4\u5982\u8eab\u4efd\u8bc1\u53f7\u7801\uff0c\u94f6\u884c\u8d26\u53f7\uff0c\u80a1\u7968\u4ee3\u7801\u7b49\u3002

    "},{"location":"reader/datareader/#_1","title":"\u4e3a\u4ec0\u4e48\u8981\u91cd\u590d\u53d1\u660e\u8f6e\u5b50","text":"

    \u8bda\u7136\uff0c\u7f51\u7edc\u4e0a\u6709\u76f8\u5f53\u591a\u7684\u4e13\u95e8\u7684\u6570\u636e\u751f\u4ea7\u5de5\u5177\uff0c\u5176\u4e2d\u4e0d\u4e4f\u529f\u80fd\u5f3a\u5927\u3001\u6027\u80fd\u4e5f\u5f3a\u608d\u3002 \u4f46\u8fd9\u4e9b\u5de5\u5177\u5927\u90e8\u5206\u662f\u8003\u8651\u5230\u4e86\u6570\u636e\u751f\u6210\u8fd9\u4e00\u6bb5\uff0c\u800c\u5ffd\u7565\u4e86\u6570\u636e\u5199\u5165\u5230\u76ee\u6807\u7aef\u7684\u95ee\u9898\uff0c\u6216\u8005\u8bf4\u6709\u4e9b\u8003\u8651\u5230\u4e86\uff0c\u4f46\u4ec5\u4ec5\u53ea\u8003\u8651\u4e86\u4e00\u79cd\u6216\u6709\u9650\u7684\u51e0\u79cd\u6570\u636e\u5e93\u3002

    \u6070\u597d Addax \u5de5\u5177\u80fd\u591f\u63d0\u4f9b\u8db3\u591f\u591a\u7684\u76ee\u6807\u7aef\u5199\u5165\u80fd\u529b\uff0c\u52a0\u4e0a\u4e4b\u524d\u7684\u5df2\u6709\u7684 Stream Reader \u5df2\u7ecf\u7b97\u662f\u4e00\u4e2a\u7b80\u5355\u7248\u7684\u6570\u636e\u751f\u6210\u5de5\u5177\uff0c\u56e0\u6b64\u5728\u6b64\u529f\u80fd\u4e0a \u589e\u52a0\u4e00\u4e9b\u7279\u5b9a\u89c4\u5219\uff0c\u518d\u5229\u7528\u5199\u5165\u7aef\u591a\u6837\u6027\u7684\u80fd\u529b\uff0c\u81ea\u7136\u5c31\u6210\u4e3a\u4e86\u4e00\u4e2a\u8f83\u597d\u7684\u6570\u636e\u751f\u6210\u5de5\u5177\u3002

    "},{"location":"reader/datareader/#_2","title":"\u914d\u7f6e\u793a\u4f8b","text":"

    \u8fd9\u91cc\u6211\u628a\u76ee\u524d\u63d2\u4ef6\u652f\u6301\u7684\u89c4\u5219\u5168\u90e8\u5217\u4e3e\u5230\u4e0b\u9762\u7684\u4f8b\u5b50\u4e2d

    datareader2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"datareader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"1,100,\",\n              \"rule\": \"random\",\n              \"type\": \"double\"\n            },\n            {\n              \"value\": \"DataX\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"1\",\n              \"rule\": \"incr\",\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989/06/04 00:00:01,-1\",\n              \"rule\": \"incr\",\n              \"type\": \"date\",\n              \"dateFormat\": \"yyyy/MM/dd hh:mm:ss\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            },\n            {\n              \"rule\": \"address\"\n            },\n            {\n              \"rule\": \"bank\"\n            },\n            {\n              \"rule\": \"company\"\n            },\n            {\n              \"rule\": \"creditCard\"\n            },\n            {\n              \"rule\": \"debitCard\"\n            },\n            {\n              \"rule\": \"idCard\"\n            },\n            {\n              \"rule\": \"lat\"\n            },\n            {\n              \"rule\": \"lng\"\n            },\n            {\n              \"rule\": \"name\"\n            },\n            {\n              \"rule\": \"job\"\n            },\n            {\n              \"rule\": \"phone\"\n            },\n            {\n              \"rule\": \"stockCode\"\n            },\n            {\n              \"rule\": \"stockAccount\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true,\n          \"encoding\": \"UTF-8\"\n        }\n      }\n    }\n  }\n}\n

    \u4fdd\u5b58\u4e0a\u8ff0\u5185\u5bb9\u5230 job/datareader2stream.json

    \u7136\u540e\u6267\u884c\u8be5\u4efb\u52a1\uff0c\u5176\u8f93\u51fa\u7ed3\u679c\u7c7b\u4f3c\u5982\u4e0b\uff1a

    $ bin/addax.sh job/datareader2stream.json\n\n  ___      _     _\n / _ \\    | |   | |\n/ /_\\ \\ __| | __| | __ ___  __\n|  _  |/ _` |/ _` |/ _` \\ \\/ /\n| | | | (_| | (_| | (_| |>  <\n\\_| |_/\\__,_|\\__,_|\\__,_/_/\\_\\\n\n:: Addax version ::    (v4.0.2-SNAPSHOT)\n\n2021-08-13 17:02:00.888 [        main] INFO  VMInfo               - VMInfo# operatingSystem class => com.sun.management.internal.OperatingSystemImpl\n2021-08-13 17:02:00.910 [        main] INFO  Engine               -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"column\":[\n                        {\n                            \"rule\":\"random\",\n                            \"type\":\"double\",\n                            \"scale\": \"2\",\n                            \"value\":\"1,100,\"\n                        },\n                        {\n                            \"type\":\"string\",\n                            \"value\":\"DataX\"\n                        },\n                        {\n                            \"rule\":\"incr\",\n                            \"type\":\"long\",\n                            \"value\":\"1\"\n                        },\n                        {\n                            \"dateFormat\":\"yyyy/MM/dd hh:mm:ss\",\n                            \"rule\":\"incr\",\n                            \"type\":\"date\",\n                            \"value\":\"1989/06/04 00:00:01,-1\"\n                        },\n                        {\n                            \"type\":\"bytes\",\n                            \"value\":\"test\"\n                        },\n                        {\n                            \"rule\":\"address\"\n                        },\n                        {\n                            \"rule\":\"bank\"\n                        },\n                        {\n                            \"rule\":\"company\"\n                        },\n                        {\n                            \"rule\":\"creditCard\"\n                        },\n                        {\n                            \"rule\":\"debitCard\"\n                        },\n                        {\n                            \"rule\":\"idCard\"\n                        },\n                        {\n                            \"rule\":\"lat\"\n                        },\n                        {\n                            \"rule\":\"lng\"\n                        },\n                        {\n                            \"rule\":\"name\"\n                        },\n                        {\n                            \"rule\":\"job\"\n                        },\n                        {\n                            \"rule\":\"phone\"\n                        },\n                        {\n                            \"rule\":\"stockCode\"\n                        },\n                        {\n                            \"rule\":\"stockAccount\"\n                        }\n                    ],\n                    \"sliceRecordCount\":10\n                },\n                \"name\":\"datareader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":true,\n                    \"encoding\":\"UTF-8\"\n                },\n                \"name\":\"streamwriter\"\n            }\n    },\n    \"setting\":{\n        \"errorLimit\":{\n            \"record\":0,\n            \"percentage\":0.02\n        },\n        \"speed\":{\n            \"byte\":-1,\n            \"channel\":1\n        }\n    }\n}\n\n2021-08-13 17:02:00.937 [        main] INFO  PerfTrace            - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-08-13 17:02:00.938 [        main] INFO  JobContainer         - Addax jobContainer starts job.\n2021-08-13 17:02:00.940 [        main] INFO  JobContainer         - Set jobId = 0\n2021-08-13 17:02:00.976 [       job-0] INFO  JobContainer         - Addax Reader.Job [datareader] do prepare work .\n2021-08-13 17:02:00.977 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do prepare work .\n2021-08-13 17:02:00.978 [       job-0] INFO  JobContainer         - Job set Channel-Number to 1 channels.\n2021-08-13 17:02:00.979 [       job-0] INFO  JobContainer         - Addax Reader.Job [datareader] splits to [1] tasks.\n2021-08-13 17:02:00.980 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] splits to [1] tasks.\n2021-08-13 17:02:01.002 [       job-0] INFO  JobContainer         - Scheduler starts [1] taskGroups.\n2021-08-13 17:02:01.009 [ taskGroup-0] INFO  TaskGroupContainer   - taskGroupId=[0] start [1] channels for [1] tasks.\n2021-08-13 17:02:01.017 [ taskGroup-0] INFO  Channel              - Channel set byte_speed_limit to -1, No bps activated.\n2021-08-13 17:02:01.017 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\n\n7.65    DataX   1   1989-06-04 00:00:01 test    \u5929\u6d25\u5e02\u5357\u4eac\u53bf\u957f\u5bff\u533a\u5149\u660e\u8def263\u53f7    \u4ea4\u901a\u94f6\u884c    \u6613\u52a8\u529b\u4fe1\u606f\u6709\u9650\u516c\u53f8   6227894836568607    6235712610856305437 450304194808316766  31.3732613  -125.3507716    \u9f9a\u519b  \u673a\u7535\u5de5\u7a0b\u5e08   13438631667 726929  8741848665\n18.58   DataX   2   1989-06-03 00:00:01 test    \u6c5f\u82cf\u7701\u592a\u539f\u5e02\u6d54\u9633\u533a\u4e1c\u5c71\u8def33\u53f7 \u4e2d\u56fd\u94f6\u884c    \u65f6\u7a7a\u76d2\u6570\u5b57\u4fe1\u606f\u6709\u9650\u516c\u53f8 4096666711928233    6217419359154239015 220301200008188547  48.6648764  104.8567048 \u5321\u98de  \u5316\u5986\u5e08 18093137306 006845  1815787371\n16.16   DataX   3   1989-06-02 00:00:01 test    \u53f0\u6e7e\u7701\u90af\u90f8\u5e02\u6e05\u6cb3\u533a\u4e07\u987a\u8def10\u53f7 \u5927\u540c\u5546\u884c    \u5f00\u53d1\u533a\u4e16\u521b\u79d1\u6280\u6709\u9650\u516c\u53f8 4096713966912225    6212977716107080594 150223196408276322  29.0134395  142.6426842 \u652f\u6ce2  \u5ba1\u6838\u5458 13013458079 020695  3545552026\n63.89   DataX   4   1989-06-01 00:00:01 test    \u4e0a\u6d77\u5e02\u8f9b\u96c6\u53bf\u516d\u679d\u7279\u533a\u7518\u56ed\u8def119\u53f7   \u4e2d\u56fd\u519c\u4e1a\u94f6\u884c  \u6cf0\u9e92\u9e9f\u4f20\u5a92\u6709\u9650\u516c\u53f8   6227893481508780    6215686558778997167 220822196208286838  -71.6484635 111.8181273 \u656c\u5764  \u623f\u5730\u4ea7\u5ba2\u670d   13384928291 174445  0799668655\n79.18   DataX   5   1989-05-31 00:00:01 test    \u9655\u897f\u7701\u5357\u4eac\u5e02\u671d\u9633\u533a\u5927\u80dc\u8def170\u53f7    \u5185\u8499\u53e4\u94f6\u884c   \u6656\u6765\u8ba1\u7b97\u673a\u4fe1\u606f\u6709\u9650\u516c\u53f8 6227535683896707    6217255315590053833 350600198508222018  -24.9783587 78.017024   \u848b\u6768  \u56fa\u5b9a\u8d44\u4ea7\u4f1a\u8ba1  18766298716 402188  9633759917\n14.97   DataX   6   1989-05-30 00:00:01 test    \u6d77\u5357\u7701\u957f\u6625\u53bf\u74a7\u5c71\u533a\u78a7\u6d77\u8857147\u53f7    \u534e\u590f\u94f6\u884c    \u6d59\u5927\u4e07\u670b\u79d1\u6280\u6709\u9650\u516c\u53f8  6224797475369912    6215680436662199846 220122199608190275  -3.5088667  -40.2634359 \u8fb9\u6768  \u7763\u5bfc/\u5de1\u5e97   13278765923 092780  2408887582\n45.49   DataX   7   1989-05-29 00:00:01 test    \u53f0\u6e7e\u7701\u6f5c\u6c5f\u53bf\u6881\u5e73\u533a\u4e03\u661f\u8857201\u53f7    \u664b\u57ce\u5546\u884c    \u5f00\u53d1\u533a\u4e16\u521b\u4fe1\u606f\u6709\u9650\u516c\u53f8 5257468530819766    6213336008535546044 141082197908244004  -72.9200596 120.6018163 \u6851\u660e  \u7cfb\u7edf\u5de5\u7a0b\u5e08   13853379719 175864  8303448618\n8.45    DataX   8   1989-05-28 00:00:01 test    \u6d77\u5357\u7701\u676d\u5dde\u53bf\u57ce\u5317\u533a\u5929\u5174\u8def11\u53f7 \u5927\u540c\u5546\u884c    \u4e07\u8fc5\u7535\u8111\u79d1\u6280\u6709\u9650\u516c\u53f8  6227639043120062    6270259717880740332 430405198908214042  -16.5115338 -39.336119  \u8983\u5065  \u4eba\u4e8b\u603b\u76d1    13950216061 687461  0216734574\n15.01   DataX   9   1989-05-27 00:00:01 test    \u4e91\u5357\u7701\u60e0\u5dde\u5e02\u548c\u5e73\u533a\u6d77\u9e25\u8857201\u53f7    \u5185\u8499\u53e4\u94f6\u884c   \u9ec4\u77f3\u91d1\u627f\u4fe1\u606f\u6709\u9650\u516c\u53f8  6200358843233005    6235730928871528500 130300195008312067  -61.646097  163.0882369 \u536b\u5efa\u534e \u7535\u8bdd\u91c7\u7f16    15292600492 001658  1045093445\n55.14   DataX   10  1989-05-26 00:00:01 test    \u8fbd\u5b81\u7701\u5170\u5dde\u5e02\u5f90\u6c47\u533a\u4e1c\u5c71\u8857176\u53f7    \u5eca\u574a\u94f6\u884c    \u521b\u6c47\u79d1\u6280\u6709\u9650\u516c\u53f8    6227605280751588    6270262330691012025 341822200908168063  77.2165746  139.5431377 \u6c60\u6d69  \u591a\u5a92\u4f53\u8bbe\u8ba1   18693948216 201678  0692522928\n\n2021-08-13 17:02:04.020 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2021-08-13 17:02:04.021 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do post work.\n2021-08-13 17:02:04.022 [       job-0] INFO  JobContainer         - Addax Reader.Job [datareader] do post work.\n2021-08-13 17:02:04.025 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2021-08-13 17:02:04.028 [       job-0] INFO  StandAloneJobContainerCommunicator - Total 10 records, 1817 bytes | Speed 605B/s, 3 records/s | Error 0 records, 0 bytes |  All Task WaitWriterTime 0.000s |  All Task WaitReaderTime 0.000s | Percentage 100.00%\n2021-08-13 17:02:04.030 [       job-0] INFO  JobContainer         -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-08-13 17:02:00\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-08-13 17:02:04\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :              605B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :              3rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                  10\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"reader/datareader/#_3","title":"\u914d\u7f6e\u8bf4\u660e","text":"

    column \u7684\u914d\u7f6e\u548c\u5176\u4ed6\u63d2\u4ef6\u7684\u914d\u7f6e\u7a0d\u6709\u4e0d\u540c\uff0c\u4e00\u4e2a\u5b57\u6bb5\u7531\u4ee5\u4e0b\u914d\u7f6e\u9879\u7ec4\u6210

    \u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u9ed8\u8ba4\u503c \u793a\u4f8b \u8bf4\u660e value \u5426 \u65e0 Addax \u6570\u636e\u503c\uff0c\u5728\u67d0\u4e9b\u60c5\u51b5\u4e0b\u4e3a\u5fc5\u9009\u9879 rule \u5426 constantidCard \u6570\u636e\u751f\u4ea7\u89c4\u5219\uff0c\u8be6\u7ec6\u4e0b\u9762\u7684\u63cf\u8ff0 type \u5426 stringdouble \u6570\u636e\u503c\u7c7b\u578b dateFormat \u5426 yyyy-MM-dd HH:mm:ssyyyy/MM/dd HH:mm:ss \u65e5\u671f\u683c\u5f0f\uff0c\u4ec5\u5728 type \u4e3a date \u65f6\u6709\u6548"},{"location":"reader/datareader/#rule","title":"rule \u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u7684\u5b57\u6bb5\u914d\u7f6e\u6838\u5fc3\u662f rule \u5b57\u6bb5\uff0c\u5b83\u7528\u6765\u6307\u793a\u5e94\u8be5\u751f\u6210\u4ec0\u4e48\u6837\u7684\u6570\u636e\uff0c\u5e76\u4f9d\u636e\u4e0d\u540c\u89c4\u5219\uff0c\u914d\u5408\u5176\u4ed6\u914d\u7f6e\u9009\u9879\u6765\u751f\u4ea7\u6ee1\u8db3\u671f\u671b\u7684\u6570\u636e\u3002 \u5f53\u524d rule \u7684\u914d\u7f6e\u5747\u4e3a\u5185\u7f6e\u652f\u6301\u7684\u89c4\u5219\uff0c\u6682\u4e0d\u652f\u6301\u81ea\u5b9a\u4e49\uff0c\u4ee5\u4e0b\u8be6\u7ec6\u8bf4\u660e

    "},{"location":"reader/datareader/#constant","title":"constant","text":"

    constant \u662f rule \u7684\u9ed8\u8ba4\u914d\u7f6e\uff0c\u8be5\u89c4\u5219\u610f\u5473\u7740\u8981\u751f\u6210\u7684\u6570\u636e\u503c\u7531 value \u914d\u7f6e\u9879\u51b3\u5b9a\uff0c\u5176\u4e0d\u505a\u4efb\u4f55\u53d8\u66f4\u3002\u6bd4\u5982

    {\n  \"value\": \"Addax\",\n  \"type\": \"string\",\n  \"rule\": \"constant\"\n}\n

    \u8868\u793a\u8be5\u5b57\u6bb5\u751f\u4ea7\u7684\u6570\u636e\u503c\u5747\u4e3a Addax

    "},{"location":"reader/datareader/#incr","title":"incr","text":"

    incr \u914d\u7f6e\u9879\u7684\u542b\u4e49\u548c streamreader \u63d2\u4ef6\u4e2d\u7684 incr \u542b\u4e49\u4e00\u81f4\uff0c\u8868\u793a\u8fd9\u662f\u4e00\u4e2a\u9012\u589e\u7684\u6570\u636e\u751f\u4ea7\u89c4\u5219\uff0c\u6bd4\u5982

    {\n  \"value\": \"1,2\",\n  \"rule\": \"incr\",\n  \"type\": \"long\"\n}\n

    \u8868\u793a\u8be5\u5b57\u6bb5\u7684\u6570\u636e\u662f\u4e00\u4e2a\u957f\u6574\u5f62\uff0c\u6570\u503c\u4ece 1 \u5f00\u59cb\uff0c\u6bcf\u6b21\u9012\u589e 2\uff0c\u4e5f\u5c31\u662f\u5f62\u6210 1 \u5f00\u59cb\uff0c\u6b65\u957f\u4e3a 2 \u7684\u9012\u589e\u6570\u5217\u3002

    \u8be5\u5b57\u6bb5\u66f4\u8be6\u7ec6\u7684\u914d\u7f6e\u89c4\u5219\u548c\u6ce8\u610f\u4e8b\u9879\uff0c\u53ef\u4ee5\u53c2\u8003 streamreader \u4e2d\u7684 incr \u8bf4\u660e\u3002

    "},{"location":"reader/datareader/#random","title":"random","text":"

    random \u914d\u7f6e\u9879\u7684\u542b\u4e49\u548c streamreader \u63d2\u4ef6\u4e2d\u7684 random \u542b\u4e49\u4e00\u81f4\uff0c\u8868\u793a\u8fd9\u662f\u4e00\u4e2a\u9012\u589e\u7684\u6570\u636e\u751f\u4ea7\u89c4\u5219\uff0c\u6bd4\u5982

    {\n  \"value\": \"1,10\",\n  \"rule\": \"random\",\n  \"type\": \"string\"\n}\n

    \u8868\u793a\u8be5\u5b57\u6bb5\u7684\u6570\u636e\u662f\u4e00\u4e2a\u957f\u5ea6\u4e3a 1 \u5230 10 \uff081 \u548c 10 \u90fd\u5305\u62ec\uff09\u968f\u673a\u5b57\u7b26\u4e32\u3002

    \u8be5\u5b57\u6bb5\u66f4\u8be6\u7ec6\u7684\u914d\u7f6e\u89c4\u5219\u548c\u6ce8\u610f\u4e8b\u9879\uff0c\u53ef\u4ee5\u53c2\u8003 streamreader \u4e2d\u7684 random \u8bf4\u660e\u3002

    \u89c4\u5219\u540d\u79f0 \u542b\u4e49 \u793a\u4f8b \u6570\u636e\u7c7b\u578b \u8bf4\u660e address \u968f\u673a\u751f\u6210\u4e00\u6761\u57fa\u672c\u6ee1\u8db3\u56fd\u5185\u5b9e\u9645\u60c5\u51b5\u7684\u5730\u5740\u4fe1\u606f \u8fbd\u5b81\u7701\u5170\u5dde\u5e02\u5f90\u6c47\u533a\u4e1c\u5c71\u8857176\u53f7 string bank \u968f\u673a\u751f\u6210\u4e00\u4e2a\u56fd\u5185\u94f6\u884c\u540d\u79f0 \u534e\u590f\u94f6\u884c string company \u968f\u673a\u751f\u6210\u4e00\u4e2a\u516c\u53f8\u7684\u540d\u79f0 \u4e07\u8fc5\u7535\u8111\u79d1\u6280\u6709\u9650\u516c\u53f8 string creditCard \u968f\u673a\u751f\u6210\u4e00\u4e2a\u4fe1\u7528\u5361\u5361\u53f7 430405198908214042 string 16 \u4f4d debitCard \u968f\u673a\u751f\u6210\u4e00\u4e2a\u50a8\u84c4\u5361\u5361\u53f7 6227894836568607 string 19 \u4f4d email \u968f\u673a\u751f\u6210\u4e00\u4e2a\u7535\u5b50\u90ae\u4ef6\u5730\u5740 ok2a@gmail.com string idCard \u968f\u673a\u751f\u6210\u4e00\u4e2a\u56fd\u5185\u8eab\u4efd\u8bc1\u53f7\u7801 350600198508222018 string 18 \u4f4d\uff0c\u8d1f\u8d23\u6821\u9a8c\u89c4\u5219\uff0c\u5934 6 \u4f4d\u7f16\u7801\u6ee1\u8db3\u884c\u653f\u533a\u5212\u8981\u6c42 lat \u968f\u673a\u751f\u6210\u7ef4\u5ea6\u6570\u636e 48.6648764 double \u56fa\u5b9a 7 \u4f4d\u5c0f\u6570 \uff0c\u4e5f\u53ef\u4ee5\u7528latitude \u8868\u793a lng \u968f\u673a\u751f\u6210\u7ecf\u5ea6\u6570\u636e 120.6018163 double \u56fa\u5b9a 7 \u4f4d\u5c0f\u6570\uff0c\u4e5f\u53ef\u4ee5\u4f7f\u7528longitude \u8868\u793a name \u968f\u673a\u751f\u6210\u4e00\u4e2a\u56fd\u5185\u540d\u5b57 \u6c60\u6d69 string \u6682\u6ca1\u8003\u8651\u59d3\u6c0f\u5728\u56fd\u5185\u7684\u5360\u6bd4\u5ea6 job \u968f\u673a\u751f\u6210\u4e00\u4e2a\u56fd\u5185\u5c97\u4f4d\u540d\u79f0 \u7cfb\u7edf\u5de5\u7a0b\u5e08 string \u6570\u636e\u6765\u6e90\u4e8e\u62db\u8058\u7f51\u7ad9 phone \u968f\u673a\u751f\u6210\u4e00\u4e2a\u56fd\u5185\u624b\u673a\u53f7\u7801 15292600492 string \u6682\u4e0d\u8003\u8651\u865a\u62df\u624b\u673a\u53f7 stockCode \u968f\u673a\u751f\u6210\u4e00\u4e2a 6 \u4f4d\u7684\u80a1\u7968\u4ee3\u7801 687461 string \u524d\u4e24\u4f4d\u6ee1\u8db3\u56fd\u5185\u80a1\u7968\u4ee3\u7801\u7f16\u53f7\u89c4\u8303 stockAccount \u968f\u673a\u751f\u6210\u4e00\u4e2a 10 \u4f4d\u7684\u80a1\u7968\u4ea4\u6613\u8d26\u6237 0692522928 string \u5b8c\u5168\u968f\u673a\uff0c\u4e0d\u6ee1\u8db3\u8d26\u6237\u89c4\u8303 uuid \u968f\u673a\u751f\u6210\u4e00\u4e2a UUID \u5b57\u7b26\u4e32 bc1cf125-929b-43b7-b324-d7c4cc5a75d2 string \u5b8c\u5168\u968f\u673a\uff0c\u4e0d\u6ee1\u8db3\u8d26\u6237\u89c4\u8303 zipCode \u968f\u673a\u751f\u4ea7\u4e00\u4e2a\u56fd\u5185\u90ae\u653f\u7f16\u53f7 411105 long \u4e0d\u5b8c\u5168\u6ee1\u8db3\u56fd\u5185\u90ae\u653f\u7f16\u53f7\u89c4\u8303

    \u6ce8\u610f\uff1a\u4e0a\u8ff0\u8868\u683c\u4e2d\u7684\u89c4\u5219\u8fd4\u56de\u7684\u6570\u636e\u7c7b\u578b\u662f\u56fa\u5b9a\u7684\uff0c\u4e14\u4e0d\u652f\u6301\u4fee\u6539\uff0c\u56e0\u6b64 type \u65e0\u9700\u914d\u7f6e\uff0c\u914d\u7f6e\u7684\u7c7b\u578b\u4e5f\u4f1a\u88ab\u5ffd\u7565\uff0c\u56e0\u4e3a\u6570\u636e\u751f\u6210\u6765\u81ea\u5185\u90e8\u89c4\u5219\uff0c\u6240\u4ee5 value \u4e5f\u65e0\u9700\u914d\u7f6e\uff0c\u914d\u7f6e\u7684\u5185\u5bb9\u4e5f\u4f1a\u88ab\u5ffd\u7565\u3002

    "},{"location":"reader/dbfreader/","title":"Dbf Reader","text":"

    DbfReader \u63d2\u4ef6\u652f\u6301\u8bfb\u53d6 DBF \u683c\u5f0f\u6587\u4ef6

    "},{"location":"reader/dbfreader/#_1","title":"\u914d\u7f6e\u8bf4\u660e","text":"

    \u4ee5\u4e0b\u662f\u8bfb\u53d6 DBF \u6587\u4ef6\u540e\u6253\u5370\u5230\u7ec8\u7aef\u7684\u914d\u7f6e\u6837\u4f8b

    jobs/dbf2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"dbfreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"long\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 3,\n              \"type\": \"boolean\"\n            },\n            {\n              \"index\": 4,\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"dbf\",\n              \"type\": \"string\"\n            }\n          ],\n          \"path\": [\n            \"/tmp/out\"\n          ],\n          \"encoding\": \"GBK\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": \"true\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/dbfreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    parameter \u914d\u7f6e\u9879\u652f\u6301\u4ee5\u4e0b\u914d\u7f6e

    \u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f \u65e0 DBF \u6587\u4ef6\u8def\u5f84\uff0c\u652f\u6301\u5199\u591a\u4e2a\u8def\u5f84\uff0c\u8be6\u7ec6\u60c5\u51b5\u89c1\u4e0b column \u662f \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u96c6\u5408, \u662f {type: value} \u6216 {type: index} \u7684\u96c6\u5408\uff0c\u8be6\u7ec6\u914d\u7f6e\u89c1\u4e0b encoding \u5426 GBK DBF \u6587\u4ef6\u7f16\u7801\uff0c\u6bd4\u5982 GBK, UTF-8 nullFormat \u5426 \\N \u5b9a\u4e49\u54ea\u4e2a\u5b57\u7b26\u4e32\u53ef\u4ee5\u8868\u793a\u4e3a null,"},{"location":"reader/dbfreader/#path","title":"path","text":"

    \u63cf\u8ff0\uff1a\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84\u3002

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0cAddax \u4f1a\u5c06\u4e00\u4e2a\u4f5c\u4e1a\u4e0b\u540c\u6b65\u7684\u6240\u6709 dbf File \u89c6\u4f5c\u540c\u4e00\u5f20\u6570\u636e\u8868\u3002\u7528\u6237\u5fc5\u987b\u81ea\u5df1\u4fdd\u8bc1\u6240\u6709\u7684 File \u80fd\u591f\u9002\u914d\u540c\u4e00\u5957 schema \u4fe1\u606f\u3002\u8bfb\u53d6\u6587\u4ef6\u7528\u6237\u5fc5\u987b\u4fdd\u8bc1\u4e3a\u7c7b dbf \u683c\u5f0f\uff0c\u5e76\u4e14\u63d0\u4f9b\u7ed9 Addax \u6743\u9650\u53ef\u8bfb\u3002

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0c\u5982\u679c Path \u6307\u5b9a\u7684\u8def\u5f84\u4e0b\u6ca1\u6709\u7b26\u5408\u5339\u914d\u7684\u6587\u4ef6\u62bd\u53d6\uff0cAddax \u5c06\u62a5\u9519\u3002

    "},{"location":"reader/dbfreader/#column","title":"column","text":"

    \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cname \u4e3a\u5b57\u6bb5\u540d,\u957f\u5ea6\u6700\u5927 8\uff0cvalue \u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece\u6e90\u5934\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636e value \u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u7528\u6237\u53ef\u4ee5\u5168\u90e8\u6309\u7167 String \u7c7b\u578b\u8bfb\u53d6\u6570\u636e\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"column\": [\"*\"]\n}\n

    \u7528\u6237\u53ef\u4ee5\u6307\u5b9a Column \u5b57\u6bb5\u4fe1\u606f\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    [\n  {\n    \"type\": \"long\",\n    \"index\": 0\n  },\n  {\n    \"type\": \"string\",\n    \"value\": \"addax\"\n  }\n]\n
    "},{"location":"reader/dbfreader/#_3","title":"\u652f\u6301\u7684\u6570\u636e\u7c7b\u578b","text":"

    \u672c\u5730\u6587\u4ef6\u672c\u8eab\u63d0\u4f9b\u6570\u636e\u7c7b\u578b\uff0c\u8be5\u7c7b\u578b\u662f Addax dbfFileReader \u5b9a\u4e49\uff1a

    Addax \u5185\u90e8\u7c7b\u578b \u672c\u5730\u6587\u4ef6 \u6570\u636e\u7c7b\u578b Long Long Double Double String String Boolean Boolean Date Date

    \u5176\u4e2d\uff1a

    "},{"location":"reader/elasticsearchreader/","title":"ElasticSearchReader","text":"

    ElasticSearchReader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Elasticsearch \u8bfb\u53d6\u7d22\u5f15\u7684\u529f\u80fd\uff0c \u5b83\u901a\u8fc7 Elasticsearch \u63d0\u4f9b\u7684 Rest API \uff08\u9ed8\u8ba4\u7aef\u53e39200\uff09\uff0c\u6267\u884c\u6307\u5b9a\u7684\u67e5\u8be2\u8bed\u53e5\u6279\u91cf\u83b7\u53d6\u6570\u636e

    "},{"location":"reader/elasticsearchreader/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u83b7\u53d6\u7684\u7d22\u5f15\u5185\u5bb9\u5982\u4e0b

    {\n  \"took\": 14,\n  \"timed_out\": false,\n  \"_shards\": {\n    \"total\": 1,\n    \"successful\": 1,\n    \"skipped\": 0,\n    \"failed\": 0\n  },\n  \"hits\": {\n    \"total\": 2,\n    \"max_score\": 1,\n    \"hits\": [\n      {\n        \"_index\": \"test-1\",\n        \"_type\": \"default\",\n        \"_id\": \"38\",\n        \"_score\": 1,\n        \"_source\": {\n          \"col_date\": \"2017-05-25T11:22:33.000+08:00\",\n          \"col_integer\": 19890604,\n          \"col_keyword\": \"hello world\",\n          \"col_ip\": \"1.1.1.1\",\n          \"col_text\": \"long text\",\n          \"col_double\": 19890604,\n          \"col_long\": 19890604,\n          \"col_geo_point\": \"41.12,-71.34\"\n        }\n      },\n      {\n        \"_index\": \"test-1\",\n        \"_type\": \"default\",\n        \"_id\": \"103\",\n        \"_score\": 1,\n        \"_source\": {\n          \"col_date\": \"2017-05-25T11:22:33.000+08:00\",\n          \"col_integer\": 19890604,\n          \"col_keyword\": \"hello world\",\n          \"col_ip\": \"1.1.1.1\",\n          \"col_text\": \"long text\",\n          \"col_double\": 19890604,\n          \"col_long\": 19890604,\n          \"col_geo_point\": \"41.12,-71.34\"\n        }\n      }\n    ]\n  }\n}\n

    \u914d\u7f6e\u4e00\u4e2a\u4ece Elasticsearch \u8bfb\u53d6\u6570\u636e\u5e76\u6253\u5370\u5230\u7ec8\u7aef\u7684\u4efb\u52a1

    job/es2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"elasticsearchreader\",\n        \"parameter\": {\n          \"endpoint\": \"http://127.0.0.1:9200\",\n          \"accessId\": \"\",\n          \"accesskey\": \"\",\n          \"index\": \"test-1\",\n          \"type\": \"default\",\n          \"searchType\": \"dfs_query_then_fetch\",\n          \"headers\": {},\n          \"scroll\": \"3m\",\n          \"search\": [\n            {\n              \"query\": {\n                \"match\": {\n                  \"col_ip\": \"1.1.1.1\"\n                }\n              },\n              \"aggregations\": {\n                \"top_10_states\": {\n                  \"terms\": {\n                    \"field\": \"col_date\",\n                    \"size\": 10\n                  }\n                }\n              }\n            }\n          ],\n          \"column\": [\n            \"col_ip\",\n            \"col_double\",\n            \"col_long\",\n            \"col_integer\",\n            \"col_keyword\",\n            \"col_text\",\n            \"col_geo_point\",\n            \"col_date\"\n          ]\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true,\n          \"encoding\": \"UTF-8\"\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u5185\u5bb9\u4fdd\u5b58\u4e3a job/es2stream.json

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91c7\u96c6

    bin/addax.sh job/es2stream.json\n

    \u5176\u8f93\u51fa\u7ed3\u679c\u7c7b\u4f3c\u5982\u4e0b\uff08\u8f93\u51fa\u8bb0\u5f55\u6570\u6709\u5220\u51cf)

    2021-02-19 13:38:15.860 [main] INFO  VMInfo - VMInfo# operatingSystem class => com.sun.management.internal.OperatingSystemImpl\n2021-02-19 13:38:15.895 [main] INFO  Engine -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"accessId\":\"\",\n                    \"headers\":{},\n                    \"endpoint\":\"http://127.0.0.1:9200\",\n                    \"search\":[\n                      {\n                        \"query\": {\n                          \"match\": {\n                            \"col_ip\": \"1.1.1.1\"\n                          }\n                        },\n                        \"aggregations\": {\n                          \"top_10_states\": {\n                            \"terms\": {\n                              \"field\": \"col_date\",\n                              \"size\": 10\n                            }\n                          }\n                        }\n                      }\n                    ],\n                    \"accesskey\":\"*****\",\n                    \"searchType\":\"dfs_query_then_fetch\",\n                    \"scroll\":\"3m\",\n                    \"column\":[\n                        \"col_ip\",\n                        \"col_double\",\n                        \"col_long\",\n                        \"col_integer\",\n                        \"col_keyword\",\n                        \"col_text\",\n                        \"col_geo_point\",\n                        \"col_date\"\n                    ],\n                    \"index\":\"test-1\",\n                    \"type\":\"default\"\n                },\n                \"name\":\"elasticsearchreader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":true,\n                    \"encoding\":\"UTF-8\"\n                },\n                \"name\":\"streamwriter\"\n            }\n        },\n    \"setting\":{\n        \"errorLimit\":{\n            \"record\":0,\n            \"percentage\":0.02\n        },\n        \"speed\":{\n            \"byte\":-1,\n            \"channel\":1\n        }\n    }\n}\n\n2021-02-19 13:38:15.934 [main] INFO  PerfTrace - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-02-19 13:38:15.934 [main] INFO  JobContainer - Addax jobContainer starts job.\n2021-02-19 13:38:15.937 [main] INFO  JobContainer - Set jobId = 0\n\n2017-05-25T11:22:33.000+08:00   19890604    hello world 1.1.1.1 long text   19890604    19890604    41.12,-71.34\n2017-05-25T11:22:33.000+08:00   19890604    hello world 1.1.1.1 long text   19890604    19890604    41.12,-71.34\n\n2021-02-19 13:38:19.845 [job-0] INFO  AbstractScheduler - Scheduler accomplished all tasks.\n2021-02-19 13:38:19.848 [job-0] INFO  JobContainer - Addax Writer.Job [streamwriter] do post work.\n2021-02-19 13:38:19.849 [job-0] INFO  JobContainer - Addax Reader.Job [elasticsearchreader] do post work.\n2021-02-19 13:38:19.855 [job-0] INFO  JobContainer - PerfTrace not enable!\n2021-02-19 13:38:19.858 [job-0] INFO  StandAloneJobContainerCommunicator - Total 95 records, 8740 bytes | Speed 2.84KB/s, 31 records/s | Error 0 records, 0 bytes |  All Task WaitWriterTime 0.000s |  All Task WaitReaderTime 0.103s | Percentage 100.00%\n2021-02-19 13:38:19.861 [job-0] INFO  JobContainer -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-02-19 13:38:15\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-02-19 13:38:19\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :            2.84KB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :             31rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                   2\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"reader/elasticsearchreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 ElasticSearch\u7684\u8fde\u63a5\u5730\u5740 accessId \u5426 string \"\" http auth\u4e2d\u7684user accessKey \u5426 string \"\" http auth\u4e2d\u7684password index \u662f string \u65e0 elasticsearch\u4e2d\u7684index\u540d type \u5426 string index\u540d elasticsearch\u4e2dindex\u7684type\u540d search \u662f list [] json\u683c\u5f0fapi\u641c\u7d22\u6570\u636e\u4f53 column \u662f list \u65e0 \u9700\u8981\u8bfb\u53d6\u7684\u5b57\u6bb5 timeout \u5426 int 60 \u5ba2\u6237\u7aef\u8d85\u65f6\u65f6\u95f4(\u5355\u4f4d\uff1a\u79d2) discovery \u5426 boolean false \u542f\u7528\u8282\u70b9\u53d1\u73b0(\u8f6e\u8be2)\u5e76\u5b9a\u671f\u66f4\u65b0\u5ba2\u6237\u673a\u4e2d\u7684\u670d\u52a1\u5668\u5217\u8868 compression \u5426 boolean true http\u8bf7\u6c42\uff0c\u5f00\u542f\u538b\u7f29 multiThread \u5426 boolean true http\u8bf7\u6c42\uff0c\u662f\u5426\u6709\u591a\u7ebf\u7a0b searchType \u5426 string dfs_query_then_fetch \u641c\u7d22\u7c7b\u578b headers \u5426 map {} http\u8bf7\u6c42\u5934 scroll \u5426 string \"\" \u6eda\u52a8\u5206\u9875\u914d\u7f6e"},{"location":"reader/elasticsearchreader/#search","title":"search","text":"

    search \u914d\u7f6e\u9879\u5141\u8bb8\u914d\u7f6e\u4e3a\u6ee1\u8db3 Elasticsearch API \u67e5\u8be2\u8981\u6c42\u7684\u5185\u5bb9\uff0c\u6bd4\u5982\u8fd9\u6837\uff1a

    {\n  \"query\": {\n    \"match\": {\n      \"message\": \"myProduct\"\n    }\n  },\n  \"aggregations\": {\n    \"top_10_states\": {\n      \"terms\": {\n        \"field\": \"state\",\n        \"size\": 10\n      }\n    }\n  }\n}\n
    "},{"location":"reader/elasticsearchreader/#searchtype","title":"searchType","text":"

    searchType \u76ee\u524d\u652f\u6301\u4ee5\u4e0b\u51e0\u79cd\uff1a

    "},{"location":"reader/excelreader/","title":"Excel Reader","text":"

    Excel Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Microsoft Excel \u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u7684\u80fd\u529b\u3002

    "},{"location":"reader/excelreader/#_1","title":"\u914d\u7f6e","text":""},{"location":"reader/excelreader/#_2","title":"\u83b7\u53d6\u6837\u4f8b\u6587\u4ef6","text":"

    \u4ece\u8fd9\u91cc\u4e0b\u8f7d\u7528\u4e8e\u6f14\u793a\u7684 Excel \u538b\u7f29\u6587\u4ef6\uff0c\u5e76\u89e3\u538b\u7f29\u653e\u7f6e\u5230 /tmp/in \u76ee\u5f55\u4e0b\u3002 \u4e09\u4e2a\u6587\u4ef6\u5939\u7684\u5185\u5bb9\u76f8\u540c\uff0c\u5176\u4e2d

    \u6587\u4ef6\u5185\u5bb9\uff0c\u5982\u4e0b\u8868\u6240\u793a\uff1a

    \u7f16\u53f7 \u6574\u6570\u7c7b\u578b \u6d6e\u70b9\u6570\u7c7b\u578b \u5b57\u7b26\u4e32\u7c7b\u578b \u65e5\u671f\u7c7b\u578b \u516c\u5f0f\u8ba1\u7b97 \u5355\u5143\u683c\u5f0f\u5316 1 11 1102.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/10 5544.17 \u00a51,102.23 2 12 1103.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/11 5552.17 \u00a51,103.23 3 13 1104.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/12 5560.17 \u00a51,104.23 4 14 1105.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/13 5568.17 \u00a51,105.23 5 15 1106.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/14 5576.17 \u00a51,106.23 6 16 1107.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/15 5584.17 \u00a51,107.23 7 17 1108.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/16 5592.17 \u00a51,108.23 8 18 1109.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/17 5600.17 \u00a51,109.23 9 19 1110.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/18 5608.17 \u00a51,110.23 10 20 1111.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/19 5616.17 \u00a51,111.23 11 21 1112.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/20 5624.17 \u00a51,112.23 12 22 1113.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/21 5632.17 \u00a51,113.23 13 23 1114.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/22 5640.17 \u00a51,114.23 14 24 1115.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/23 5648.17 \u00a51,115.23 15 25 1116.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/24 5656.17 \u00a51,116.23 16 26 1117.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/25 5664.17 \u00a51,117.23 17 27 1118.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/26 5672.17 \u00a51,118.23 18 28 1119.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/27 5680.17 \u00a51,119.23 19 29 1120.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/28 5688.17 \u00a51,120.23 20 30 1121.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/29 5696.17 \u00a51,121.23

    \u8868\u5934\u5927\u81f4\u8bf4\u660e\u4e86\u5355\u5143\u6570\u636e\u7684\u7279\u5f81

    "},{"location":"reader/excelreader/#job","title":"\u521b\u5efa job \u6587\u4ef6","text":"

    \u521b\u5efa\u5982\u4e0b json \u6587\u4ef6

    excel2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"excelreader\",\n        \"parameter\": {\n          \"path\": [\n            \"/tmp/in\"\n          ],\n          \"header\": true,\n          \"skipRows\": 0\n        }\n      },\n      \"writer\": {\n        \"parameter\": {\n          \"print\": true\n        },\n        \"name\": \"streamwriter\"\n      }\n    }\n  }\n}\n

    \u5c06\u8f93\u51fa\u5185\u5bb9\u5b58\u4fdd\u4e3a\u5230 job/excel2stream.json \u6587\u4ef6\u4e2d\uff0c\u6267\u884c\u91c7\u96c6\u547d\u4ee4\uff1a

    $ bin/addax.sh job/excel2stream.json\n

    \u5982\u679c\u6ca1\u6709\u62a5\u9519\uff0c\u5e94\u8be5\u5f97\u5230\u5982\u4e0b\u8f93\u51fa\uff1a

    \u70b9\u51fb\u5c55\u5f00 
     ___      _     _\n / _ \\    | |   | |\n/ /_\\ \\ __| | __| | __ ___  __\n|  _  |/ _` |/ _` |/ _` \\ \\/ /\n| | | | (_| | (_| | (_| |>  <\n\\_| |_/\\__,_|\\__,_|\\__,_/_/\\_\\\n\n:: Addax version ::    (v4.0.3)\n\n2021-09-09 14:43:42.579 [        main] INFO  VMInfo               - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl\n2021-09-09 14:43:42.621 [        main] INFO  Engine               -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"path\":[\n                        \"/tmp/in\"\n                    ],\n                    \"column\":[\n                        {\n                            \"name\":\"no\",\n                            \"type\":\"long\"\n                        },\n                        {\n                            \"name\":\"birth\",\n                            \"format\":\"yyyy-MM-dd HH:mm:ss\",\n                            \"type\":\"date\"\n                        },\n                        {\n                            \"name\":\"kk\",\n                            \"type\":\"string\"\n                        }\n                    ],\n                    \"header\":true,\n                    \"skipHeader\":true,\n                    \"encoding\":\"UTF-8\",\n                    \"fieldDelimiter\":\",\"\n                },\n                \"name\":\"excelreader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":true\n                },\n                \"name\":\"streamwriter\"\n            }\n        },\n    \"setting\":{\n        \"speed\":{\n            \"bytes\":-1,\n            \"channel\":2\n        }\n    }\n}\n\n2021-09-09 14:43:42.653 [        main] INFO  PerfTrace            - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-09-09 14:43:42.653 [        main] INFO  JobContainer         - Addax jobContainer starts job.\n2021-09-09 14:43:42.655 [        main] INFO  JobContainer         - Set jobId = 0\n2021-09-09 14:43:42.669 [       job-0] INFO  ExcelReader$Job      - add file [/tmp/in/demo_old.xls] as a candidate to be read.\n2021-09-09 14:43:42.669 [       job-0] INFO  ExcelReader$Job      - add file [/tmp/in/demo_gbk.xlsx] as a candidate to be read.\n2021-09-09 14:43:42.670 [       job-0] INFO  ExcelReader$Job      - add file [/tmp/in/demo.xlsx] as a candidate to be read.\n2021-09-09 14:43:42.670 [       job-0] INFO  ExcelReader$Job      - The number of files to read is: [3]\n2021-09-09 14:43:42.677 [       job-0] INFO  JobContainer         - Addax Reader.Job [excelreader] do prepare work .\n2021-09-09 14:43:42.678 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do prepare work .\n2021-09-09 14:43:42.679 [       job-0] INFO  JobContainer         - Job set Channel-Number to 2 channels.\n2021-09-09 14:43:42.681 [       job-0] INFO  JobContainer         - Addax Reader.Job [excelreader] splits to [3] tasks.\n2021-09-09 14:43:42.682 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] splits to [3] tasks.\n2021-09-09 14:43:42.727 [       job-0] INFO  JobContainer         - Scheduler starts [1] taskGroups.\n2021-09-09 14:43:42.736 [ taskGroup-0] INFO  TaskGroupContainer   - taskGroupId=[0] start [2] channels for [3] tasks.\n2021-09-09 14:43:42.741 [ taskGroup-0] INFO  Channel              - Channel set byte_speed_limit to -1, No bps activated.\n2021-09-09 14:43:42.742 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\n2021-09-09 14:43:42.755 [0-0-1-reader] INFO  ExcelReader$Task     - The first row is skipped as a table header\n2021-09-09 14:43:42.755 [0-0-1-reader] INFO  ExcelReader$Task     - begin read file /tmp/in/demo.xlsx\n2021-09-09 14:43:42.757 [0-0-0-reader] INFO  ExcelReader$Task     - The first row is skipped as a table header\n2021-09-09 14:43:42.758 [0-0-0-reader] INFO  ExcelReader$Task     - begin read file /tmp/in/demo_gbk.xlsx\n1   11  1102.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-10 00:00:00 5544.17 1102.234\n1   12  1103.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-11 00:00:00 5552.17 1103.234\n1   13  1104.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-12 00:00:00 5560.17 1104.234\n1   14  1105.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-13 00:00:00 5568.17 1105.234\n1   15  1106.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-14 00:00:00 5576.17 1106.234\n1   16  1107.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-15 00:00:00 5584.17 1107.234\n1   17  1108.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-16 00:00:00 5592.17 1108.234\n1   18  1109.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-17 00:00:00 5600.17 1109.234\n1   19  1110.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-18 00:00:00 5608.17 1110.234\n1   20  1111.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-19 00:00:00 5616.17 1111.234\n1   21  1112.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-20 00:00:00 5624.17 1112.234\n1   22  1113.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-21 00:00:00 5632.17 1113.234\n1   23  1114.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-22 00:00:00 5640.17 1114.234\n1   24  1115.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-23 00:00:00 5648.17 1115.234\n1   25  1116.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-24 00:00:00 5656.17 1116.234\n1   26  1117.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-25 00:00:00 5664.17 1117.234\n1   27  1118.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-26 00:00:00 5672.17 1118.234\n1   28  1119.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-27 00:00:00 5680.17 1119.234\n1   29  1120.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-28 00:00:00 5688.17 1120.234\n1   30  1121.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-29 00:00:00 5696.17 1121.234\n1   11  1102.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-10 00:00:00 5544.17 1102.234\n2   12  1103.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-11 00:00:00 5552.17 1103.234\n3   13  1104.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-12 00:00:00 5560.17 1104.234\n4   14  1105.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-13 00:00:00 5568.17 1105.234\n5   15  1106.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-14 00:00:00 5576.17 1106.234\n6   16  1107.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-15 00:00:00 5584.17 1107.234\n7   17  1108.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-16 00:00:00 5592.17 1108.234\n8   18  1109.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-17 00:00:00 5600.17 1109.234\n9   19  1110.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-18 00:00:00 5608.17 1110.234\n10  20  1111.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-19 00:00:00 5616.17 1111.234\n11  21  1112.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-20 00:00:00 5624.17 1112.234\n12  22  1113.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-21 00:00:00 5632.17 1113.234\n13  23  1114.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-22 00:00:00 5640.17 1114.234\n14  24  1115.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-23 00:00:00 5648.17 1115.234\n15  25  1116.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-24 00:00:00 5656.17 1116.234\n16  26  1117.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-25 00:00:00 5664.17 1117.234\n17  27  1118.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-26 00:00:00 5672.17 1118.234\n18  28  1119.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-27 00:00:00 5680.17 1119.234\n19  29  1120.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-28 00:00:00 5688.17 1120.234\n20  30  1121.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-29 00:00:00 5696.17 1121.234\n2021-09-09 14:43:43.894 [0-0-2-reader] INFO  ExcelReader$Task     - The first row is skipped as a table header\n2021-09-09 14:43:43.894 [0-0-2-reader] INFO  ExcelReader$Task     - begin read file /tmp/in/demo_old.xls\n1   11  1102.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-10 00:00:00 5544.17 1102.234\n2   12  1103.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-11 00:00:00 5552.17 1103.234\n3   13  1104.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-12 00:00:00 5560.17 1104.234\n4   14  1105.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-13 00:00:00 5568.17 1105.234\n5   15  1106.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-14 00:00:00 5576.17 1106.234\n6   16  1107.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-15 00:00:00 5584.17 1107.234\n7   17  1108.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-16 00:00:00 5592.17 1108.234\n8   18  1109.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-17 00:00:00 5600.17 1109.234\n9   19  1110.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-18 00:00:00 5608.17 1110.234\n10  20  1111.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-19 00:00:00 5616.17 1111.234\n11  21  1112.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-20 00:00:00 5624.17 1112.234\n12  22  1113.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-21 00:00:00 5632.17 1113.234\n13  23  1114.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-22 00:00:00 5640.17 1114.234\n14  24  1115.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-23 00:00:00 5648.17 1115.234\n15  25  1116.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-24 00:00:00 5656.17 1116.234\n16  26  1117.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-25 00:00:00 5664.17 1117.234\n17  27  1118.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-26 00:00:00 5672.17 1118.234\n18  28  1119.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-27 00:00:00 5680.17 1119.234\n19  29  1120.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-28 00:00:00 5688.17 1120.234\n20  30  1121.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-29 00:00:00 5696.17 1121.234\n2021-09-09 14:43:45.753 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2021-09-09 14:43:45.754 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do post work.\n2021-09-09 14:43:45.756 [       job-0] INFO  JobContainer         - Addax Reader.Job [excelreader] do post work.\n2021-09-09 14:43:45.761 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2021-09-09 14:43:45.762 [       job-0] INFO  StandAloneJobContainerCommunicator - Total 60 records, 3360 bytes | Speed 1.09KB/s, 20 records/s | Error 0 records, 0 bytes |  All Task WaitWriterTime 0.000s |  All Task WaitReaderTime 0.993s | Percentage 100.00%\n2021-09-09 14:43:45.764 [       job-0] INFO  JobContainer         -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-09-09 14:43:42\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-09-09 14:43:45\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :            1.09KB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :             20rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                  60\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"reader/excelreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f string/list \u65e0 \u6307\u5b9a\u8981\u8bfb\u53d6\u7684\u6587\u4ef6\u5939\uff0c\u53ef\u4ee5\u6307\u5b9a\u591a\u4e2a header \u5426 boolean false \u6587\u4ef6\u662f\u5426\u5305\u542b\u5934 skipRows \u5426 int 0 \u8981\u8df3\u8fc7\u524d\u591a\u5c11\u884c"},{"location":"reader/excelreader/#header","title":"header","text":"

    Excel \u6587\u4ef6\u662f\u5426\u5305\u542b\u5934\uff0c\u5982\u679c\u5305\u542b\uff0c\u5219\u8df3\u8fc7

    "},{"location":"reader/excelreader/#skiprows","title":"skipRows","text":"

    \u6307\u5b9a\u8981\u8df3\u8fc7\u7684\u884c\u6570\uff0c \u9ed8\u8ba4\u4e3a 0\uff0c\u8868\u793a\u4e0d\u8df3\u8fc7\u3002\u8fd9\u91cc\u8981\u6ce8\u610f\u7684\u662f\uff0c\u5047\u5b9a \u8bbe\u7f6e\u4e86 header \u4e3a true\uff0c\u540c\u65f6\u8bbe\u7f6e skipRows \u4e3a 2\u3002\u5219\u8868\u793a\u524d\u4e09\u884c\u90fd\u8df3\u8fc7\u3002 \u5982\u679c header \u4e3a false\uff0c \u5219\u8868\u793a\u8df3\u8fc7\u524d\u4e24\u884c\u3002

    "},{"location":"reader/excelreader/#_4","title":"\u652f\u6301\u7684\u6570\u636e\u7c7b\u578b","text":"

    Excel \u8bfb\u53d6\u529f\u80fd\u7684\u5b9e\u73b0\u4f9d\u8d56\u4e8e Apache POI \u9879\u76ee\uff0c\u8be5\u5b9e\u73b0\u5bf9\u5355\u5143\u683c\u7684\u6570\u636e\u7c7b\u578b\u5b9a\u4e49\u5f88\u5bbd\u6cdb\u3002 \u4ec5\u5b9a\u4e49\u4e86\u5e03\u5c14\u578b(Boolean)\uff0c\u6570\u503c\u578b (Double)\uff0c\u5b57\u7b26\u4e32\u578b(String) \u4e09\u79cd\u3002\u5176\u4e2d\u6570\u503c\u578b\u5305\u62ec\u6240\u6709\u6574\u6570\uff0c\u5c0f\u6570\u548c\u65e5\u671f\u3002 \u76ee\u524d\u5bf9\u4e8e\u6570\u503c\u7c7b\u578b\u505a\u4e86\u7b80\u5355\u533a\u5206

    1. \u4f7f\u7528\u5e93\u5de5\u5177\u7c7b\u63a2\u6d4b\u662f\u5426\u4e3a\u65e5\u671f\u7c7b\u578b\uff0c\u5982\u679c\u662f\uff0c\u5219\u5224\u65ad\u4e3a\u65e5\u671f\u7c7b\u578b
    2. \u5c06\u6570\u503c\u8f6c\u6362\u4e3a\u957f\u6574\u5f62\uff0c\u5e76\u548c\u539f\u503c\u6bd4\u8f83\uff0c\u5982\u679c\u5927\u5c0f\u76f8\u7b49\uff0c\u5219\u5224\u65ad\u4e3a\u957f\u6574\u578b(Long)
    3. \u5426\u5219\u5224\u65ad\u4e3a\u6d6e\u70b9\u578b\uff08Double\uff09
    "},{"location":"reader/excelreader/#_5","title":"\u9650\u5236","text":"
    1. \u5f53\u524d\u4ec5\u8bfb\u53d6\u6587\u4ef6\u7684\u7b2c\u4e00\u4e2a Sheet \u800c\u5ffd\u7565\u5176\u4ed6 Sheets
    2. \u6682\u4e0d\u652f\u6301\u6307\u5b9a\u5217\u8bfb\u53d6
    3. \u6682\u4e0d\u652f\u6301\u8df3\u8fc7\u5c3e\u90e8\u884c\u6570\uff08\u6bd4\u5982\u6709\u603b\u7ed3\u7684\u5c3e\u884c\u53ef\u80fd\u5e76\u4e0d\u7b26\u5408\u8981\u6c42\uff09
    4. \u6682\u4e0d\u5224\u65ad\u6bcf\u4e00\u884c\u7684\u5217\u6570\u662f\u5426\u76f8\u7b49\uff0c\u9700\u8981 Excel \u81ea\u884c\u4fdd\u8bc1
    5. \u4ec5\u4f1a\u8bfb\u53d6\u6307\u5b9a\u76ee\u5f55\u4e0b\u6587\u4ef6\u540e\u7f00\u4e3a xlsx \u6216 xls \u7684\u6587\u4ef6\uff0c\u5176\u4ed6\u540e\u7f00\u6587\u4ef6\u5c06\u4f1a\u5ffd\u7565\u5e76\u7ed9\u51fa\u544a\u8b66\u6d88\u606f
    "},{"location":"reader/ftpreader/","title":"Ftp Reader","text":"

    Ftp Reader \u63d0\u4f9b\u4e86\u8bfb\u53d6\u8fdc\u7a0b FTP/SFTP \u6587\u4ef6\u7cfb\u7edf\u6570\u636e\u5b58\u50a8\u7684\u80fd\u529b\u3002

    "},{"location":"reader/ftpreader/#_1","title":"\u529f\u80fd\u8bf4\u660e","text":""},{"location":"reader/ftpreader/#_2","title":"\u914d\u7f6e\u6837\u4f8b","text":"job/ftp2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"ftpreader\",\n        \"parameter\": {\n          \"protocol\": \"sftp\",\n          \"host\": \"127.0.0.1\",\n          \"port\": 22,\n          \"username\": \"xx\",\n          \"password\": \"xxx\",\n          \"path\": [\n            \"/var/ftp/test.txt\",\n            \"/var/tmp/*.txt\",\n            \"/public/ftp\",\n            \"/public/a??.txt\"\n          ],\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"long\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"boolean\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"double\"\n            },\n            {\n              \"index\": 3,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 4,\n              \"type\": \"date\",\n              \"format\": \"yyyy.MM.dd\"\n            }\n          ],\n          \"encoding\": \"UTF-8\",\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"ftpWriter\",\n        \"parameter\": {\n          \"path\": \"/var/ftp/FtpWriter/result\",\n          \"fileName\": \"shihf\",\n          \"writeMode\": \"truncate\",\n          \"format\": \"yyyy-MM-dd\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/ftpreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 protocol \u662f string \u65e0 \u670d\u52a1\u5668\u534f\u8bae\uff0c\u76ee\u524d\u652f\u6301\u4f20\u8f93\u534f\u8bae\u6709 ftp \u548c sftp host \u662f string \u65e0 \u670d\u52a1\u5668\u5730\u5740 port \u5426 int 22/21 \u82e5\u4f20\u8f93\u534f\u8bae\u662f sftp \u534f\u8bae\uff0c\u9ed8\u8ba4\u503c\u662f 22\uff1b\u82e5\u4f20\u8f93\u534f\u8bae\u662f\u6807\u51c6 ftp \u534f\u8bae\uff0c\u9ed8\u8ba4\u503c\u662f 21 timeout \u5426 int 60000 \u8fde\u63a5 ftp \u670d\u52a1\u5668\u8fde\u63a5\u8d85\u65f6\u65f6\u95f4\uff0c\u5355\u4f4d\u6beb\u79d2(ms) connectPattern \u5426 string PASV \u8fde\u63a5\u6a21\u5f0f\uff0c\u4ec5\u652f\u6301 PORT, PASV \u6a21\u5f0f\u3002\u8be5\u53c2\u6570\u4ec5\u5728 ftp \u534f\u8bae\u65f6\u4f7f\u7528 username \u662f string \u65e0 ftp \u670d\u52a1\u5668\u8bbf\u95ee\u7528\u6237\u540d password \u5426 string \u65e0 ftp \u670d\u52a1\u5668\u8bbf\u95ee\u5bc6\u7801 useKey \u5426 boolean false \u662f\u5426\u4f7f\u7528\u79c1\u94a5\u767b\u5f55\uff0c\u4ec5\u9488\u5bf9 sftp \u767b\u5f55\u6709\u6548 keyPath \u5426 string ~/.ssh/id_rsa \u79c1\u94a5\u5730\u5740 keyPass \u5426 string \u65e0 \u79c1\u94a5\u5bc6\u7801\uff0c\u82e5\u6ca1\u6709\u8bbe\u7f6e\u79c1\u94a5\u5bc6\u7801\uff0c\u5219\u65e0\u9700\u914d\u7f6e\u8be5\u9879 path \u662f list \u65e0 \u8fdc\u7a0b FTP \u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1\u4e0b column \u662f list<map> \u65e0 \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0c\u8be6\u89c1\u4e0b\u6587 fieldDelimiter \u662f string , \u63cf\u8ff0\uff1a\u8bfb\u53d6\u7684\u5b57\u6bb5\u5206\u9694\u7b26 compress \u5426 string \u65e0 \u6587\u672c\u538b\u7f29\u7c7b\u578b\uff0c\u9ed8\u8ba4\u4e0d\u586b\u5199\u610f\u5473\u7740\u6ca1\u6709\u538b\u7f29\u3002\u652f\u6301\u538b\u7f29\u7c7b\u578b\u4e3a zip\u3001gz\u3001bzip2 encoding \u5426 string utf-8 \u8bfb\u53d6\u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e skipHeader \u5426 boolean false \u7c7b CSV \u683c\u5f0f\u6587\u4ef6\u53ef\u80fd\u5b58\u5728\u8868\u5934\u4e3a\u6807\u9898\u60c5\u51b5\uff0c\u9700\u8981\u8df3\u8fc7\u3002\u9ed8\u8ba4\u4e0d\u8df3\u8fc7 nullFormat \u5426 char \\N \u5b9a\u4e49\u54ea\u4e9b\u5b57\u7b26\u4e32\u53ef\u4ee5\u8868\u793a\u4e3a null maxTraversalLevel \u5426 int 100 \u5141\u8bb8\u904d\u5386\u6587\u4ef6\u5939\u7684\u6700\u5927\u5c42\u6570 csvReaderConfig \u5426 map \u65e0 \u8bfb\u53d6 CSV \u7c7b\u578b\u6587\u4ef6\u53c2\u6570\u914d\u7f6e\uff0cMap \u7c7b\u578b\u3002\u4e0d\u914d\u7f6e\u5219\u4f7f\u7528\u9ed8\u8ba4\u503c,\u8be6\u89c1\u4e0b\u6587"},{"location":"reader/ftpreader/#path","title":"path","text":"

    \u8fdc\u7a0b FTP \u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84\uff0c\u793a\u4f8b\u914d\u7f6e\u4e2d\u6f14\u793a\u4e86\u5982\u4f55\u586b\u5199\u591a\u4e2a\u8def\u5f84\u3002

    {\n  \"path\": [\n    \"/var/ftp/test.txt\", // \u8bfb\u53d6 /var/ftp \u76ee\u5f55\u4e0b\u7684 test.txt \u6587\u4ef6\n    \"/var/tmp/*.txt\", // \u8bfb\u53d6 /var/tmp \u76ee\u5f55\u4e0b\u6240\u6709 txt \u6587\u4ef6\n    \"/public/ftp\", // \u8bfb\u53d6 /public/ftp \u76ee\u5f55\u4e0b\u6240\u6709\u6587\u4ef6, \u5982\u679c ftp \u662f\u6587\u4ef6\u7684\u8bdd\uff0c\u5219\u76f4\u63a5\u8bfb\u53d6\n    \"/public/a??.txt\" // \u8bfb\u53d6 /public \u76ee\u5f55\u4e0b\u6240\u6709 a \u5f00\u5934\uff0c\u540e\u9762\u8ddf\u4e24\u4e2a\u5b57\u7b26\uff0c\u6700\u540e\u662f txt \u7ed3\u5c3e\u7684\u6587\u4ef6\n  ]\n}\n

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0cAddax \u4f1a\u5c06\u4e00\u4e2a\u4f5c\u4e1a\u4e0b\u540c\u6b65\u7684\u6240\u6709 Text File \u89c6\u4f5c\u540c\u4e00\u5f20\u6570\u636e\u8868\u3002\u7528\u6237\u5fc5\u987b\u81ea\u5df1\u4fdd\u8bc1\u6240\u6709\u7684 File \u80fd\u591f\u9002\u914d\u540c\u4e00\u5957 schema \u4fe1\u606f\u3002\u8bfb\u53d6\u6587\u4ef6\u7528\u6237\u5fc5\u987b\u4fdd\u8bc1\u4e3a\u7c7b CSV \u683c\u5f0f\uff0c\u5e76\u4e14\u63d0\u4f9b\u7ed9 Addax \u6743\u9650\u53ef\u8bfb\u3002

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0c\u5982\u679c Path \u6307\u5b9a\u7684\u8def\u5f84\u4e0b\u6ca1\u6709\u7b26\u5408\u5339\u914d\u7684\u6587\u4ef6\u62bd\u53d6\uff0cAddax \u5c06\u62a5\u9519\u3002

    "},{"location":"reader/ftpreader/#column","title":"column","text":"

    \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cindex \u6307\u5b9a\u5f53\u524d\u5217\u6765\u81ea\u4e8e\u6587\u672c\u7b2c\u51e0\u5217(\u4ee5 0 \u5f00\u59cb)\uff0cvalue \u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece\u6e90\u5934\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636e value \u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u7528\u6237\u53ef\u4ee5\u5168\u90e8\u6309\u7167 String \u7c7b\u578b\u8bfb\u53d6\u6570\u636e\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"column\": [\"*\"]\n}\n

    \u7528\u6237\u53ef\u4ee5\u6307\u5b9a Column \u5b57\u6bb5\u4fe1\u606f\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    [\n  {\n    \"type\": \"long\",\n    \"index\": 0,\n    \"description\": \"\u4ece\u8fdc\u7a0bFTP\u6587\u4ef6\u6587\u672c\u7b2c\u4e00\u5217\u83b7\u53d6int\u5b57\u6bb5\"\n  },\n  {\n    \"type\": \"string\",\n    \"value\": \"addax\",\n    \"description\": \"\u4eceFtpReader\u5185\u90e8\u751f\u6210alibaba\u7684\u5b57\u7b26\u4e32\u5b57\u6bb5\u4f5c\u4e3a\u5f53\u524d\u5b57\u6bb5\"\n  }\n]\n

    \u5bf9\u4e8e\u7528\u6237\u6307\u5b9a Column \u4fe1\u606f\uff0ctype \u5fc5\u987b\u586b\u5199\uff0cindex/value \u5fc5\u987b\u9009\u62e9\u5176\u4e00\u3002

    "},{"location":"reader/ftpreader/#csvreaderconfig","title":"csvReaderConfig","text":"

    \u5e38\u89c1\u914d\u7f6e\uff1a

    {\n  \"csvReaderConfig\": {\n    \"safetySwitch\": false,\n    \"skipEmptyRecords\": false,\n    \"useTextQualifier\": false\n  }\n}\n

    \u6240\u6709\u914d\u7f6e\u9879\u53ca\u9ed8\u8ba4\u503c,\u914d\u7f6e\u65f6 csvReaderConfig \u7684 map \u4e2d\u8bf7 \u4e25\u683c\u6309\u7167\u4ee5\u4e0b\u5b57\u6bb5\u540d\u5b57\u8fdb\u884c\u914d\u7f6e\uff1a

    boolean caseSensitive = true;\nchar textQualifier = 34;\nboolean trimWhitespace = true;\nboolean useTextQualifier = true;//\u662f\u5426\u4f7f\u7528csv\u8f6c\u4e49\u5b57\u7b26\nchar delimiter = 44;//\u5206\u9694\u7b26\nchar recordDelimiter = 0;\nchar comment = 35;\nboolean useComments = false;\nint escapeMode = 1;\nboolean safetySwitch = true;//\u5355\u5217\u957f\u5ea6\u662f\u5426\u9650\u5236100000\u5b57\u7b26\nboolean skipEmptyRecords = true;//\u662f\u5426\u8df3\u8fc7\u7a7a\u884c\nboolean captureRawRecord = true;\n
    "},{"location":"reader/ftpreader/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u8fdc\u7a0b FTP \u6587\u4ef6\u672c\u8eab\u4e0d\u63d0\u4f9b\u6570\u636e\u7c7b\u578b\uff0c\u8be5\u7c7b\u578b\u662f Addax FtpReader \u5b9a\u4e49\uff1a

    Addax \u5185\u90e8\u7c7b\u578b \u8fdc\u7a0b FTP \u6587\u4ef6 \u6570\u636e\u7c7b\u578b Long Long Double Double String String Boolean Boolean Date Date

    \u5176\u4e2d\uff1a

    "},{"location":"reader/ftpreader/#_5","title":"\u9650\u5236","text":"
    1. \u5355\u4e2a File \u652f\u6301\u591a\u7ebf\u7a0b\u5e76\u53d1\u8bfb\u53d6\uff0c\u8fd9\u91cc\u6d89\u53ca\u5230\u5355\u4e2a File \u5185\u90e8\u5207\u5206\u7b97\u6cd5
    2. \u5355\u4e2a File \u5728\u538b\u7f29\u60c5\u51b5\u4e0b\uff0c\u4ece\u6280\u672f\u4e0a\u65e0\u6cd5\u652f\u6301\u591a\u7ebf\u7a0b\u5e76\u53d1\u8bfb\u53d6\u3002
    "},{"location":"reader/hanareader/","title":"HANA Reader","text":"

    HANA Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece SAP HANA \u8bfb\u53d6\u6570\u636e\u7684\u80fd\u529b

    "},{"location":"reader/hanareader/#_1","title":"\u793a\u4f8b","text":"

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/hanareader.json 
    {\n  \"job\": {\n    \"content\": [\n      {\n        \"reader\": {\n          \"name\": \"hanareader\",\n          \"parameter\": {\n            \"column\": [\n              \"*\"\n            ],\n            \"connection\": {\n              \"jdbcUrl\": \"jdbc:sap://wgzhao-pc:39017/system\",\n              \"table\": [\n                \"addax_tbl\"\n              ]\n            },\n            \"username\": \"system\",\n            \"password\": \"HXEHana1\"\n          }\n        },\n        \"writer\": {\n          \"name\": \"streamwriter\",\n          \"parameter\": {\n            \"print\": true\n          }\n        }\n      }\n    ],\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/hana2stream.json

    "},{"location":"reader/hanareader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/hana2stream.json\n
    "},{"location":"reader/hanareader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u53c2\u6570\u3002

    "},{"location":"reader/hbase11xreader/","title":"HBase11X Reader","text":"

    HBase11X Reader \u63d2\u4ef6\u652f\u6301\u4ece HBase 1.x \u7248\u672c\u8bfb\u53d6\u6570\u636e\uff0c \u5176\u5b9e\u73b0\u65b9\u5f0f\u4e3a \u901a\u8fc7 HBase \u7684 Java \u5ba2\u6237\u7aef\u8fde\u63a5\u8fdc\u7a0b HBase \u670d\u52a1\uff0c\u5e76\u901a\u8fc7 Scan \u65b9\u5f0f\u8bfb\u53d6\u4f60\u6307\u5b9a rowkey \u8303\u56f4\u5185\u7684\u6570\u636e\u3002

    "},{"location":"reader/hbase11xreader/#_1","title":"\u914d\u7f6e","text":""},{"location":"reader/hbase11xreader/#_2","title":"\u5efa\u8868\u4ee5\u53ca\u586b\u5145\u6570\u636e","text":"

    \u4ee5\u4e0b\u6f14\u793a\u57fa\u4e8e\u4e0b\u9762\u521b\u5efa\u7684\u8868\u4ee5\u53ca\u6570\u636e

    create 'users', 'address','info'\nput 'users', 'lisi', 'address:country', 'china'\nput 'users', 'lisi', 'address:province',    'beijing'\nput 'users', 'lisi', 'info:age',        27\nput 'users', 'lisi', 'info:birthday',   '1987-06-17'\nput 'users', 'lisi', 'info:company',    'baidu'\nput 'users', 'xiaoming', 'address:city',    'hangzhou'\nput 'users', 'xiaoming', 'address:country', 'china'\nput 'users', 'xiaoming', 'address:province',    'zhejiang'\nput 'users', 'xiaoming', 'info:age',        29\nput 'users', 'xiaoming', 'info:birthday',   '1987-06-17'\nput 'users', 'xiaoming', 'info:company',    'alibaba'\n
    "},{"location":"reader/hbase11xreader/#normal","title":"normal \u6a21\u5f0f","text":"

    \u628a HBase \u4e2d\u7684\u8868\uff0c\u5f53\u6210\u666e\u901a\u4e8c\u7ef4\u8868\uff08\u6a2a\u8868\uff09\u8fdb\u884c\u8bfb\u53d6,\u8bfb\u53d6\u6700\u65b0\u7248\u672c\u6570\u636e\u3002\u5982\uff1a

    hbase(main):017:0> scan 'users'\nROW           COLUMN+CELL\n lisi         column=address:city, timestamp=1457101972764, value=beijing\n lisi         column=address:country, timestamp=1457102773908, value=china\n lisi         column=address:province, timestamp=1457101972736, value=beijing\n lisi         column=info:age, timestamp=1457101972548, value=27\n lisi         column=info:birthday, timestamp=1457101972604, value=1987-06-17\n lisi         column=info:company, timestamp=1457101972653, value=baidu\n xiaoming     column=address:city, timestamp=1457082196082, value=hangzhou\n xiaoming     column=address:country, timestamp=1457082195729, value=china\n xiaoming     column=address:province, timestamp=1457082195773, value=zhejiang\n xiaoming     column=info:age, timestamp=1457082218735, value=29\n xiaoming     column=info:birthday, timestamp=1457082186830, value=1987-06-17\n xiaoming     column=info:company, timestamp=1457082189826, value=alibaba\n2 row(s) in 0.0580 seconds\n

    \u8bfb\u53d6\u540e\u6570\u636e

    rowKey addres:city address:country address:province info:age info:birthday info:company lisi beijing china beijing 27 1987-06-17 baidu xiaoming hangzhou china zhejiang 29 1987-06-17 alibaba"},{"location":"reader/hbase11xreader/#multiversionfixedcolumn","title":"multiVersionFixedColumn \u6a21\u5f0f","text":"

    \u628a HBase \u4e2d\u7684\u8868\uff0c\u5f53\u6210\u7ad6\u8868\u8fdb\u884c\u8bfb\u53d6\u3002\u8bfb\u51fa\u7684\u6bcf\u6761\u8bb0\u5f55\u4e00\u5b9a\u662f\u56db\u5217\u5f62\u5f0f\uff0c\u4f9d\u6b21\u4e3a\uff1arowKey\uff0cfamily:qualifier\uff0ctimestamp\uff0cvalue\u3002

    \u8bfb\u53d6\u65f6\u9700\u8981\u660e\u786e\u6307\u5b9a\u8981\u8bfb\u53d6\u7684\u5217\uff0c\u628a\u6bcf\u4e00\u4e2a cell \u4e2d\u7684\u503c\uff0c\u4f5c\u4e3a\u4e00\u6761\u8bb0\u5f55\uff08record\uff09\uff0c\u82e5\u6709\u591a\u4e2a\u7248\u672c\u5c31\u6709\u591a\u6761\u8bb0\u5f55\uff08record\uff09\u3002\u5982\uff1a

    hbase(main):018:0> scan 'users',{VERSIONS=>5}\nROW               COLUMN+CELL\n lisi             column=address:city, timestamp=1457101972764, value=beijing\n lisi             column=address:contry, timestamp=1457102773908, value=china\n lisi             column=address:province, timestamp=1457101972736, value=beijing\n lisi             column=info:age, timestamp=1457101972548, value=27\n lisi             column=info:birthday, timestamp=1457101972604, value=1987-06-17\n lisi             column=info:company, timestamp=1457101972653, value=baidu\n xiaoming         column=address:city, timestamp=1457082196082, value=hangzhou\n xiaoming         column=address:contry, timestamp=1457082195729, value=china\n xiaoming         column=address:province, timestamp=1457082195773, value=zhejiang\n xiaoming         column=info:age, timestamp=1457082218735, value=29\n xiaoming         column=info:age, timestamp=1457082178630, value=24\n xiaoming         column=info:birthday, timestamp=1457082186830, value=1987-06-17\n xiaoming         column=info:company, timestamp=1457082189826, value=alibaba\n2 row(s) in 0.0260 seconds\n

    \u8bfb\u53d6\u540e\u6570\u636e(4 \u5217)

    rowKey column:qualifier timestamp value lisi address:city 1457101972764 beijing lisi address:contry 1457102773908 china lisi address:province 1457101972736 beijing lisi info:age 1457101972548 27 lisi info:birthday 1457101972604 1987-06-17 lisi info:company 1457101972653 beijing xiaoming address:city 1457082196082 hangzhou xiaoming address:contry 1457082195729 china xiaoming address:province 1457082195773 zhejiang xiaoming info:age 1457082218735 29 xiaoming info:age 1457082178630 24 xiaoming info:birthday 1457082186830 1987-06-17 xiaoming info:company 1457082189826 alibaba"},{"location":"reader/hbase11xreader/#_3","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4ece HBase \u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a\uff0c\u5206\u522b\u4e3a\u6807\u51c6\u6a21\u5f0f\u548c\u591a\u7248\u672c\u6a21\u5f0f

    job/hbase11x2stream_normal.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"hbase11xreader\",\n        \"parameter\": {\n          \"hbaseConfig\": {\n            \"hbase.zookeeper.quorum\": \"xxxf\"\n          },\n          \"table\": \"users\",\n          \"encoding\": \"utf-8\",\n          \"mode\": \"normal\",\n          \"column\": [\n            {\n              \"name\": \"rowkey\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: age\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: birthday\",\n              \"type\": \"date\",\n              \"format\": \"yyyy-MM-dd\"\n            },\n            {\n              \"name\": \"info: company\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: country\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: province\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: city\",\n              \"type\": \"string\"\n            }\n          ],\n          \"range\": {\n            \"startRowkey\": \"\",\n            \"endRowkey\": \"\",\n            \"isBinaryRowkey\": true\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"txtfilewriter\",\n        \"parameter\": {\n          \"path\": \"/Users/shf/workplace/addax_test/hbase11xreader/result\",\n          \"fileName\": \"qiran\",\n          \"writeMode\": \"truncate\"\n        }\n      }\n    }\n  }\n}\n
    job/hbase11x2srtream_version.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"hbase11xreader\",\n        \"parameter\": {\n          \"hbaseConfig\": {\n            \"hbase.zookeeper.quorum\": \"127.0.0.1:2181\"\n          },\n          \"table\": \"users\",\n          \"encoding\": \"utf-8\",\n          \"mode\": \"multiVersionFixedColumn\",\n          \"maxVersion\": \"-1\",\n          \"column\": [\n            {\n              \"name\": \"rowkey\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: age\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: birthday\",\n              \"type\": \"date\",\n              \"format\": \"yyyy-MM-dd\"\n            },\n            {\n              \"name\": \"info: company\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: contry\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: province\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: city\",\n              \"type\": \"string\"\n            }\n          ],\n          \"range\": {\n            \"startRowkey\": \"\",\n            \"endRowkey\": \"\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"txtfilewriter\",\n        \"parameter\": {\n          \"path\": \"/Users/shf/workplace/addax_test/hbase11xreader/result\",\n          \"fileName\": \"qiran\",\n          \"writeMode\": \"truncate\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/hbase11xreader/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 hbaseConfig \u662f map \u65e0 \u8fde\u63a5 HBase \u96c6\u7fa4\u9700\u8981\u7684\u914d\u7f6e\u4fe1\u606f, hbase.zookeeper.quorum \u4e3a\u5fc5\u586b\u9879\uff0c\u5176\u4ed6 client \u7684\u914d\u7f6e\u4e3a\u53ef\u9009\u9879 mode \u662f string \u65e0 \u8bfb\u53d6 HBase \u7684\u6a21\u5f0f\uff0c\u53ef\u586b\u5199 normal \u6216 multiVersionFixedColumn table \u662f string \u65e0 \u8981\u8bfb\u53d6\u7684 hbase \u8868\u540d\uff08\u5927\u5c0f\u5199\u654f\u611f\uff09 encoding \u5426 string UTF-8 \u7f16\u7801\u65b9\u5f0f\uff0cUTF-8 \u6216\u662f GBK\uff0c\u7528\u4e8e\u5bf9\u4e8c\u8fdb\u5236\u5b58\u50a8\u7684 HBase byte[] \u8f6c\u4e3a String \u65f6\u7684\u7f16\u7801 column \u662f list<map> \u65e0 \u8981\u8bfb\u53d6\u7684 hbase \u5b57\u6bb5\uff0cnormal \u6a21\u5f0f\u4e0e multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b\u9879, \u8be6\u7ec6\u8bf4\u660e\u89c1\u4e0b\u6587 maxVersion \u662f string \u65e0 \u6307\u5b9a\u5728\u591a\u7248\u672c\u6a21\u5f0f\u4e0b\u8bfb\u53d6\u7684\u7248\u672c\u6570\uff0c-1 \u8868\u793a\u8bfb\u53d6\u6240\u6709\u7248\u672c, multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b range \u5426 string \u65e0 \u6307\u5b9a\u8bfb\u53d6\u7684rowkey \u8303\u56f4, \u8be6\u89c1\u4e0b\u6587 scanCacheSize \u5426 int 256 \u6bcf\u6b21\u4ece\u670d\u52a1\u5668\u7aef\u8bfb\u53d6\u7684\u884c\u6570 scanBatchSize \u5426 int 100 \u6bcf\u6b21\u4ece\u670d\u52a1\u5668\u7aef\u8bfb\u53d6\u7684\u5217\u6570"},{"location":"reader/hbase11xreader/#column","title":"column","text":"

    \u63cf\u8ff0\uff1a\u8981\u8bfb\u53d6\u7684 hbase \u5b57\u6bb5\uff0cnormal \u6a21\u5f0f\u4e0e multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b\u9879\u3002

    "},{"location":"reader/hbase11xreader/#normal_1","title":"normal \u6a21\u5f0f","text":"

    name \u6307\u5b9a\u8bfb\u53d6\u7684 hbase \u5217\uff0c\u9664\u4e86 rowkey \u5916\uff0c\u5fc5\u987b\u4e3a \u5217\u65cf:\u5217\u540d \u7684\u683c\u5f0f\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cformat\u6307\u5b9a\u65e5\u671f\u7c7b\u578b\u7684\u683c\u5f0f\uff0c value \u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece hbase \u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636e value \u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002\u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"column\": [\n    {\n      \"name\": \"rowkey\",\n      \"type\": \"string\"\n    },\n    {\n      \"value\": \"test\",\n      \"type\": \"string\"\n    }\n  ]\n}\n

    normal \u6a21\u5f0f\u4e0b\uff0c\u5bf9\u4e8e\u7528\u6237\u6307\u5b9a Column \u4fe1\u606f\uff0ctype \u5fc5\u987b\u586b\u5199\uff0cname/value \u5fc5\u987b\u9009\u62e9\u5176\u4e00\u3002

    "},{"location":"reader/hbase11xreader/#multiversionfixedcolumn_1","title":"multiVersionFixedColumn \u6a21\u5f0f","text":"

    name \u6307\u5b9a\u8bfb\u53d6\u7684 hbase \u5217\uff0c\u9664\u4e86 rowkey \u5916\uff0c\u5fc5\u987b\u4e3a \u5217\u65cf:\u5217\u540d \u7684\u683c\u5f0f\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cformat\u6307\u5b9a\u65e5\u671f\u7c7b\u578b\u7684\u683c\u5f0f \u3002 multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u4e0d\u652f\u6301\u5e38\u91cf\u5217\u3002\u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"column\": [\n    {\n      \"name\": \"rowkey\",\n      \"type\": \"string\"\n    },\n    {\n      \"name\": \"info: age\",\n      \"type\": \"string\"\n    }\n  ]\n}\n
    "},{"location":"reader/hbase11xreader/#range","title":"range","text":"

    \u6307\u5b9a\u8bfb\u53d6\u7684 rowkey \u8303\u56f4

    \u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"range\": {\n    \"startRowkey\": \"aaa\",\n    \"endRowkey\": \"ccc\",\n    \"isBinaryRowkey\": false\n  }\n}\n
    "},{"location":"reader/hbase11xreader/#_5","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u4e0b\u9762\u5217\u51fa\u652f\u6301\u7684\u8bfb\u53d6 HBase \u6570\u636e\u7c7b\u578b\uff0cHbaseReader \u9488\u5bf9 HBase \u7c7b\u578b\u8f6c\u6362\u5217\u8868:

    Addax \u5185\u90e8\u7c7b\u578b HBase \u6570\u636e\u7c7b\u578b Long int, short ,long Double float, double String string, binarystring Date date Boolean boolean

    \u8bf7\u6ce8\u610f:

    \u9664\u4e0a\u8ff0\u7f57\u5217\u5b57\u6bb5\u7c7b\u578b\u5916\uff0c\u5176\u4ed6\u7c7b\u578b\u5747\u4e0d\u652f\u6301

    "},{"location":"reader/hbase11xreader/#_6","title":"\u9650\u5236","text":"
    1. \u76ee\u524d\u4e0d\u652f\u6301\u52a8\u6001\u5217\u7684\u8bfb\u53d6\u3002\u8003\u8651\u7f51\u7edc\u4f20\u8f93\u6d41\u91cf\uff08\u652f\u6301\u52a8\u6001\u5217\uff0c\u9700\u8981\u5148\u5c06 hbase \u6240\u6709\u5217\u7684\u6570\u636e\u8bfb\u53d6\u51fa\u6765\uff0c\u518d\u6309\u89c4\u5219\u8fdb\u884c\u8fc7\u6ee4\uff09\uff0c\u73b0\u652f\u6301\u7684\u4e24\u79cd\u8bfb\u53d6\u6a21\u5f0f\u4e2d\u9700\u8981\u7528\u6237\u660e\u786e\u6307\u5b9a\u8981\u8bfb\u53d6\u7684\u5217\u3002
    2. \u5173\u4e8e\u540c\u6b65\u4f5c\u4e1a\u7684\u5207\u5206\uff1a\u76ee\u524d\u7684\u5207\u5206\u65b9\u5f0f\u662f\u6839\u636e\u7528\u6237 hbase \u8868\u6570\u636e\u7684 region \u5206\u5e03\u8fdb\u884c\u5207\u5206\u3002\u5373\uff1a\u5728\u7528\u6237\u586b\u5199\u7684 [startrowkey\uff0cendrowkey\uff3d \u8303\u56f4\u5185\uff0c\u4e00\u4e2a region \u4f1a\u5207\u5206\u6210\u4e00\u4e2a task\uff0c\u5355\u4e2a region \u4e0d\u8fdb\u884c\u5207\u5206\u3002
    3. multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u4e0d\u652f\u6301\u589e\u52a0\u5e38\u91cf\u5217
    "},{"location":"reader/hbase11xsqlreader/","title":"HBase11x SQL Reader","text":"

    HBase11x SQL Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Phoenix(HBase SQL)\u8bfb\u53d6\u6570\u636e, \u652f\u6301\u7684 HBase \u7248\u672c\u4e3a 1.x

    "},{"location":"reader/hbase11xsqlreader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4ecePhoenix\u540c\u6b65\u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a:

    {\n    \"job\": {\n        \"setting\": {\n            \"speed\": {\n                \"byte\":-1,\n              \"channel\": 1\n            }\n        },\n        \"content\": [ {\n                \"reader\": {\n                    \"name\": \"hbase11xsqlreader\",\n                    \"parameter\": {\n                        \"hbaseConfig\": {\n                            \"hbase.zookeeper.quorum\": \"node1,node2,node3\"\n                        },\n                        \"table\": \"US_POPULATION\",\n                        \"column\": [],\n                        \"where\": \"1=1\",\n                        \"querySql\": \"\"\n                    }\n                },\n                \"writer\": {\n                    \"name\": \"streamwriter\",\n                    \"parameter\": {\n                        \"print\":true,\n                        \"encoding\": \"UTF-8\"\n                    }\n                }\n            }\n        ]\n    }\n}\n
    "},{"location":"reader/hbase11xsqlreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 hbaseConfig \u662f map \u65e0 \u9700\u8981\u901a\u8fc7 Phoenix \u5ba2\u6237\u7aef\u53bb\u8fde\u63a5 hbase \u96c6\u7fa4\uff0c\u56e0\u6b64\u8fd9\u91cc\u9700\u8981\u586b\u5199\u5bf9\u5e94 hbase \u96c6\u7fa4\u7684 zkurl\u5730\u5740 table \u662f string \u65e0 \u6307\u5b9a Phoenix \u4e2d\u7684\u8868\u540d,\u5982\u679c\u6709 namespace\uff0c\u8be5\u503c\u8bbe\u7f6e\u4e3a namespace.tablename querySql \u5426 string \u65e0 \u4e0d\u662f\u76f4\u63a5\u67e5\u8be2\u8868\uff0c\u800c\u662f\u63d0\u4f9b\u5177\u4f53\u7684\u67e5\u8be2\u8bed\u53e5\uff0c\u5982\u679c\u8be5\u53c2\u6570\u548c table \u53c2\u6570\u540c\u65f6\u5b58\u5728\uff0c\u5219\u4f18\u5148\u4f7f\u7528\u8be5\u53c2\u6570 column \u662f list<map> \u65e0 \u586b\u5199\u9700\u8981\u4ecephoenix\u8868\u4e2d\u8bfb\u53d6\u7684\u5217\u540d\u96c6\u5408\uff0c\u4f7f\u7528JSON\u7684\u6570\u7ec4\u63cf\u8ff0\u5b57\u6bb5\u4fe1\u606f\uff0c\u7a7a\u503c\u6216 \"*\" \u8868\u793a\u8bfb\u53d6\u6240\u6709\u5217 where \u5426 string \u65e0 where \u6761\u4ef6"},{"location":"reader/hbase11xsqlreader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u76ee\u524d\u652f\u6301\u5927\u90e8\u5206 Phoenix\u7c7b\u578b\uff0c\u4f46\u4e5f\u5b58\u5728\u90e8\u5206\u4e2a\u522b\u7c7b\u578b\u6ca1\u6709\u652f\u6301\u7684\u60c5\u51b5\uff0c\u8bf7\u6ce8\u610f\u68c0\u67e5\u4f60\u7684\u7c7b\u578b\u3002

    \u4e0b\u9762\u5217\u51fa\u7c7b\u578b\u8f6c\u6362\u5217\u8868:

    Addax \u5185\u90e8\u7c7b\u578b Phoenix \u6570\u636e\u7c7b\u578b String CHAR, VARCHAR Bytes BINARY, VARBINARY Bool BOOLEAN Long INTEGER, TINYINT, SMALLINT, BIGINT Double FLOAT, DECIMAL, DOUBLE, Date DATE, TIME, TIMESTAMP"},{"location":"reader/hbase20xreader/","title":"HBase20 Reader","text":"

    HBase20 Reader \u63d2\u4ef6\u652f\u6301\u4ece HBase 2.x \u7248\u672c\u8bfb\u53d6\u6570\u636e\uff0c \u5176\u5b9e\u73b0\u65b9\u5f0f\u4e3a \u901a\u8fc7 HBase \u7684 Java \u5ba2\u6237\u7aef\u8fde\u63a5\u8fdc\u7a0b HBase \u670d\u52a1\uff0c\u5e76\u901a\u8fc7 Scan \u65b9\u5f0f\u8bfb\u53d6\u4f60\u6307\u5b9a rowkey \u8303\u56f4\u5185\u7684\u6570\u636e\u3002

    "},{"location":"reader/hbase20xreader/#_1","title":"\u914d\u7f6e","text":"

    \u4ee5\u4e0b\u6f14\u793a\u57fa\u4e8e\u4e0b\u9762\u521b\u5efa\u7684\u8868\u4ee5\u53ca\u6570\u636e

    create 'users', {NAME=>'address', VERSIONS=>100},{NAME=>'info',VERSIONS=>1000}\nput 'users', 'lisi', 'address:country', 'china1', 20200101\nput 'users', 'lisi', 'address:province',    'beijing1', 20200101\nput 'users', 'lisi', 'info:age',        27, 20200101\nput 'users', 'lisi', 'info:birthday',   '1987-06-17', 20200101\nput 'users', 'lisi', 'info:company',    'baidu1', 20200101\nput 'users', 'xiaoming', 'address:city',    'hangzhou1', 20200101\nput 'users', 'xiaoming', 'address:country', 'china1', 20200101\nput 'users', 'xiaoming', 'address:province',    'zhejiang1',20200101\nput 'users', 'xiaoming', 'info:age',        29, 20200101\nput 'users', 'xiaoming', 'info:birthday',   '1987-06-17',20200101\nput 'users', 'xiaoming', 'info:company',    'alibaba1', 20200101\nput 'users', 'lisi', 'address:country', 'china2', 20200102\nput 'users', 'lisi', 'address:province',    'beijing2', 20200102\nput 'users', 'lisi', 'info:age',        27, 20200102\nput 'users', 'lisi', 'info:birthday',   '1987-06-17', 20200102\nput 'users', 'lisi', 'info:company',    'baidu2', 20200102\nput 'users', 'xiaoming', 'address:city',    'hangzhou2', 20200102\nput 'users', 'xiaoming', 'address:country', 'china2', 20200102\nput 'users', 'xiaoming', 'address:province',    'zhejiang2', 20200102\nput 'users', 'xiaoming', 'info:age',        29, 20200102\nput 'users', 'xiaoming', 'info:birthday',   '1987-06-17', 20200102\nput 'users', 'xiaoming', 'info:company',    'alibaba2', 20200102\n
    "},{"location":"reader/hbase20xreader/#normal","title":"normal \u6a21\u5f0f","text":"

    \u628aHBase\u4e2d\u7684\u8868\uff0c\u5f53\u6210\u666e\u901a\u4e8c\u7ef4\u8868\uff08\u6a2a\u8868\uff09\u8fdb\u884c\u8bfb\u53d6,\u8bfb\u53d6\u6700\u65b0\u7248\u672c\u6570\u636e\u3002\u5982\uff1a

    hbase(main):017:0> scan 'users'\nROW           COLUMN+CELL\n lisi         column=address:city, timestamp=1457101972764, value=beijing\n lisi         column=address:country, timestamp=1457102773908, value=china\n lisi         column=address:province, timestamp=1457101972736, value=beijing\n lisi         column=info:age, timestamp=1457101972548, value=27\n lisi         column=info:birthday, timestamp=1457101972604, value=1987-06-17\n lisi         column=info:company, timestamp=1457101972653, value=baidu\n xiaoming     column=address:city, timestamp=1457082196082, value=hangzhou\n xiaoming     column=address:country, timestamp=1457082195729, value=china\n xiaoming     column=address:province, timestamp=1457082195773, value=zhejiang\n xiaoming     column=info:age, timestamp=1457082218735, value=29\n xiaoming     column=info:birthday, timestamp=1457082186830, value=1987-06-17\n xiaoming     column=info:company, timestamp=1457082189826, value=alibaba\n2 row(s) in 0.0580 seconds\n

    \u8bfb\u53d6\u540e\u6570\u636e

    rowKey addres:city address:country address:province info:age info:birthday info:company lisi beijing china beijing 27 1987-06-17 baidu xiaoming hangzhou china zhejiang 29 1987-06-17 alibaba"},{"location":"reader/hbase20xreader/#multiversionfixedcolumn","title":"multiVersionFixedColumn \u6a21\u5f0f","text":"

    \u628aHBase\u4e2d\u7684\u8868\uff0c\u5f53\u6210\u7ad6\u8868\u8fdb\u884c\u8bfb\u53d6\u3002\u8bfb\u51fa\u7684\u6bcf\u6761\u8bb0\u5f55\u4e00\u5b9a\u662f\u56db\u5217\u5f62\u5f0f\uff0c\u4f9d\u6b21\u4e3a\uff1arowKey\uff0cfamily:qualifier\uff0ctimestamp\uff0cvalue\u3002

    \u8bfb\u53d6\u65f6\u9700\u8981\u660e\u786e\u6307\u5b9a\u8981\u8bfb\u53d6\u7684\u5217\uff0c\u628a\u6bcf\u4e00\u4e2a cell \u4e2d\u7684\u503c\uff0c\u4f5c\u4e3a\u4e00\u6761\u8bb0\u5f55\uff08record\uff09\uff0c\u82e5\u6709\u591a\u4e2a\u7248\u672c\u5c31\u6709\u591a\u6761\u8bb0\u5f55\uff08record\uff09\u3002\u5982\uff1a

    hbase(main):018:0> scan 'users',{VERSIONS=>5}\nROW              COLUMN+CELL\n lisi            column=address:city, timestamp=1457101972764, value=beijing\n lisi            column=address:contry, timestamp=1457102773908, value=china\n lisi            column=address:province, timestamp=1457101972736, value=beijing\n lisi            column=info:age, timestamp=1457101972548, value=27\n lisi            column=info:birthday, timestamp=1457101972604, value=1987-06-17\n lisi            column=info:company, timestamp=1457101972653, value=baidu\n xiaoming        column=address:city, timestamp=1457082196082, value=hangzhou\n xiaoming        column=address:contry, timestamp=1457082195729, value=china\n xiaoming        column=address:province, timestamp=1457082195773, value=zhejiang\n xiaoming        column=info:age, timestamp=1457082218735, value=29\n xiaoming        column=info:age, timestamp=1457082178630, value=24\n xiaoming        column=info:birthday, timestamp=1457082186830, value=1987-06-17\n xiaoming        column=info:company, timestamp=1457082189826, value=alibaba\n2 row(s) in 0.0260 seconds\n

    \u8bfb\u53d6\u540e\u6570\u636e(4\u5217)

    rowKey column:qualifier timestamp value lisi address:city 1457101972764 beijing lisi address:contry 1457102773908 china lisi address:province 1457101972736 beijing lisi info:age 1457101972548 27 lisi info:birthday 1457101972604 1987-06-17 lisi info:company 1457101972653 beijing xiaoming address:city 1457082196082 hangzhou xiaoming address:contry 1457082195729 china xiaoming address:province 1457082195773 zhejiang xiaoming info:age 1457082218735 29 xiaoming info:age 1457082178630 24 xiaoming info:birthday 1457082186830 1987-06-17 xiaoming info:company 1457082189826 alibaba

    \u914d\u7f6e\u4e00\u4e2a\u4ece HBase \u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a\uff0c\u5206\u522b\u4e3a\u6807\u51c6\u6a21\u5f0f\u548c\u591a\u7248\u672c\u6a21\u5f0f

    hbase20x2stream_normal.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"hbase11xreader\",\n        \"parameter\": {\n          \"hbaseConfig\": {\n            \"hbase.zookeeper.quorum\": \"xxxf\"\n          },\n          \"table\": \"users\",\n          \"encoding\": \"utf-8\",\n          \"mode\": \"normal\",\n          \"column\": [\n            {\n              \"name\": \"rowkey\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: age\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: birthday\",\n              \"type\": \"date\",\n              \"format\": \"yyyy-MM-dd\"\n            },\n            {\n              \"name\": \"info: company\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: country\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: province\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: city\",\n              \"type\": \"string\"\n            }\n          ],\n          \"range\": {\n            \"startRowkey\": \"\",\n            \"endRowkey\": \"\",\n            \"isBinaryRowkey\": true\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"txtfilewriter\",\n        \"parameter\": {\n          \"path\": \"/Users/shf/workplace/addax_test/hbase11xreader/result\",\n          \"fileName\": \"qiran\",\n          \"writeMode\": \"truncate\"\n        }\n      }\n    }\n  }\n}\n
    hbase20x2stream_version.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"hbase11xreader\",\n        \"parameter\": {\n          \"hbaseConfig\": {\n            \"hbase.zookeeper.quorum\": \"xxx\"\n          },\n          \"table\": \"users\",\n          \"encoding\": \"utf-8\",\n          \"mode\": \"multiVersionFixedColumn\",\n          \"maxVersion\": \"-1\",\n          \"column\": [\n            {\n              \"name\": \"rowkey\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: age\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: birthday\",\n              \"type\": \"date\",\n              \"format\": \"yyyy-MM-dd\"\n            },\n            {\n              \"name\": \"info: company\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: contry\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: province\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: city\",\n              \"type\": \"string\"\n            }\n          ],\n          \"range\": {\n            \"startRowkey\": \"\",\n            \"endRowkey\": \"\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"txtfilewriter\",\n        \"parameter\": {\n          \"path\": \"/Users/shf/workplace/addax_test/hbase11xreader/result\",\n          \"fileName\": \"qiran\",\n          \"writeMode\": \"truncate\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/hbase20xreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 hbaseConfig \u662f map \u65e0 \u8fde\u63a5HBase\u96c6\u7fa4\u9700\u8981\u7684\u914d\u7f6e\u4fe1\u606f, hbase.zookeeper.quorum \u4e3a\u5fc5\u586b\u9879\uff0c\u5176\u4ed6\u4e3a\u53ef\u9009\u9879 mode \u662f string \u65e0 \u8bfb\u53d6hbase\u7684\u6a21\u5f0f\uff0c\u53ef\u586b\u5199 normal \u6216 multiVersionFixedColumn table \u662f string \u65e0 \u8981\u8bfb\u53d6\u7684 hbase \u8868\u540d\uff08\u5927\u5c0f\u5199\u654f\u611f\uff09 encoding \u5426 string UTF-8 \u7f16\u7801\u65b9\u5f0f\uff0cUTF-8 \u6216\u662f GBK\uff0c\u7528\u4e8e\u5bf9\u4e8c\u8fdb\u5236\u5b58\u50a8\u7684 HBase byte[] \u8f6c\u4e3a String \u65f6\u7684\u7f16\u7801 column \u662f list<map> \u65e0 \u8981\u8bfb\u53d6\u7684\u5b57\u6bb5\uff0cnormal \u6a21\u5f0f\u4e0e multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b\u9879, \u8be6\u7ec6\u8bf4\u660e\u89c1\u4e0b\u6587 maxVersion \u662f string \u65e0 \u6307\u5b9a\u5728\u591a\u7248\u672c\u6a21\u5f0f\u4e0b\u8bfb\u53d6\u7684\u7248\u672c\u6570\uff0c-1 \u8868\u793a\u8bfb\u53d6\u6240\u6709\u7248\u672c, multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b range \u5426 string \u65e0 \u6307\u5b9a\u8bfb\u53d6\u7684 rowkey \u8303\u56f4, \u8be6\u89c1\u4e0b\u6587 scanCacheSize \u5426 int 256 \u6bcf\u6b21\u4ece\u670d\u52a1\u5668\u7aef\u8bfb\u53d6\u7684\u884c\u6570 scanBatchSize \u5426 int 100 \u6bcf\u6b21\u4ece\u670d\u52a1\u5668\u7aef\u8bfb\u53d6\u7684\u5217\u6570"},{"location":"reader/hbase20xreader/#column","title":"column","text":"

    \u63cf\u8ff0\uff1a\u8981\u8bfb\u53d6\u7684\u5b57\u6bb5\uff0cnormal \u6a21\u5f0f\u4e0e multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b\u9879\u3002

    "},{"location":"reader/hbase20xreader/#normal_1","title":"normal \u6a21\u5f0f","text":"

    name \u6307\u5b9a\u8bfb\u53d6\u7684 hbase \u5217\uff0c\u9664\u4e86 rowkey \u5916\uff0c\u5fc5\u987b\u4e3a \u5217\u65cf:\u5217\u540d \u7684\u683c\u5f0f\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cformat\u6307\u5b9a\u65e5\u671f\u7c7b\u578b\u7684\u683c\u5f0f\uff0c value \u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece hbase \u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636e value \u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002\u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"column\": [\n    {\n      \"name\": \"rowkey\",\n      \"type\": \"string\"\n    },\n    {\n      \"value\": \"test\",\n      \"type\": \"string\"\n    }\n  ]\n}\n

    normal \u6a21\u5f0f\u4e0b\uff0c\u5bf9\u4e8e\u7528\u6237\u6307\u5b9aColumn\u4fe1\u606f\uff0ctype\u5fc5\u987b\u586b\u5199\uff0cname/value\u5fc5\u987b\u9009\u62e9\u5176\u4e00\u3002

    "},{"location":"reader/hbase20xreader/#multiversionfixedcolumn_1","title":"multiVersionFixedColumn \u6a21\u5f0f","text":"

    name \u6307\u5b9a\u8bfb\u53d6\u7684 hbase \u5217\uff0c\u9664\u4e86 rowkey \u5916\uff0c\u5fc5\u987b\u4e3a \u5217\u65cf:\u5217\u540d \u7684\u683c\u5f0f\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cformat\u6307\u5b9a\u65e5\u671f\u7c7b\u578b\u7684\u683c\u5f0f \u3002 multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u4e0d\u652f\u6301\u5e38\u91cf\u5217\u3002 \u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"mode\": \"multiVersionFixedColumn\",\n  \"maxVersion\": 3,\n  \"column\": [\n    {\n      \"name\": \"rowkey\",\n      \"type\": \"string\"\n    },\n    {\n      \"name\": \"info: age\",\n      \"type\": \"string\"\n    }\n  ]\n}\n
    "},{"location":"reader/hbase20xreader/#range","title":"range","text":"

    \u6307\u5b9a\u8bfb\u53d6\u7684 rowkey \u8303\u56f4

    \u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"range\": {\n    \"startRowkey\": \"aaa\",\n    \"endRowkey\": \"ccc\",\n    \"isBinaryRowkey\": false\n  }\n}\n
    "},{"location":"reader/hbase20xreader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u4e0b\u9762\u5217\u51fa\u652f\u6301\u7684\u8bfb\u53d6HBase\u6570\u636e\u7c7b\u578b:

    Addax \u5185\u90e8\u7c7b\u578b HBase \u6570\u636e\u7c7b\u578b Long int, short ,long Double float, double String string, binarystring Date date Boolean boolean

    \u8bf7\u6ce8\u610f:

    \u9664\u4e0a\u8ff0\u7f57\u5217\u5b57\u6bb5\u7c7b\u578b\u5916\uff0c\u5176\u4ed6\u7c7b\u578b\u5747\u4e0d\u652f\u6301

    "},{"location":"reader/hbase20xreader/#_4","title":"\u9650\u5236","text":"
    1. \u76ee\u524d\u4e0d\u652f\u6301\u52a8\u6001\u5217\u7684\u8bfb\u53d6\u3002\u8003\u8651\u7f51\u7edc\u4f20\u8f93\u6d41\u91cf\uff08\u652f\u6301\u52a8\u6001\u5217\uff0c\u9700\u8981\u5148\u5c06hbase\u6240\u6709\u5217\u7684\u6570\u636e\u8bfb\u53d6\u51fa\u6765\uff0c\u518d\u6309\u89c4\u5219\u8fdb\u884c\u8fc7\u6ee4\uff09\uff0c\u73b0\u652f\u6301\u7684\u4e24\u79cd\u8bfb\u53d6\u6a21\u5f0f\u4e2d\u9700\u8981\u7528\u6237\u660e\u786e\u6307\u5b9a\u8981\u8bfb\u53d6\u7684\u5217\u3002
    2. \u5173\u4e8e\u540c\u6b65\u4f5c\u4e1a\u7684\u5207\u5206\uff1a\u76ee\u524d\u7684\u5207\u5206\u65b9\u5f0f\u662f\u6839\u636e\u7528\u6237hbase\u8868\u6570\u636e\u7684region\u5206\u5e03\u8fdb\u884c\u5207\u5206\u3002\u5373\uff1a\u5728\u7528\u6237\u586b\u5199\u7684 [startrowkey\uff0cendrowkey] \u8303\u56f4\u5185\uff0c\u4e00\u4e2aregion\u4f1a\u5207\u5206\u6210\u4e00\u4e2atask\uff0c\u5355\u4e2aregion\u4e0d\u8fdb\u884c\u5207\u5206\u3002
    3. multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u4e0d\u652f\u6301\u589e\u52a0\u5e38\u91cf\u5217
    "},{"location":"reader/hbase20xsqlreader/","title":"HBase20 SQL Reader","text":"

    HBase20 SQL Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Phoenix(HBase SQL) \u8bfb\u53d6\u6570\u636e\uff0c\u5bf9\u5e94\u7248\u672c\u4e3a HBase2.X \u548c Phoenix5.X\u3002

    "},{"location":"reader/hbase20xsqlreader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4ece Phoenix \u540c\u6b65\u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a:

    {\n  \"job\": {\n    \"content\": [\n      {\n        \"reader\": {\n          \"name\": \"hbase20xsqlreader\",\n          \"parameter\": {\n            \"queryServerAddress\": \"http://127.0.0.1:8765\",\n            \"serialization\": \"PROTOBUF\",\n            \"table\": \"TEST\",\n            \"column\": [\"ID\", \"NAME\"],\n            \"splitKey\": \"ID\"\n          }\n        },\n        \"writer\": {\n          \"name\": \"streamwriter\",\n          \"parameter\": {\n            \"encoding\": \"UTF-8\",\n            \"print\": true\n          }\n        }\n      }\n    ],\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3,\n        \"bytes\": -1\n      }\n    }\n  }\n}\n
    "},{"location":"reader/hbase20xsqlreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 queryServerAddress \u662f string \u65e0 Phoenix QueryServer \u5730\u5740, \u8be5\u63d2\u4ef6\u901a\u8fc7 PQS \u8fdb\u884c\u8fde\u63a5 serialization \u5426 string PROTOBUF QueryServer \u4f7f\u7528\u7684\u5e8f\u5217\u5316\u534f\u8bae table \u662f string \u65e0 \u6240\u8981\u8bfb\u53d6\u8868\u540d schema \u5426 string \u65e0 \u8868\u6240\u5728\u7684 schema column \u5426 list `` \u586b\u5199\u9700\u8981\u4ece phoenix \u8868\u4e2d\u8bfb\u53d6\u7684\u5217\u540d\u96c6\u5408\uff0c\u7a7a\u503c\u8868\u793a\u8bfb\u53d6\u6240\u6709\u5217 splitKey \u662f string \u65e0 \u6839\u636e\u6570\u636e\u7279\u5f81\u52a8\u6001\u6307\u5b9a\u5207\u5206\u70b9\uff0c\u5bf9\u8868\u6570\u636e\u6309\u7167\u6307\u5b9a\u7684\u5217\u7684\u6700\u5927\u3001\u6700\u5c0f\u503c\u8fdb\u884c\u5207\u5206,\u4ec5\u652f\u6301\u6574\u578b\u548c\u5b57\u7b26\u4e32\u7c7b\u578b splitPoints \u5426 string \u65e0 \u6309\u7167\u8868\u7684 split \u8fdb\u884c\u5207\u5206 where \u5426 string \u65e0 \u652f\u6301\u5bf9\u8868\u67e5\u8be2\u589e\u52a0\u8fc7\u6ee4\u6761\u4ef6\uff0c\u6bcf\u4e2a\u5207\u5206\u90fd\u4f1a\u643a\u5e26\u8be5\u8fc7\u6ee4\u6761\u4ef6 querySql \u5426 string \u65e0 \u652f\u6301\u6307\u5b9a\u591a\u4e2a\u67e5\u8be2\u8bed\u53e5\uff0c\u4f46\u67e5\u8be2\u5217\u7c7b\u578b\u548c\u6570\u76ee\u5fc5\u987b\u4fdd\u6301\u4e00\u81f4"},{"location":"reader/hbase20xsqlreader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u76ee\u524d\u652f\u6301\u5927\u90e8\u5206 Phoenix \u7c7b\u578b\uff0c\u4f46\u4e5f\u5b58\u5728\u90e8\u5206\u4e2a\u522b\u7c7b\u578b\u6ca1\u6709\u652f\u6301\u7684\u60c5\u51b5\uff0c\u8bf7\u6ce8\u610f\u68c0\u67e5\u4f60\u7684\u7c7b\u578b\u3002

    Addax \u5185\u90e8\u7c7b\u578b Phoenix \u6570\u636e\u7c7b\u578b String CHAR, VARCHAR Bytes BINARY, VARBINARY Bool BOOLEAN Long INTEGER, TINYINT, SMALLINT, BIGINT Double FLOAT, DECIMAL, DOUBLE, Date DATE, TIME, TIMESTAMP"},{"location":"reader/hbase20xsqlreader/#_4","title":"\u7ea6\u675f\u9650\u5236","text":""},{"location":"reader/hdfsreader/","title":"HDFS Reader","text":"

    HDFS Reader \u63d0\u4f9b\u4e86\u8bfb\u53d6\u5206\u5e03\u5f0f\u6587\u4ef6\u7cfb\u7edf Hadoop HDFS \u6570\u636e\u5b58\u50a8\u7684\u80fd\u529b\u3002

    \u76ee\u524d HdfsReader \u652f\u6301\u7684\u6587\u4ef6\u683c\u5f0f\u5982\u4e0b\uff1a

    "},{"location":"reader/hdfsreader/#_1","title":"\u529f\u80fd\u4e0e\u9650\u5236","text":"
    1. \u652f\u6301 textfile, orcfile, parquet, rcfile, sequence file \u548c csv \u683c\u5f0f\u7684\u6587\u4ef6\uff0c\u4e14\u8981\u6c42\u6587\u4ef6\u5185\u5bb9\u5b58\u653e\u7684\u662f\u4e00\u5f20\u903b\u8f91\u610f\u4e49\u4e0a\u7684\u4e8c\u7ef4\u8868\u3002
    2. \u652f\u6301\u591a\u79cd\u7c7b\u578b\u6570\u636e\u8bfb\u53d6(\u4f7f\u7528 String \u8868\u793a)\uff0c\u652f\u6301\u5217\u88c1\u526a\uff0c\u652f\u6301\u5217\u5e38\u91cf
    3. \u652f\u6301\u9012\u5f52\u8bfb\u53d6\u3001\u652f\u6301\u6b63\u5219\u8868\u8fbe\u5f0f\uff08*\u548c ?\uff09\u3002
    4. \u652f\u6301\u5e38\u89c1\u7684\u538b\u7f29\u7b97\u6cd5\uff0c\u5305\u62ec GZIP\uff0c SNAPPY\uff0c ZLIB \u7b49\u3002
    5. \u591a\u4e2a File \u53ef\u4ee5\u652f\u6301\u5e76\u53d1\u8bfb\u53d6\u3002
    6. \u652f\u6301 sequence file \u6570\u636e\u538b\u7f29\uff0c\u76ee\u524d\u652f\u6301 lzo \u538b\u7f29\u65b9\u5f0f\u3002
    7. csv \u7c7b\u578b\u652f\u6301\u538b\u7f29\u683c\u5f0f\u6709\uff1agzip\u3001bz2\u3001zip\u3001lzo\u3001lzo_deflate\u3001snappy\u3002
    8. \u76ee\u524d\u63d2\u4ef6\u4e2d Hive \u7248\u672c\u4e3a 3.1.1\uff0cHadoop \u7248\u672c\u4e3a3.1.1, \u5728 Hadoop 2.7.x, Hadoop 3.1.x \u548c Hive 2.x, hive 3.1.x \u6d4b\u8bd5\u73af\u5883\u4e2d\u5199\u5165\u6b63\u5e38\uff1b\u5176\u5b83\u7248\u672c\u7406\u8bba\u4e0a\u90fd\u652f\u6301\uff0c\u4f46\u5728\u751f\u4ea7\u73af\u5883\u4f7f\u7528\u524d\uff0c\u8bf7\u8fdb\u4e00\u6b65\u6d4b\u8bd5\uff1b
    9. \u652f\u6301kerberos \u8ba4\u8bc1
    "},{"location":"reader/hdfsreader/#_2","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"hdfsreader\",\n        \"parameter\": {\n          \"path\": \"/user/hive/warehouse/mytable01/*\",\n          \"defaultFS\": \"hdfs://xxx:port\",\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"long\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"boolean\"\n            },\n            {\n              \"type\": \"string\",\n              \"value\": \"hello\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"double\"\n            }\n          ],\n          \"fileType\": \"orc\",\n          \"encoding\": \"UTF-8\",\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/hdfsreader/#_3","title":"\u914d\u7f6e\u9879\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e path \u662f string \u65e0 \u8981\u8bfb\u53d6\u7684\u6587\u4ef6\u8def\u5f84 defaultFS \u662f string \u65e0 HDFS NAMENODE \u8282\u70b9\u5730\u5740\uff0c\u5982\u679c\u914d\u7f6e\u4e86 HA \u6a21\u5f0f\uff0c\u5219\u4e3a defaultFS \u7684\u503c fileType \u662f string \u65e0 \u6587\u4ef6\u7684\u7c7b\u578b column \u662f list<map> \u65e0 \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868 fieldDelimiter \u5426 char , \u6307\u5b9a\u6587\u672c\u6587\u4ef6\u7684\u5b57\u6bb5\u5206\u9694\u7b26\uff0c\u4e8c\u8fdb\u5236\u6587\u4ef6\u4e0d\u9700\u8981\u6307\u5b9a\u8be5\u9879 encoding \u5426 string utf-8 \u8bfb\u53d6\u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e\uff0c \u76ee\u524d\u4ec5\u652f\u6301 utf-8 nullFormat \u5426 string \u65e0 \u53ef\u4ee5\u8868\u793a\u4e3a\u7a7a\u7684\u5b57\u7b26\uff0c\u5982\u679c\u7528\u6237\u914d\u7f6e: \"\\\\N\" \uff0c\u90a3\u4e48\u5982\u679c\u6e90\u5934\u6570\u636e\u662f \"\\N\" \uff0c\u89c6\u4f5c null \u5b57\u6bb5 haveKerberos \u5426 boolean \u65e0 \u662f\u5426\u542f\u7528 Kerberos \u8ba4\u8bc1\uff0c\u5982\u679c\u542f\u7528\uff0c\u5219\u9700\u8981\u540c\u65f6\u914d\u7f6e\u4e0b\u9762\u4e24\u9879 kerberosKeytabFilePath \u5426 string \u65e0 Kerberos \u8ba4\u8bc1\u7684\u51ed\u8bc1\u6587\u4ef6\u8def\u5f84, \u6bd4\u5982 /your/path/addax.service.keytab kerberosPrincipal \u5426 string \u65e0 Kerberos \u8ba4\u8bc1\u7684\u51ed\u8bc1\u4e3b\u4f53, \u6bd4\u5982 addax/node1@WGZHAO.COM compress \u5426 string \u65e0 \u6307\u5b9a\u8981\u8bfb\u53d6\u7684\u6587\u4ef6\u7684\u538b\u7f29\u683c\u5f0f hadoopConfig \u5426 map \u65e0 \u91cc\u53ef\u4ee5\u914d\u7f6e\u4e0e Hadoop \u76f8\u5173\u7684\u4e00\u4e9b\u9ad8\u7ea7\u53c2\u6570\uff0c\u6bd4\u5982 HA \u7684\u914d\u7f6e"},{"location":"reader/hdfsreader/#path","title":"path","text":"

    \u8981\u8bfb\u53d6\u7684\u6587\u4ef6\u8def\u5f84\uff0c\u5982\u679c\u8981\u8bfb\u53d6\u591a\u4e2a\u6587\u4ef6\uff0c\u53ef\u4ee5\u4f7f\u7528\u6b63\u5219\u8868\u8fbe\u5f0f *\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84\uff1a

    1. \u5f53\u6307\u5b9a\u5355\u4e2a Hdfs \u6587\u4ef6\uff0cHdfsReader \u6682\u65f6\u53ea\u80fd\u4f7f\u7528\u5355\u7ebf\u7a0b\u8fdb\u884c\u6570\u636e\u62bd\u53d6\u3002\u4e8c\u671f\u8003\u8651\u5728\u975e\u538b\u7f29\u6587\u4ef6\u60c5\u51b5\u4e0b\u9488\u5bf9\u5355\u4e2a File \u53ef\u4ee5\u8fdb\u884c\u591a\u7ebf\u7a0b\u5e76\u53d1\u8bfb\u53d6\u3002
    2. \u5f53\u6307\u5b9a\u591a\u4e2a Hdfs \u6587\u4ef6\uff0cHdfsReader \u652f\u6301\u4f7f\u7528\u591a\u7ebf\u7a0b\u8fdb\u884c\u6570\u636e\u62bd\u53d6\u3002\u7ebf\u7a0b\u5e76\u53d1\u6570\u901a\u8fc7\u901a\u9053\u6570\u6307\u5b9a\u3002
    3. \u5f53\u6307\u5b9a\u901a\u914d\u7b26\uff0cHdfsReader \u5c1d\u8bd5\u904d\u5386\u51fa\u591a\u4e2a\u6587\u4ef6\u4fe1\u606f\u3002\u4f8b\u5982: \u6307\u5b9a /* \u4ee3\u8868\u8bfb\u53d6 / \u76ee\u5f55\u4e0b\u6240\u6709\u7684\u6587\u4ef6\uff0c\u6307\u5b9a /bazhen/* \u4ee3\u8868\u8bfb\u53d6 bazhen \u76ee\u5f55\u4e0b\u6e38\u6240\u6709\u7684\u6587\u4ef6\u3002HdfsReader \u76ee\u524d\u53ea\u652f\u6301 *\u548c ? \u4f5c\u4e3a\u6587\u4ef6\u901a\u914d\u7b26\u3002

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0cAddax \u4f1a\u5c06\u4e00\u4e2a\u4f5c\u4e1a\u4e0b\u540c\u6b65\u7684\u6240\u6709\u7684\u6587\u4ef6\u89c6\u4f5c\u540c\u4e00\u5f20\u6570\u636e\u8868\u3002\u7528\u6237\u5fc5\u987b\u81ea\u5df1\u4fdd\u8bc1\u6240\u6709\u7684 File \u80fd\u591f\u9002\u914d\u540c\u4e00\u5957 schema \u4fe1\u606f\u3002\u5e76\u4e14\u63d0\u4f9b\u7ed9 Addax \u6743\u9650\u53ef\u8bfb\u3002

    "},{"location":"reader/hdfsreader/#filetype","title":"fileType","text":"

    \u63cf\u8ff0\uff1a\u6587\u4ef6\u7684\u7c7b\u578b\uff0c\u76ee\u524d\u53ea\u652f\u6301\u7528\u6237\u914d\u7f6e\u4e3a

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0cHdfsReader \u80fd\u591f\u81ea\u52a8\u8bc6\u522b\u6587\u4ef6\u662f orcfile\u3001textfile \u6216\u8005\u8fd8\u662f\u5176\u5b83\u7c7b\u578b\u7684\u6587\u4ef6\uff0c\u4f46\u8be5\u9879\u662f\u5fc5\u586b\u9879\uff0cHdfsReader \u5219\u4f1a\u53ea\u8bfb\u53d6\u7528\u6237\u914d\u7f6e\u7684\u7c7b\u578b\u7684\u6587\u4ef6\uff0c\u5ffd\u7565\u8def\u5f84\u4e0b\u5176\u4ed6\u683c\u5f0f\u7684\u6587\u4ef6

    \u53e6\u5916\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0c\u7531\u4e8e textfile \u548c orcfile \u662f\u4e24\u79cd\u5b8c\u5168\u4e0d\u540c\u7684\u6587\u4ef6\u683c\u5f0f\uff0c\u6240\u4ee5 HdfsReader \u5bf9\u8fd9\u4e24\u79cd\u6587\u4ef6\u7684\u89e3\u6790\u65b9\u5f0f\u4e5f\u5b58\u5728\u5dee\u5f02\uff0c\u8fd9\u79cd\u5dee\u5f02\u5bfc\u81f4 hive \u652f\u6301\u7684\u590d\u6742\u590d\u5408\u7c7b\u578b(\u6bd4\u5982 map,array,struct,union)\u5728\u8f6c\u6362\u4e3a\u652f\u6301\u7684 String \u7c7b\u578b\u65f6\uff0c\u8f6c\u6362\u7684\u7ed3\u679c\u683c\u5f0f\u7565\u6709\u5dee\u5f02\uff0c\u6bd4\u5982\u4ee5 map \u7c7b\u578b\u4e3a\u4f8b\uff1a

    \u4ece\u4e0a\u9762\u7684\u8f6c\u6362\u7ed3\u679c\u53ef\u4ee5\u770b\u51fa\uff0c\u6570\u636e\u672c\u8eab\u6ca1\u6709\u53d8\u5316\uff0c\u4f46\u662f\u8868\u793a\u7684\u683c\u5f0f\u7565\u6709\u5dee\u5f02\uff0c\u6240\u4ee5\u5982\u679c\u7528\u6237\u914d\u7f6e\u7684\u6587\u4ef6\u8def\u5f84\u4e2d\u8981\u540c\u6b65\u7684\u5b57\u6bb5\u5728 Hive \u4e2d\u662f\u590d\u5408\u7c7b\u578b\u7684\u8bdd\uff0c\u5efa\u8bae\u914d\u7f6e\u7edf\u4e00\u7684\u6587\u4ef6\u683c\u5f0f\u3002

    \u5982\u679c\u9700\u8981\u7edf\u4e00\u590d\u5408\u7c7b\u578b\u89e3\u6790\u51fa\u6765\u7684\u683c\u5f0f\uff0c\u6211\u4eec\u5efa\u8bae\u7528\u6237\u5728 hive \u5ba2\u6237\u7aef\u5c06 textfile \u683c\u5f0f\u7684\u8868\u5bfc\u6210 orcfile \u683c\u5f0f\u7684\u8868

    "},{"location":"reader/hdfsreader/#column","title":"column","text":"

    \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cindex \u6307\u5b9a\u5f53\u524d\u5217\u6765\u81ea\u4e8e\u6587\u672c\u7b2c\u51e0\u5217(\u4ee5 0 \u5f00\u59cb)\uff0cvalue \u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece\u6e90\u5934\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636e value \u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u7528\u6237\u53ef\u4ee5\u5168\u90e8\u6309\u7167 String \u7c7b\u578b\u8bfb\u53d6\u6570\u636e\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"column\": [\"*\"]\n}\n

    \u7528\u6237\u53ef\u4ee5\u6307\u5b9a Column \u5b57\u6bb5\u4fe1\u606f\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    [\n  {\n    \"type\": \"long\",\n    \"index\": 0,\n    \"description\": \"\u4ece\u672c\u5730\u6587\u4ef6\u6587\u672c\u7b2c\u4e00\u5217\u83b7\u53d6int\u5b57\u6bb5\"\n  },\n  {\n    \"type\": \"string\",\n    \"value\": \"addax\",\n    \"description\": \"HdfsReader\u5185\u90e8\u751f\u6210alibaba\u7684\u5b57\u7b26\u4e32\u5b57\u6bb5\u4f5c\u4e3a\u5f53\u524d\u5b57\u6bb5\"\n  }\n]\n

    \u5bf9\u4e8e\u7528\u6237\u6307\u5b9a Column \u4fe1\u606f\uff0ctype \u5fc5\u987b\u586b\u5199\uff0cindex/value \u5fc5\u987b\u9009\u62e9\u5176\u4e00\u3002

    "},{"location":"reader/hdfsreader/#compress","title":"compress","text":"

    \u5f53 fileType\uff08\u6587\u4ef6\u7c7b\u578b\uff09\u4e3a csv \u4e0b\u7684\u6587\u4ef6\u538b\u7f29\u65b9\u5f0f\uff0c\u76ee\u524d\u4ec5\u652f\u6301 gzip\u3001bz2\u3001zip\u3001lzo\u3001lzo_deflate\u3001hadoop-snappy\u3001framing-snappy \u538b\u7f29\uff1b \u503c\u5f97\u6ce8\u610f\u7684\u662f\uff0clzo \u5b58\u5728\u4e24\u79cd\u538b\u7f29\u683c\u5f0f\uff1alzo \u548c lzo_deflate\uff0c\u7528\u6237\u5728\u914d\u7f6e\u7684\u65f6\u5019\u9700\u8981\u7559\u5fc3\uff0c\u4e0d\u8981\u914d\u9519\u4e86\uff1b

    \u53e6\u5916\uff0c\u7531\u4e8e snappy \u76ee\u524d\u6ca1\u6709\u7edf\u4e00\u7684 stream format\uff0caddax \u76ee\u524d\u53ea\u652f\u6301\u6700\u4e3b\u6d41\u7684\u4e24\u79cd\uff1ahadoop-snappy\uff08hadoop \u4e0a\u7684 snappy stream format\uff09 \u548c framing-snappy\uff08google \u5efa\u8bae\u7684 snappy stream format\uff09;

    "},{"location":"reader/hdfsreader/#hadoopconfig","title":"hadoopConfig","text":"

    hadoopConfig \u91cc\u53ef\u4ee5\u914d\u7f6e\u4e0e Hadoop \u76f8\u5173\u7684\u4e00\u4e9b\u9ad8\u7ea7\u53c2\u6570\uff0c\u6bd4\u5982 HA \u7684\u914d\u7f6e

    {\n  \"hadoopConfig\": {\n    \"dfs.nameservices\": \"cluster\",\n    \"dfs.ha.namenodes.cluster\": \"nn1,nn2\",\n    \"dfs.namenode.rpc-address.cluster.nn1\": \"node1.example.com:8020\",\n    \"dfs.namenode.rpc-address.cluster.nn2\": \"node2.example.com:8020\",\n    \"dfs.client.failover.proxy.provider.cluster\": \"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\"\n  }\n}\n

    \u8fd9\u91cc\u7684 cluster \u8868\u793a HDFS \u914d\u7f6e\u6210 HA \u65f6\u7684\u540d\u5b57\uff0c\u4e5f\u662f defaultFS \u914d\u7f6e\u9879\u4e2d\u7684\u540d\u5b57 \u5982\u679c\u5b9e\u9645\u73af\u5883\u4e2d\u7684\u540d\u5b57\u4e0d\u662f cluster \uff0c\u5219\u4e0a\u8ff0\u914d\u7f6e\u4e2d\u6240\u6709\u5199\u6709 cluster \u90fd\u9700\u8981\u66ff\u6362

    "},{"location":"reader/hdfsreader/#csvreaderconfig","title":"csvReaderConfig","text":"

    \u8bfb\u53d6 CSV \u7c7b\u578b\u6587\u4ef6\u53c2\u6570\u914d\u7f6e\uff0cMap \u7c7b\u578b\u3002\u8bfb\u53d6 CSV \u7c7b\u578b\u6587\u4ef6\u4f7f\u7528\u7684 CsvReader \u8fdb\u884c\u8bfb\u53d6\uff0c\u4f1a\u6709\u5f88\u591a\u914d\u7f6e\uff0c\u4e0d\u914d\u7f6e\u5219\u4f7f\u7528\u9ed8\u8ba4\u503c\u3002

    \u5e38\u89c1\u914d\u7f6e\uff1a

    {\n  \"csvReaderConfig\": {\n    \"safetySwitch\": false,\n    \"skipEmptyRecords\": false,\n    \"useTextQualifier\": false\n  }\n}\n

    \u6240\u6709\u914d\u7f6e\u9879\u53ca\u9ed8\u8ba4\u503c,\u914d\u7f6e\u65f6 csvReaderConfig \u7684 map \u4e2d\u8bf7 \u4e25\u683c\u6309\u7167\u4ee5\u4e0b\u5b57\u6bb5\u540d\u5b57\u8fdb\u884c\u914d\u7f6e\uff1a

    boolean caseSensitive = true;\nchar textQualifier = 34;\nboolean trimWhitespace = true;\nboolean useTextQualifier = true;//\u662f\u5426\u4f7f\u7528csv\u8f6c\u4e49\u5b57\u7b26\nchar delimiter = 44;//\u5206\u9694\u7b26\nchar recordDelimiter = 0;\nchar comment = 35;\nboolean useComments = false;\nint escapeMode = 1;\nboolean safetySwitch = true;//\u5355\u5217\u957f\u5ea6\u662f\u5426\u9650\u5236100000\u5b57\u7b26\nboolean skipEmptyRecords = true;//\u662f\u5426\u8df3\u8fc7\u7a7a\u884c\nboolean captureRawRecord = true;\n
    "},{"location":"reader/hdfsreader/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b Hive \u8868 \u6570\u636e\u7c7b\u578b Long TINYINT, SMALLINT, INT, BIGINT Double FLOAT, DOUBLE String String, CHAR, VARCHAR, STRUCT, MAP, ARRAY, UNION, BINARY Boolean BOOLEAN Date Date, TIMESTAMP Bytes BINARY

    \u5176\u4e2d\uff1a

    \u7279\u522b\u63d0\u9192\uff1a

    "},{"location":"reader/hdfsreader/#faq","title":"FAQ","text":"

    Q: \u5982\u679c\u62a5 java.io.IOException: Maximum column length of 100,000 exceeded in column...\u5f02\u5e38\u4fe1\u606f\uff0c\u8bf4\u660e\u6570\u636e\u6e90 column \u5b57\u6bb5\u957f\u5ea6\u8d85\u8fc7\u4e86 100000 \u5b57\u7b26\u3002

    A: \u9700\u8981\u5728 json \u7684 reader \u91cc\u589e\u52a0\u5982\u4e0b\u914d\u7f6e

    {\n  \"csvReaderConfig\": {\n    \"safetySwitch\": false,\n    \"skipEmptyRecords\": false,\n    \"useTextQualifier\": false\n  }\n}\n

    safetySwitch = false \u8868\u793a\u5355\u5217\u957f\u5ea6\u4e0d\u9650\u5236 100000 \u5b57\u7b26

    "},{"location":"reader/hivereader/","title":"Hive Reader","text":"

    Hive Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Apache Hive \u6570\u636e\u5e93\u8bfb\u53d6\u6570\u636e\u7684\u80fd\u529b

    \u65b0\u589e\u8be5\u63d2\u4ef6\u7684\u4e3b\u8981\u76ee\u7684\u662f\u89e3\u51b3\u4f7f\u7528 RDBMS Reader \u63d2\u4ef6\u8bfb\u53d6 Hive \u6570\u636e\u5e93\u65f6\u4e0d\u80fd\u89e3\u51b3 Kerberos \u8ba4\u8bc1\u7684\u95ee\u9898\uff0c \u5982\u679c\u4f60\u7684 Hive \u6570\u636e\u5e93\u6ca1\u6709\u542f\u7528 Kerberos \u8ba4\u8bc1\uff0c\u90a3\u4e48\u76f4\u63a5\u4f7f\u7528 RDBMS Reader \u4e5f\u53ef\u4ee5\u3002 \u5982\u679c\u542f\u7528\u4e86 Kerberos \u8ba4\u8bc1\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u8be5\u63d2\u4ef6\u3002

    "},{"location":"reader/hivereader/#_1","title":"\u793a\u4f8b","text":"

    \u6211\u4eec\u5728 Hive \u7684 test \u5e93\u4e0a\u521b\u5efa\u5982\u4e0b\u8868\uff0c\u5e76\u63d2\u5165\u4e00\u6761\u8bb0\u5f55

    create table default.hive_reader\n(\n    col1 int,\n    col2 string,\n    col3 timestamp\n)\nstored as orc;\n\n\ninsert into hive_reader values(1, 'hello', current_timestamp()), (2, 'world', current_timestamp());\n

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/hive2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"hivereader\",\n        \"parameter\": {\n          \"column\": [\n            \"*\"\n          ],\n          \"username\": \"hive\",\n          \"password\": \"\",\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:hive2://localhost:10000/default;principal=hive/_HOST@EXAMPLE.COM\",\n            \"table\": [\n              \"hive_reader\"\n            ]\n          },\n          \"where\": \"logdate='20211013'\",\n          \"haveKerberos\": true,\n          \"kerberosKeytabFilePath\": \"/etc/security/keytabs/hive.headless.keytab\",\n          \"kerberosPrincipal\": \"hive@EXAMPLE.COM\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/hive2stream.json

    "},{"location":"reader/hivereader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/hive2stream.json\n
    "},{"location":"reader/hivereader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 jdbcUrl \u662f list \u65e0 \u5bf9\u7aef\u6570\u636e\u5e93\u7684 JDBC \u8fde\u63a5\u4fe1\u606f driver \u5426 string \u65e0 \u81ea\u5b9a\u4e49\u9a71\u52a8\u7c7b\u540d\uff0c\u89e3\u51b3\u517c\u5bb9\u6027\u95ee\u9898\uff0c\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0 username \u662f string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801\uff0c\u82e5\u65e0\u5bc6\u7801\uff0c\u53ef\u4e0d\u6307\u5b9a table \u662f list \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d,\u4f7f\u7528 JSON \u6570\u636e\u683c\u5f0f column \u662f list<map> \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1 rdbmreader splitPk \u5426 string \u65e0 \u4f7f\u7528 splitPk \u4ee3\u8868\u7684\u5b57\u6bb5\u8fdb\u884c\u6570\u636e\u5206\u7247\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1 rdbmreader where \u5426 string \u65e0 \u9488\u5bf9\u8868\u7684\u7b5b\u9009\u6761\u4ef6 querySql \u5426 list \u65e0 \u4f7f\u7528 SQL \u6765\u83b7\u53d6\u6570\u636e\uff0c\u5f53\u914d\u7f6e\u4e86\u8fd9\u4e00\u9879\u4e4b\u540e\uff0c table\uff0ccolumn \u914d\u7f6e\u9879\u65e0\u6548 haveKerberos \u5426 string \u65e0 \u662f\u5426\u542f\u7528 Kerberos \u8ba4\u8bc1\uff0c\u5982\u679c\u542f\u7528\uff0c\u5219\u9700\u8981\u540c\u65f6\u914d\u7f6e\u4e0b\u9762\u4e24\u9879 kerberosKeytabFilePath \u5426 string \u65e0 Kerberos \u8ba4\u8bc1\u7684\u51ed\u8bc1\u6587\u4ef6\u8def\u5f84, \u6bd4\u5982 /your/path/addax.service.keytab kerberosPrincipal \u5426 string \u65e0 Kerberos \u8ba4\u8bc1\u7684\u51ed\u8bc1\u4e3b\u4f53, \u6bd4\u5982 addax/node1@EXAMPLE.COM"},{"location":"reader/hivereader/#jdbcurl","title":"jdbcUrl","text":"

    \u8fde\u63a5 Hive \u7684 JDBC URL \u6709\u591a\u79cd\u5199\u6cd5\uff0c\u4e00\u79cd\u662f\u76f4\u63a5\u6307\u5b9a HiveServer/HiveServer2 \u670d\u52a1\u7684\u4e3b\u673a\u540d\u548c\u7aef\u53e3\u5373\u53ef\uff0c\u6bd4\u5982\uff1a jdbc:hive2://node1:10000/default

    \u5982\u679c\u4f60\u6709\u591a\u4e2a HiveServer/HiveServer2 \u670d\u52a1\uff0c\u5e76\u91c7\u53d6\u7528\u4e86\u670d\u52a1\u53d1\u73b0\uff0c\u5219\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9a zookeeper \u7684\u65b9\u5f0f\u6765\u83b7\u5f97\u6545\u969c\u8f6c\u79fb\u529f\u80fd\uff0c\u7c7b\u4f3c\u5982\u4e0b\uff1a

    jdbc:hive2://node1:2181,node2:2181,node3:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2\n

    \u5982\u679c\u4f60\u7684 Hive \u542f\u7528\u4e86 Kerberos \u8ba4\u8bc1\uff0c\u8fd8\u9700\u8981\u5728 URL \u540e\u6307\u5b9a principal \u53c2\u6570\uff0c\u4e00\u822c\u4e3a principal=hive/_HOST@EXAMPLE.COM\uff0c\u5176\u4e2d EXAMPLE.COM \u4e3a realm \u503c\u3002

    "},{"location":"reader/hivereader/#driver","title":"driver","text":"

    \u5f53\u524d Addax \u91c7\u7528\u7684 Hive JDBC \u9a71\u52a8\u4e3a 3.1.0 \u4ee5\u4e0a\u7248\u672c\uff0c\u9a71\u52a8\u7c7b\u540d\u4f7f\u7528\u7684 org.apache.hive.jdbc.HiveDriver\uff0c \u5982\u679c\u5f53\u524d\u7684 Hive JDBC \u9a71\u52a8\u4e0d\u517c\u5bb9 Hive \u6570\u636e\u5e93\uff0c \u5219\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u6b65\u9aa4\u66ff\u6362\u9a71\u52a8\u3002

    \u66ff\u6362\u63d2\u4ef6\u5185\u7f6e\u7684\u9a71\u52a8

    rm -f plugin/reader/hivereader/libs/hive-jdbc-*.jar

    \u62f7\u8d1d\u517c\u5bb9\u9a71\u52a8\u5230\u63d2\u4ef6\u76ee\u5f55

    cp hive-jdbc-<version>.jar plugin/reader/hivereader/libs/

    \u6307\u5b9a\u9a71\u52a8\u7c7b\u540d\u79f0

    \u5728\u4f60\u7684 json \u6587\u4ef6\u7c7b\uff0c\u914d\u7f6e \"driver\": \"<your jdbc class name>\"

    "},{"location":"reader/hivereader/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u76ee\u524d HiveReader \u652f\u6301\u5927\u90e8\u5206 Hive \u7c7b\u578b\uff0c\u4f46\u4e5f\u5b58\u5728\u90e8\u5206\u4e2a\u522b\u7c7b\u578b\u6ca1\u6709\u652f\u6301\u7684\u60c5\u51b5\uff0c\u8bf7\u6ce8\u610f\u68c0\u67e5\u4f60\u7684\u7c7b\u578b\u3002

    \u4e0b\u9762\u5217\u51fa HiveReader \u9488\u5bf9 Hive \u7c7b\u578b\u8f6c\u6362\u5217\u8868:

    Addax \u5185\u90e8\u7c7b\u578b Hive \u6570\u636e\u7c7b\u578b Long int, tinyint, smallint, mediumint, int, bigint Double float, double, decimal String varchar, char, string Date date, timestamp Boolean boolean Bytes binary"},{"location":"reader/httpreader/","title":"HTTP Reader","text":"

    HTTP Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u8bfb\u53d6 Restful API \u6570\u636e\u7684\u80fd\u529b

    "},{"location":"reader/httpreader/#_1","title":"\u793a\u4f8b","text":""},{"location":"reader/httpreader/#_2","title":"\u793a\u4f8b\u63a5\u53e3\u4e0e\u6570\u636e","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u6f14\u793a\u4e86\u5982\u4f55\u4ece\u4e00\u4e2a\u6307\u5b9a\u7684 API \u4e2d\u83b7\u53d6\u6570\u636e\uff0c\u5047\u5b9a\u8bbf\u95ee\u7684\u63a5\u53e3\u4e3a\uff1a

    http://127.0.0.1:9090/mock/17/LDJSC/ASSET

    \u63a5\u53e3\u63a5\u53d7 GET \u8bf7\u6c42\uff0c\u8bf7\u6c42\u7684\u53c2\u6570\u6709

    \u53c2\u6570\u540d\u79f0 \u53c2\u6570\u503c\u793a\u4f8b CURR_DATE 2021-01-17 DEPT 9400 USERNAME andi

    \u4ee5\u4e0b\u662f\u8bbf\u95ee\u7684\u6570\u636e\u6837\u4f8b\uff0c\uff08\u5b9e\u9645\u8fd4\u56de\u6570\u636e\u7565\u6709\u4e0d\u540c\uff09

    {\n  \"result\": [\n    {\n      \"CURR_DATE\": \"2019-12-09\",\n      \"DEPT\": \"9700\",\n      \"TOTAL_MANAGED_MARKET_VALUE\": 1581.03,\n      \"TOTAL_MANAGED_MARKET_VALUE_GROWTH\": 36.75,\n      \"TMMARKET_VALUE_DOD_GROWTH_RATE\": -0.009448781026677719,\n      \"TMMARKET_VALUE_GROWTH_MON\": -0.015153586011995693,\n      \"TMMARKET_VALUE_GROWTH_YEAR\": 0.0652347643813081,\n      \"TMMARKET_VALUE_SHARECOM\": 0.024853621341525287,\n      \"TMMARKET_VALUE_SHARE_GROWTH_RATE\": -0.005242133578517903,\n      \"AVERAGE_NEW_ASSETS_DAYINMON\": 1645.1193961136973,\n      \"YEAR_NEW_ASSET_SSHARECOM\": 0.16690149257388515,\n      \"YN_ASSET_SSHARECOM_GROWTH_RATE\": 0.017886267801303465,\n      \"POTENTIAL_LOST_ASSETS\": 56.76,\n      \"TOTAL_LIABILITIES\": 57.81,\n      \"TOTAL_ASSETS\": 1306.33,\n      \"TOTAL_ASSETS_DOD_GROWTH\": 4.79,\n      \"TOTAL_ASSETS_DOD_GROWTH_RATE\": -0.006797058194980485,\n      \"NEW_ASSETS_DAY\": 14.92,\n      \"NEW_ASSETS_MON\": 90.29,\n      \"NEW_ASSETS_YEAR\": 297.32,\n      \"NEW_ASSETS_DOD_GROWTH_RATE\": -0.04015576541561927,\n      \"NEW_FUNDS_DAY\": 18.16,\n      \"INFLOW_FUNDS_DAY\": 2.12,\n      \"OUTFLOW_FUNDS_DAY\": 9.73,\n      \"OVERALL_POSITION\": 0.810298404938773,\n      \"OVERALL_POSITION_DOD_GROWTH_RATE\": -0.03521615634095476,\n      \"NEW_CUST_FUNDS_MON\": 69.44,\n      \"INFLOW_FUNDS_MONTH\": 62.26,\n      \"OUTFLOW_FUNDS_MONTH\": 32.59\n    },\n    {\n      \"CURR_DATE\": \"2019-08-30\",\n      \"DEPT\": \"8700\",\n      \"TOTAL_MANAGED_MARKET_VALUE\": 1596.74,\n      \"TOTAL_MANAGED_MARKET_VALUE_GROWTH\": 41.86,\n      \"TMMARKET_VALUE_DOD_GROWTH_RATE\": 0.03470208565515685,\n      \"TMMARKET_VALUE_GROWTH_MON\": 0.07818120801111743,\n      \"TMMARKET_VALUE_GROWTH_YEAR\": -0.05440250244736409,\n      \"TMMARKET_VALUE_SHARECOM\": 0.09997733019626448,\n      \"TMMARKET_VALUE_SHARE_GROWTH_RATE\": -0.019726478499825697,\n      \"AVERAGE_NEW_ASSETS_DAYINMON\": 1007.9314679742108,\n      \"YEAR_NEW_ASSET_SSHARECOM\": 0.15123738798885086,\n      \"YN_ASSET_SSHARECOM_GROWTH_RATE\": 0.04694052069678048,\n      \"POTENTIAL_LOST_ASSETS\": 52.48,\n      \"TOTAL_LIABILITIES\": 55.28,\n      \"TOTAL_ASSETS\": 1366.72,\n      \"TOTAL_ASSETS_DOD_GROWTH\": 10.12,\n      \"TOTAL_ASSETS_DOD_GROWTH_RATE\": 0.009708491982487952,\n      \"NEW_ASSETS_DAY\": 12.42,\n      \"NEW_ASSETS_MON\": 41.14,\n      \"NEW_ASSETS_YEAR\": 279.32,\n      \"NEW_ASSETS_DOD_GROWTH_RATE\": -0.025878627161898062,\n      \"NEW_FUNDS_DAY\": 3.65,\n      \"INFLOW_FUNDS_DAY\": 14.15,\n      \"OUTFLOW_FUNDS_DAY\": 17.08,\n      \"OVERALL_POSITION\": 0.9098432997243932,\n      \"OVERALL_POSITION_DOD_GROWTH_RATE\": 0.02111922282868306,\n      \"NEW_CUST_FUNDS_MON\": 57.21,\n      \"INFLOW_FUNDS_MONTH\": 61.16,\n      \"OUTFLOW_FUNDS_MONTH\": 15.83\n    },\n    {\n      \"CURR_DATE\": \"2019-06-30\",\n      \"DEPT\": \"6501\",\n      \"TOTAL_MANAGED_MARKET_VALUE\": 1506.72,\n      \"TOTAL_MANAGED_MARKET_VALUE_GROWTH\": -13.23,\n      \"TMMARKET_VALUE_DOD_GROWTH_RATE\": -0.0024973354204176554,\n      \"TMMARKET_VALUE_GROWTH_MON\": -0.015530793150701896,\n      \"TMMARKET_VALUE_GROWTH_YEAR\": -0.08556724628979398,\n      \"TMMARKET_VALUE_SHARECOM\": 0.15000077963967678,\n      \"TMMARKET_VALUE_SHARE_GROWTH_RATE\": -0.049629446804825755,\n      \"AVERAGE_NEW_ASSETS_DAYINMON\": 1250.1040863177336,\n      \"YEAR_NEW_ASSET_SSHARECOM\": 0.19098445630488178,\n      \"YN_ASSET_SSHARECOM_GROWTH_RATE\": -0.007881179708853471,\n      \"POTENTIAL_LOST_ASSETS\": 50.53,\n      \"TOTAL_LIABILITIES\": 56.62,\n      \"TOTAL_ASSETS\": 1499.53,\n      \"TOTAL_ASSETS_DOD_GROWTH\": 29.56,\n      \"TOTAL_ASSETS_DOD_GROWTH_RATE\": -0.02599813232345556,\n      \"NEW_ASSETS_DAY\": 28.81,\n      \"NEW_ASSETS_MON\": 123.24,\n      \"NEW_ASSETS_YEAR\": 263.63,\n      \"NEW_ASSETS_DOD_GROWTH_RATE\": 0.0073986669331394875,\n      \"NEW_FUNDS_DAY\": 18.52,\n      \"INFLOW_FUNDS_DAY\": 3.26,\n      \"OUTFLOW_FUNDS_DAY\": 6.92,\n      \"OVERALL_POSITION\": 0.8713692113306709,\n      \"OVERALL_POSITION_DOD_GROWTH_RATE\": 0.02977644553289545,\n      \"NEW_CUST_FUNDS_MON\": 85.14,\n      \"INFLOW_FUNDS_MONTH\": 23.35,\n      \"OUTFLOW_FUNDS_MONTH\": 92.95\n    },\n    {\n      \"CURR_DATE\": \"2019-12-07\",\n      \"DEPT\": \"8705\",\n      \"TOTAL_MANAGED_MARKET_VALUE\": 1575.85,\n      \"TOTAL_MANAGED_MARKET_VALUE_GROWTH\": 8.94,\n      \"TMMARKET_VALUE_DOD_GROWTH_RATE\": -0.04384846980627058,\n      \"TMMARKET_VALUE_GROWTH_MON\": -0.022962456288549656,\n      \"TMMARKET_VALUE_GROWTH_YEAR\": -0.005047009316021089,\n      \"TMMARKET_VALUE_SHARECOM\": 0.07819484815809447,\n      \"TMMARKET_VALUE_SHARE_GROWTH_RATE\": -0.008534369960890256,\n      \"AVERAGE_NEW_ASSETS_DAYINMON\": 1340.0339240689955,\n      \"YEAR_NEW_ASSET_SSHARECOM\": 0.19019952857677042,\n      \"YN_ASSET_SSHARECOM_GROWTH_RATE\": 0.01272353909992914,\n      \"POTENTIAL_LOST_ASSETS\": 54.63,\n      \"TOTAL_LIABILITIES\": 53.17,\n      \"TOTAL_ASSETS\": 1315.08,\n      \"TOTAL_ASSETS_DOD_GROWTH\": 49.31,\n      \"TOTAL_ASSETS_DOD_GROWTH_RATE\": 0.0016538407028265922,\n      \"NEW_ASSETS_DAY\": 29.17,\n      \"NEW_ASSETS_MON\": 44.75,\n      \"NEW_ASSETS_YEAR\": 172.87,\n      \"NEW_ASSETS_DOD_GROWTH_RATE\": 0.045388692595736746,\n      \"NEW_FUNDS_DAY\": 18.46,\n      \"INFLOW_FUNDS_DAY\": 12.93,\n      \"OUTFLOW_FUNDS_DAY\": 10.38,\n      \"OVERALL_POSITION\": 0.8083127036694828,\n      \"OVERALL_POSITION_DOD_GROWTH_RATE\": -0.02847453515632541,\n      \"NEW_CUST_FUNDS_MON\": 49.74,\n      \"INFLOW_FUNDS_MONTH\": 81.93,\n      \"OUTFLOW_FUNDS_MONTH\": 18.17\n    }\n  ]\n}\n

    \u6211\u4eec\u9700\u8981\u628a result \u7ed3\u679c\u4e2d\u7684\u90e8\u5206 key \u503c\u6570\u636e\u83b7\u53d6

    "},{"location":"reader/httpreader/#_3","title":"\u914d\u7f6e","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u5b9e\u73b0\u4ece\u63a5\u53e3\u83b7\u53d6\u6570\u636e\u5e76\u6253\u5370\u5230\u7ec8\u7aef

    job/httpreader2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"httpreader\",\n        \"parameter\": {\n          \"connection\": [\n            {\n              \"url\": \"http://127.0.0.1:9090/mock/17/LDJSC/ASSET\",\n              \"proxy\": {\n                \"host\": \"http://127.0.0.1:3128\",\n                \"auth\": \"user:pass\"\n              }\n            }\n          ],\n          \"reqParams\": {\n            \"CURR_DATE\": \"2021-01-18\",\n            \"DEPT\": \"9700\"\n          },\n          \"resultKey\": \"result\",\n          \"method\": \"GET\",\n          \"column\": [\n            \"CURR_DATE\",\n            \"DEPT\",\n            \"TOTAL_MANAGED_MARKET_VALUE\",\n            \"TOTAL_MANAGED_MARKET_VALUE_GROWTH\"\n          ],\n          \"username\": \"user\",\n          \"password\": \"passw0rd\",\n          \"headers\": {\n            \"X-Powered-by\": \"Addax\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": \"true\"\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u5185\u5bb9\u4fdd\u5b58\u4e3a job/httpreader2stream.json \u6587\u4ef6\u3002

    "},{"location":"reader/httpreader/#_4","title":"\u6267\u884c","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8fdb\u884c\u91c7\u96c6

    bin/addax.sh job/httpreader2stream.json\n

    \u4e0a\u8ff0\u547d\u4ee4\u7684\u8f93\u51fa\u7ed3\u679c\u5927\u81f4\u5982\u4e0b\uff1a

    2021-01-20 09:07:41.864 [main] INFO  VMInfo - VMInfo# operatingSystem class => com.sun.management.internal.OperatingSystemImpl\n2021-01-20 09:07:41.877 [main] INFO  Engine - the machine info  =>\n\n    osInfo:     Mac OS X x86_64 10.15.1\n    jvmInfo:    AdoptOpenJDK 14 14.0.2+12\n    cpu num:    8\n\n    totalPhysicalMemory:    -0.00G\n    freePhysicalMemory: -0.00G\n    maxFileDescriptorCount: -1\n    currentOpenFileDescriptorCount: -1\n\n    GC Names    [G1 Young Generation, G1 Old Generation]\n\n    MEMORY_NAME                    | allocation_size                | init_size\n    CodeHeap 'profiled nmethods'   | 117.21MB                       | 2.44MB\n    G1 Old Gen                     | 2,048.00MB                     | 39.00MB\n    G1 Survivor Space              | -0.00MB                        | 0.00MB\n    CodeHeap 'non-profiled nmethods' | 117.21MB                       | 2.44MB\n    Compressed Class Space         | 1,024.00MB                     | 0.00MB\n    Metaspace                      | -0.00MB                        | 0.00MB\n    G1 Eden Space                  | -0.00MB                        | 25.00MB\n    CodeHeap 'non-nmethods'        | 5.57MB                         | 2.44MB\n\n\n2021-01-20 09:07:41.903 [main] INFO  Engine -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"reqParams\":{\n                        \"CURR_DATE\":\"2021-01-18\",\n                        \"DEPT\":\"9700\"\n                    },\n                    \"method\":\"GET\",\n                    \"column\":[\n                        \"CURR_DATE\",\n                        \"DEPT\",\n                        \"TOTAL_MANAGED_MARKET_VALUE\",\n                        \"TOTAL_MANAGED_MARKET_VALUE_GROWTH\"\n                    ],\n                    \"resultKey\":\"result\",\n                    \"connection\":[\n                        {\n                            \"url\":\"http://127.0.0.1:9090/mock/17/LDJSC/ASSET\"\n                        }\n                    ]\n                },\n                \"name\":\"httpreader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":\"true\"\n                },\n                \"name\":\"streamwriter\"\n            }\n    },\n    \"setting\":{\n        \"speed\":{\n            \"bytes\":-1,\n            \"channel\":1\n        }\n    }\n}\n\n2021-01-20 09:07:41.926 [main] INFO  PerfTrace - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-01-20 09:07:41.927 [main] INFO  JobContainer - Addax jobContainer starts job.\n2021-01-20 09:07:41.928 [main] INFO  JobContainer - Set jobId = 0\n2021-01-20 09:07:42.002 [taskGroup-0] INFO  TaskGroupContainer - taskGroup[0] taskId[0] attemptCount[1] is started\n\n2019-08-30  9700    1539.85 -14.78\n2019-10-01  9700    1531.71 47.66\n2020-12-03  9700    1574.38 7.34\n2020-11-31  9700    1528.13 41.62\n2019-03-01  9700    1554.28 -9.29\n\n2021-01-20 09:07:45.006 [job-0] INFO  JobContainer -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-01-20 09:07:41\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-01-20 09:07:44\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :               42B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :              1rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                   5\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"reader/httpreader/#_5","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e url \u662f string \u65e0 \u8981\u8bbf\u95ee\u7684 HTTP \u5730\u5740 reqParams \u5426 map \u65e0 \u63a5\u53e3\u8bf7\u6c42\u53c2\u6570 resultKey \u5426 string \u65e0 \u8981\u83b7\u53d6\u7ed3\u679c\u7684\u90a3\u4e2a key \u503c\uff0c\u5982\u679c\u662f\u83b7\u53d6\u6574\u4e2a\u8fd4\u56de\u503c\uff0c\u5219\u53ef\u4ee5\u4e0d\u7528\u586b\u5199 method \u5426 string get \u8bf7\u6c42\u6a21\u5f0f\uff0c\u4ec5\u652f\u6301 GET\uff0cPOST \u4e24\u79cd\uff0c\u4e0d\u533a\u5206\u5927\u5c0f\u5199 column \u662f list \u65e0 \u8981\u83b7\u53d6\u7684 key\uff0c\u914d\u7f6e\u4e3a \"*\" \u8868\u793a\u83b7\u53d6\u6240\u6709 key \u503c username \u5426 string \u65e0 \u63a5\u53e3\u8bf7\u6c42\u9700\u8981\u7684\u8ba4\u8bc1\u5e10\u53f7(\u5982\u6709) password \u5426 string \u65e0 \u63a5\u53e3\u8bf7\u6c42\u9700\u8981\u7684\u5bc6\u7801(\u5982\u6709) proxy \u5426 map \u65e0 \u4ee3\u7406\u5730\u5740,\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0 headers \u5426 map \u65e0 \u5b9a\u5236\u7684\u8bf7\u6c42\u5934\u4fe1\u606f isPage \u5426 boolean \u65e0 \u63a5\u53e3\u662f\u5426\u5206\u652f\u5206\u9875 pageParams \u5426 map \u65e0 \u5206\u9875\u53c2\u6570"},{"location":"reader/httpreader/#proxy","title":"proxy","text":"

    \u5982\u679c\u8bbf\u95ee\u7684\u63a5\u53e3\u9700\u8981\u901a\u8fc7\u4ee3\u7406\uff0c\u5219\u53ef\u4ee5\u914d\u7f6e proxy \u914d\u7f6e\u9879\uff0c\u8be5\u914d\u7f6e\u9879\u662f\u4e00\u4e2a json \u5b57\u5178\uff0c\u5305\u542b\u4e00\u4e2a\u5fc5\u9009\u7684 host \u5b57\u6bb5\u548c\u4e00\u4e2a\u53ef\u9009\u7684 auth \u5b57\u6bb5\u3002

    {\n  \"proxy\": {\n    \"host\": \"http://127.0.0.1:8080\",\n    \"auth\": \"user:pass\"\n  }\n}\n

    \u5982\u679c\u662f sock \u4ee3\u7406 (V4,v5)\uff0c\u5219\u53ef\u4ee5\u5199

    {\n  \"proxy\": {\n    \"host\": \"socks://127.0.0.1:8080\",\n    \"auth\": \"user:pass\"\n  }\n}\n

    host \u662f\u4ee3\u7406\u5730\u5740\uff0c\u5305\u542b\u4ee3\u7406\u7c7b\u578b\uff0c\u76ee\u524d\u4ec5\u652f\u6301 http \u4ee3\u7406\u548c socks(V4, V5 \u5747\u53ef) \u4ee3\u7406\u3002 \u5982\u679c\u4ee3\u7406\u9700\u8981\u8ba4\u8bc1\uff0c\u5219\u53ef\u4ee5\u914d\u7f6e auth , \u5b83\u7531\u7528\u6237\u540d\u548c\u5bc6\u7801\u7ec4\u6210\uff0c\u4e24\u8005\u4e4b\u95f4\u7528\u5192\u53f7(:) \u9694\u5f00\u3002

    "},{"location":"reader/httpreader/#column","title":"column","text":"

    column \u9664\u4e86\u76f4\u63a5\u6307\u5b9a key \u4e4b\u5916\uff0c\u8fd8\u5141\u8bb8\u7528 JSON Xpath \u98ce\u683c\u6765\u6307\u5b9a\u9700\u8981\u83b7\u53d6\u7684 key \u503c\uff0c\u5047\u5b9a\u4f60\u8981\u8bfb\u53d6\u7684 JSON \u6587\u4ef6\u5982\u4e0b\uff1a

    {\n  \"result\": [\n    {\n      \"CURR_DATE\": \"2019-12-09\",\n      \"DEPT\": {\n        \"ID\": \"9700\"\n      },\n      \"KK\": [\n        {\n          \"COL1\": 1\n        },\n        {\n          \"COL2\": 2\n        }\n      ]\n    },\n    {\n      \"CURR_DATE\": \"2021-11-09\",\n      \"DEPT\": {\n        \"ID\": \"6500\"\n      },\n      \"KK\": [\n        {\n          \"COL1\": 3\n        },\n        {\n          \"COL2\": 4\n        }\n      ]\n    }\n  ]\n}\n

    \u6211\u4eec\u5e0c\u671b\u628a CURR_DATE, ID, COL1, COL2 \u5f53\u4f5c\u56db\u4e2a\u5b57\u6bb5\u8bfb\u53d6\uff0c\u90a3\u4e48\u4f60\u7684 column \u53ef\u4ee5\u8fd9\u6837\u914d\u7f6e\uff1a

    {\n  \"column\": [\n    \"CURR_DATE\",\n    \"DEPT.ID\",\n    \"KK[0].COL1\",\n    \"KK[1].COL2\"\n  ]\n}\n

    \u5176\u6267\u884c\u7ed3\u679c\u5982\u4e0b\uff1a

    ...\n2021-10-30 14:01:50.273 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\n\n2019-12-09  9700    1   2\n2021-11-09  6500    3   4\n\n2021-10-30 14:01:53.283 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2021-10-30 14:01:53.284 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do post work.\n2021-10-30 14:01:53.284 [       job-0] INFO  JobContainer         - Addax Reader.Job [httpreader] do post work.\n2021-10-30 14:01:53.286 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2021-10-30 14:01:53.289 [       job-0] INFO  JobContainer         -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-10-30 14:01:50\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-10-30 14:01:53\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :               10B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :              0rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                   2\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n

    \u6ce8\u610f\uff1a \u5982\u679c\u4f60\u6307\u5b9a\u4e86\u4e0d\u5b58\u5728\u7684 Key\uff0c\u5219\u76f4\u63a5\u8fd4\u56de\u4e3a NULL \u503c\u3002

    "},{"location":"reader/httpreader/#ispage","title":"isPage","text":"

    isPage \u53c2\u6570\u7528\u6765\u6307\u5b9a\u63a5\u53e3\u662f\u5426\u5206\u9875\uff0c\u5b83\u662f\u4e00\u4e2a\u5e03\u5c14\u503c\uff0c\u5982\u679c\u4e3a true \u5219\u8868\u793a\u63a5\u53e3\u5206\u9875\uff0c\u5426\u5219\u8868\u793a\u4e0d\u5206\u9875\u3002

    \u5f53\u63a5\u53e3\u652f\u6301\u5206\u9875\u65f6\uff0c\u8be5\u76f4\u63a5\u4f1a\u81ea\u52a8\u5206\u9875\u8bfb\u53d6\uff0c\u76f4\u5230\u63a5\u53e3\u8fd4\u56de\u7684\u6700\u540e\u4e00\u6b21\u8fd4\u56de\u7684\u6570\u636e\u7684\u8bb0\u5f55\u6570\u5c0f\u4e8e\u6bcf\u9875\u7684\u8bb0\u5f55\u6570\u4e3a\u6b62\u3002

    "},{"location":"reader/httpreader/#pageparams","title":"pageParams","text":"

    pageParams \u53c2\u6570\u4ec5\u5728 isPage \u53c2\u6570\u4e3a true \u65f6\u751f\u6548\uff0c\u5b83\u662f\u4e00\u4e2a JSON \u5b57\u5178\uff0c\u5305\u542b\u4e24\u4e2a\u53ef\u9009\u5b57\u6bb5 pageIndex \u548c pageSize \u3002

    pageIndex \u7528\u6765\u8868\u793a\u7528\u4e8e\u5206\u9875\u6307\u793a\u7684\u5f53\u524d\u9875\u9762\uff0c\u4ed6\u662f\u4e00\u4e2a JSON \u5b57\u6bb5\uff0c\u5305\u542b\u4e24\u4e2a\u53ef\u9009\u5b57\u6bb5 key \u548c value \uff0c\u5176\u4e2d key \u7528\u6765\u6307\u5b9a\u8868\u793a\u9875\u7801\u7684\u53c2\u6570\u540d\uff0cvalue \u7528\u6765\u6307\u5b9a\u5f53\u524d\u9875\u7801\u7684\u503c\u3002

    pageSize \u7528\u6765\u8868\u793a\u7528\u4e8e\u5206\u9875\u6307\u793a\u7684\u6bcf\u9875\u5927\u5c0f\uff0c\u4ed6\u662f\u4e00\u4e2a JSON \u5b57\u6bb5\uff0c\u5305\u542b\u4e24\u4e2a\u53ef\u9009\u5b57\u6bb5 key \u548c value \uff0c\u5176\u4e2d key \u7528\u6765\u6307\u5b9a\u8868\u793a\u6bcf\u9875\u5927\u5c0f\u7684\u53c2\u6570\u540d\uff0cvalue \u7528\u6765\u6307\u5b9a\u6bcf\u9875\u5927\u5c0f\u7684\u503c\u3002

    \u8fd9\u4e24\u4e2a\u53c2\u6570\u7684\u9ed8\u8ba4\u503c\u5982\u4e0b\uff1a

    {\n  \"pageParams\": {\n    \"pageIndex\": {\n      \"key\": \"pageIndex\",\n      \"value\": 1\n    },\n    \"pageSize\": {\n      \"key\": \"pageSize\",\n      \"value\": 100\n    }\n  }\n}\n

    \u5982\u679c\u4f60\u7684\u63a5\u53e3\u5206\u9875\u53c2\u6570\u4e0d\u662f pageIndex \u548c pageSize \uff0c\u5219\u53ef\u4ee5\u901a\u8fc7 pageParams \u53c2\u6570\u6765\u6307\u5b9a\u3002\u6bd4\u5982

    {\n  \"isPage\": true,\n  \"pageParams\": {\n    \"pageIndex\": {\n      \"key\": \"page\",\n      \"value\": 1\n    },\n    \"pageSize\": {\n      \"key\": \"size\",\n      \"value\": 100\n    }\n  }\n}\n

    \u8fd9\u8868\u793a\u4f60\u4f20\u9012\u7ed9\u63a5\u53e3\u7684\u5206\u9875\u53c2\u6570\u4e3a page=1&size=100 \u3002

    "},{"location":"reader/httpreader/#_6","title":"\u9650\u5236\u8bf4\u660e","text":"
    1. \u8fd4\u56de\u7684\u7ed3\u679c\u5fc5\u987b\u662f JSON \u7c7b\u578b
    2. \u5f53\u524d\u6240\u6709 key \u7684\u503c\u5747\u5f53\u4f5c\u5b57\u7b26\u4e32\u7c7b\u578b
    3. \u6682\u4e0d\u652f\u6301\u63a5\u53e3 Token \u9274\u6743\u6a21\u5f0f
    4. \u6682\u4e0d\u652f\u6301\u5206\u9875\u83b7\u53d6
    5. \u4ee3\u7406\u4ec5\u652f\u6301 http \u6a21\u5f0f
    "},{"location":"reader/influxdb2reader/","title":"InfluxDB2 Reader","text":"

    InfluxDB2 Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece InfluxDB 2.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u8bfb\u53d6\u6570\u636e\u3002

    \u6ce8\u610f\uff0c\u5982\u679c\u4f60\u7684 InfluxDB \u662f 1.8\u53ca\u4ee5\u4e0b\u7248\u672c\uff0c\u5219\u5e94\u8be5\u4f7f\u7528 InfluxDBReader \u63d2\u4ef6

    "},{"location":"reader/influxdb2reader/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u793a\u4f8b\u7528\u6765\u6f14\u793a\u8be5\u63d2\u4ef6\u5982\u4f55\u4ece\u6307\u5b9a\u8868(\u5373\u6307\u6807)\u4e0a\u8bfb\u53d6\u6570\u636e\u5e76\u8f93\u51fa\u5230\u7ec8\u7aef

    "},{"location":"reader/influxdb2reader/#job","title":"\u521b\u5efa job \u6587\u4ef6","text":"

    \u521b\u5efa job/influx2stream.json \u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    job/influx2stream.json 
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"influxdb2reader\",\n        \"parameter\": {\n          \"column\": [\n            \"location\",\n            \"height\",\n            \"wet\"\n          ],\n          \"connection\": {\n            \"endpoint\": \"http://localhost:8086\",\n            \"bucket\": \"test\",\n            \"table\": [\n              \"temperature\"\n            ],\n            \"org\": \"com.wgzhao\"\n          },\n          \"token\": \"YOUR_SECURE_TOKEN\",\n          \"range\": [\n            \"-1h\",\n            \"-5m\"\n          ],\n          \"limit\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": \"true\"\n        }\n      }\n    },\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    }\n  }\n}\n
    "},{"location":"reader/influxdb2reader/#_2","title":"\u8fd0\u884c","text":"

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/influx2stream.json\n
    "},{"location":"reader/influxdb2reader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 InfluxDB \u8fde\u63a5\u4e32 \uff5c token \u662f string \u65e0 \u8bbf\u95ee\u6570\u636e\u5e93\u7684 token table \u5426 list \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d(\u5373\u6307\u6807) org \u662f string \u65e0 \u6307\u5b9a InfluxDB \u7684 org \u540d\u79f0 bucket \u662f string \u65e0 \u6307\u5b9a InfluxDB \u7684 bucket \u540d\u79f0 column \u5426 list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1 rdbmreader range \u662f list \u65e0 \u8bfb\u53d6\u6570\u636e\u7684\u65f6\u95f4\u8303\u56f4 limit \u5426 int \u65e0 \u9650\u5236\u83b7\u53d6\u8bb0\u5f55\u6570"},{"location":"reader/influxdb2reader/#column","title":"column","text":"

    \u5982\u679c\u4e0d\u6307\u5b9a column, \u6216\u8005\u6307\u5b9a column \u4e3a [\"*\"] \uff0c \u5219\u4f1a\u8bfb\u53d6\u6240\u6709\u6709\u6548\u7684 _field \u5b57\u6bb5\u4ee5\u53ca _time \u5b57\u6bb5 \u5426\u5219\u6309\u7167\u6307\u5b9a\u5b57\u6bb5\u8bfb\u53d6

    "},{"location":"reader/influxdb2reader/#range","title":"range","text":"

    range \u7528\u6765\u6307\u5b9a\u8bfb\u53d6\u6307\u6807\u7684\u65f6\u95f4\u8303\u56f4\uff0c\u5176\u683c\u5f0f\u5982\u4e0b:

    {\n  \"range\": [\"start_time\", \"end_time\"]\n}\n

    range \u7531\u4e24\u4e2a\u5b57\u7b26\u4e32\u7ec4\u6210\u7684\u5217\u8868\u7ec4\u6210\uff0c\u7b2c\u4e00\u4e2a\u5b57\u7b26\u4e32\u8868\u793a\u5f00\u59cb\u65f6\u95f4\uff0c\u7b2c\u4e8c\u4e2a\u8868\u793a\u7ed3\u675f\u65f6\u95f4\u3002\u5176\u65f6\u95f4\u8868\u8fbe\u65b9\u5f0f\u8981\u6c42\u7b26\u5408 Flux \u683c\u5f0f\u8981\u6c42,\u7c7b\u4f3c\u8fd9\u6837:

    {\n  \"range\": [\"-15h\", \"-2h\"]\n}\n

    \u5176\u4e2d\u7b2c\u4e8c\u4e2a\u7ed3\u675f\u65f6\u95f4\u5982\u679c\u4e0d\u60f3\u6307\u5b9a\uff0c\u53ef\u4ee5\u4e0d\u5199\uff0c\u7c7b\u4f3c\u8fd9\u6837\uff1a

    {\n  \"range\": [\"-15h\"]\n}\n
    "},{"location":"reader/influxdb2reader/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u5f53\u524d\u5b9e\u73b0\u662f\u5c06\u6240\u6709\u5b57\u6bb5\u5f53\u4f5c\u5b57\u7b26\u4e32\u5904\u7406

    "},{"location":"reader/influxdb2reader/#_5","title":"\u9650\u5236","text":"
    1. \u5f53\u524d\u63d2\u4ef6\u4ec5\u652f\u6301 2.0 \u53ca\u4ee5\u4e0a\u7248\u672c
    "},{"location":"reader/influxdbreader/","title":"InfluxDB Reader","text":"

    InfluxDBReader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece InfluxDB \u8bfb\u53d6\u6570\u636e\u3002\u5e95\u5c42\u5b9e\u73b0\u4e0a\uff0c\u662f\u901a\u8fc7\u8c03\u7528 InfluQL \u8bed\u8a00\u67e5\u8be2\u8868\uff0c\u7136\u540e\u83b7\u5f97\u8fd4\u56de\u6570\u636e\u3002

    "},{"location":"reader/influxdbreader/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u793a\u4f8b\u7528\u6765\u6f14\u793a\u8be5\u63d2\u4ef6\u5982\u4f55\u4ece\u6307\u5b9a\u8868(\u5373\u6307\u6807)\u4e0a\u8bfb\u53d6\u6570\u636e\u5e76\u8f93\u51fa\u5230\u7ec8\u7aef

    "},{"location":"reader/influxdbreader/#_2","title":"\u521b\u5efa\u9700\u8981\u7684\u5e93\u8868\u548c\u6570\u636e","text":"

    \u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u6765\u521b\u5efa\u9700\u8981\u8bfb\u53d6\u7684\u8868\u4ee5\u53ca\u6570\u636e

    # create database\ninflux --execute \"CREATE DATABASE NOAA_water_database\"\n# download sample data\ncurl https://s3.amazonaws.com/noaa.water-database/NOAA_data.txt -o NOAA_data.txt\n# import data via influx-cli\ninflux -import -path=NOAA_data.txt -precision=s -database=NOAA_water_database\n
    "},{"location":"reader/influxdbreader/#job","title":"\u521b\u5efa job \u6587\u4ef6","text":"

    \u521b\u5efa job/influxdb2stream.json \u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    job/influxdb2stream.json 
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"influxdbreader\",\n        \"parameter\": {\n          \"column\": [\n            \"*\"\n          ],\n          \"where\": \"1=1\",\n          \"connection\": {\n            \"endpoint\": \"http://localhost:8086\",\n            \"database\": \"NOAA_water_database\",\n            \"table\": \"h2o_feet\"\n          },\n          \"connTimeout\": 15,\n          \"readTimeout\": 20,\n          \"writeTimeout\": 20,\n          \"username\": \"influx\",\n          \"password\": \"influx123\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": \"true\"\n        }\n      }\n    },\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    }\n  }\n}\n
    "},{"location":"reader/influxdbreader/#_3","title":"\u8fd0\u884c","text":"

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/influxdb2stream.json\n
    "},{"location":"reader/influxdbreader/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 InfluxDB \u8fde\u63a5\u4e32 username \u662f string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801 database \u662f string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7684\u6570\u636e\u5e93 table \u662f string \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1 rdbmreader connTimeout \u5426 int 15 \u8bbe\u7f6e\u8fde\u63a5\u8d85\u65f6\u503c\uff0c\u5355\u4f4d\u4e3a\u79d2 readTimeout \u5426 int 20 \u8bbe\u7f6e\u8bfb\u53d6\u8d85\u65f6\u503c\uff0c\u5355\u4f4d\u4e3a\u79d2 writeTimeout \u5426 int 20 \u8bbe\u7f6e\u5199\u5165\u8d85\u65f6\u503c\uff0c\u5355\u4f4d\u4e3a\u79d2 where \u5426 string \u65e0 \u9488\u5bf9\u8868\u7684\u7b5b\u9009\u6761\u4ef6 querySql \u5426 string \u65e0 \u4f7f\u7528 SQL \u67e5\u8be2\u83b7\u53d6\u6570\u636e\uff0c\u5982\u914d\u7f6e\u8be5\u9879\uff0c\u5219 table\uff0ccolumn \u914d\u7f6e\u9879\u65e0\u6548"},{"location":"reader/influxdbreader/#_5","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u5f53\u524d\u5b9e\u73b0\u662f\u5c06\u6240\u6709\u5b57\u6bb5\u5f53\u4f5c\u5b57\u7b26\u4e32\u5904\u7406

    "},{"location":"reader/influxdbreader/#_6","title":"\u9650\u5236","text":"
    1. \u5f53\u524d\u63d2\u4ef6\u4ec5\u652f\u6301 1.x \u7248\u672c\uff0c2.0 \u53ca\u4ee5\u4e0a\u5e76\u4e0d\u652f\u6301
    "},{"location":"reader/jsonfilereader/","title":"JSON File Reader","text":"

    JSON File Reader \u63d0\u4f9b\u4e86\u8bfb\u53d6\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u6570\u636e\u5b58\u50a8\u7684\u80fd\u529b\u3002

    "},{"location":"reader/jsonfilereader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"job/json2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": \"true\"\n        }\n      },\n      \"reader\": {\n        \"name\": \"jsonfilereader\",\n        \"parameter\": {\n          \"path\": [\n            \"/tmp/test*.json\"\n          ],\n          \"column\": [\n            {\n              \"index\": \"$.id\",\n              \"type\": \"long\"\n            },\n            {\n              \"index\": \"$.name\",\n              \"type\": \"string\"\n            },\n            {\n              \"index\": \"$.age\",\n              \"type\": \"long\"\n            },\n            {\n              \"index\": \"$.score.math\",\n              \"type\": \"double\"\n            },\n            {\n              \"index\": \"$.score.english\",\n              \"type\": \"double\"\n            },\n            {\n              \"index\": \"$.pubdate\",\n              \"type\": \"date\"\n            },\n            {\n              \"type\": \"string\",\n              \"value\": \"constant string\"\n            }\n          ]\n        }\n      }\n    }\n  }\n}\n

    \u5176\u4e2d /tmp/test*.json \u4e3a\u540c\u4e00\u4e2a json \u6587\u4ef6\u7684\u591a\u4e2a\u590d\u5236\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    {\"name\": \"zhangshan\",\"id\": 19890604,\"age\": 12,\"score\": {\"math\": 92.5,\"english\": 97.5,\"chinese\": 95},\"pubdate\": \"2020-09-05\"}\n{\"name\": \"lisi\",\"id\": 19890605,\"age\": 12,\"score\": {\"math\": 90.5,\"english\": 77.5,\"chinese\": 90},\"pubdate\": \"2020-09-05\"}\n{\"name\": \"wangwu\",\"id\": 19890606,\"age\": 12,\"score\": {\"math\": 89,\"english\": 100,\"chinese\": 92},\"pubdate\": \"2020-09-05\"}\n
    "},{"location":"reader/jsonfilereader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f list \u65e0 \u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84,\u8be6\u7ec6\u63cf\u8ff0\u89c1\u4e0b\u6587 column \u662f list \u65e0 \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype\u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0c\u8be6\u89c1\u4e0b\u6587 fieldDelimiter \u662f string , \u63cf\u8ff0\uff1a\u8bfb\u53d6\u7684\u5b57\u6bb5\u5206\u9694\u7b26 compress \u5426 string \u65e0 \u6587\u672c\u538b\u7f29\u7c7b\u578b\uff0c\u9ed8\u8ba4\u4e0d\u586b\u5199\u610f\u5473\u7740\u6ca1\u6709\u538b\u7f29\u3002\u652f\u6301\u538b\u7f29\u7c7b\u578b\u4e3azip\u3001gzip\u3001bzip2 encoding \u5426 string utf-8 \u8bfb\u53d6\u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e singleLine \u5426 boolean true \u6bcf\u6761\u6570\u636e\u662f\u5426\u4e3a\u4e00\u884c\uff0c \u8be6\u89c1\u4e0b\u6587"},{"location":"reader/jsonfilereader/#path","title":"path","text":"

    \u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84\uff0c\u6bd4\u5982\uff1a

    {\n  \"path\": [\n    \"/var/ftp/test.json\", // \u8bfb\u53d6 /var/ftp \u76ee\u5f55\u4e0b\u7684 test.json \u6587\u4ef6\n    \"/var/tmp/*.json\", // \u8bfb\u53d6 /var/tmp \u76ee\u5f55\u4e0b\u6240\u6709 json \u6587\u4ef6\n    \"/public/ftp\", // \u8bfb\u53d6 /public/ftp \u76ee\u5f55\u4e0b\u6240\u6709\u6587\u4ef6, \u5982\u679c ftp \u662f\u6587\u4ef6\u7684\u8bdd\uff0c\u5219\u76f4\u63a5\u8bfb\u53d6\n    \"/public/a??.json\" // \u8bfb\u53d6 /public \u76ee\u5f55\u4e0b\u6240\u6709 a \u5f00\u5934\uff0c\u540e\u9762\u8ddf\u4e24\u4e2a\u5b57\u7b26\uff0c\u6700\u540e\u662f json \u7ed3\u5c3e\u7684\u6587\u4ef6\n  ]\n}\n

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0c\u5982\u679cPath\u6307\u5b9a\u7684\u8def\u5f84\u4e0b\u6ca1\u6709\u7b26\u5408\u5339\u914d\u7684\u6587\u4ef6\u62bd\u53d6\uff0cAddax\u5c06\u62a5\u9519\u3002

    "},{"location":"reader/jsonfilereader/#column","title":"column","text":"

    \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype\u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cindex\u6307\u5b9a\u5f53\u524d\u5217\u6765\u81ea\u4e8ejson\u7684\u6307\u5b9a\uff0c\u8bed\u6cd5\u4e3a Jayway JsonPath \u7684\u8bed\u6cd5\uff0cvalue\u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece\u6e90\u5934\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636evalue\u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002 \u7528\u6237\u5fc5\u987b\u6307\u5b9aColumn\u5b57\u6bb5\u4fe1\u606f

    \u5bf9\u4e8e\u7528\u6237\u6307\u5b9aColumn\u4fe1\u606f\uff0ctype\u5fc5\u987b\u586b\u5199\uff0cindex/value \u5fc5\u987b\u9009\u62e9\u5176\u4e00

    "},{"location":"reader/jsonfilereader/#singleline","title":"singleLine","text":"

    \u4f7f\u7528 JSON \u683c\u5f0f\u5b58\u50a8\u6570\u636e\uff0c\u4e1a\u754c\u6709\u4e24\u79cd\u65b9\u5f0f\uff0c\u4e00\u79cd\u662f\u6bcf\u884c\u4e00\u4e2a JSON \u5bf9\u8c61\uff0c\u4e5f\u5c31\u662f Single Line JSON(aka. JSONL or JSON Lines); \u53e6\u4e00\u79cd\u662f\u6574\u4e2a\u6587\u4ef6\u662f\u4e00\u4e2a JSON \u6570\u7ec4\uff0c\u6bcf\u4e2a\u5143\u7d20\u662f\u4e00\u4e2a JSON \u5bf9\u8c61\uff0c\u4e5f\u5c31\u662f Multiline JSON\u3002

    Addax \u9ed8\u8ba4\u652f\u6301\u6bcf\u884c\u4e00\u4e2a JSON \u5bf9\u8c61\u7684\u683c\u5f0f\uff0c\u5373 singeLine = true, \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u8981\u6ce8\u610f\u7684\u662f\uff1a

    1. \u6bcf\u884c JSON \u5bf9\u8c61\u7684\u672b\u5c3e\u4e0d\u80fd\u6709\u9017\u53f7\uff0c\u5426\u5219\u4f1a\u89e3\u6790\u5931\u8d25\u3002
    2. \u4e00\u4e2aJSON \u5bf9\u8c61\u4e0d\u80fd\u8de8\u884c\uff0c\u5426\u5219\u4f1a\u89e3\u6790\u5931\u8d25\u3002

    \u5982\u679c\u6570\u636e\u662f\u6574\u4e2a\u6587\u4ef6\u662f\u4e00\u4e2a JSON \u6570\u7ec4\uff0c\u6bcf\u4e2a\u5143\u7d20\u662f\u4e00\u4e2a JSON \u5bf9\u8c61\uff0c\u9700\u8981\u8bbe\u7f6e singeLine \u4e3a false\u3002 \u5047\u8bbe\u4e0a\u8ff0\u5217\u5b50\u4e2d\u7684\u6570\u636e\u7528\u4e0b\u9762\u7684\u683c\u5f0f\u8868\u793a\uff1a

    {\n  \"result\": [\n    {\n      \"name\": \"zhangshan\",\n      \"id\": 19890604,\n      \"age\": 12,\n      \"score\": {\n        \"math\": 92.5,\n        \"english\": 97.5,\n        \"chinese\": 95\n      },\n      \"pubdate\": \"2020-09-05\"\n    },\n    {\n      \"name\": \"lisi\",\n      \"id\": 19890605,\n      \"age\": 12,\n      \"score\": {\n        \"math\": 90.5,\n        \"english\": 77.5,\n        \"chinese\": 90\n      },\n      \"pubdate\": \"2020-09-05\"\n    },\n    {\n      \"name\": \"wangwu\",\n      \"id\": 19890606,\n      \"age\": 12,\n      \"score\": {\n        \"math\": 89,\n        \"english\": 100,\n        \"chinese\": 92\n      },\n      \"pubdate\": \"2020-09-05\"\n    }\n  ]\n}\n

    \u56e0\u4e3a\u8fd9\u79cd\u683c\u5f0f\u662f\u5408\u6cd5\u7684 JSON \u683c\u5f0f\uff0c\u56e0\u6b64\u6bcf\u4e2a JSON \u5bf9\u8c61\u53ef\u4ee5\u8de8\u884c\u3002\u76f8\u5e94\u7684\uff0c\u8fd9\u7c7b\u6570\u636e\u8bfb\u53d6\u65f6\uff0c\u5176 path \u914d\u7f6e\u5e94\u8be5\u5982\u4e0b\u586b\u5199\uff1a

    {\n  \"singleLine\": false,\n  \"column\": [\n    {\n      \"index\": \"$.result[*].id\",\n      \"type\": \"long\"\n    },\n    {\n      \"index\": \"$.result[*].name\",\n      \"type\": \"string\"\n    },\n    {\n      \"index\": \"$.result[*].age\",\n      \"type\": \"long\"\n    },\n    {\n      \"index\": \"$.result[*].score.math\",\n      \"type\": \"double\"\n    },\n    {\n      \"index\": \"$.result[*].score.english\",\n      \"type\": \"double\"\n    },\n    {\n      \"index\": \"$..result[*].pubdate\",\n      \"type\": \"date\"\n    },\n    {\n      \"type\": \"string\",\n      \"value\": \"constant string\"\n    }\n  ]\n}\n

    \u66f4\u8be6\u7ec6\u7684\u4f7f\u7528\u8bf4\u660e\u8bf7\u53c2\u8003 Jayway JsonPath \u7684\u8bed\u6cd5\u3002

    \u6ce8\u610f: \u8fd9\u79cd\u6570\u636e\u5728\u4e00\u4e2a JSON \u6570\u7ec4\u91cc\u65f6\uff0c\u7a0b\u5e8f\u53ea\u80fd\u91c7\u53d6\u5c06\u6574\u4e2a\u6587\u4ef6\u8bfb\u53d6\u5230\u5185\u5b58\u4e2d\uff0c\u7136\u540e\u89e3\u6790\u7684\u65b9\u5f0f\uff0c\u56e0\u6b64\u4e0d\u9002\u5408\u5927\u6587\u4ef6\u7684\u8bfb\u53d6\u3002 \u5bf9\u4e8e\u5927\u6587\u4ef6\u7684\u8bfb\u53d6\uff0c\u5efa\u8bae\u4f7f\u7528\u6bcf\u884c\u4e00\u4e2a JSON \u5bf9\u8c61\u7684\u683c\u5f0f\uff0c\u4e5f\u5c31\u662f Single Line JSON \u7684\u683c\u5f0f\uff0c\u8fd9\u79cd\u683c\u5f0f\u53ef\u4ee5\u91c7\u53d6\u9010\u884c\u8bfb\u53d6\u7684\u65b9\u5f0f\uff0c\u4e0d\u4f1a\u5360\u7528\u592a\u591a\u5185\u5b58\u3002

    "},{"location":"reader/jsonfilereader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b \u672c\u5730\u6587\u4ef6 \u6570\u636e\u7c7b\u578b Long Long Double Double String String Boolean Boolean Date Date"},{"location":"reader/kafkareader/","title":"Kafka Reader","text":"

    Kafka Reader \u63d2\u4ef6\u5b9e\u73b0\u4ece Kafka \u961f\u5217\u4e2d\u8bfb\u53d6 JSON \u683c\u5f0f\u6d88\u606f\u7684\u529f\u80fd\u3002 \u8be5\u63d2\u4ef6\u5728 4.0.10 \u7248\u672c\u4e2d\u5f15\u5165\u3002

    "},{"location":"reader/kafkareader/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u6f14\u793a\u4e86\u5982\u4f55\u4ece\u4ece kafka \u7684\u8bfb\u53d6\u6307\u5b9a topic \u4e2d\uff0c\u5e76\u8f93\u51fa\u5230\u7ec8\u7aef\u4e0a\u3002

    "},{"location":"reader/kafkareader/#_2","title":"\u521b\u5efa\u4efb\u52a1\u6587\u4ef6","text":"

    \u9996\u5148\u521b\u5efa\u4e00\u4e2a\u4efb\u52a1\u6587\u4ef6 kafka2stream.json , \u5185\u5bb9\u5982\u4e0b\uff1a

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1\n      }\n    },\n    \"content\": [\n      {\n        \"writer\": {\n          \"name\": \"streamwriter\",\n          \"parameter\": {\n            \"print\": true\n          }\n        },\n        \"reader\": {\n          \"name\": \"kafkareader\",\n          \"parameter\": {\n            \"brokerList\": \"wgzhao-laptop:9092\",\n            \"topic\": \"test-1\",\n            \"column\": [\n              \"col1\",\n              \"col3\",\n              \"col0\",\n              \"col9\"\n            ],\n            \"missingKeyValue\": \"\\\\N\",\n            \"properties\": {\n              \"auto.offset.reset\": \"earliest\"\n            }\n          }\n        }\n      }\n    ]\n  }\n}\n
    "},{"location":"reader/kafkareader/#_3","title":"\u8fd0\u884c","text":"

    \u6267\u884c bin/addax.sh kafka2stream.json \u547d\u4ee4\u3002

    "},{"location":"reader/kafkareader/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 brokerList \u662f string \u65e0 \u8fde\u63a5 kafka \u670d\u52a1\u7684 broker \u914d\u7f6e\uff0c\u7c7b\u4f3c localhost:9092 \uff0c\u591a\u4e2a broker\u4e4b\u95f4\u7528\u9017\u53f7(,)\u5206\u9694 topic \u662f string \u65e0 \u8981\u5199\u5165\u7684 topic column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u4ee5\u4e0b\u8be6\u8ff0 missingKeyValue \u5426 string \u65e0 \u5b57\u6bb5\u4e0d\u5b58\u5728\u65f6\u7528\u4ec0\u4e48\u503c\u586b\u5145\uff0c\u4ee5\u4e0b\u8be6\u8ff0 properties \u5426 map \u65e0 \u9700\u8981\u8bbe\u7f6e\u7684\u5176\u4ed6 kafka \u8fde\u63a5\u53c2\u6570"},{"location":"reader/kafkareader/#column","title":"column","text":"

    column \u7528\u6765\u6307\u5b9a\u8981\u8bfb\u53d6\u7684 JSON \u6d88\u606f\u4e2d\u7684 key\uff0c\u5982\u679c\u586b\u5199\u4e3a * \uff0c\u5219\u8868\u793a\u8bfb\u53d6\u6d88\u606f\u4e2d\u7684\u6240\u6709 key\u3002\u4f46\u8981\u6ce8\u610f\uff0c\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u4e00\u662f\u8f93\u51fa\u4e0d\u4f1a\u6709\u6392\u5e8f\uff0c\u4e5f\u5c31\u662f\u8bf4\u7b2c\u6bcf\u6761\u8bb0\u5f55\u7684 key \u7684 \u8f93\u51fa\u987a\u5e8f\u4e0d\u786e\u4fdd\u4e00\u81f4\u3002

    \u4e5f\u53ef\u4ee5\u6307\u5b9a key \u6765\u8fdb\u884c\u8bfb\u53d6\uff0c\u6bd4\u5982

    {\n  \"column\": [\"col1\", \"col2\", \"col3\"]\n}\n

    \u8fd9\u6837\uff0c\u63d2\u4ef6\u4f1a\u5c1d\u8bd5\u6309\u7167\u7ed9\u5b9a\u7684\u987a\u5e8f\u53bb\u8bfb\u53d6\u76f8\u5e94\u7684 key\uff0c\u5982\u679c\u4e00\u6761\u6d88\u606f\u4e2d\u8981\u8bfb\u53d6\u7684 key \u4e0d\u5b58\u5728\uff0c\u63d2\u4ef6\u4f1a\u62a5\u9519\u5e76\u9000\u51fa\u3002\u5982\u679c\u5e0c\u671b\u4e0d\u9000\u51fa\uff0c\u5219\u53ef\u4ee5\u8bbe\u7f6e missingKeyValue \u4ed6\u8868\u793a\u5f53\u8981\u8bfb\u53d6\u7684 key \u4e0d\u5b58\u5728\u65f6\uff0c\u7528\u8be5\u914d\u7f6e\u7684\u503c\u6765\u586b\u5145\u3002

    \u53e6\u5916\uff0c\u8bfb\u53d6\u7684 key \u7684\u503c\u7684\u7c7b\u578b\uff0c\u63d2\u4ef6\u4f1a\u81ea\u52a8\u53bb\u731c\u6d4b\uff0c\u5982\u679c\u7c7b\u578b\u65e0\u6cd5\u731c\u6d4b\uff0c\u5219\u4f1a\u5f53\u4f5c String \u7c7b\u578b\u3002

    "},{"location":"reader/kafkareader/#_5","title":"\u9650\u5236","text":"
    1. \u4ec5\u652f\u6301 Kafka 1.0 \u53ca\u4ee5\u4e0a\u7248\u672c\uff0c\u4f4e\u4e8e\u8be5\u7248\u672c\u7684\u65e0\u6cd5\u786e\u5b9a\u662f\u5426\u80fd\u5199\u5165
    2. \u5f53\u524d\u4e0d\u652f\u6301\u542f\u7528\u4e86 kerberos \u8ba4\u8bc1\u7684 kafka \u670d\u52a1
    "},{"location":"reader/kudureader/","title":"Kudu Reader","text":"

    Kudu Reader \u63d2\u4ef6\u5229\u7528 Kudu \u7684 java\u5ba2\u6237\u7aef KuduClient \u8fdb\u884c Kudu \u7684\u8bfb\u64cd\u4f5c\u3002

    "},{"location":"reader/kudureader/#_1","title":"\u914d\u7f6e\u793a\u4f8b","text":"

    \u6211\u4eec\u901a\u8fc7 Trino \u7684 kudu connector \u8fde\u63a5 kudu \u670d\u52a1\uff0c\u7136\u540e\u8fdb\u884c\u8868\u521b\u5efa\u4ee5\u53ca\u6570\u636e\u63d2\u5165

    "},{"location":"reader/kudureader/#_2","title":"\u5efa\u8868\u8bed\u53e5\u4ee5\u53ca\u6570\u636e\u63d2\u5165\u8bed\u53e5","text":"
    CREATE TABLE kudu.default.users (\n  user_id int WITH (primary_key = true),\n  user_name varchar with (nullable=true),\n  age int with (nullable=true),\n  salary double with (nullable=true),\n  longtitue decimal(18,6) with (nullable=true),\n  latitude decimal(18,6) with (nullable=true),\n  p decimal(21,20) with (nullable=true),\n  mtime timestamp with (nullable=true)\n) WITH (\n  partition_by_hash_columns = ARRAY['user_id'],\n  partition_by_hash_buckets = 2\n);\n\ninsert into kudu.default.users \nvalues \n(1, cast('wgzhao' as varchar), 18, cast(18888.88 as double), \n cast(123.282424 as decimal(18,6)), cast(23.123456 as decimal(18,6)),\n cast(1.12345678912345678912 as decimal(21,20)), \n timestamp '2021-01-10 14:40:41'),\n(2, cast('anglina' as varchar), 16, cast(23456.12 as double), \n cast(33.192123 as decimal(18,6)), cast(56.654321 as decimal(18,6)), \n cast(1.12345678912345678912 as decimal(21,20)), \n timestamp '2021-01-10 03:40:41');\n-- ONLY insert primary key value\n insert into kudu.default.users(user_id) values  (3);\n
    "},{"location":"reader/kudureader/#_3","title":"\u914d\u7f6e","text":"

    \u4ee5\u4e0b\u662f\u8bfb\u53d6kudu\u8868\u5e76\u8f93\u51fa\u5230\u7ec8\u7aef\u7684\u914d\u7f6e

    job/kudu2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"kudureader\",\n        \"parameter\": {\n          \"masterAddress\": \"localhost:7051,localhost:7151,localhost:7251\",\n          \"table\": \"users\",\n          \"splitPk\": \"user_id\",\n          \"lowerBound\": 1,\n          \"upperBound\": 100,\n          \"readTimeout\": 5,\n          \"scanTimeout\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u628a\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/kudu2stream.json

    "},{"location":"reader/kudureader/#_4","title":"\u6267\u884c","text":"

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91c7\u96c6

    bin/addax.sh job/kudu2stream.json\n
    "},{"location":"reader/kudureader/#_5","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 masterAddress \u662f string \u65e0 Kudu Master \u96c6\u7fa4RPC\u5730\u5740,\u591a\u4e2a\u5730\u5740\u7528\u9017\u53f7(,)\u5206\u9694 table \u662f string \u65e0 kudu \u8868\u540d splitPk \u5426 string \u65e0 \u5e76\u884c\u8bfb\u53d6\u6570\u636e\u5206\u7247\u5b57\u6bb5 lowerBound \u5426 string \u65e0 \u5e76\u884c\u8bfb\u53d6\u6570\u636e\u5206\u7247\u8303\u56f4\u4e0b\u754c upperBound \u5426 string \u65e0 \u5e76\u884c\u8bfb\u53d6\u6570\u636e\u5206\u7247\u8303\u56f4\u4e0a\u754c readTimeout \u5426 int 10 \u8bfb\u53d6\u6570\u636e\u8d85\u65f6(\u79d2) scanTimeout \u5426 int 20 \u6570\u636e\u626b\u63cf\u8bf7\u6c42\u8d85\u65f6(\u79d2) column \u5426 list \u65e0 \u6307\u5b9a\u8981\u83b7\u53d6\u7684\u5b57\u6bb5 where \u5426 list \u65e0 \u6307\u5b9a\u5176\u4ed6\u8fc7\u6ee4\u6761\u4ef6\uff0c\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0"},{"location":"reader/kudureader/#where","title":"where","text":"

    where \u7528\u6765\u5b9a\u5236\u66f4\u591a\u7684\u8fc7\u6ee4\u6761\u4ef6\uff0c\u4ed6\u662f\u4e00\u4e2a\u6570\u7ec4\u7c7b\u578b\uff0c\u6570\u7ec4\u7684\u6bcf\u4e2a\u5143\u7d20\u90fd\u662f\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u6bd4\u5982

    {\n  \"where\": [\"age > 1\", \"user_name = 'wgzhao'\"] \n}\n

    \u4e0a\u8ff0\u5b9a\u4e49\u4e86\u4e24\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u6bcf\u4e2a\u8fc7\u6ee4\u6761\u4ef6\u7531\u4e09\u90e8\u5206\u7ec4\u6210\uff0c\u683c\u5f0f\u4e3a column operator value

    \u8fd9\u91cc\u8fd8\u6709\u5176\u4ed6\u4e00\u4e9b\u9650\u5b9a\uff0c\u5728\u4f7f\u7528\u65f6\uff0c\u8981\u7279\u522b\u6ce8\u610f\uff1a

    1. \u591a\u4e2a\u8fc7\u6ee4\u6761\u4ef6\u4e4b\u95f4\u7684\u903b\u8f91\u4e0e\u5173\u7cfb(AND)\uff0c\u6682\u4e0d\u652f\u6301\u903b\u8f91\u6216(OR)\u5173\u7cfb
    "},{"location":"reader/kudureader/#_6","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b Kudu \u6570\u636e\u7c7b\u578b Long byte, short, int, long Double float, double, decimal String string Date timestamp Boolean boolean Bytes binary"},{"location":"reader/mongodbreader/","title":"MongoDB Reader","text":"

    MongoDBReader \u63d2\u4ef6\u5229\u7528 MongoDB \u7684java\u5ba2\u6237\u7aefMongoClient\u8fdb\u884cMongoDB\u7684\u8bfb\u64cd\u4f5c\u3002

    "},{"location":"reader/mongodbreader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u8be5\u793a\u4f8b\u4eceMongoDB\u4e2d\u8bfb\u4e00\u5f20\u8868\u5e76\u6253\u5370\u5230\u7ec8\u7aef

    job/mongo2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"mongodbreader\",\n        \"parameter\": {\n          \"connection\": {\n            \"address\": [\n              \"127.0.0.1:27017\"\n            ],\n            \"database\": \"tag_per_data\",\n            \"collection\": \"tag_data\",\n            \"authDb\": \"admin\"\n          },\n          \"username\": \"\",\n          \"password\": \"\",\n          \"column\": [\n            \"unique_id\",\n            \"sid\",\n            \"user_id\",\n            \"auction_id\",\n            \"content_type\",\n            \"pool_type\",\n            \"frontcat_id\",\n            \"catagoryid\",\n            \"gmt_create\",\n            \"taglist\",\n            \"property\",\n            \"scorea\",\n            \"scoreb\",\n            \"scorec\"\n          ]\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": \"true\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/mongodbreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 address \u662f list \u65e0 MongoDB \u7684\u6570\u636e\u5730\u5740\u4fe1\u606f, \u53ef\u5199\u591a\u4e2a username \u5426 string \u65e0 MongoDB \u7528\u6237\u540d password \u5426 string \u65e0 MongoDB \u5bc6\u7801 database \u662f string \u65e0 MongoDB \u6570\u636e\u5e93 collection \u662f string \u65e0 MongoDB \u7684\u96c6\u5408\u540d column \u662f list \u65e0 MongoDB \u7684\u6587\u6863\u5217\u540d\uff0c\u4e0d\u652f\u6301 [\"*\"] \u83b7\u53d6\u6240\u6709\u5217\u65b9\u5f0f query \u5426 string \u65e0 \u81ea\u5b9a\u4e49\u67e5\u8be2\u6761\u4ef6 fetchSize \u5426 int 2048 \u6279\u91cf\u83b7\u53d6\u7684\u8bb0\u5f55\u6570"},{"location":"reader/mongodbreader/#collection","title":"collection","text":"

    \u8fd9\u91cc\u7684 collection \u76ee\u524d\u53ea\u652f\u6301\u5355\u4e00 collection\uff0c\u56e0\u6b64\u8bbe\u7f6e\u7c7b\u578b\u4e3a\u5b57\u7b26\u4e32\uff0c\u800c\u4e0d\u662f\u5176\u4ed6\u63d2\u4ef6\u5e38\u89c1\u7684\u6570\u7ec4\u7c7b\u578b\uff0c\u8fd9\u4e00\u70b9\u5c24\u4e3a\u6ce8\u610f\u3002

    "},{"location":"reader/mongodbreader/#column","title":"column","text":"

    column \u7528\u6765\u6307\u5b9a\u9700\u8981\u8bfb\u53d6\u7684\u5b57\u6bb5\u540d\u79f0\uff0c\u8fd9\u91cc\u6211\u4eec\u505a\u4e86\u5b57\u6bb5\u540d\u79f0\u7684\u7ec4\u6210\u4e24\u4e2a\u5047\u5b9a\uff1a

    \u57fa\u4e8e\u4ee5\u4e0a\u5047\u5b9a\uff0c\u6211\u4eec\u53ef\u4ee5\u5728\u7b80\u5316 column \u914d\u7f6e\u7684\u540c\u65f6\uff0c\u8fd8\u53ef\u4ee5\u6307\u5b9a\u4e00\u4e9b\u5e38\u91cf\u4f5c\u4e3a\u8865\u5145\u5b57\u6bb5\uff0c\u6bd4\u5982\u4e00\u822c\u91c7\u96c6\u4e00\u5f20\u8868\uff0c\u6211\u4eec\u9700\u8981\u589e\u52a0\u91c7\u96c6\u65f6\u95f4\uff0c\u91c7\u96c6\u6e90\u7b49\u5e38\u91cf\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u914d\u7f6e

    {\n  \"column\": [\n    \"col1\",\n    \"col2\",\n    \"col3\",\n    \"'source_mongodb'\",\n    \"20211026\",\n    \"123.12\"\n  ]\n}\n

    \u4e0a\u8ff0\u914d\u7f6e\u7684\u540e\u4e09\u4e2a\u5b57\u6bb5\u5c31\u662f\u5e38\u91cf\uff0c\u5206\u522b\u5f53\u4f5c\u5b57\u7b26\u7c7b\u578b\uff0c\u6574\u578b\u548c\u6d6e\u70b9\u578b\u5904\u7406\u3002

    "},{"location":"reader/mongodbreader/#query","title":"query","text":"

    query \u662f\u53ea\u7b26\u5408 MongoDB \u67e5\u8be2\u683c\u5f0f\u7684 BSON \u5b57\u7b26\u4e32\uff0c\u6bd4\u5982\uff1a

    {\n  \"query\": \"{amount: {$gt: 140900}, oc_date: {$gt: 20190110}}\"\n}\n

    \u4e0a\u8ff0\u67e5\u8be2\u7c7b\u4f3c SQL \u4e2d\u7684 where amount > 140900 and oc_date > 20190110

    "},{"location":"reader/mongodbreader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b MongoDB \u6570\u636e\u7c7b\u578b Long int, Long Double double String string, array Date date Boolean boolean Bytes bytes"},{"location":"reader/mysqlreader/","title":"MySQL Reader","text":"

    MysqlReader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece MySQL \u8bfb\u53d6\u6570\u636e\u7684\u80fd\u529b

    "},{"location":"reader/mysqlreader/#_1","title":"\u793a\u4f8b","text":"

    \u6211\u4eec\u5728 MySQL \u7684 test \u5e93\u4e0a\u521b\u5efa\u5982\u4e0b\u8868\uff0c\u5e76\u63d2\u5165\u4e00\u6761\u8bb0\u5f55

    CREATE TABLE addax_reader\n(\n    c_bigint     bigint,\n    c_varchar    varchar(100),\n    c_timestamp  timestamp,\n    c_text       text,\n    c_decimal    decimal(8, 3),\n    c_mediumtext mediumtext,\n    c_longtext   longtext,\n    c_int        int,\n    c_time       time,\n    c_datetime   datetime,\n    c_enum       enum('one', 'two', 'three'),\n    c_float      float,\n    c_smallint   smallint,\n    c_bit        bit,\n    c_double     double,\n    c_blob       blob,\n    c_char       char(5),\n    c_varbinary  varbinary(100),\n    c_tinyint    tinyint,\n    c_json       json,\n    c_set SET ('a', 'b', 'c', 'd'),\n    c_binary     binary,\n    c_longblob   longblob,\n    c_mediumblob mediumblob\n);\nINSERT INTO addax_reader\nVALUES (2E18,\n        'a varchar data',\n        '2021-12-12 12:12:12',\n        'a long text',\n        12345.122,\n        'a medium text',\n        'a long text',\n         2 ^ 32 - 1,\n        '12:13:14',\n        '2021-12-12 12:13:14',\n        'one',\n        17.191,\n        126,\n        0,\n        1114.1114,\n        'blob',\n        'a123b',\n        'a var binary content',\n        126,\n        '{\"k1\":\"val1\",\"k2\":\"val2\"}',\n        'b',\n        binary(1),\n        x'89504E470D0A1A0A0000000D494844520000001000000010080200000090916836000000017352474200',\n        x'89504E470D0A1A0A0000000D');\n

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/mysql2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"mysqlreader\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"root\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"addax_reader\"\n            ],\n            \"jdbcUrl\": \"jdbc:mysql://127.0.0.1:3306/test\",\n            \"driver\": \"com.mysql.jdbc.Driver\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/mysql2stream.json

    "},{"location":"reader/mysqlreader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/mysql2stream.json\n
    "},{"location":"reader/mysqlreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/mysqlreader/#driver","title":"driver","text":"

    \u5f53\u524d Addax \u91c7\u7528\u7684 MySQL JDBC \u9a71\u52a8\u4e3a 8.0 \u4ee5\u4e0a\u7248\u672c\uff0c\u9a71\u52a8\u7c7b\u540d\u4f7f\u7528\u7684 com.mysql.cj.jdbc.Driver\uff0c\u800c\u4e0d\u662f com.mysql.jdbc.Driver\u3002 \u5982\u679c\u4f60\u9700\u8981\u91c7\u96c6\u7684 MySQL \u670d\u52a1\u4f4e\u4e8e 5.6\uff0c\u9700\u8981\u4f7f\u7528\u5230 Connector/J 5.1 \u9a71\u52a8\uff0c\u5219\u53ef\u4ee5\u91c7\u53d6\u4e0b\u9762\u7684\u6b65\u9aa4\uff1a

    \u66ff\u6362\u63d2\u4ef6\u5185\u7f6e\u7684\u9a71\u52a8

    rm -f plugin/reader/mysqlreader/libs/mysql-connector-java-*.jar

    \u62f7\u8d1d\u8001\u7684\u9a71\u52a8\u5230\u63d2\u4ef6\u76ee\u5f55

    cp mysql-connector-java-5.1.48.jar plugin/reader/mysqlreader/libs/

    \u6307\u5b9a\u9a71\u52a8\u7c7b\u540d\u79f0

    \u5728\u4f60\u7684 json \u6587\u4ef6\u7c7b\uff0c\u914d\u7f6e \"driver\": \"com.mysql.jdbc.Driver\"

    "},{"location":"reader/mysqlreader/#_4","title":"\u7c7b\u578b\u8f6c\u6362\u6ce8\u610f\u4e8b\u9879","text":""},{"location":"reader/oraclereader/","title":"Oracle Reader","text":"

    Oracle Reader \u63d2\u4ef6\u7528\u4e8e\u4ece Oracle \u8bfb\u53d6\u6570\u636e

    "},{"location":"reader/oraclereader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4eceOracle\u6570\u636e\u5e93\u540c\u6b65\u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a:

    job/oracle2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": 1048576,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"oraclereader\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"root\",\n          \"column\": [\n            \"id\",\n            \"name\"\n          ],\n          \"splitPk\": \"db_id\",\n          \"connection\": {\n            \"table\": [\n              \"table\"\n            ],\n            \"jdbcUrl\": \"jdbc:oracle:thin:@<HOST_NAME>:PORT:<DATABASE_NAME>\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/oraclereader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/oraclereader/#geometry","title":"\u5bf9 GEOMETRY \u7c7b\u578b\u7684\u652f\u6301","text":"

    \u4ece Addax 4.0.13 \u5f00\u59cb\uff0c\u5b9e\u9a8c\u6027\u7684\u652f\u6301 Oracle GEOMETRY \u7c7b\u578b\uff0c\u8be5\u63d2\u4ef6\u4f1a\u628a\u8be5\u7c7b\u578b\u7684\u6570\u636e\u8f6c\u4e3a JSON \u6570\u7ec4\u5b57\u7b26\u4e32\u3002

    \u5047\u5b9a\u4f60\u6709\u8fd9\u6837\u7684\u7684\u8868\u548c\u6570\u636e

    --8<-- \"assets/sql/oracle_geom.sql\n

    \u8bfb\u53d6\u8868\u8be5\u7684\u6570\u636e\u7684\u6700\u540e\u8f93\u51fa\u7ed3\u679c\u7c7b\u4f3c\u5982\u4e0b\uff1a

    --8<-- \"assets/output/oracle_geom_reader.txt\n

    \u6ce8\u610f\uff1a\u8be5\u6570\u636e\u7c7b\u578b\u76ee\u524d\u8fd8\u5904\u4e8e\u5b9e\u9a8c\u652f\u6301\u9636\u6bb5\uff0c\u4f5c\u8005\u5bf9\u6b64\u6570\u636e\u7c7b\u578b\u7684\u7406\u89e3\u5e76\u4e0d\u6df1\u523b\uff0c\u4e5f\u672a\u7ecf\u8fc7\u5168\u9762\u7684\u6d4b\u8bd5\uff0c\u8bf7\u52ff\u76f4\u63a5\u5728\u751f\u4ea7\u73af\u5883\u4f7f\u7528\u3002

    "},{"location":"reader/postgresqlreader/","title":"PostgreSQL Reader","text":"

    PostgreSQL Reader \u63d2\u4ef6\u7528\u4e8e\u4ece PostgreSQL \u8bfb\u53d6\u6570\u636e

    "},{"location":"reader/postgresqlreader/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u5efa\u8868\u8bed\u53e5\u4ee5\u53ca\u8f93\u5165\u63d2\u5165\u8bed\u53e5\u5982\u4e0b\uff1a

    create table if not exists addax_tbl\n(\n    c_bigint bigint,\n    c_bit bit(3),\n    c_bool boolean,\n    c_byte bytea,\n    c_char char(10),\n    c_varchar varchar(20),\n    c_date date,\n    c_double float8,\n    c_int integer,\n    c_json json,\n    c_number decimal(8, 3),\n    c_real real,\n    c_small smallint,\n    c_text text,\n    c_ts timestamp,\n    c_uuid uuid,\n    c_xml xml,\n    c_money money,\n    c_inet inet,\n    c_cidr cidr,\n    c_macaddr macaddr\n    );\n\ninsert into addax_tbl\nvalues (999988887777,\n        b'101',\n        TRUE,\n        '\\xDEADBEEF',\n        'hello',\n        'hello, world',\n        '2021-01-04',\n        999888.9972,\n        9876542,\n        '{\"bar\": \"baz\", \"balance\": 7.77, \"active\": false}'::json,\n        12345.123,\n        123.123,\n        126,\n        'this is a long text ',\n        '2020-01-04 12:13:14',\n        'A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11'::uuid,\n        '<foo>bar</foo>'::xml,\n        '52093.89'::money,\n        '192.168.1.1'::inet,\n        '192.168.1/24'::cidr,\n        '08002b:010203'::macaddr);\n

    \u914d\u7f6e\u4e00\u4e2a\u4ecePostgreSQL\u6570\u636e\u5e93\u540c\u6b65\u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a:

    job/postgres2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"postgresqlreader\",\n        \"parameter\": {\n          \"username\": \"pgtest\",\n          \"password\": \"pgtest\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"addax_tbl\"\n            ],\n            \"jdbcUrl\": \"jdbc:postgresql://127.0.0.1:5432/pgtest\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/postgres2stream.json

    "},{"location":"reader/postgresqlreader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/postgres2stream.json\n
    "},{"location":"reader/postgresqlreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/rdbmsreader/","title":"RDBMS Reader","text":"

    RDBMS Reader \u63d2\u4ef6\u652f\u6301\u4ece\u4f20\u7edf RDBMS \u8bfb\u53d6\u6570\u636e\u3002\u8fd9\u662f\u4e00\u4e2a\u901a\u7528\u5173\u7cfb\u6570\u636e\u5e93\u8bfb\u53d6\u63d2\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u6ce8\u518c\u6570\u636e\u5e93\u9a71\u52a8\u7b49\u65b9\u5f0f\u652f\u6301\u66f4\u591a\u5173\u7cfb\u6570\u636e\u5e93\u8bfb\u53d6\u3002

    \u540c\u65f6 RDBMS Reader \u53c8\u662f\u5176\u4ed6\u5173\u7cfb\u578b\u6570\u636e\u5e93\u8bfb\u53d6\u63d2\u4ef6\u7684\u7684\u57fa\u7840\u7c7b\u3002\u4ee5\u4e0b\u8bfb\u53d6\u63d2\u4ef6\u5747\u4f9d\u8d56\u8be5\u63d2\u4ef6

    \u6ce8\u610f\uff0c \u5982\u679c\u5df2\u7ecf\u63d0\u4f9b\u4e86\u4e13\u95e8\u7684\u6570\u636e\u5e93\u8bfb\u53d6\u63d2\u4ef6\u7684\uff0c\u63a8\u8350\u4f7f\u7528\u4e13\u7528\u63d2\u4ef6\uff0c\u5982\u679c\u4f60\u9700\u8981\u8bfb\u53d6\u7684\u6570\u636e\u5e93\u6ca1\u6709\u4e13\u95e8\u63d2\u4ef6\uff0c\u5219\u8003\u8651\u4f7f\u7528\u8be5\u901a\u7528\u63d2\u4ef6\u3002 \u5728\u4f7f\u7528\u4e4b\u524d\uff0c\u8fd8\u9700\u8981\u6267\u884c\u4ee5\u4e0b\u64cd\u4f5c\u624d\u53ef\u4ee5\u6b63\u5e38\u8fd0\u884c\uff0c\u5426\u5219\u8fd0\u884c\u4f1a\u51fa\u73b0\u5f02\u5e38\u3002

    "},{"location":"reader/rdbmsreader/#_1","title":"\u914d\u7f6e\u9a71\u52a8","text":"

    \u5047\u5b9a\u4f60\u9700\u8981\u8bfb\u53d6 IBM DB2 \u7684\u6570\u636e\uff0c\u56e0\u4e3a\u6ca1\u6709\u63d0\u4f9b\u4e13\u95e8\u7684\u8bfb\u53d6\u63d2\u4ef6\uff0c\u6240\u4ee5\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528\u8be5\u63d2\u4ef6\u6765\u5b9e\u73b0\uff0c\u5728\u4f7f\u7528\u4e4b\u524d\uff0c\u9700\u8981\u4e0b\u8f7d\u5bf9\u5e94\u7684 JDBC \u9a71\u52a8\uff0c\u5e76\u62f7\u8d1d\u5230 plugin/reader/rdbmsreader/libs \u76ee\u5f55\u3002 \u5982\u679c\u4f60\u7684\u9a71\u52a8\u7c7b\u540d\u6bd4\u8f83\u7279\u6b8a\uff0c\u5219\u9700\u8981\u5728\u4efb\u52a1\u914d\u7f6e\u6587\u4ef6\u4e2d\u627e\u5230 driver \u4e00\u9879\uff0c\u586b\u5199\u6b63\u786e\u7684 JDBC \u9a71\u52a8\u540d\uff0c\u6bd4\u5982 DB2 \u7684\u9a71\u52a8\u540d\u4e3a com.ibm.db2.jcc.DB2Driver\u3002\u5982\u679c\u4e0d\u586b\u5199\uff0c\u5219\u63d2\u4ef6\u4f1a\u81ea\u52a8\u731c\u6d4b\u9a71\u52a8\u540d\u3002

    \u4ee5\u4e0b\u5217\u51fa\u5e38\u89c1\u7684\u6570\u636e\u5e93\u4ee5\u53ca\u5bf9\u5e94\u7684\u9a71\u52a8\u540d\u79f0

    "},{"location":"reader/rdbmsreader/#_2","title":"\u914d\u7f6e\u8bf4\u660e","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u5c55\u793a\u4e86\u5982\u4f55\u4ece Presto \u6570\u636e\u5e93\u8bfb\u53d6\u6570\u636e\u5230\u7ec8\u7aef

    job/rdbms2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": 1048576,\n        \"channel\": 1\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"rdbmsreader\",\n        \"parameter\": {\n          \"username\": \"hive\",\n          \"password\": \"\",\n          \"column\": [\n            \"*\"\n          ],\n          \"driver\": \"io.prestosql.jdbc.PrestoDriver\",\n          \"connection\": {\n            \"table\": [\n              \"default.table\"\n            ],\n            \"jdbcUrl\": \"jdbc:presto://127.0.0.1:8080/hive\"\n          },\n          \"fetchSize\": 1024,\n          \"where\": \"1 = 1\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/rdbmsreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 jdbcUrl \u662f list \u65e0 \u5bf9\u7aef\u6570\u636e\u5e93\u7684JDBC\u8fde\u63a5\u4fe1\u606f\uff0cjdbcUrl\u6309\u7167RDBMS\u5b98\u65b9\u89c4\u8303\uff0c\u5e76\u53ef\u4ee5\u586b\u5199\u8fde\u63a5\u9644\u4ef6\u63a7\u5236\u4fe1\u606f driver \u5426 string \u65e0 \u81ea\u5b9a\u4e49\u9a71\u52a8\u7c7b\u540d\uff0c\u89e3\u51b3\u517c\u5bb9\u6027\u95ee\u9898\uff0c\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0 username \u662f string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801 table \u662f list \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d,\u4f7f\u7528JSON\u6570\u636e\u683c\u5f0f\uff0c\u5f53\u914d\u7f6e\u4e3a\u591a\u5f20\u8868\u65f6\uff0c\u7528\u6237\u81ea\u5df1\u9700\u4fdd\u8bc1\u591a\u5f20\u8868\u662f\u540c\u4e00\u8868\u7ed3\u6784 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1\u540e splitPk \u5426 string \u65e0 \u4f7f\u7528splitPk\u4ee3\u8868\u7684\u5b57\u6bb5\u8fdb\u884c\u6570\u636e\u5206\u7247\uff0c\u8fd9\u6837\u53ef\u4ee5\u5927\u5927\u63d0\u4f9b\u6570\u636e\u540c\u6b65\u7684\u6548\u80fd\uff0c\u6ce8\u610f\u4e8b\u9879\u89c1\u540e autoPk \u5426 boolean false \u662f\u5426\u81ea\u52a8\u731c\u6d4b\u5206\u7247\u4e3b\u952e\uff0c3.2.6 \u7248\u672c\u5f15\u5165\uff0c\u8be6\u89c1\u540e\u9762\u63cf\u8ff0 where \u5426 string \u65e0 \u9488\u5bf9\u8868\u7684\u7b5b\u9009\u6761\u4ef6 session \u662f\u5426 list \u65e0 \u9488\u5bf9\u672c\u5730\u8fde\u63a5,\u4fee\u6539\u4f1a\u8bdd\u914d\u7f6e,\u8be6\u89c1\u4e0b\u6587 querySql \u5426 string \u65e0 \u4f7f\u7528\u81ea\u5b9a\u4e49\u7684SQL\u800c\u4e0d\u662f\u6307\u5b9a\u8868\u6765\u83b7\u53d6\u6570\u636e\uff0c\u5f53\u914d\u7f6e\u4e86\u8fd9\u4e00\u9879\u4e4b\u540e\uff0c\u5ffd\u7565 table\uff0ccolumn\u914d\u7f6e\u9879\u5ffd\u7565 fetchSize \u5426 int 1024 \u5b9a\u4e49\u4e86\u63d2\u4ef6\u548c\u6570\u636e\u5e93\u670d\u52a1\u5668\u7aef\u6bcf\u6b21\u6279\u91cf\u6570\u636e\u83b7\u53d6\u6761\u6570\uff0c\u8c03\u9ad8\u8be5\u503c\u53ef\u80fd\u5bfc\u81f4 Addax \u51fa\u73b0OOM"},{"location":"reader/rdbmsreader/#jdbcurl","title":"jdbcUrl","text":"

    jdbcUrl \u914d\u7f6e\u9664\u4e86\u914d\u7f6e\u5fc5\u8981\u7684\u4fe1\u606f\u5916\uff0c\u6211\u4eec\u8fd8\u53ef\u4ee5\u5728\u589e\u52a0\u6bcf\u79cd\u7279\u5b9a\u9a71\u52a8\u7684\u7279\u5b9a\u914d\u7f6e\u5c5e\u6027\uff0c\u8fd9\u91cc\u7279\u522b\u63d0\u5230\u6211\u4eec\u53ef\u4ee5\u5229\u7528\u914d\u7f6e\u5c5e\u6027\u5bf9\u4ee3\u7406\u7684\u652f\u6301\u4ece\u800c\u5b9e\u73b0\u901a\u8fc7\u4ee3\u7406\u8bbf\u95ee\u6570\u636e\u5e93\u7684\u529f\u80fd\u3002 \u6bd4\u5982\u5bf9\u4e8e PrestoSQL \u6570\u636e\u5e93\u7684 JDBC \u9a71\u52a8\u800c\u8a00\uff0c\u652f\u6301 socksProxy \u53c2\u6570\uff0c\u4e8e\u662f\u4e0a\u8ff0\u914d\u7f6e\u7684 jdbcUrl \u53ef\u4ee5\u4fee\u6539\u4e3a

    jdbc:presto://127.0.0.1:8080/hive?socksProxy=192.168.1.101:1081

    \u5927\u90e8\u5206\u5173\u7cfb\u578b\u6570\u636e\u5e93\u7684 JDBC \u9a71\u52a8\u652f\u6301 socksProxyHost,socksProxyPort \u53c2\u6570\u6765\u652f\u6301\u4ee3\u7406\u8bbf\u95ee\u3002\u4e5f\u6709\u4e00\u4e9b\u7279\u522b\u7684\u60c5\u51b5\u3002

    \u4ee5\u4e0b\u662f\u5404\u7c7b\u6570\u636e\u5e93 JDBC \u9a71\u52a8\u6240\u652f\u6301\u7684\u4ee3\u7406\u7c7b\u578b\u4ee5\u53ca\u914d\u7f6e\u65b9\u5f0f

    \u6570\u636e\u5e93 \u4ee3\u7406\u7c7b\u578b \u4ee3\u7406\u914d\u7f6e \u4f8b\u5b50 MySQL socks socksProxyHost,socksProxyPort socksProxyHost=192.168.1.101&socksProxyPort=1081 Presto socks socksProxy socksProxy=192.168.1.101:1081 Presto http httpProxy httpProxy=192.168.1.101:3128"},{"location":"reader/rdbmsreader/#driver","title":"driver","text":"

    \u5927\u90e8\u5206\u60c5\u51b5\u4e0b\uff0c\u4e00\u4e2a\u6570\u636e\u5e93\u7684JDBC\u9a71\u52a8\u662f\u56fa\u5b9a\u7684\uff0c\u4f46\u6709\u4e9b\u56e0\u4e3a\u7248\u672c\u7684\u4e0d\u540c\uff0c\u6240\u5efa\u8bae\u7684\u9a71\u52a8\u7c7b\u540d\u4e0d\u540c\uff0c\u6bd4\u5982 MySQL\u3002 \u65b0\u7684 MySQL JDBC \u9a71\u52a8\u7c7b\u578b\u63a8\u8350\u4f7f\u7528 com.mysql.cj.jdbc.Driver \u800c\u4e0d\u662f\u4ee5\u524d\u7684 com.mysql.jdbc.Drver \u3002\u5982\u679c\u60f3\u8981\u4f7f\u7528\u5c31\u7684\u9a71\u52a8\u540d\u79f0\uff0c\u5219\u53ef\u4ee5\u914d\u7f6e driver \u914d\u7f6e\u9879\u3002\u5426\u5219\u63d2\u4ef6\u4f1a\u81ea\u52a8\u4f9d\u636e jdbcUrl \u4e2d\u7684\u5b57\u7b26\u4e32\u6765\u731c\u6d4b\u9a71\u52a8\u540d\u79f0.

    "},{"location":"reader/rdbmsreader/#column","title":"column","text":"

    \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u4f7f\u7528JSON\u7684\u6570\u7ec4\u63cf\u8ff0\u5b57\u6bb5\u4fe1\u606f\u3002\u7528\u6237\u4f7f\u7528 * \u4ee3\u8868\u9ed8\u8ba4\u4f7f\u7528\u6240\u6709\u5217\u914d\u7f6e\uff0c\u4f8b\u5982 [\"*\"]\u3002

    \u652f\u6301\u5217\u88c1\u526a\uff0c\u5373\u5217\u53ef\u4ee5\u6311\u9009\u90e8\u5206\u5217\u8fdb\u884c\u5bfc\u51fa\u3002

    \u652f\u6301\u5217\u6362\u5e8f\uff0c\u5373\u5217\u53ef\u4ee5\u4e0d\u6309\u7167\u8868schema\u4fe1\u606f\u8fdb\u884c\u5bfc\u51fa\u3002

    \u652f\u6301\u5e38\u91cf\u914d\u7f6e\uff0c\u7528\u6237\u9700\u8981\u6309\u7167JSON\u683c\u5f0f:

    [\"id\", \"`table`\", \"1\", \"'bazhen.csy'\", \"null\", \"to_char(a + 1)\", \"2.3\" , \"true\"]

    Column\u5fc5\u987b\u663e\u793a\u586b\u5199\uff0c\u4e0d\u5141\u8bb8\u4e3a\u7a7a\uff01

    "},{"location":"reader/rdbmsreader/#splitpk","title":"splitPk","text":"

    \u5982\u679c\u6307\u5b9a splitPk\uff0c\u8868\u793a\u7528\u6237\u5e0c\u671b\u4f7f\u7528 splitPk \u4ee3\u8868\u7684\u5b57\u6bb5\u8fdb\u884c\u6570\u636e\u5206\u7247\uff0c\u56e0\u6b64\u4f1a\u542f\u52a8\u5e76\u53d1\u4efb\u52a1\u8fdb\u884c\u6570\u636e\u540c\u6b65\uff0c\u8fd9\u6837\u53ef\u4ee5\u5927\u5927\u63d0\u4f9b\u6570\u636e\u540c\u6b65\u7684\u6548\u80fd\u3002

    \u63a8\u8350 splitPk \u7528\u6237\u4f7f\u7528\u8868\u4e3b\u952e\uff0c\u56e0\u4e3a\u8868\u4e3b\u952e\u901a\u5e38\u60c5\u51b5\u4e0b\u6bd4\u8f83\u5747\u5300\uff0c\u56e0\u6b64\u5207\u5206\u51fa\u6765\u7684\u5206\u7247\u4e5f\u4e0d\u5bb9\u6613\u51fa\u73b0\u6570\u636e\u70ed\u70b9\u3002

    \u76ee\u524d splitPk \u4ec5\u652f\u6301\u6574\u5f62\u3001\u5b57\u7b26\u4e32\u578b\u6570\u636e(ASCII\u7c7b\u578b) \u5207\u5206\uff0c\u4e0d\u652f\u6301\u6d6e\u70b9\u3001\u65e5\u671f\u7b49\u5176\u4ed6\u7c7b\u578b\u3002 \u5982\u679c\u7528\u6237\u6307\u5b9a\u5176\u4ed6\u975e\u652f\u6301\u7c7b\u578b\uff0cRDBMSReader \u5c06\u62a5\u9519\uff01

    splitPk \u5982\u679c\u4e0d\u586b\u5199\uff0c\u5c06\u89c6\u4f5c\u7528\u6237\u4e0d\u5bf9\u5355\u8868\u8fdb\u884c\u5207\u5206\uff0c\u800c\u4f7f\u7528\u5355\u901a\u9053\u540c\u6b65\u5168\u91cf\u6570\u636e\u3002

    "},{"location":"reader/rdbmsreader/#autopk","title":"autoPk","text":"

    \u4ece 3.2.6 \u7248\u672c\u5f00\u59cb\uff0c\u652f\u6301\u81ea\u52a8\u83b7\u53d6\u8868\u4e3b\u952e\u6216\u552f\u4e00\u7d22\u5f15\uff0c\u5982\u679c\u8bbe\u7f6e\u4e3a true \uff0c\u5c06\u5c1d\u8bd5\u901a\u8fc7\u67e5\u8be2\u6570\u636e\u5e93\u7684\u5143\u6570\u636e\u4fe1\u606f\u83b7\u53d6\u6307\u5b9a\u8868\u7684\u4e3b\u952e\u5b57\u6bb5\u6216\u552f\u4e00\u7d22\u5f15\u5b57\u6bb5\uff0c\u5982\u679c\u83b7\u53d6\u53ef\u7528\u4e8e\u5206\u9694\u7684 \u5b57\u6bb5\u4e0d\u6b62\u4e00\u4e2a\uff0c\u5219\u9ed8\u8ba4\u53d6\u7b2c\u4e00\u4e2a\u3002

    \u8be5\u7279\u6027\u76ee\u524d\u652f\u6301\u7684\u6570\u636e\u5e93\u6709\uff1a

    "},{"location":"reader/rdbmsreader/#session","title":"session","text":"

    \u63a7\u5236\u5199\u5165\u6570\u636e\u7684\u65f6\u95f4\u683c\u5f0f\uff0c\u65f6\u533a\u7b49\u7684\u914d\u7f6e\uff0c\u76ee\u524d\u4ec5\u5bf9 MySQL, Oracle, SQLServer \u6709\u6548\u3002\u4e0b\u9762\u662f\u4e00\u4e2a\u9488\u5bf9 Oracle \u6570\u636e\u5e93\u914d\u7f6e session \u7684\u4f8b\u5b50\u3002

    {\n  \"session\": [\n    \"alter session set NLS_DATE_FORMAT='yyyy-mm-dd hh24:mi:ss'\",\n    \"alter session set NLS_TIMESTAMP_FORMAT='yyyy-mm-dd hh24:mi:ss'\",\n    \"alter session set NLS_TIMESTAMP_TZ_FORMAT='yyyy-mm-dd hh24:mi:ss'\",\n    \"alter session set TIME_ZONE='Asia/Chongqing'\"\n  ]\n}\n

    \u6ce8\u610f &quot;\u662f \" \u7684\u8f6c\u4e49\u5b57\u7b26\u4e32

    "},{"location":"reader/rdbmsreader/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b RDBMS \u6570\u636e\u7c7b\u578b Long int, tinyint, smallint, mediumint, int, bigint Double float, double, decimal String varchar, char, tinytext, text, mediumtext, longtext, year,xml Date date, datetime, timestamp, time Boolean bit, bool Bytes tinyblob, mediumblob, blob, longblob, varbinary"},{"location":"reader/redisreader/","title":"Redis Reader","text":"

    Redis Reader \u63d2\u4ef6\u7528\u4e8e\u8bfb\u53d6 Redis RDB \u6570\u636e

    "},{"location":"reader/redisreader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"redisreader\",\n        \"parameter\": {\n          \"connection\": {\n            \"uri\": [\"tcp://127.0.0.1:6379\", \"file:///data/dump.rdb\", \"http://localhost/dump.rdb\"],\n            \"auth\": \"password\"\n          },\n          \"include\": [\n            \"^user\"\n          ],\n          \"exclude\": [\n            \"^password\"\n          ],\n          \"db\": [\n            0,\n            1\n          ]\n        }\n      },\n      \"writer\": {\n        \"name\": \"rediswriter\",\n        \"parameter\": {\n          \"connection\": {\n            \"uri\": \"tcp://127.0.0.1:6379\",\n            \"auth\": \"123456\"\n          },\n          \"timeout\": 60000\n        }\n      }\n    },\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1\n      }\n    }\n  }\n}\n
    "},{"location":"reader/redisreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u9ed8\u8ba4\u503c \u63cf\u8ff0 uri \u662f \u5426 redis\u94fe\u63a5,\u652f\u6301\u591a\u4e2a\u672c\u5730rdb\u6587\u4ef6/\u7f51\u7edcrdb\u6587\u4ef6,\u5982\u679c\u662f\u96c6\u7fa4,\u586b\u5199\u6240\u6709master\u8282\u70b9\u5730\u5740 db \u5426 \u65e0 \u9700\u8981\u8bfb\u53d6\u7684db\u7d22\u5f15,\u82e5\u4e0d\u586b\u5199,\u5219\u8bfb\u53d6\u6240\u6709db include \u5426 \u65e0 \u8981\u5305\u542b\u7684 key, \u652f\u6301\u6b63\u5219\u8868\u8fbe\u5f0f exclude \u5426 \u65e0 \u8981\u6392\u9664\u7684 key,\u652f\u6301\u6b63\u5219\u8868\u8fbe\u5f0f"},{"location":"reader/redisreader/#_3","title":"\u7ea6\u675f\u9650\u5236","text":"
    1. \u4e0d\u652f\u6301\u76f4\u63a5\u8bfb\u53d6\u4efb\u4f55\u4e0d\u652f\u6301 sync \u547d\u4ee4\u7684 redis server\uff0c\u5982\u679c\u9700\u8981\u8bf7\u5907\u4efd\u7684rdb\u6587\u4ef6\u8fdb\u884c\u8bfb\u53d6\u3002
    2. \u5982\u679c\u662f\u539f\u751fredis cluster\u96c6\u7fa4\uff0c\u8bf7\u586b\u5199\u6240\u6709master\u8282\u70b9\u7684tcp\u5730\u5740\uff0credisreader \u63d2\u4ef6\u4f1a\u81ea\u52a8dump \u6240\u6709\u8282\u70b9\u7684rdb\u6587\u4ef6\u3002
    3. \u4ec5\u89e3\u6790 String \u6570\u636e\u7c7b\u578b\uff0c\u5176\u4ed6\u590d\u5408\u7c7b\u578b(Sets, List \u7b49\u4f1a\u5ffd\u7565)
    "},{"location":"reader/s3reader/","title":"S3 Reader","text":"

    S3 Reader \u63d2\u4ef6\u7528\u4e8e\u8bfb\u53d6 Amazon AWS S3 \u5b58\u50a8\u4e0a\u7684\u6570\u636e\u3002\u5728\u5b9e\u73b0\u4e0a\uff0c\u672c\u63d2\u4ef6\u57fa\u4e8e S3 \u5b98\u65b9\u7684 SDK 2.0 \u7f16\u5199\u3002

    \u540c\u65f6\u672c\u63d2\u4ef6\u4e5f\u652f\u6301\u8bfb\u53d6\u517c\u5bb9 S3 \u534f\u8bae\u7684\u5b58\u50a8\u670d\u52a1\uff0c\u6bd4\u5982 MinIO

    "},{"location":"reader/s3reader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u4ee5\u4e0b\u6837\u4f8b\u914d\u7f6e\u7528\u4e8e\u4ece S3 \u5b58\u50a8\u4e0a\u8bfb\u53d6\u4e24\u4e2a\u6587\u4ef6\uff0c\u5e76\u6253\u5370\u51fa\u6765

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"s3reader\",\n        \"parameter\": {\n          \"endpoint\": \"https://s3.amazonaws.com\",\n          \"accessId\": \"xxxxxxxxxxxx\",\n          \"accessKey\": \"xxxxxxxxxxxxxxxxxxxxxxx\",\n          \"bucket\": \"test\",\n          \"object\": [\n            \"1.csv\",\n            \"aa.csv\",\n            \"upload_*.csv\",\n            \"bb_??.csv\"\n          ],\n          \"column\": [\n            \"*\"\n          ],\n          \"region\": \"ap-northeast-1\",\n          \"fileFormat\": \"csv\",\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/s3reader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 S3 Server\u7684 EndPoint\u5730\u5740\uff0c\u4f8b\u5982 s3.xx.amazonaws.com region \u662f string \u65e0 S3 Server\u7684 Region \u5730\u5740\uff0c\u4f8b\u5982 ap-southeast-1 accessId \u662f string \u65e0 \u8bbf\u95ee ID accessKey \u662f string \u65e0 \u8bbf\u95ee Key bucket \u662f string \u65e0 \u8981\u8bfb\u53d6\u7684 bucket object \u662f list \u65e0 \u8981\u8bfb\u53d6\u7684 object\uff0c\u53ef\u4ee5\u586b\u5199\u591a\u4e2a\u4ee5\u53ca\u901a\u914d\u7b26\u65b9\u5f0f\uff0c\u8be6\u89c1\u4e0b\u9762\u8bf4\u660e column \u662f list \u65e0 \u8bfb\u53d6\u7684 object \u7684\u5217\u4fe1\u606f\uff0c\u586b\u5199\u65b9\u5f0f\u89c1RDBMS Reader \u4e2d column \u63cf\u8ff0 fieldDelimiter \u5426 string , \u8bfb\u53d6\u7684\u5b57\u6bb5\u5206\u9694\u7b26\uff0c\u4ec5\u652f\u6301\u5355\u5b57\u7b26 compress \u5426 string \u65e0 \u6587\u4ef6\u538b\u7f29\u683c\u5f0f\uff0c\u9ed8\u8ba4\u4e0d\u538b\u7f29 encoding \u5426 string utf8 \u6587\u4ef6\u7f16\u7801\u683c\u5f0f writeMode \u5426 string nonConflict"},{"location":"reader/s3reader/#object","title":"object","text":"

    \u5f53\u6307\u5b9a\u5355\u4e2a object\uff0c\u63d2\u4ef6\u6682\u65f6\u53ea\u80fd\u4f7f\u7528\u5355\u7ebf\u7a0b\u8fdb\u884c\u6570\u636e\u62bd\u53d6\u3002

    \u5f53\u6307\u5b9a\u591a\u4e2a object\uff0c\u63d2\u4ef6\u652f\u6301\u4f7f\u7528\u591a\u7ebf\u7a0b\u8fdb\u884c\u6570\u636e\u62bd\u53d6\u3002\u7ebf\u7a0b\u5e76\u53d1\u6570\u901a\u8fc7\u901a\u9053\u6570\u6307\u5b9a\u3002

    \u5f53\u6307\u5b9a\u901a\u914d\u7b26\uff0c\u63d2\u4ef6\u5c1d\u8bd5\u904d\u5386\u51fa\u591a\u4e2a object \u4fe1\u606f\u3002

    \u4f8b\u5982: \u6307\u5b9a /* \u4ee3\u8868\u8bfb\u53d6 bucket \u4e0b\u6240\u6709\u7684 object\uff0c\u6307\u5b9a /foo/* \u4ee3\u8868\u8bfb\u53d6 foo \u76ee\u5f55\u4e0b\u6240\u6709\u7684 object\u3002

    "},{"location":"reader/s3reader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b S3 \u6570\u636e\u7c7b\u578b Long int, tinyint, smallint, mediumint, int, bigint Double float, double, decimal String varchar, char, tinytext, text, mediumtext, longtext, year,xml Date date, datetime, timestamp, time Boolean bit, bool Bytes tinyblob, mediumblob, blob, longblob, varbinary"},{"location":"reader/s3reader/#_4","title":"\u9650\u5236\u8bf4\u660e","text":"
    1. \u4ec5\u652f\u6301\u8bfb\u53d6\u6587\u672c\u6587\u4ef6
    "},{"location":"reader/sqlitereader/","title":"SQLite Reader","text":"

    SQLite Reader \u63d2\u4ef6\u7528\u4e8e\u8bfb\u53d6\u6307\u5b9a\u76ee\u5f55\u4e0b\u7684 sqlite \u6587\u4ef6\uff0c \u4ed6\u7ee7\u627f\u4e8e RDBMS Reader

    "},{"location":"reader/sqlitereader/#_1","title":"\u793a\u4f8b","text":"

    \u6211\u4eec\u521b\u5efa\u793a\u4f8b\u6587\u4ef6\uff1a

    $ sqlite3  /tmp/test.sqlite3\nSQLite version 3.7.17 2013-05-20 00:56:22\nEnter \".help\" for instructions\nEnter SQL statements terminated with a \";\"\nsqlite> create table test(id int, name varchar(10), salary double);\nsqlite> insert into test values(1,'foo', 12.13),(2,'bar',202.22);\nsqlite> .q\n

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/sqlite2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"sqlitereader\",\n        \"parameter\": {\n          \"username\": \"fakeuser\",\n          \"password\": \"\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:sqlite:/tmp/test.sqlite3\",\n            \"table\": [\n              \"test\"\n            ]\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/sqlite2stream.json

    "},{"location":"reader/sqlitereader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/sqlite2stream.json\n
    "},{"location":"reader/sqlitereader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/sqlserverreader/","title":"SQLServer Reader","text":"

    SqlServerReader \u63d2\u4ef6\u7528\u4e8e\u4ece\u4ece SQLServer \u8bfb\u53d6\u6570\u636e\u3002

    "},{"location":"reader/sqlserverreader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4ece SQLServer \u6570\u636e\u5e93\u540c\u6b65\u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a:

    job/sqlserver2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"sqlserverreader\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"root\",\n          \"column\": [\n            \"*\"\n          ],\n          \"splitPk\": \"db_id\",\n          \"connection\": {\n            \"table\": [\n              \"table\"\n            ],\n            \"jdbcUrl\": \"jdbc:sqlserver://localhost:3433;DatabaseName=dbname\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true,\n          \"encoding\": \"UTF-8\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/sqlserverreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/streamreader/","title":"Stream Reader","text":"

    Stream Reader \u662f\u4e00\u4e2a\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\u7684\u63d2\u4ef6\uff0c \u4ed6\u4e3b\u8981\u7528\u6765\u5feb\u901f\u751f\u6210\u671f\u671b\u7684\u6570\u636e\u5e76\u5bf9\u5199\u5165\u63d2\u4ef6\u8fdb\u884c\u6d4b\u8bd5

    \u4e00\u4e2a\u5b8c\u6574\u7684 StreamReader \u914d\u7f6e\u6587\u4ef6\u5982\u4e0b\uff1a

    {\n  \"reader\": {\n    \"name\": \"streamreader\",\n    \"parameter\": {\n      \"column\": [\n        {\n          \"value\": \"unique_id\",\n          \"type\": \"string\"\n        },\n        {\n          \"value\": \"1989-06-04 08:12:13\",\n          \"type\": \"date\",\n          \"dateFormat\": \"yyyy-MM-dd HH:mm:ss\"\n        },\n        {\n          \"value\": 1984,\n          \"type\": \"long\"\n        },\n        {\n          \"value\": 1989.64,\n          \"type\": \"double\"\n        },\n        {\n          \"value\": true,\n          \"type\": \"bool\"\n        },\n        {\n          \"value\": \"a long text\",\n          \"type\": \"bytes\"\n        }\n      ],\n      \"sliceRecordCount\": 10\n    }\n  }\n}\n

    \u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u5c06\u4f1a\u751f\u6210 10\u6761\u8bb0\u5f55\uff08\u5047\u5b9achannel\u4e3a1\uff09\uff0c\u6bcf\u6761\u8bb0\u5f55\u7684\u5185\u5bb9\u5982\u4e0b\uff1a

    unique_id,'1989-06-04 08:12:13',1984,1989.64,true,'a long text'

    \u76ee\u524d StreamReader \u652f\u6301\u7684\u8f93\u51fa\u6570\u636e\u7c7b\u578b\u5168\u90e8\u5217\u5728\u4e0a\u9762\uff0c\u5206\u522b\u662f\uff1a

    \u5176\u4e2d date \u7c7b\u578b\u8fd8\u652f\u6301 dateFormat \u914d\u7f6e\uff0c\u7528\u6765\u6307\u5b9a\u8f93\u5165\u7684\u65e5\u671f\u7684\u683c\u5f0f\uff0c\u9ed8\u8ba4\u4e3a yyyy-MM-dd HH:mm:ss\u3002\u6bd4\u5982\u4f60\u7684\u8f93\u5165\u53ef\u4ee5\u8fd9\u6837\uff1a

    {\n  \"value\": \"1989/06/04 12:13:14\",\n  \"type\": \"date\",\n  \"dateFormat\": \"yyyy/MM/dd HH:mm:ss\"\n}\n

    \u6ce8\u610f\uff0c\u65e5\u671f\u7c7b\u578b\u4e0d\u7ba1\u8f93\u5165\u662f\u4f55\u79cd\u683c\u5f0f\uff0c\u5185\u90e8\u90fd\u8f6c\u4e3a yyyy-MM-dd HH:mm:ss \u683c\u5f0f\u3002

    StreamReader \u8fd8\u652f\u6301\u968f\u673a\u8f93\u5165\u529f\u80fd\uff0c\u6bd4\u5982\u6211\u4eec\u8981\u968f\u673a\u5f97\u52300-10\u4e4b\u95f4\u7684\u4efb\u610f\u4e00\u4e2a\u6574\u6570\uff0c\u6211\u4eec\u53ef\u4ee5\u8fd9\u6837\u914d\u7f6e\u5217\uff1a

    {\n  \"random\": \"0,10\",\n  \"type\": \"long\"\n}\n

    \u83b7\u5f97\u4e00\u4e2a 0 \u81f3 100 \u4e4b\u95f4\u7684\u968f\u673a\u6d6e\u70b9\u6570\uff0c\u53ef\u4ee5\u8fd9\u6837\u914d\u7f6e\uff1a

    {\n  \"random\": \"0,100\",\n  \"type\": \"double\"\n}\n

    \u5982\u679c\u8981\u6307\u5b9a\u6d6e\u70b9\u6570\u7684\u5c0f\u6570\u4f4d\u6570\uff0c\u6bd4\u5982\u6307\u5b9a\u5c0f\u6570\u4f4d\u4e3a2\u4f4d\uff0c\u5219\u53ef\u4ee5\u8fd9\u6837\u8bbe\u5b9a

    {\n  \"random\": \"0,100,2\",\n  \"type\": \"double\"\n}\n

    \u6ce8\u610f\uff1a \u5e76\u4e0d\u80fd\u4fdd\u8bc1\u6bcf\u6b21\u751f\u6210\u7684\u5c0f\u6570\u6070\u597d\u662f2\u4f4d\uff0c\u5982\u679c\u5c0f\u6570\u4e3a\u6570\u4e3a0 \uff0c\u5219\u5c0f\u6570\u4f4d\u6570\u4f1a\u5c11\u4e8e\u6307\u5b9a\u7684\u4f4d\u6570\u3002

    \u8fd9\u91cc\u4f7f\u7528 random \u8fd9\u4e2a\u5173\u952e\u5b57\u6765\u8868\u793a\u5176\u503c\u4e3a\u968f\u673a\u503c\uff0c\u5176\u503c\u7684\u8303\u56f4\u4e3a\u5de6\u53f3\u95ed\u533a\u95f4\u3002

    \u5176\u4ed6\u7c7b\u578b\u7684\u968f\u673a\u7c7b\u578b\u914d\u7f6e\u5982\u4e0b\uff1a

    StreamReader \u8fd8\u652f\u6301\u9012\u589e\u51fd\u6570\uff0c\u6bd4\u5982\u6211\u4eec\u8981\u5f97\u5230\u4e00\u4e2a\u4ece1\u5f00\u59cb\uff0c\u6bcf\u6b21\u52a05\u7684\u7b49\u5dee\u6570\u5217\uff0c\u53ef\u4ee5\u8fd9\u6837\u914d\u7f6e\uff1a

    {\n  \"incr\": \"1,5\",\n  \"type\": \"long\"\n}\n

    \u5982\u679c\u9700\u8981\u83b7\u5f97\u4e00\u4e2a\u9012\u51cf\u7684\u6570\u5217\uff0c\u5219\u628a\u7b2c\u4e8c\u4e2a\u53c2\u6570\u7684\u6b65\u957f\uff08\u4e0a\u4f8b\u4e2d\u76845\uff09\u6539\u4e3a\u8d1f\u6570\u5373\u53ef\u3002\u6b65\u957f\u9ed8\u8ba4\u503c\u4e3a1\u3002

    \u9012\u589e\u8fd8\u652f\u6301\u65e5\u671f\u7c7b\u578b( 4.0.1 \u7248\u672c\u5f15\u5165)\uff0c\u6bd4\u5982\u4e0b\u9762\u7684\u914d\u7f6e\uff1a

    {\n  \"incr\": \"1989-06-04 09:01:02,2,d\",\n  \"type\": \"date\"\n}\n

    incr \u7531\u4e09\u90e8\u5206\u7ec4\u6210\uff0c\u5206\u522b\u662f\u5f00\u59cb\u65e5\u671f\uff0c\u6b65\u957f\u4ee5\u53ca\u6b65\u957f\u5355\u4f4d\uff0c\u4e2d\u95f4\u7528\u82f1\u6587\u9017\u53f7(,)\u5206\u9694\u3002

    \u914d\u7f6e\u9879 sliceRecordCount \u7528\u6765\u6307\u5b9a\u8981\u751f\u6210\u7684\u6570\u636e\u6761\u6570\uff0c\u5982\u679c\u6307\u5b9a\u7684 channel\uff0c\u5219\u5b9e\u9645\u751f\u6210\u7684\u8bb0\u5f55\u6570\u4e3a sliceRecordCount * channel

    "},{"location":"reader/sybasereader/","title":"Sybase Reader","text":"

    SybaseReader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Sybase \u8bfb\u53d6\u6570\u636e

    "},{"location":"reader/sybasereader/#_1","title":"\u793a\u4f8b","text":"

    \u6211\u4eec\u53ef\u4ee5\u7528 Docker \u5bb9\u5668\u6765\u542f\u52a8\u4e00\u4e2a Sybase \u6570\u636e\u5e93

    docker run -tid --rm  -h dksybase --name sybase  -p 5000:5000  ifnazar/sybase_15_7 bash /sybase/start\n

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/sybasereader.json 
    {\n  \"job\": {\n    \"content\": [\n      {\n        \"reader\": {\n          \"name\": \"sybasereader\",\n          \"parameter\": {\n            \"column\": [\n              \"*\"\n            ],\n            \"connection\": {\n              \"jdbcUrl\": \"jdbc:sybase:Tds:127.0.0.1:5000/master\",\n              \"table\": [\n                \"dbo.ijdbc_function_escapes\"\n              ]\n            },\n            \"username\": \"sa\",\n            \"password\": \"password\"\n          }\n        },\n        \"writer\": {\n          \"name\": \"streamwriter\",\n          \"parameter\": {\n            \"print\": \"true\"\n          }\n        }\n      }\n    ],\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/sybase2stream.json

    "},{"location":"reader/sybasereader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/sybase2stream.json\n
    "},{"location":"reader/sybasereader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/tdenginereader/","title":"TDengine Reader","text":"

    TDengine Reader \u63d2\u4ef6\u7528\u4e8e\u4ece\u6d9b\u601d\u516c\u53f8\u7684 TDengine \u8bfb\u53d6\u6570\u636e\u3002

    "},{"location":"reader/tdenginereader/#_1","title":"\u524d\u7f6e\u6761\u4ef6","text":"

    \u8003\u8651\u5230\u6027\u80fd\u95ee\u9898\uff0c\u8be5\u63d2\u4ef6\u4f7f\u7528\u4e86 TDengine \u7684 JDBC-JNI \u9a71\u52a8\uff0c \u8be5\u9a71\u52a8\u76f4\u63a5\u8c03\u7528\u5ba2\u6237\u7aef API\uff08libtaos.so \u6216 taos.dll\uff09\u5c06\u5199\u5165\u548c\u67e5\u8be2\u8bf7\u6c42\u53d1\u9001\u5230 taosd \u5b9e\u4f8b\u3002\u56e0\u6b64\u5728\u4f7f\u7528\u4e4b\u524d\u9700\u8981\u914d\u7f6e\u597d\u52a8\u6001\u5e93\u94fe\u63a5\u6587\u4ef6\u3002

    \u9996\u5148\u5c06 plugin/reader/tdenginereader/libs/libtaos.so.2.0.16.0 \u62f7\u8d1d\u5230 /usr/lib64 \u76ee\u5f55\uff0c\u7136\u540e\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u521b\u5efa\u8f6f\u94fe\u63a5

    ln -sf /usr/lib64/libtaos.so.2.0.16.0 /usr/lib64/libtaos.so.1\nln -sf /usr/lib64/libtaos.so.1 /usr/lib64/libtaos.so\n
    "},{"location":"reader/tdenginereader/#_2","title":"\u793a\u4f8b","text":"

    TDengine \u6570\u636e\u81ea\u5e26\u4e86\u4e00\u4e2a\u6f14\u793a\u6570\u636e\u5e93 taosdemo , \u6211\u4eec\u4ece\u6f14\u793a\u6570\u636e\u5e93\u8bfb\u53d6\u90e8\u5206\u6570\u636e\u5e76\u6253\u5370\u5230\u7ec8\u7aef

    \u4ee5\u4e0b\u662f\u914d\u7f6e\u6587\u4ef6

    job/tdengine2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"tdenginereader\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"taosdata\",\n          \"beginDateTime\": \"2017-07-14 10:40:00\",\n          \"endDateTime\": \"2017-08-14 10:40:00\",\n          \"splitInterval\": \"1d\",\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:TAOS://127.0.0.1:6030/test\",\n            \"querySql\": [\n              \"select * from test.meters where ts <'2017-07-14 10:40:02' and  loc='beijing' limit 10\"\n            ]\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/tdengine2stream.json

    "},{"location":"reader/tdenginereader/#_3","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/tdengine2stream.json\n

    \u547d\u4ee4\u8f93\u51fa\u7c7b\u4f3c\u5982\u4e0b\uff1a

    2021-02-20 15:32:23.161 [main] INFO  VMInfo - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl\n2021-02-20 15:32:23.229 [main] INFO  Engine -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"password\":\"*****\",\n                    \"connection\":[\n                        {\n                            \"querySql\":[\n                                \"select * from test.meters where ts <'2017-07-14 10:40:02' and  loc='beijing' limit 100\"\n                            ],\n                            \"jdbcUrl\":[\n                                \"jdbc:TAOS://127.0.0.1:6030/test\"\n                            ]\n                        }\n                    ],\n                    \"username\":\"root\"\n                },\n                \"name\":\"tdenginereader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":true\n                },\n                \"name\":\"streamwriter\"\n            }\n    },\n    \"setting\":{\n        \"errorLimit\":{\n            \"record\":0,\n            \"percentage\":0.02\n        },\n        \"speed\":{\n            \"channel\":3\n        }\n    }\n}\n\n2021-02-20 15:32:23.277 [main] INFO  PerfTrace - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-02-20 15:32:23.278 [main] INFO  JobContainer - Addax jobContainer starts job.\n2021-02-20 15:32:23.281 [main] INFO  JobContainer - Set jobId = 0\njava.library.path:/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib\n....\n2021-02-20 15:32:23.687 [0-0-0-reader] INFO  CommonRdbmsReader$Task - Begin to read record by Sql: [select * from test.meters where ts <'2017-07-14 10:40:02' and  loc='beijing' limit 100\n] jdbcUrl:[jdbc:TAOS://127.0.0.1:6030/test].\n2021-02-20 15:32:23.692 [0-0-0-reader] WARN  DBUtil - current database does not supoort TYPE_FORWARD_ONLY/CONCUR_READ_ONLY\n2021-02-20 15:32:23.740 [0-0-0-reader] INFO  CommonRdbmsReader$Task - Finished read record by Sql: [select * from test.meters where ts <'2017-07-14 10:40:02' and  loc='beijing' limit 100\n] jdbcUrl:[jdbc:TAOS://127.0.0.1:6030/test].\n\n1500000001000   5   5   0   1   beijing\n1500000001000   0   6   2   1   beijing\n1500000001000   7   0   0   1   beijing\n1500000001000   8   9   6   1   beijing\n1500000001000   9   9   1   1   beijing\n1500000001000   8   2   0   1   beijing\n1500000001000   4   5   5   3   beijing\n1500000001000   3   3   3   3   beijing\n1500000001000   5   4   8   3   beijing\n1500000001000   9   4   6   3   beijing\n\n2021-02-20 15:32:26.689 [job-0] INFO  JobContainer -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-02-20 15:32:23\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-02-20 15:32:26\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :              800B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :             33rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                 100\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"reader/tdenginereader/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 jdbcUrl \u662f list \u65e0 \u5bf9\u7aef\u6570\u636e\u5e93\u7684JDBC\u8fde\u63a5\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u7684 TAOS \u5fc5\u987b\u5927\u5199 username \u662f string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801 table \u662f list \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d,\u4f7f\u7528JSON\u6570\u636e\u683c\u5f0f\uff0c\u5f53\u914d\u7f6e\u4e3a\u591a\u5f20\u8868\u65f6\uff0c\u7528\u6237\u81ea\u5df1\u9700\u4fdd\u8bc1\u591a\u5f20\u8868\u662f\u540c\u4e00\u8868\u7ed3\u6784 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0rdbmreader where \u5426 string \u65e0 \u9488\u5bf9\u8868\u7684\u7b5b\u9009\u6761\u4ef6 querySql \u5426 list \u65e0 \u4f7f\u7528\u81ea\u5b9a\u4e49\u7684SQL\u800c\u4e0d\u662f\u6307\u5b9a\u8868\u6765\u83b7\u53d6\u6570\u636e\uff0c\u5f53\u914d\u7f6e\u4e86\u8fd9\u4e00\u9879\u4e4b\u540e\uff0cAddax\u7cfb\u7edf\u5c31\u4f1a\u5ffd\u7565 table\uff0ccolumn\u8fd9\u4e9b\u914d\u7f6e\u9879 beginDateTime \u662f string \u65e0 \u6570\u636e\u7684\u5f00\u59cb\u65f6\u95f4\uff0cJob\u8fc1\u79fb\u4ece begineDateTime \u5230 endDateTime \u7684\u6570\u636e\uff0c\u683c\u5f0f\u4e3a yyyy-MM-dd HH:mm:ss endDateTime \u662f string \u65e0 \u6570\u636e\u7684\u7ed3\u675f\u65f6\u95f4\uff0cJob\u8fc1\u79fb\u4ece begineDateTime \u5230 endDateTime \u7684\u6570\u636e\uff0c\u683c\u5f0f\u4e3a yyyy-MM-dd HH:mm:ss splitInterval \u662f string \u65e0 \u6309\u7167 splitInterval \u6765\u5212\u5206 task, \u6bcf splitInterval \u521b\u5efa\u4e00\u4e2a task"},{"location":"reader/tdenginereader/#splitinterval","title":"splitInterval","text":"

    \u7528\u6765\u5212\u5206 task\u3002 \u4f8b\u5982\uff0c20d \u4ee3\u8868\u6309\u7167\u6bcf 20 \u5929\u7684\u6570\u636e\u5212\u5206\u4e3a 1 \u4e2a task\u3002 \u53ef\u4ee5\u914d\u7f6e\u7684\u65f6\u95f4

    "},{"location":"reader/tdenginereader/#jdbc-restful","title":"\u4f7f\u7528 JDBC-RESTful \u63a5\u53e3","text":"

    \u5982\u679c\u4e0d\u60f3\u4f9d\u8d56\u672c\u5730\u5e93\uff0c\u6216\u8005\u6ca1\u6709\u6743\u9650\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528 JDBC-RESTful \u63a5\u53e3\u6765\u5199\u5165\u8868\uff0c\u76f8\u6bd4 JDBC-JNI \u800c\u8a00\uff0c\u914d\u7f6e\u533a\u522b\u662f\uff1a

    \u6240\u4ee5\u4e0a\u8ff0\u914d\u7f6e\u4e2d\u7684 connection \u5e94\u8be5\u4fee\u6539\u4e3a\u5982\u4e0b\uff1a

    {\n  \"connection\": [\n    {\n      \"querySql\": [\n        \"select * from test.meters where ts <'2017-07-14 10:40:02' and  loc='beijing' limit 100\"\n      ],\n      \"jdbcUrl\": [\n        \"jdbc:TAOS-RS://127.0.0.1:6041/test\"\n      ],\n      \"driver\": \"com.taosdata.jdbc.rs.RestfulDriver\"\n    }\n  ]\n}\n
    "},{"location":"reader/tdenginereader/#_5","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b TDengine \u6570\u636e\u7c7b\u578b Long SMALLINT, TINYINT, INT, BIGINT, TIMESTAMP Double FLOAT, DOUBLE String BINARY, NCHAR Boolean BOOL"},{"location":"reader/tdenginereader/#_6","title":"\u5f53\u524d\u652f\u6301\u7248\u672c","text":"

    TDengine 2.0.16

    "},{"location":"reader/tdenginereader/#_7","title":"\u6ce8\u610f\u4e8b\u9879","text":""},{"location":"reader/txtfilereader/","title":"TxtFile Reader","text":"

    TxtFile Reader \u63d0\u4f9b\u4e86\u8bfb\u53d6\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u6570\u636e\u5b58\u50a8\u7684\u80fd\u529b\u3002

    "},{"location":"reader/txtfilereader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"job/txtfile2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"txtfilereader\",\n        \"parameter\": {\n          \"path\": [\n            \"/tmp/data\"\n          ],\n          \"encoding\": \"UTF-8\",\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"long\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"boolean\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"double\"\n            },\n            {\n              \"index\": 3,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 4,\n              \"type\": \"date\",\n              \"format\": \"yyyy.MM.dd\"\n            }\n          ],\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"txtfilewriter\",\n        \"parameter\": {\n          \"path\": \"/tmp/result\",\n          \"fileName\": \"txt_\",\n          \"writeMode\": \"truncate\",\n          \"format\": \"yyyy-MM-dd\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/txtfilereader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f list \u65e0 \u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84,\u8be6\u7ec6\u63cf\u8ff0\u89c1\u4e0b\u6587 column \u662f list<map>\\|* \u65e0 \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype\u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0c\u8be6\u89c1\u4e0b\u6587 fieldDelimiter \u662f string , \u63cf\u8ff0\uff1a\u8bfb\u53d6\u7684\u5b57\u6bb5\u5206\u9694\u7b26 encoding \u5426 string utf-8 \u8bfb\u53d6\u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e skipHeader \u5426 boolean false \u7c7bCSV\u683c\u5f0f\u6587\u4ef6\u53ef\u80fd\u5b58\u5728\u8868\u5934\u4e3a\u6807\u9898\u60c5\u51b5\uff0c\u9700\u8981\u8df3\u8fc7\u3002\u9ed8\u8ba4\u4e0d\u8df3\u8fc7 csvReaderConfig \u5426 string \u65e0 \u8bfb\u53d6CSV\u7c7b\u578b\u6587\u4ef6\u53c2\u6570\u914d\u7f6e\uff0cMap\u7c7b\u578b\u3002\u4e0d\u914d\u7f6e\u5219\u4f7f\u7528\u9ed8\u8ba4\u503c,\u8be6\u89c1\u4e0b\u6587"},{"location":"reader/txtfilereader/#path","title":"path","text":"

    \u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84\u3002

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0cAddax\u4f1a\u5c06\u4e00\u4e2a\u4f5c\u4e1a\u4e0b\u540c\u6b65\u7684\u6240\u6709Text File\u89c6\u4f5c\u540c\u4e00\u5f20\u6570\u636e\u8868\u3002\u7528\u6237\u5fc5\u987b\u81ea\u5df1\u4fdd\u8bc1\u6240\u6709\u7684File\u80fd\u591f\u9002\u914d\u540c\u4e00\u5957schema\u4fe1\u606f\u3002\u8bfb\u53d6\u6587\u4ef6\u7528\u6237\u5fc5\u987b\u4fdd\u8bc1\u4e3a\u7c7bCSV\u683c\u5f0f\uff0c\u5e76\u4e14\u63d0\u4f9b\u7ed9Addax\u6743\u9650\u53ef\u8bfb\u3002

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0c\u5982\u679cPath\u6307\u5b9a\u7684\u8def\u5f84\u4e0b\u6ca1\u6709\u7b26\u5408\u5339\u914d\u7684\u6587\u4ef6\u62bd\u53d6\uff0cAddax\u5c06\u62a5\u9519\u3002

    \u4ece 3.2.3 \u7248\u672c\u8d77\uff0c path \u4e0b\u5141\u8bb8\u6df7\u5408\u4e0d\u540c\u538b\u7f29\u683c\u5f0f\u7684\u6587\u4ef6\uff0c\u63d2\u4ef6\u4f1a\u5c1d\u8bd5\u81ea\u52a8\u731c\u6d4b\u538b\u7f29\u683c\u5f0f\u5e76\u81ea\u52a8\u89e3\u538b\uff0c\u76ee\u524d\u652f\u6301\u7684\u538b\u7f29\u683c\u5f0f\u6709\uff1a

    "},{"location":"reader/txtfilereader/#column","title":"column","text":"

    \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype\u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cindex\u6307\u5b9a\u5f53\u524d\u5217\u6765\u81ea\u4e8e\u6587\u672c\u7b2c\u51e0\u5217(\u4ee50\u5f00\u59cb)\uff0cvalue\u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece\u6e90\u5934\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636evalue\u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u7528\u6237\u53ef\u4ee5\u5168\u90e8\u6309\u7167String\u7c7b\u578b\u8bfb\u53d6\u6570\u636e\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"column\": [\n    \"*\"\n  ]\n}\n

    \u7528\u6237\u53ef\u4ee5\u6307\u5b9aColumn\u5b57\u6bb5\u4fe1\u606f\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    [\n  {\n    \"type\": \"long\",\n    \"index\": 0\n  },\n  {\n    \"type\": \"string\",\n    \"value\": \"alibaba\"\n  }\n]\n

    \u5bf9\u4e8e\u7528\u6237\u6307\u5b9aColumn\u4fe1\u606f\uff0ctype\u5fc5\u987b\u586b\u5199\uff0cindex/value\u5fc5\u987b\u9009\u62e9\u5176\u4e00\u3002

    \u4ece 4.0.1 \u5f00\u59cb\uff0c\u8868\u793a\u5b57\u6bb5\u9664\u4e86\u4f7f\u7528 index \u6765\u6307\u5b9a\u5b57\u6bb5\u7684\u987a\u5e8f\u5916\uff0c\u8fd8\u652f\u6301 name \u65b9\u5f0f\uff0c\u8fd9\u9700\u8981\u6240\u8bfb\u53d6\u7684\u6587\u4ef6\u7684\u90fd\u5305\u542b\u4e86\u6587\u4ef6\u5934\uff0c\u63d2\u4ef6\u4f1a\u5c1d\u8bd5\u5c06\u6307\u5b9a\u7684 name \u53bb\u5339\u914d\u4ece\u6587\u4ef6\u8bfb\u53d6\u7684\u6587\u4ef6\u5934\uff0c \u7136\u540e\u5f97\u5230\u5bf9\u5e94\u7684 index \u503c\uff0c\u5e76\u56de\u5199\u5230 \u914d\u7f6e\u6587\u4ef6\u4e2d\u3002\u540c\u65f6\uff0cindex \u548c name \u53ef\u4ee5\u5728\u4e0d\u540c\u7684\u5217\u4e0a\u8fdb\u884c\u6df7\u5408\u4f7f\u7528\uff0c\u6bd4\u5982\u4e0b\u9762\u8fd9\u6837\uff1a

    [\n  {\n    \"type\": \"long\",\n    \"index\": 0\n  },\n  {\n    \"name\": \"region\",\n    \"type\": \"string\"\n  },\n  {\n    \"type\": \"string\",\n    \"value\": \"alibaba\"\n  }\n]\n

    \u6ce8\uff1a \u8fd9\u79cd\u65b9\u5f0f\u4ee5\u4e3a\u8fd9\u5728\u51c6\u5907\u9636\u6bb5\u5c31\u8981\u5c1d\u8bd5\u8bfb\u53d6\u6587\u4ef6\uff0c\u56e0\u4e3a\u4f1a\u6709\u4e00\u5b9a\u7684\u6027\u80fd\u635f\u5931\uff0c\u5982\u975e\u5fc5\u8981\uff0c\u4e0d\u5efa\u8bae\u914d\u7f6e name \u65b9\u5f0f\u3002

    "},{"location":"reader/txtfilereader/#csvreaderconfig","title":"csvReaderConfig","text":"

    \u8bfb\u53d6CSV\u7c7b\u578b\u6587\u4ef6\u53c2\u6570\u914d\u7f6e\uff0cMap\u7c7b\u578b\u3002\u8bfb\u53d6CSV\u7c7b\u578b\u6587\u4ef6\u4f7f\u7528\u7684CsvReader\u8fdb\u884c\u8bfb\u53d6\uff0c\u4f1a\u6709\u5f88\u591a\u914d\u7f6e\uff0c\u4e0d\u914d\u7f6e\u5219\u4f7f\u7528\u9ed8\u8ba4\u503c\u3002

    \u5e38\u89c1\u914d\u7f6e\uff1a

    {\n  \"csvReaderConfig\": {\n    \"safetySwitch\": false,\n    \"skipEmptyRecords\": false,\n    \"useTextQualifier\": false\n  }\n}\n

    \u6240\u6709\u914d\u7f6e\u9879\u53ca\u9ed8\u8ba4\u503c,\u914d\u7f6e\u65f6 csvReaderConfig \u7684map\u4e2d\u8bf7**\u4e25\u683c\u6309\u7167\u4ee5\u4e0b\u5b57\u6bb5\u540d\u5b57\u8fdb\u884c\u914d\u7f6e**\uff1a

    boolean caseSensitive = true;\nchar textQualifier = 34;\nboolean trimWhitespace = true;\nboolean useTextQualifier = true;//\u662f\u5426\u4f7f\u7528csv\u8f6c\u4e49\u5b57\u7b26\nchar delimiter = 44;//\u5206\u9694\u7b26\nchar recordDelimiter = 0;\nchar comment = 35;\nboolean useComments = false;\nint escapeMode = 1;\nboolean safetySwitch = true;//\u5355\u5217\u957f\u5ea6\u662f\u5426\u9650\u5236100000\u5b57\u7b26\nboolean skipEmptyRecords = true;//\u662f\u5426\u8df3\u8fc7\u7a7a\u884c\nboolean captureRawRecord = true;\n
    "},{"location":"reader/txtfilereader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b \u672c\u5730\u6587\u4ef6 \u6570\u636e\u7c7b\u578b Long Long Double Double String String Boolean Boolean Date Date"},{"location":"writer/accesswriter/","title":"Access Writer","text":"

    Access Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 Access \u76ee\u7684\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/accesswriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684 Access \u8868\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    create table tbl_test(name varchar(20), file_size int, file_date date, file_open boolean, memo blob);\n

    \u8fd9\u91cc\u4f7f\u7528\u4e00\u4efd\u4ece\u5185\u5b58\u4ea7\u751f\u5230 Access \u5bfc\u5165\u7684\u6570\u636e\u3002

    job/stream2access.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"accesswriter\",\n        \"parameter\": {\n          \"username\": \"wgzhao\",\n          \"password\": \"\",\n          \"column\": [\n            \"name\",\n            \"file_size\",\n            \"file_date\",\n            \"file_open\",\n            \"memo\"\n          ],\n          \"ddl\": \"create table tbl_test(name varchar(20), file_size int, file_date date, file_open boolean, memo blob);\",\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:ucanaccess:////Users/wgzhao/Downloads/AccessThemeDemo.mdb\",\n            \"table\": [\n              \"tbl_test\"\n            ]\n          }\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2access.json

    "},{"location":"writer/accesswriter/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/stream2access.json\n
    "},{"location":"writer/accesswriter/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/cassandrawriter/","title":"Cassandra Writer","text":"

    Cassandra Writer \u63d2\u4ef6\u7528\u4e8e\u5411 Cassandra \u5199\u5165\u6570\u636e\u3002

    "},{"location":"writer/cassandrawriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4ece\u5185\u5b58\u4ea7\u751f\u5230 Cassandra \u5bfc\u5165\u7684\u4f5c\u4e1a:

    jobs/stream2cassandra.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 5,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"name\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"false\",\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": \"addr\",\n              \"type\": \"bytes\"\n            },\n            {\n              \"value\": 1.234,\n              \"type\": \"double\"\n            },\n            {\n              \"value\": 12345678,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": 2.345,\n              \"type\": \"double\"\n            },\n            {\n              \"value\": 3456789,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"4a0ef8c0-4d97-11d0-db82-ebecdb03ffa5\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"value\",\n              \"type\": \"bytes\"\n            },\n            {\n              \"value\": \"-838383838,37377373,-383883838,27272772,393993939,-38383883,83883838,-1350403181,817650816,1630642337,251398784,-622020148\",\n              \"type\": \"string\"\n            }\n          ],\n          \"sliceRecordCount\": 10000000\n        }\n      },\n      \"writer\": {\n        \"name\": \"cassandrawriter\",\n        \"parameter\": {\n          \"host\": \"localhost\",\n          \"port\": 9042,\n          \"useSSL\": false,\n          \"keyspace\": \"stresscql\",\n          \"table\": \"dst\",\n          \"batchSize\": 10,\n          \"column\": [\n            \"name\",\n            \"choice\",\n            \"date\",\n            \"address\",\n            \"dbl\",\n            \"lval\",\n            \"fval\",\n            \"ival\",\n            \"uid\",\n            \"value\",\n            \"listval\"\n          ]\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/cassandrawriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 host \u662f string \u65e0 \u8fde\u63a5\u70b9\u7684\u57df\u540d\u6216 ip\uff0c\u591a\u4e2a node \u4e4b\u95f4\u7528\u9017\u53f7\u5206\u9694 port \u662f int 9042 Cassandra \u7aef\u53e3 username \u5426 string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801 useSSL \u5426 boolean false \u662f\u5426\u4f7f\u7528 SSL \u8fde\u63a5 connectionsPerHost \u5426 int 8 \u5ba2\u6237\u7aef\u8fde\u63a5\u6c60\u914d\u7f6e\uff1a\u4e0e\u670d\u52a1\u5668\u6bcf\u4e2a\u8282\u70b9\u5efa\u591a\u5c11\u4e2a\u8fde\u63a5 maxPendingPerConnection \u5426 int 128 \u5ba2\u6237\u7aef\u8fde\u63a5\u6c60\u914d\u7f6e\uff1a\u6bcf\u4e2a\u8fde\u63a5\u6700\u5927\u8bf7\u6c42\u6570 keyspace \u662f string \u65e0 \u9700\u8981\u540c\u6b65\u7684\u8868\u6240\u5728\u7684 keyspace table \u662f string \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u96c6\u5408 consistancyLevel \u5426 string LOCAL_QUORUM \u6570\u636e\u4e00\u81f4\u6027\u7ea7\u522b, batchSize \u5426 int 1 \u4e00\u6b21\u6279\u91cf\u63d0\u4ea4(UNLOGGED BATCH)\u7684\u8bb0\u5f55\u6570\u5927\u5c0f\uff08\u6761\u6570\uff09"},{"location":"writer/cassandrawriter/#column","title":"column","text":"

    \u5185\u5bb9\u53ef\u4ee5\u662f\u5217\u7684\u540d\u79f0\u6216 writetime()\u3002\u5982\u679c\u5c06\u5217\u540d\u914d\u7f6e\u4e3a writetime()\uff0c\u4f1a\u5c06\u8fd9\u4e00\u5217\u7684\u5185\u5bb9\u4f5c\u4e3a\u65f6\u95f4\u6233

    "},{"location":"writer/cassandrawriter/#consistancylevel","title":"consistancyLevel","text":"

    \u53ef\u9009 ONE, QUORUM, LOCAL_QUORUM, EACH_QUORUM, ALL, ANY, TWO, THREE, LOCAL_ONE

    "},{"location":"writer/cassandrawriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b Cassandra \u6570\u636e\u7c7b\u578b Long int, tinyint, smallint,varint,bigint,time Double float, double, decimal String ascii,varchar, text,uuid,timeuuid,duration,list,map,set,tuple,udt,inet Date date, timestamp Boolean bool Bytes blob

    \u8bf7\u6ce8\u610f:

    \u76ee\u524d\u4e0d\u652f\u6301 counter \u7c7b\u578b\u548c custom \u7c7b\u578b\u3002

    "},{"location":"writer/cassandrawriter/#_4","title":"\u7ea6\u675f\u9650\u5236","text":""},{"location":"writer/cassandrawriter/#batchsize","title":"batchSize","text":"
    1. \u4e0d\u80fd\u8d85\u8fc7 65535
    2. batch \u4e2d\u7684\u5185\u5bb9\u5927\u5c0f\u53d7\u5230\u670d\u52a1\u5668\u7aef batch_size_fail_threshold_in_kb \u7684\u9650\u5236\u3002
    3. \u5982\u679c batch \u4e2d\u7684\u5185\u5bb9\u8d85\u8fc7\u4e86 batch_size_warn_threshold_in_kb \u7684\u9650\u5236\uff0c\u4f1a\u6253\u51fa warn \u65e5\u5fd7\uff0c\u4f46\u5e76\u4e0d\u5f71\u54cd\u5199\u5165\uff0c\u5ffd\u7565\u5373\u53ef\u3002
    4. \u5982\u679c\u6279\u91cf\u63d0\u4ea4\u5931\u8d25\uff0c\u4f1a\u628a\u8fd9\u4e2a\u6279\u91cf\u7684\u6240\u6709\u5185\u5bb9\u91cd\u65b0\u9010\u6761\u5199\u5165\u4e00\u904d\u3002
    "},{"location":"writer/clickhousewriter/","title":"ClickHouse Writer","text":"

    ClickHouse Writer \u63d2\u4ef6\u7528\u4e8e\u5411 ClickHouse \u5199\u5165\u6570\u636e\u3002

    "},{"location":"writer/clickhousewriter/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u793a\u4f8b\u6211\u4eec\u6f14\u793a\u4ece clickhouse \u4e2d\u8bfb\u53d6\u4e00\u5f20\u8868\u7684\u5185\u5bb9\uff0c\u5e76\u5199\u5165\u5230\u76f8\u540c\u8868\u7ed3\u6784\u7684\u53e6\u5916\u4e00\u5f20\u8868\u4e2d\uff0c\u7528\u6765\u6d4b\u8bd5\u63d2\u4ef6\u6240\u652f\u6301\u7684\u6570\u636e\u7ed3\u6784

    "},{"location":"writer/clickhousewriter/#_2","title":"\u8868\u7ed3\u6784\u4ee5\u6570\u636e","text":"

    \u5047\u5b9a\u8981\u8bfb\u53d6\u7684\u8868\u7ed3\u6784\u53ca\u6570\u636e\u5982\u4e0b\uff1a

    CREATE TABLE ck_addax (\n    c_int8 Int8,\n    c_int16 Int16,\n    c_int32 Int32,\n    c_int64 Int64,\n    c_uint8 UInt8,\n    c_uint16 UInt16,\n    c_uint32 UInt32,\n    c_uint64 UInt64,\n    c_float32 Float32,\n    c_float64 Float64,\n    c_decimal Decimal(38,10),\n    c_string String,\n    c_fixstr FixedString(36),\n    c_uuid UUID,\n    c_date Date,\n    c_datetime DateTime('Asia/Chongqing'),\n    c_datetime64 DateTime64(3, 'Asia/Chongqing'),\n    c_enum Enum('hello' = 1, 'world'=2)\n) ENGINE = MergeTree() ORDER BY (c_int8, c_int16) SETTINGS index_granularity = 8192;\n\ninsert into ck_addax values(\n    127,\n    -32768,\n    2147483647,\n    -9223372036854775808,\n    255,\n    65535,\n    4294967295,\n    18446744073709551615,\n    0.999999999999,\n    0.99999999999999999,\n    1234567891234567891234567891.1234567891,\n    'Hello String',\n    '2c:16:db:a3:3a:4f',\n    '5F042A36-5B0C-4F71-ADFD-4DF4FCA1B863',\n    '2021-01-01',\n    '2021-01-01 11:22:33',\n    '2021-01-01 10:33:23.123',\n    'hello'\n);\n

    \u8981\u5199\u5165\u7684\u8868\u91c7\u53d6\u548c\u8bfb\u53d6\u8868\u7ed3\u6784\u76f8\u540c\uff0c\u5176\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    create table ck_addax_writer as ck_addax;\n
    "},{"location":"writer/clickhousewriter/#_3","title":"\u914d\u7f6e","text":"

    \u4ee5\u4e0b\u4e3a\u914d\u7f6e\u6587\u4ef6

    job/clickhouse2clickhouse.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"writer\": {\n        \"name\": \"clickhousewriter\",\n        \"parameter\": {\n          \"username\": \"default\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"ck_addax_writer\"\n            ],\n            \"jdbcUrl\": \"jdbc:clickhouse://127.0.0.1:8123/default\"\n          },\n          \"preSql\": [\n            \"alter table @table delete where 1=1\"\n          ]\n        }\n      },\n      \"reader\": {\n        \"name\": \"clickhousereader\",\n        \"parameter\": {\n          \"username\": \"default\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:clickhouse://127.0.0.1:8123/\",\n            \"table\": [\n              \"ck_addax\"\n            ]\n          }\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/clickhouse2clickhouse.json

    "},{"location":"writer/clickhousewriter/#_4","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/clickhouse2clickhouse.json\n
    "},{"location":"writer/clickhousewriter/#_5","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/databendwriter/","title":"DatabendWriter","text":"

    Databend \u63d2\u4ef6\u7528\u4e8e\u5411 Databend \u6570\u636e\u5e93\u4ee5 JDBC \u65b9\u5f0f\u5199\u5165\u6570\u636e\u3002

    Databend \u662f\u4e00\u4e2a\u517c\u5bb9 MySQL \u534f\u8bae\u7684\u6570\u636e\u5e93\u540e\u7aef\uff0c\u56e0\u6b64 Databend \u5199\u5165\u53ef\u4ee5\u4f7f\u7528 MySQLWriter \u8fdb\u884c\u8bbf\u95ee\u3002

    "},{"location":"writer/databendwriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684\u8868\u7684\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    CREATE\nDATABASE example_db;\nCREATE TABLE `example_db`.`table1`\n(\n    `siteid`   INT DEFAULT CAST(10 AS INT),\n    `citycode` INT,\n    `username` VARCHAR,\n    `pv`       BIGINT\n);\n

    \u4e0b\u9762\u914d\u7f6e\u4e00\u4e2a\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\uff0c\u7136\u540e\u5199\u5165\u5230 databend \u8868\u7684\u914d\u7f6e\u6587\u4ef6

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2\n      }\n    },\n    \"content\": {\n      \"writer\": {\n        \"name\": \"databendwriter\",\n        \"parameter\": {\n          \"preSql\": [\n            \"truncate table @table\"\n          ],\n          \"postSql\": [],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:databend://localhost:8000/addax\",\n            \"table\": [\n              \"table1\"\n            ]\n          },\n          \"username\": \"u1\",\n          \"password\": \"123\",\n          \"column\": [\n            \"*\"\n          ]\n        }\n      },\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"1,500\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"1,127\",\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"this is a text\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"5,200\",\n              \"type\": \"long\"\n            }\n          ],\n          \"sliceRecordCount\": 100\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2databend.json

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4

    bin/addax.sh job/stream2Databend.json\n
    "},{"location":"writer/databendwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879, \u5e76\u589e\u52a0\u4e86\u5982\u4e0b\u914d\u7f6e\u9879\uff1a

    \u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 writeMode \u5426 string insert \u5199\u5165\u6a21\u5f0f\uff0c\u652f\u6301 insert \u548c replace \u4e24\u79cd\u6a21\u5f0f onConflictColumn \u5426 string \u65e0 \u51b2\u7a81\u5217\uff0c\u5f53 writeMode \u4e3a replace \u65f6\uff0c\u5fc5\u987b\u6307\u5b9a\u51b2\u7a81\u5217\uff0c\u5426\u5219\u4f1a\u5bfc\u81f4\u5199\u5165\u5931\u8d25\u3002"},{"location":"writer/databendwriter/#writemode","title":"writeMode","text":"

    \u7528\u6765\u652f\u6301 Databend \u7684 replace into \u8bed\u6cd5\uff0c\u5f53\u8be5\u53c2\u6570\u8bbe\u5b9a\u4e3a replace \u65f6\uff0c\u5fc5\u987b\u540c\u65f6\u6307\u5b9a onConflictColumn \u53c2\u6570\uff0c\u7528\u6765\u5224\u65ad\u6570\u636e\u662f\u63d2\u5165\u8fd8\u662f\u66f4\u65b0\u7684\u4f9d\u636e\u3002

    \u4e24\u4e2a\u53c2\u6570\u7684\u793a\u4f8b\u5982\u4e0b\uff1a

    {\n  \"writeMode\": \"replace\",\n  \"onConflictColumn\": [\n    \"id\"\n  ]\n}\n
    "},{"location":"writer/dbfwriter/","title":"DBF Writer","text":"

    Dbf Writer \u63d0\u4f9b\u4e86\u5411\u672c\u5730\u6587\u4ef6\u5199\u5165\u7c7bdbf\u683c\u5f0f\u7684\u4e00\u4e2a\u6216\u8005\u591a\u4e2a\u8868\u6587\u4ef6\u3002

    "},{"location":"writer/dbfwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 00:00:00\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"\u4e2d\u6587\u6d4b\u8bd5\",\n              \"type\": \"string\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"dbfwriter\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"name\": \"col1\",\n              \"type\": \"char\",\n              \"length\": 100\n            },\n            {\n              \"name\": \"col2\",\n              \"type\": \"numeric\",\n              \"length\": 18,\n              \"scale\": 0\n            },\n            {\n              \"name\": \"col3\",\n              \"type\": \"date\"\n            },\n            {\n              \"name\": \"col4\",\n              \"type\": \"logical\"\n            },\n            {\n              \"name\": \"col5\",\n              \"type\": \"char\",\n              \"length\": 100\n            }\n          ],\n          \"fileName\": \"test.dbf\",\n          \"path\": \"/tmp/out\",\n          \"writeMode\": \"truncate\",\n          \"encoding\": \"GBK\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/dbfwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f string \u65e0 \u6587\u4ef6\u76ee\u5f55\uff0c\u6ce8\u610f\u8fd9\u91cc\u662f\u6587\u4ef6\u5939\uff0c\u4e0d\u662f\u6587\u4ef6 column \u662f list<map> \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u96c6\u5408\uff0c\u8be6\u89c1\u793a\u4f8b\u914d\u7f6e fileName \u662f string \u65e0 \u5199\u5165\u7684\u6587\u4ef6\u540d writeMode \u662f string \u65e0 \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff0c\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0 encoding \u5426 string UTF-8 \u6587\u4ef6\u7f16\u7801\uff0c\u6bd4\u5982 GBK, UTF-8 nullFormat \u5426 string \\N \u5b9a\u4e49\u54ea\u4e2a\u5b57\u7b26\u4e32\u53ef\u4ee5\u8868\u793a\u4e3anull, dateFormat \u5426 string \u65e0 \u65e5\u671f\u7c7b\u578b\u7684\u6570\u636e\u5e8f\u5217\u5316\u5230\u6587\u4ef6\u4e2d\u65f6\u7684\u683c\u5f0f\uff0c\u4f8b\u5982 \"yyyy-MM-dd\""},{"location":"writer/dbfwriter/#writemode","title":"writeMode","text":"

    \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff1a

    "},{"location":"writer/dbfwriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u5f53\u524d\u8be5\u63d2\u4ef6\u652f\u6301\u5199\u5165\u7684\u7c7b\u578b\u4ee5\u53ca\u5bf9\u5e94\u5173\u7cfb\u5982\u4e0b\uff1a

    XBase Type XBase Symbol Java Type used in JavaDBF Character C java.lang.String Numeric N java.math.BigDecimal Floating Point F java.math.BigDecimal Logical L java.lang.Boolean Date D java.util.Date

    \u5176\u4e2d\uff1a

    "},{"location":"writer/doriswriter/","title":"Doris Writer","text":"

    DorisWriter \u63d2\u4ef6\u7528\u4e8e\u5411 Doris \u6570\u636e\u5e93\u4ee5\u6d41\u5f0f\u65b9\u5f0f\u5199\u5165\u6570\u636e\u3002 \u5176\u5b9e\u73b0\u4e0a\u662f\u901a\u8fc7\u8bbf\u95ee Doris http \u8fde\u63a5(8030)\uff0c\u7136\u540e\u901a\u8fc7 stream load \u52a0\u8f7d\u6570\u636e\u5230\u6570\u636e\u4e2d\uff0c\u76f8\u6bd4 insert into \u65b9\u5f0f\u6548\u7387\u8981\u9ad8\u4e0d\u5c11\uff0c\u4e5f\u662f\u5b98\u65b9\u63a8\u8350\u7684\u751f\u4ea7\u73af\u5883\u4e0b\u7684\u6570\u636e\u52a0\u8f7d\u65b9\u5f0f\u3002

    Doris \u662f\u4e00\u4e2a\u517c\u5bb9 MySQL \u534f\u8bae\u7684\u6570\u636e\u5e93\u540e\u7aef\uff0c\u56e0\u6b64 Doris \u8bfb\u53d6\u53ef\u4ee5\u4f7f\u7528 MySQL Reader \u8fdb\u884c\u8bbf\u95ee\u3002

    "},{"location":"writer/doriswriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684\u8868\u7684\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    CREATE\nDATABASE example_db;\nCREATE TABLE example_db.table1\n(\n    siteid   INT         DEFAULT '10',\n    citycode SMALLINT,\n    username VARCHAR(32) DEFAULT '',\n    pv       BIGINT SUM DEFAULT '0'\n) AGGREGATE KEY(siteid, citycode, username)\nDISTRIBUTED BY HASH(siteid) BUCKETS 10\nPROPERTIES(\"replication_num\" = \"1\");\n

    \u4e0b\u9762\u914d\u7f6e\u4e00\u4e2a\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\uff0c\u7136\u540e\u5199\u5165\u5230 doris \u8868\u7684\u914d\u7f6e\u6587\u4ef6

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2\n      }\n    },\n    \"content\": {\n      \"writer\": {\n        \"name\": \"doriswriter\",\n        \"parameter\": {\n          \"loadUrl\": [\n            \"127.0.0.1:8030\"\n          ],\n          \"username\": \"test\",\n          \"password\": \"123456\",\n          \"batchSize\": 1024,\n          \"column\": [\n            \"siteid\",\n            \"citycode\",\n            \"username\",\n            \"pv\"\n          ],\n          \"connection\": {\n            \"table\": \"table1\",\n            \"database\": \"example_db\",\n            \"jdbcUrl\": \"jdbc:mysql://localhost:9030/example_db\"\n          },\n          \"loadProps\": {\n            \"format\": \"json\",\n            \"strip_outer_array\": true\n          }\n        }\n      },\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"1,500\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"1,127\",\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"this is a text\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"5,200\",\n              \"type\": \"long\"\n            }\n          ],\n          \"sliceRecordCount\": 100\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2doris.json

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4

    bin/addax.sh job/stream2doris.json\n

    \u8f93\u51fa\u7c7b\u4f3c\u5982\u4e0b\uff1a

    2021-02-23 15:22:57.851 [main] INFO  VMInfo - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl\n2021-02-23 15:22:57.871 [main] INFO  Engine -\n{\n\"content\":{\n\"reader\":{\n    \"parameter\":{\n            \"column\":[\n                    {\n                            \"random\":\"1,500\",\n                            \"type\":\"long\"\n                    },\n                    {\n                            \"random\":\"1,127\",\n                            \"type\":\"long\"\n                    },\n                    {\n                            \"type\":\"string\",\n                            \"value\":\"username\"\n                    }\n            ],\n            \"sliceRecordCount\":100\n    },\n    \"name\":\"streamreader\"\n},\n\"writer\":{\n    \"parameter\":{\n            \"password\":\"*****\",\n            \"batchSize\":1024,\n            \"connection\":[\n                    {\n                            \"database\":\"example_db\",\n                            \"endpoint\":\"http://127.0.0.1:8030/\",\n                            \"table\":\"table1\"\n                    }\n            ],\n            \"username\":\"test\"\n    },\n    \"name\":\"doriswriter\"\n }\n},\n\"setting\":{\n\"speed\":{\n\"channel\":2\n}\n}\n}\n\n2021-02-23 15:22:57.886 [main] INFO  PerfTrace - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-02-23 15:22:57.886 [main] INFO  JobContainer - Addax jobContainer starts job.\n2021-02-23 15:22:57.920 [job-0] INFO  JobContainer - Scheduler starts [1] taskGroups.\n2021-02-23 15:22:57.928 [taskGroup-0] INFO  TaskGroupContainer - taskGroupId=[0] start [2] channels for [2] tasks.\n2021-02-23 15:22:57.935 [taskGroup-0] INFO  Channel - Channel set byte_speed_limit to -1, No bps activated.\n2021-02-23 15:22:57.936 [taskGroup-0] INFO  Channel - Channel set record_speed_limit to -1, No tps activated.\n2021-02-23 15:22:57.970 [0-0-1-writer] INFO  DorisWriterTask - connect DorisDB with http://127.0.0.1:8030//api/example_db/table1/_stream_load\n2021-02-23 15:22:57.970 [0-0-0-writer] INFO  DorisWriterTask - connect DorisDB with http://127.0.0.1:8030//api/example_db/table1/_stream_load\n\n2021-02-23 15:23:00.941 [job-0] INFO  JobContainer - PerfTrace not enable!\n2021-02-23 15:23:00.946 [job-0] INFO  JobContainer -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-02-23 15:22:57\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-02-23 15:23:00\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :            1.56KB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :             66rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                 200\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"writer/doriswriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 loadUrl \u662f string \u65e0 Stream Load \u7684\u8fde\u63a5\u76ee\u6807 \uff5c username \u662f string \u65e0 \u8bbf\u95eeDoris\u6570\u636e\u5e93\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u8bbf\u95eeDoris\u6570\u636e\u5e93\u7684\u5bc6\u7801 flushInterval \u5426 int 3000 \u6570\u636e\u5199\u5165\u5230\u76ee\u6807\u8868\u7684\u95f4\u9694\u65f6\u95f4\uff0c\u5355\u4f4d\u4e3a\u6beb\u79d2\uff0c\u5373\u6bcf\u9694\u591a\u5c11\u6beb\u79d2\u5199\u5165\u4e00\u6b21\u6570\u636e flushQueueLength \u5426 int 1 \u4e0a\u4f20\u6570\u636e\u7684\u961f\u5217\u957f\u5ea6 table \u662f string \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1 RBDMS Writer batchSize \u5426 int 2048 \u6bcf\u6279\u6b21\u5bfc\u5165\u6570\u636e\u7684\u6700\u5927\u884c\u6570 loadProps \u5426 map csv streamLoad \u7684\u8bf7\u6c42\u53c2\u6570\uff0c\u8be6\u60c5\u53c2\u7167StreamLoad\u4ecb\u7ecd\u9875\u9762 preSql \u5426 list \u5199\u5165\u6570\u636e\u5230\u76ee\u6807\u8868\u524d\u8981\u6267\u884c\u7684 SQL \u8bed\u53e5 postSql \u5426 list \u6570\u636e\u5199\u5b8c\u540e\u8981\u6267\u884c\u7684 SQL \u8bed\u53e5"},{"location":"writer/doriswriter/#loadurl","title":"loadUrl","text":"

    \u4f5c\u4e3a Stream Load \u7684\u8fde\u63a5\u76ee\u6807\u3002\u683c\u5f0f\u4e3a \"ip:port\"\u3002\u5176\u4e2d IP \u662f FE \u8282\u70b9 IP\uff0cport \u662f FE \u8282\u70b9\u7684 http_port\u3002\u53ef\u4ee5\u586b\u5199\u591a\u4e2a\uff0c\u5f53\u586b\u5199\u591a\u4e2a\u65f6\uff0c\u63d2\u4ef6\u4f1a\u6bcf\u4e2a\u6279\u6b21\u968f\u673a\u9009\u62e9\u4e00\u4e2a\u6709\u6548 FE \u8282\u70b9\u8fdb\u884c\u8fde\u63a5\u3002

    "},{"location":"writer/doriswriter/#column","title":"column","text":"

    \u5141\u8bb8\u914d\u7f6e\u4e3a [\"*\"] \uff0c \u5982\u679c\u662f \"*\" , \u5219\u5c1d\u8bd5\u4ece Doris \u6570\u636e\u5e93\u4e2d\u76f4\u63a5\u8bfb\u53d6\u8868\u5b57\u6bb5\uff0c\u7136\u540e\u8fdb\u884c\u62fc\u88c5\u3002

    "},{"location":"writer/doriswriter/#loadprops","title":"loadProps","text":"

    StreamLoad \u7684\u8bf7\u6c42\u53c2\u6570\uff0c\u8be6\u60c5\u53c2\u7167StreamLoad\u4ecb\u7ecd\u9875\u9762\u3002Stream load - Apache Doris

    \u8fd9\u91cc\u5305\u62ec\u5bfc\u5165\u7684\u6570\u636e\u683c\u5f0f\uff1aformat\u7b49\uff0c\u5bfc\u5165\u6570\u636e\u683c\u5f0f\u9ed8\u8ba4\u6211\u4eec\u4f7f\u7528csv\uff0c\u652f\u6301JSON\uff0c\u5177\u4f53\u53ef\u4ee5\u53c2\u7167\u4e0b\u9762\u7c7b\u578b\u8f6c\u6362\u90e8\u5206\uff0c\u4e5f\u53ef\u4ee5\u53c2\u7167\u4e0a\u9762Stream load \u5b98\u65b9\u4fe1\u606f

    "},{"location":"writer/doriswriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u9ed8\u8ba4\u4f20\u5165\u7684\u6570\u636e\u5747\u4f1a\u88ab\u8f6c\u4e3a\u5b57\u7b26\u4e32\uff0c\u5e76\u4ee5\\t\u4f5c\u4e3a\u5217\u5206\u9694\u7b26\uff0c\\n\u4f5c\u4e3a\u884c\u5206\u9694\u7b26\uff0c\u7ec4\u6210csv\u6587\u4ef6\u8fdb\u884cStreamLoad\u5bfc\u5165\u64cd\u4f5c\u3002

    \u9ed8\u8ba4\u662fcsv\u683c\u5f0f\u5bfc\u5165\uff0c\u5982\u9700\u66f4\u6539\u5217\u5206\u9694\u7b26\uff0c \u5219\u6b63\u786e\u914d\u7f6e loadProps \u5373\u53ef

    {\n  \"loadProps\": {\n    \"column_separator\": \"\\\\x01\",\n    \"line_delimiter\": \"\\\\x02\"\n  }\n}\n

    \u5982\u9700\u66f4\u6539\u5bfc\u5165\u683c\u5f0f\u4e3ajson\uff0c \u5219\u6b63\u786e\u914d\u7f6e loadProps \u5373\u53ef\uff1a

    {\n  \"loadProps\": {\n    \"format\": \"json\",\n    \"strip_outer_array\": true\n  }\n}\n
    "},{"location":"writer/elasticsearchwriter/","title":"ElasticSearch Writer","text":"

    ElasticSearch Writer \u63d2\u4ef6\u7528\u4e8e\u5411 ElasticSearch \u5199\u5165\u6570\u636e\u3002 \u5176\u5b9e\u73b0\u662f\u901a\u8fc7 elasticsearch \u7684 rest api \u63a5\u53e3\uff0c \u6279\u91cf\u628a\u636e\u5199\u5165 elasticsearch

    "},{"location":"writer/elasticsearchwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"job/stream2es.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"10,1000\",\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1.1.1.1\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"double\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"hello world\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"long text\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"41.12,-71.34\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"2017-05-25 11:22:33\",\n              \"type\": \"string\"\n            }\n          ],\n          \"sliceRecordCount\": 100\n        }\n      },\n      \"writer\": {\n        \"name\": \"elasticsearchwriter\",\n        \"parameter\": {\n          \"endpoint\": \"http://localhost:9200\",\n          \"index\": \"test-1\",\n          \"type\": \"default\",\n          \"cleanup\": true,\n          \"settings\": {\n            \"index\": {\n              \"number_of_shards\": 1,\n              \"number_of_replicas\": 0\n            }\n          },\n          \"discovery\": false,\n          \"batchSize\": 1000,\n          \"splitter\": \",\",\n          \"column\": [\n            {\n              \"name\": \"pk\",\n              \"type\": \"id\"\n            },\n            {\n              \"name\": \"col_ip\",\n              \"type\": \"ip\"\n            },\n            {\n              \"name\": \"col_double\",\n              \"type\": \"double\"\n            },\n            {\n              \"name\": \"col_long\",\n              \"type\": \"long\"\n            },\n            {\n              \"name\": \"col_integer\",\n              \"type\": \"integer\"\n            },\n            {\n              \"name\": \"col_keyword\",\n              \"type\": \"keyword\"\n            },\n            {\n              \"name\": \"col_text\",\n              \"type\": \"text\",\n              \"analyzer\": \"ik_max_word\"\n            },\n            {\n              \"name\": \"col_geo_point\",\n              \"type\": \"geo_point\"\n            },\n            {\n              \"name\": \"col_date\",\n              \"type\": \"date\",\n              \"format\": \"yyyy-MM-dd HH:mm:ss\"\n            },\n            {\n              \"name\": \"col_nested1\",\n              \"type\": \"nested\"\n            },\n            {\n              \"name\": \"col_nested2\",\n              \"type\": \"nested\"\n            },\n            {\n              \"name\": \"col_object1\",\n              \"type\": \"object\"\n            },\n            {\n              \"name\": \"col_object2\",\n              \"type\": \"object\"\n            },\n            {\n              \"name\": \"col_integer_array\",\n              \"type\": \"integer\",\n              \"array\": true\n            },\n            {\n              \"name\": \"col_geo_shape\",\n              \"type\": \"geo_shape\",\n              \"tree\": \"quadtree\",\n              \"precision\": \"10m\"\n            }\n          ]\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/elasticsearchwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 ElasticSearch \u7684\u8fde\u63a5\u5730\u5740,\u5982\u679c\u662f\u96c6\u7fa4\uff0c\u5219\u591a\u4e2a\u5730\u5740\u7528\u9017\u53f7(,)\u5206\u5272 accessId \u5426 string \u7a7a http auth \u4e2d\u7684 user, \u9ed8\u8ba4\u4e3a\u7a7a accessKey \u5426 string \u7a7a http auth \u4e2d\u7684 password index \u662f string \u65e0 index \u540d type \u5426 string default index \u7c7b\u578b cleanup \u5426 boolean false \u662f\u5426\u5220\u9664\u539f\u8868 batchSize \u5426 int 1000 \u6bcf\u6b21\u6279\u91cf\u6570\u636e\u7684\u6761\u6570 trySize \u5426 int 30 \u5931\u8d25\u540e\u91cd\u8bd5\u7684\u6b21\u6570 timeout \u5426 int 600000 \u5ba2\u6237\u7aef\u8d85\u65f6\u65f6\u95f4\uff0c\u5355\u4f4d\u4e3a\u6beb\u79d2(ms) discovery \u5426 boolean false \u542f\u7528\u8282\u70b9\u53d1\u73b0\u5c06(\u8f6e\u8be2)\u5e76\u5b9a\u671f\u66f4\u65b0\u5ba2\u6237\u673a\u4e2d\u7684\u670d\u52a1\u5668\u5217\u8868 compression \u5426 boolean true \u5426\u662f\u5f00\u542f http \u8bf7\u6c42\u538b\u7f29 multiThread \u5426 boolean true \u662f\u5426\u5f00\u542f\u591a\u7ebf\u7a0b http \u8bf7\u6c42 ignoreWriteError \u5426 boolean false \u5199\u5165\u9519\u8bef\u65f6\uff0c\u662f\u5426\u91cd\u8bd5\uff0c\u5982\u679c\u662f true \u5219\u8868\u793a\u4e00\u76f4\u91cd\u8bd5\uff0c\u5426\u5219\u5ffd\u7565\u8be5\u6761\u6570\u636e ignoreParseError \u5426 boolean true \u89e3\u6790\u6570\u636e\u683c\u5f0f\u9519\u8bef\u65f6\uff0c\u662f\u5426\u7ee7\u7eed\u5199\u5165 alias \u5426 string \u65e0 \u6570\u636e\u5bfc\u5165\u5b8c\u6210\u540e\u5199\u5165\u522b\u540d aliasMode \u5426 string append \u6570\u636e\u5bfc\u5165\u5b8c\u6210\u540e\u589e\u52a0\u522b\u540d\u7684\u6a21\u5f0f\uff0cappend(\u589e\u52a0\u6a21\u5f0f), exclusive(\u53ea\u7559\u8fd9\u4e00\u4e2a) settings \u5426 map \u65e0 \u521b\u5efa index \u65f6\u5019\u7684 settings, \u4e0e elasticsearch \u5b98\u65b9\u76f8\u540c splitter \u5426 string , \u5982\u679c\u63d2\u5165\u6570\u636e\u662f array\uff0c\u5c31\u4f7f\u7528\u6307\u5b9a\u5206\u9694\u7b26 column \u662f list<map> \u65e0 \u5b57\u6bb5\u7c7b\u578b\uff0c\u6587\u6863\u4e2d\u7ed9\u51fa\u7684\u6837\u4f8b\u4e2d\u5305\u542b\u4e86\u5168\u90e8\u652f\u6301\u7684\u5b57\u6bb5\u7c7b\u578b dynamic \u5426 boolean false \u4e0d\u4f7f\u7528 addax \u7684 mappings\uff0c\u4f7f\u7528 es \u81ea\u5df1\u7684\u81ea\u52a8 mappings"},{"location":"writer/elasticsearchwriter/#_3","title":"\u7ea6\u675f\u9650\u5236","text":""},{"location":"writer/excelwriter/","title":"Excel Writer","text":"

    Excel Writer \u5b9e\u73b0\u4e86\u5c06\u6570\u636e\u5199\u5165\u5230 Excel \u6587\u4ef6\u7684\u529f\u80fd

    "},{"location":"writer/excelwriter/#_1","title":"\u914d\u7f6e\u793a\u4f8b","text":"

    \u6211\u4eec\u5047\u5b9a\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\uff0c\u5e76\u5199\u5165\u5230 Excel \u6587\u4ef6\u4e2d

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"DataX\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 11:22:33\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"excelwriter\",\n        \"parameter\": {\n          \"path\": \"/tmp/out\",\n          \"fileName\": \"test\",\n          \"header\": [\n            \"str\",\n            \"\u957f\u5ea6\",\n            \"\u65e5\u671f\",\n            \"\u662f\u5426\u4e3a\u771f\",\n            \"\u5b57\u8282\u7c7b\u578b\"\n          ]\n        }\n      }\n    }\n  }\n}\n

    \u8bb2\u4e0a\u8ff0\u5185\u5bb9\u4fdd\u5b58\u4e3a job/stream2excel.json

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\uff1a

    bin/addax.sh job/stream2excel.sh\n

    \u5e94\u8be5\u5f97\u5230\u7c7b\u4f3c\u5982\u4e0b\u7684\u8f93\u51fa

    \u70b9\u51fb\u5c55\u5f00 
      ___      _     _\n / _ \\    | |   | |\n/ /_\\ \\ __| | __| | __ ___  __\n|  _  |/ _` |/ _` |/ _` \\ \\/ /\n| | | | (_| | (_| | (_| |>  <\n\\_| |_/\\__,_|\\__,_|\\__,_/_/\\_\\\n\n:: Addax version ::    (v4.0.3-SNAPSHOT)\n\n2021-09-10 22:16:38.247 [        main] INFO  VMInfo               - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl\n2021-09-10 22:16:38.269 [        main] INFO  Engine               -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"column\":[\n                        {\n                            \"type\":\"string\",\n                            \"value\":\"DataX\"\n                        },\n                        {\n                            \"type\":\"long\",\n                            \"value\":19890604\n                        },\n                        {\n                            \"type\":\"date\",\n                            \"value\":\"1989-06-04 11:22:33\"\n                        },\n                        {\n                            \"type\":\"bool\",\n                            \"value\":true\n                        },\n                        {\n                            \"type\":\"bytes\",\n                            \"value\":\"test\"\n                        }\n                    ],\n                    \"sliceRecordCount\":1000\n                },\n                \"name\":\"streamreader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"path\":\"/tmp/out\",\n                    \"fileName\":\"test\",\n                    \"header\":[\n                        \"str\",\n                        \"\u957f\u5ea6\",\n                        \"\u65e5\u671f\",\n                        \"\u662f\u5426\u4e3a\u771f\",\n                        \"\u5b57\u8282\u7c7b\u578b\"\n                    ],\n                    \"writeMode\":\"truncate\"\n                },\n                \"name\":\"excelwriter\"\n            }\n        },\n    \"setting\":{\n        \"speed\":{\n            \"byte\":-1,\n            \"channel\":1\n        }\n    }\n}\n\n2021-09-10 22:16:38.287 [        main] INFO  PerfTrace            - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-09-10 22:16:38.287 [        main] INFO  JobContainer         - Addax jobContainer starts job.\n2021-09-10 22:16:38.289 [        main] INFO  JobContainer         - Set jobId = 0\n2021-09-10 22:16:38.303 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do prepare work .\n2021-09-10 22:16:38.304 [       job-0] INFO  JobContainer         - Addax Writer.Job [excelwriter] do prepare work .\n2021-09-10 22:16:38.304 [       job-0] INFO  JobContainer         - Job set Channel-Number to 1 channels.\n2021-09-10 22:16:38.304 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] splits to [1] tasks.\n2021-09-10 22:16:38.305 [       job-0] INFO  JobContainer         - Addax Writer.Job [excelwriter] splits to [1] tasks.\n2021-09-10 22:16:38.325 [       job-0] INFO  JobContainer         - Scheduler starts [1] taskGroups.\n2021-09-10 22:16:38.332 [ taskGroup-0] INFO  TaskGroupContainer   - taskGroupId=[0] start [1] channels for [1] tasks.\n2021-09-10 22:16:38.335 [ taskGroup-0] INFO  Channel              - Channel set byte_speed_limit to -1, No bps activated.\n2021-09-10 22:16:38.336 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\n2021-09-10 22:16:41.345 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2021-09-10 22:16:41.346 [       job-0] INFO  JobContainer         - Addax Writer.Job [excelwriter] do post work.\n2021-09-10 22:16:41.346 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do post work.\n2021-09-10 22:16:41.348 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2021-09-10 22:16:41.349 [       job-0] INFO  StandAloneJobContainerCommunicator - Total 1000 records, 26000 bytes | Speed 8.46KB/s, 333 records/s | Error 0 records, 0 bytes |  All Task WaitWriterTime 0.528s |  All Task WaitReaderTime 0.000s | Percentage 100.00%\n2021-09-10 22:16:41.350 [       job-0] INFO  JobContainer         -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-09-10 22:16:38\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-09-10 22:16:41\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :            8.46KB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :            333rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                1000\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"writer/excelwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f string \u65e0 \u6307\u5b9a\u6587\u4ef6\u4fdd\u5b58\u7684\u76ee\u5f55, \u6307\u5b9a\u7684\u76ee\u5f55\u5982\u679c\u4e0d\u5b58\u5728\uff0c\u5219\u5c1d\u8bd5\u521b\u5efa fileName \u662f string \u65e0 \u8981\u751f\u6210\u7684excel \u6587\u4ef6\u540d\uff0c\u8be6\u8ff0\u5982\u4e0b header \u5426 list \u65e0 Excel \u8868\u5934"},{"location":"writer/excelwriter/#filename","title":"fileName","text":"

    \u5982\u679c\u914d\u7f6e\u7684 fileName \u6ca1\u6709\u540e\u7f00\uff0c\u5219\u81ea\u52a8\u52a0\u4e0a .xlsx\uff1b \u5982\u679c\u540e\u7f00\u4e3a .xls\uff0c\u5219\u62a5\u9519\uff0c\u56e0\u4e3a\u5f53\u524d\u4ec5\u751f\u6210 Excel 97 \u4ee5\u540e\u7684\u6587\u4ef6\u683c\u5f0f\uff0c\u5373 .xlsx \u540e\u7f00\u7684\u6587\u4ef6

    "},{"location":"writer/excelwriter/#header","title":"header","text":"

    \u5982\u679c\u4e0d\u6307\u5b9a header \uff0c\u5219\u751f\u6210\u7684 Excel \u6587\u4ef6\u6ca1\u6709\u8868\u5934\uff0c\u53ea\u6709\u6570\u636e\u3002 \u6ce8\u610f\uff0c\u63d2\u4ef6\u4e0d\u5173\u5fc3 header \u7684\u6570\u91cf\u662f\u5426\u5339\u914d\u6570\u636e\u4e2d\u7684\u5217\u6570\uff0c\u4e5f\u5c31\u662f\u8bf4\u8868\u5934\u7684\u5217\u6570\u5e76\u4e0d\u8981\u6c42\u548c\u63a5\u4e0b\u6765\u7684\u6570\u636e\u7684\u5217\u6570\u76f8\u7b49\u3002

    "},{"location":"writer/excelwriter/#_3","title":"\u9650\u5236","text":"
    1. \u5f53\u524d\u4ec5\u751f\u6210\u4e00\u4e2a Excel \u6587\u4ef6\uff0c\u4e14\u6ca1\u6709\u8003\u8651\u884c\u6570\u548c\u5217\u6570\u662f\u5426\u8d85\u8fc7\u4e86 Excel \u7684\u9650\u5b9a
    2. \u5982\u679c\u6307\u5b9a\u7684\u76ee\u5f55\u4e0b\u6709\u540c\u540d\u6587\u4ef6\uff0c\u5f53\u524d\u4f1a\u88ab\u8986\u76d6\uff0c\u540e\u7eed\u4f1a\u7edf\u4e00\u5904\u7406\u76ee\u6807\u76ee\u5f55\u7684\u95ee\u9898
    3. \u5f53\u524d\u65e5\u671f\u683c\u5f0f\u7684\u6570\u636e\uff0c\u8bbe\u7f6e\u5355\u5143\u683c\u6837\u5f0f\u4e3a yyyy-MM-dd HH:mm:ss\uff0c\u4e14\u4e0d\u80fd\u5b9a\u5236
    4. \u4e0d\u652f\u6301\u4e8c\u8fdb\u5236\u7c7b\u578b\u7684\u6570\u636e\u5199\u5165
    "},{"location":"writer/ftpwriter/","title":"Ftp Writer","text":"

    Ftp Writer \u63d0\u4f9b\u4e86\u5411\u8fdc\u7a0b FTP/SFTP \u670d\u52a1\u5199\u5165\u6587\u4ef6\u7684\u80fd\u529b\uff0c\u5f53\u524d\u4ec5\u652f\u6301\u5199\u5165\u6587\u672c\u6587\u4ef6\u3002

    "},{"location":"writer/ftpwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"job/stream2ftp.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {},\n      \"writer\": {\n        \"name\": \"ftpwriter\",\n        \"parameter\": {\n          \"protocol\": \"sftp\",\n          \"host\": \"***\",\n          \"port\": 22,\n          \"username\": \"xxx\",\n          \"password\": \"xxx\",\n          \"timeout\": \"60000\",\n          \"connectPattern\": \"PASV\",\n          \"path\": \"/tmp/data/\",\n          \"fileName\": \"test\",\n          \"writeMode\": \"truncate|append|nonConflict\",\n          \"fieldDelimiter\": \",\",\n          \"encoding\": \"UTF-8\",\n          \"nullFormat\": \"null\",\n          \"dateFormat\": \"yyyy-MM-dd\",\n          \"fileFormat\": \"csv\",\n          \"useKey\": false,\n          \"keyPath\": \"\",\n          \"keyPass\": \"\",\n          \"header\": []\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/ftpwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 protocol \u662f string ftp \u670d\u52a1\u5668\u534f\u8bae\uff0c\u76ee\u524d\u652f\u6301\u4f20\u8f93\u534f\u8bae\u6709 ftp \u548c sftp host \u662f string \u65e0 \u670d\u52a1\u5668\u5730\u5740 port \u5426 int 22/21 ftp \u9ed8\u8ba4\u4e3a 21\uff0csftp \u9ed8\u8ba4\u4e3a 22 timeout \u5426 int 60000 \u8fde\u63a5ftp\u670d\u52a1\u5668\u8fde\u63a5\u8d85\u65f6\u65f6\u95f4\uff0c\u5355\u4f4d\u6beb\u79d2(ms) connectPattern \u5426 string PASV \u8fde\u63a5\u6a21\u5f0f\uff0c\u4ec5\u652f\u6301 PORT, PASV \u6a21\u5f0f\u3002ftp\u534f\u8bae\u65f6\u4f7f\u7528 \uff5c username \u662f string \u65e0 \u7528\u6237\u540d password \u662f string \u65e0 \u8bbf\u95ee\u5bc6\u7801 useKey \u5426 boolean false \u662f\u5426\u4f7f\u7528\u79c1\u94a5\u767b\u5f55\uff0c\u4ec5\u9488\u5bf9 sftp \u767b\u5f55\u6709\u6548 keyPath \u5426 string ~/.ssh/id_rsa \u79c1\u94a5\u5730\u5740 keyPass \u5426 string \u65e0 \u79c1\u94a5\u5bc6\u7801\uff0c\u82e5\u6ca1\u6709\u8bbe\u7f6e\u79c1\u94a5\u5bc6\u7801\uff0c\u5219\u65e0\u9700\u914d\u7f6e\u8be5\u9879 path \u662f string \u65e0 \u8fdc\u7a0bFTP\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0cFtpWriter\u4f1a\u5199\u5165Path\u76ee\u5f55\u4e0b\u5c5e\u591a\u4e2a\u6587\u4ef6 fileName \u662f string \u65e0 \u5199\u5165\u7684\u6587\u4ef6\u540d\uff0c\u8be5\u6587\u4ef6\u540d\u4f1a\u6dfb\u52a0\u968f\u673a\u7684\u540e\u7f00\u4f5c\u4e3a\u6bcf\u4e2a\u7ebf\u7a0b\u5199\u5165\u5b9e\u9645\u6587\u4ef6\u540d writeMode \u662f string \u65e0 \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff0c\u8be6\u89c1\u4e0b\u6587 fieldDelimiter \u662f string , \u63cf\u8ff0\uff1a\u8bfb\u53d6\u7684\u5b57\u6bb5\u5206\u9694\u7b26 compress \u5426 string \u65e0 \u6587\u672c\u538b\u7f29\u7c7b\u578b\uff0c\u6682\u4e0d\u652f\u6301 encoding \u5426 string utf-8 \u8bfb\u53d6\u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e dateFormat \u5426 string \u65e0 \u65e5\u671f\u7c7b\u578b\u7684\u6570\u636e\u5e8f\u5217\u5316\u5230\u6587\u4ef6\u4e2d\u65f6\u7684\u683c\u5f0f\uff0c\u4f8b\u5982 \"yyyy-MM-dd\" fileFormat \u5426 string text \u6587\u4ef6\u5199\u51fa\u7684\u683c\u5f0f\uff0c\u5305\u62ec csv, text \u4e24\u79cd\uff0c header \u5426 list \u65e0 text\u5199\u51fa\u65f6\u7684\u8868\u5934\uff0c\u793a\u4f8b ['id', 'name', 'age'] nullFormat \u5426 string \\N \u5b9a\u4e49\u54ea\u4e9b\u5b57\u7b26\u4e32\u53ef\u4ee5\u8868\u793a\u4e3anull maxTraversalLevel \u5426 int 100 \u5141\u8bb8\u904d\u5386\u6587\u4ef6\u5939\u7684\u6700\u5927\u5c42\u6570 csvReaderConfig \u5426 map \u65e0 \u8bfb\u53d6CSV\u7c7b\u578b\u6587\u4ef6\u53c2\u6570\u914d\u7f6e\uff0c\u8be6\u89c1\u4e0b\u6587"},{"location":"writer/ftpwriter/#writemode","title":"writeMode","text":"

    \u63cf\u8ff0\uff1aFtpWriter\u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff1a

    1. truncate\uff0c\u5199\u5165\u524d\u6e05\u7406\u76ee\u5f55\u4e0b\u4e00fileName\u524d\u7f00\u7684\u6240\u6709\u6587\u4ef6\u3002
    2. append\uff0c\u5199\u5165\u524d\u4e0d\u505a\u4efb\u4f55\u5904\u7406\uff0cAddax FtpWriter\u76f4\u63a5\u4f7f\u7528filename\u5199\u5165\uff0c\u5e76\u4fdd\u8bc1\u6587\u4ef6\u540d\u4e0d\u51b2\u7a81\u3002
    3. nonConflict\uff0c\u5982\u679c\u76ee\u5f55\u4e0b\u6709fileName\u524d\u7f00\u7684\u6587\u4ef6\uff0c\u76f4\u63a5\u62a5\u9519\u3002
    "},{"location":"writer/ftpwriter/#_3","title":"\u8ba4\u8bc1","text":"

    \u4ece 4.0.2 \u7248\u672c\u5f00\u59cb\uff0c \u652f\u6301\u79c1\u94a5\u8ba4\u8bc1\u65b9\u5f0f\u767b\u5f55 SFTP \u670d\u52a1\u5668\uff0c\u5982\u679c\u5bc6\u7801\u548c\u79c1\u6709\u90fd\u586b\u5199\u4e86\uff0c\u5219\u4e24\u8005\u8ba4\u8bc1\u65b9\u5f0f\u90fd\u4f1a\u5c1d\u8bd5\u3002 \u6ce8\u610f\uff0c\u5982\u679c\u586b\u5199\u4e86 keyPath, keyPass \u9879\uff0c\u4f46 useKey \u8bbe\u7f6e\u4e3a false \uff0c\u63d2\u4ef6\u4f9d\u7136\u4e0d\u4f1a\u5c1d\u8bd5\u7528\u79c1\u94a5\u8fdb\u884c\u767b\u5f55\u3002

    "},{"location":"writer/ftpwriter/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    FTP\u6587\u4ef6\u672c\u8eab\u4e0d\u63d0\u4f9b\u6570\u636e\u7c7b\u578b\uff0c\u8be5\u7c7b\u578b\u662f Addax FtpWriter \u5b9a\u4e49\uff1a

    Addax \u5185\u90e8\u7c7b\u578b FTP\u6587\u4ef6 \u6570\u636e\u7c7b\u578b Long Long -> \u5b57\u7b26\u4e32\u5e8f\u5217\u5316\u8868\u793a Double Double -> \u5b57\u7b26\u4e32\u5e8f\u5217\u5316\u8868\u793a String String -> \u5b57\u7b26\u4e32\u5e8f\u5217\u5316\u8868\u793a Boolean Boolean -> \u5b57\u7b26\u4e32\u5e8f\u5217\u5316\u8868\u793a Date Date -> \u5b57\u7b26\u4e32\u5e8f\u5217\u5316\u8868\u793a"},{"location":"writer/greenplumwriter/","title":"Greenplum Writer","text":"

    Greenplum Writer \u63d2\u4ef6\u4f7f\u7528 COPY FROM \u8bed\u6cd5 \u5c06\u6570\u636e\u5199\u5165 Greenplum \u6570\u636e\u5e93\u3002

    "},{"location":"writer/greenplumwriter/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u6f14\u793a\u4ece greenplum \u6307\u5b9a\u7684\u8868\u8bfb\u53d6\u6570\u636e\uff0c\u5e76\u63d2\u5165\u5230\u5177\u6709\u76f8\u540c\u8868\u7ed3\u6784\u7684\u53e6\u5916\u4e00\u5f20\u8868\u4e2d\uff0c\u7528\u6765\u6d4b\u8bd5\u8be5\u63d2\u4ef6\u6240\u652f\u6301\u7684\u6570\u636e\u7c7b\u578b\u3002

    create table if not exists addax_tbl\n(\n    c_bigint\n    bigint,\n    c_bit\n    bit(3),\n    c_bool boolean,\n    c_byte bytea,\n    c_char char(10),\n    c_varchar varchar(20),\n    c_date date,\n    c_double float8,\n    c_int integer,\n    c_json json,\n    c_number decimal(8, 3),\n    c_real real,\n    c_small smallint,\n    c_text text,\n    c_ts timestamp,\n    c_uuid uuid,\n    c_xml xml,\n    c_money money,\n    c_inet inet,\n    c_cidr cidr,\n    c_macaddr macaddr\n    );\ninsert into addax_tbl\nvalues (999988887777,\n        B'101',\n        TRUE,\n        '\\xDEADBEEF',\n        'hello',\n        'hello, world',\n        '2021-01-04',\n        999888.9972,\n        9876542,\n        '{\"bar\": \"baz\", \"balance\": 7.77, \"active\": false}'::json,\n        12345.123,\n        123.123,\n        126,\n        'this is a long text ',\n        '2020-01-04 12:13:14',\n        'A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11'::uuid,\n        '<foo>bar</foo>'::xml,\n        '52093.89'::money,\n        '192.168.1.1'::inet,\n        '192.168.1/24'::cidr,\n        '08002b:010203'::macaddr);\n

    \u521b\u5efa\u9700\u8981\u63d2\u5165\u7684\u8868\u7684\u8bed\u53e5\u5982\u4e0b:

    create table gp_test like addax_tbl;\n
    "},{"location":"writer/greenplumwriter/#_2","title":"\u4efb\u52a1\u914d\u7f6e","text":"

    \u4ee5\u4e0b\u662f\u914d\u7f6e\u6587\u4ef6

    job/pg2gp.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"postgresqlreader\",\n        \"parameter\": {\n          \"username\": \"wgzhao\",\n          \"password\": \"wgzhao\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"addax_tbl\"\n            ],\n            \"jdbcUrl\": \"jdbc:postgresql://localhost:5432/wgzhao\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"greenplumwriter\",\n        \"parameter\": {\n          \"username\": \"wgzhao\",\n          \"password\": \"wgzhao\",\n          \"column\": [\n            \"*\"\n          ],\n          \"preSql\": [\n            \"truncate table @table\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:postgresql://localhost:5432/wgzhao\",\n            \"table\": [\n              \"gp_test\"\n            ]\n          }\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/pg2gp.json

    "},{"location":"writer/greenplumwriter/#_3","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/pg2gp.json\n
    "},{"location":"writer/greenplumwriter/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/hanawriter/","title":"HANA Writer","text":"

    HANA Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 SAP HANA \u76ee\u7684\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/hanawriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684 HANA \u8868\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    create table system.addax_tbl\n(\ncol1 varchar(200) ,\ncol2 int(4),\ncol3 date,\ncol4 boolean,\ncol5 clob\n);\n

    \u8fd9\u91cc\u4f7f\u7528\u4e00\u4efd\u4ece\u5185\u5b58\u4ea7\u751f\u5230 HANA \u5bfc\u5165\u7684\u6570\u636e\u3002

    job/hanawriter.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"hanawriter\",\n        \"parameter\": {\n          \"connection\": [\n            {\n              \"jdbcUrl\": \"jdbc:sap://wgzhao-pc:39017/system\",\n              \"table\": [\n                \"addax_tbl\"\n              ]\n            }\n          ],\n          \"username\": \"system\",\n          \"password\": \"HXEHana1\",\n          \"column\": [\n            \"*\"\n          ]\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/hana2stream.json

    "},{"location":"writer/hanawriter/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/hana2stream.json\n
    "},{"location":"writer/hanawriter/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/hbase11xsqlwriter/","title":"HBase11x SQL Writer","text":"

    HBase11x SQL Writer \u63d2\u4ef6\u5229\u7528 Phoniex\uff0c \u7528\u4e8e\u5411 HBase 1.x \u7248\u672c\u7684\u6570\u636e\u5e93\u5199\u5165\u6570\u636e\u3002

    \u5982\u679c\u4f60\u5e0c\u671b\u901a\u8fc7\u8c03\u7528\u539f\u751f\u63a5\u53e3\u5199\u5165\u6570\u636e\uff0c\u5219\u9700\u8981\u4f7f\u7528HBase11xWriter \u63d2\u4ef6

    \u5982\u679c HBase \u662f 2.X \u7248\u672c\uff0c\u5219\u9700\u8981\u4f7f\u7528 HBase20xsqlwriter \u63d2\u4ef6

    "},{"location":"writer/hbase11xsqlwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":""},{"location":"writer/hbase11xsqlwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 hbaseConfig \u662f map \u65e0 hbase \u96c6\u7fa4\u5730\u5740\uff0c\u8be6\u89c1\u793a\u4f8b\u914d\u7f6e table \u662f string \u65e0 \u8981\u5bfc\u5165\u7684\u8868\u540d\uff0c\u5927\u5c0f\u5199\u654f\u611f\uff0c\u901a\u5e38 phoenix \u8868\u90fd\u662f \u5927\u5199 \u8868\u540d column \u662f list \u65e0 \u5217\u540d\uff0c\u5927\u5c0f\u5199\u654f\u611f\uff0c\u901a\u5e38 phoenix \u7684\u5217\u540d\u90fd\u662f \u5927\u5199 batchSize \u5426 int 256 \u4e00\u6b21\u5199\u5165\u7684\u6700\u5927\u8bb0\u5f55\u6570 nullMode \u5426 string skip \u8bfb\u53d6\u5230\u7684\u5217\u503c\u4e3a null \u65f6\uff0c\u5982\u4f55\u5904\u7406\u3002 haveKerberos \u5426 bolean false \u662f\u5426\u542f\u7528 Kerberos \u8ba4\u8bc1, true \u8868\u793a\u542f\u7528, false \u8868\u793a\u4e0d\u542f\u7528 kerberosPrincipal \u5426 string \u65e0 kerberos \u51ed\u8bc1\u4fe1\u606f\uff0c\u4ec5\u5f53 havekerberos \u542f\u7528\u540e\u6709\u6548 kerberosKeytabFilePath \u5426 string \u65e0 kerberos \u51ed\u8bc1\u6587\u4ef6\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u4ec5\u5f53 havekerberos \u542f\u7528\u540e\u6709\u6548"},{"location":"writer/hbase11xsqlwriter/#nullmode","title":"nullMode","text":"

    \u652f\u6301 skip, empty,\u524d\u8005\u8868\u793a\u8df3\u8fc7\u8be5\u5217,\u540e\u8005\u8868\u793a\u63d2\u5165\u7a7a\u503c,\u6570\u503c\u7c7b\u578b\u4e3a 0,\u5b57\u7b26\u7c7b\u578b\u4e3a null

    \u6ce8\u610f\uff1a\u542f\u7528 kerberos \u8ba4\u8bc1\u540e\uff0c\u7a0b\u5e8f\u9700\u8981\u77e5\u9053hbase-site.xml \u6240\u5728\u7684\u8def\u5f84\uff0c\u4e00\u79cd\u529e\u6cd5\u662f\u8fd0\u884c\u6267\u884c\u5728\u73af\u5883\u53d8\u91cf CLASSPATH \u4e2d\u589e\u52a0\u8be5\u6587\u4ef6\u7684\u6240\u5728\u8def\u5f84\u3002

    \u53e6\u5916\u4e00\u4e2a\u89e3\u51b3\u529e\u6cd5\u662f\u5c06 hbase-site.xml \u6587\u4ef6\u62f7\u8d1d\u5230\u63d2\u4ef6\u7684 libs \u76ee\u5f55\u91cc\u3002

    "},{"location":"writer/hbase11xwriter/","title":"HBase11X Writer","text":"

    Hbase11X Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece\u5411 HBase \u5199\u6570\u636e\u7684\u80fd\u529b\u3002\u5728\u5e95\u5c42\u5b9e\u73b0\u4e0a\uff0cHBase11X Writer \u901a\u8fc7 HBase \u7684 Java \u5ba2\u6237\u7aef\u8fde\u63a5\u8fdc\u7a0b HBase \u670d\u52a1\uff0c\u5e76\u901a\u8fc7 put \u65b9\u5f0f\u5199\u5165Hbase\u3002

    \u5982\u679c HBase \u662f 2.X \u7248\u672c\uff0c\u5219\u9700\u8981\u4f7f\u7528 HBase20xsqlwriter \u63d2\u4ef6

    "},{"location":"writer/hbase11xwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4ece\u672c\u5730\u5199\u5165hbase1.1.x\u7684\u4f5c\u4e1a\uff1a

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 5,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"txtfilereader\",\n        \"parameter\": {\n          \"path\": \"/tmp/normal.txt\",\n          \"charset\": \"UTF-8\",\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"String\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 3,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 4,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 5,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 6,\n              \"type\": \"string\"\n            }\n          ],\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"hbase11xwriter\",\n        \"parameter\": {\n          \"hbaseConfig\": {\n            \"hbase.zookeeper.quorum\": \"***\"\n          },\n          \"table\": \"writer\",\n          \"mode\": \"normal\",\n          \"rowkeyColumn\": [\n            {\n              \"index\": 0,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": -1,\n              \"type\": \"string\",\n              \"value\": \"_\"\n            }\n          ],\n          \"column\": [\n            {\n              \"index\": 1,\n              \"name\": \"cf1:q1\",\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 2,\n              \"name\": \"cf1:q2\",\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 3,\n              \"name\": \"cf1:q3\",\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 4,\n              \"name\": \"cf2:q1\",\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 5,\n              \"name\": \"cf2:q2\",\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 6,\n              \"name\": \"cf2:q3\",\n              \"type\": \"string\"\n            }\n          ],\n          \"versionColumn\": {\n            \"index\": -1,\n            \"value\": \"123456789\"\n          },\n          \"encoding\": \"utf-8\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/hbase11xwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 hbaseConfig \u662f map \u65e0 \u8fde\u63a5 HBase \u96c6\u7fa4\u9700\u8981\u7684\u914d\u7f6e\u4fe1\u606f,\u8be6\u89c1\u793a\u4f8b\u914d\u7f6e mode \u662f string \u65e0 \u5199\u5165 HBase \u7684\u6a21\u5f0f\uff0c\u76ee\u524d\u4ec5\u652f\u6301 normal \u6a21\u5f0f table \u662f string \u65e0 HBase \u8868\u540d\uff08\u5927\u5c0f\u5199\u654f\u611f\uff09 encoding \u5426 string UTF-8 \u7f16\u7801\u65b9\u5f0f\uff0cUTF-8 \u6216\u662f GBK\uff0c\u7528\u4e8e\u5bf9\u4e8c\u8fdb\u5236\u5b58\u50a8\u7684 HBase byte[] \u8f6c\u4e3a String \u65f6\u7684\u7f16\u7801 column \u662f list<map> \u65e0 \u8981\u5199\u5165\u7684\u5b57\u6bb5\uff0cnormal \u6a21\u5f0f\u4e0e multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b\u9879, \u8be6\u7ec6\u8bf4\u660e\u89c1\u4e0b\u6587 rowkeyColumn \u662f list<map> \u65e0 \u8981\u5199\u5165\u7684 rowkey \u5217, \u8be6\u7ec6\u8bf4\u660e\u89c1\u4e0b\u6587 versionColumn \u5426 string \u65e0 \u6307\u5b9a\u5199\u5165\u7684\u65f6\u95f4\u6233,\u8be6\u89c1\u4e0b\u6587 nullMode \u5426 string skip \u8bfb\u53d6\u7684null\u503c\u65f6\uff0c\u5982\u4f55\u5904\u7406, walFlag \u5426 boolean false \u662f\u5426\u5199 WAL, true \u8868\u793a\u5199\u5165, false \u8868\u793a\u4e0d\u5199 writeBufferSize \u5426 int 8388608 \u8bbe\u7f6e\u5199 buffer \u5927\u5c0f\uff0c\u5355\u4f4d\u5b57\u8282, \u9ed8\u8ba48M maxVersion \u662f string \u65e0 \u6307\u5b9a\u5728\u591a\u7248\u672c\u6a21\u5f0f\u4e0b\u8bfb\u53d6\u7684\u7248\u672c\u6570\uff0c-1 \u8868\u793a\u8bfb\u53d6\u6240\u6709\u7248\u672c, multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b range \u5426 string \u65e0 \u6307\u5b9a\u8bfb\u53d6\u7684 rowkey \u8303\u56f4, \u8be6\u89c1\u4e0b\u6587 scanCacheSize \u5426 string 256 \u6bcf\u6b21\u4ece\u670d\u52a1\u5668\u7aef\u8bfb\u53d6\u7684\u884c\u6570 scanBatchSize \u5426 string 100 \u6bcf\u6b21\u4ece\u670d\u52a1\u5668\u7aef\u8bfb\u53d6\u7684\u5217\u6570"},{"location":"writer/hbase11xwriter/#column","title":"column","text":"

    \u8981\u5199\u5165\u7684hbase\u5b57\u6bb5\u3002

    \u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"column\": [\n    {\n      \"index\": 1,\n      \"name\": \"cf1:q1\",\n      \"type\": \"string\"\n    },\n    {\n      \"index\": 2,\n      \"name\": \"cf1:q2\",\n      \"type\": \"string\"\n    }\n  ]\n}\n
    "},{"location":"writer/hbase11xwriter/#rowkey","title":"rowkey","text":"

    \u8981\u5199\u5165\u7684 rowkey \u5217\u3002

    \u63d2\u4ef6\u4f1a\u5c06 rowkeyColumn \u4e2d\u6240\u6709\u5217\u6309\u7167\u914d\u7f6e\u987a\u5e8f\u8fdb\u884c\u62fc\u63a5\u4f5c\u4e3a\u5199\u5165 hbase \u7684 rowkey\uff0c\u4e0d\u80fd\u5168\u4e3a\u5e38\u91cf\u3002\u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"rowkeyColumn\": [\n    {\n      \"index\": 0,\n      \"type\": \"string\"\n    },\n    {\n      \"index\": -1,\n      \"type\": \"string\",\n      \"value\": \"_\"\n    }\n  ]\n}\n
    "},{"location":"writer/hbase11xwriter/#versioncolumn","title":"versionColumn","text":"

    \u6307\u5b9a\u5199\u5165\u7684\u65f6\u95f4\u6233\u3002\u652f\u6301\uff1a\u5f53\u524d\u65f6\u95f4\u3001\u6307\u5b9a\u65f6\u95f4\u5217\uff0c\u6307\u5b9a\u65f6\u95f4\uff0c\u4e09\u8005\u9009\u4e00\u3002\u82e5\u4e0d\u914d\u7f6e\u8868\u793a\u7528\u5f53\u524d\u65f6\u95f4\u3002

    index\uff1a\u6307\u5b9a\u5bf9\u5e94 reader \u7aef column \u7684\u7d22\u5f15\uff0c\u4ece 0 \u5f00\u59cb\uff0c\u9700\u4fdd\u8bc1\u80fd\u8f6c\u6362\u4e3a long,\u82e5\u662f Date \u7c7b\u578b\uff0c \u4f1a\u5c1d\u8bd5\u7528 yyyy-MM-dd HH:mm:ss \u548c yyyy-MM-dd HH:mm:ss SSS \u53bb\u89e3\u6790\uff1b \u82e5\u4e3a\u6307\u5b9a\u65f6\u95f4 index \u4e3a \uff0d1\uff0c\u5219 value \u4e3a\u6307\u5b9a\u65f6\u95f4\u7684\u503c\u3002\u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"versionColumn\": {\n    \"index\": 1\n  }\n}\n

    \u6216\u8005

    {\n  \"versionColumn\": {\n    \"index\": -1,\n    \"value\": 123456789\n  }\n}\n
    "},{"location":"writer/hbase11xwriter/#nullmode","title":"nullMode","text":"

    skip \u8868\u793a\u4e0d\u5411hbase\u5199\u8fd9\u5217\uff1bempty\uff1a\u5199\u5165 HConstants.EMPTY_BYTE_ARRAY\uff0c\u5373new byte [0]

    "},{"location":"writer/hbase11xwriter/#_3","title":"\u652f\u6301\u7684\u5217\u7c7b\u578b","text":"

    \u8bf7\u6ce8\u610f: \u9664\u4e0a\u8ff0\u7f57\u5217\u5b57\u6bb5\u7c7b\u578b\u5916\uff0c\u5176\u4ed6\u7c7b\u578b\u5747\u4e0d\u652f\u6301

    "},{"location":"writer/hbase20xsqlwriter/","title":"HBase20x SQL Writer","text":"

    HBase20x SQL Writer \u63d2\u4ef6\u5229\u7528 Phoenix \u5411 HBase 2.x \u5199\u5165\u6570\u636e\u3002

    \u5982\u679c HBase \u662f 1.X \u7248\u672c\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528 HBase11xsqlWriter \u6216HBase11xWriter \u63d2\u4ef6

    "},{"location":"writer/hbase20xsqlwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"txtfilereader\",\n        \"parameter\": {\n          \"path\": \"/tmp/normal.txt\",\n          \"charset\": \"UTF-8\",\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"String\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 3,\n              \"type\": \"string\"\n            }\n          ],\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"hbase20xsqlwriter\",\n        \"parameter\": {\n          \"batchSize\": \"100\",\n          \"column\": [\n            \"UID\",\n            \"TS\",\n            \"EVENTID\",\n            \"CONTENT\"\n          ],\n          \"queryServerAddress\": \"http://127.0.0.1:8765\",\n          \"nullMode\": \"skip\",\n          \"table\": \"TEST_TBL\"\n        }\n      }\n    },\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 5,\n        \"bytes\": -1\n      }\n    }\n  }\n}\n
    "},{"location":"writer/hbase20xsqlwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 jdbcUrl \u662f string \u65e0 Phoenix \u8fde\u63a5\u5730\u5740 table \u662f string \u65e0 \u6240\u8981\u8bfb\u53d6\u8868\u540d schema \u5426 string \u65e0 \u8868\u6240\u5728\u7684 schema batchSize \u5426 int 256 \u4e00\u6b21\u6279\u91cf\u5199\u5165\u7684\u6700\u5927\u884c\u6570 column \u5426 list \u65e0 \u5217\u540d\uff0c\u5927\u5c0f\u5199\u654f\u611f\uff0c\u901a\u5e38phoenix\u7684\u5217\u540d\u90fd\u662f**\u5927\u5199** nullMode \u5426 string skip \u8bfb\u53d6\u7684 null \u503c\u65f6\uff0c\u5982\u4f55\u5904\u7406, \u8be6\u8ff0\u89c1\u4e0b haveKerberos \u5426 boolean false \u662f\u5426\u542f\u7528Kerberos\u8ba4\u8bc1, true \u8868\u793a\u542f\u7528, false \u8868\u793a\u4e0d\u542f\u7528 kerberosPrincipal \u5426 string \u65e0 kerberos \u51ed\u8bc1\u4fe1\u606f\uff0c\u4ec5\u5f53 havekerberos \u542f\u7528\u540e\u6709\u6548 kerberosKeytabFilePath \u5426 string \u65e0 kerberos \u51ed\u8bc1\u6587\u4ef6\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u4ec5\u5f53 havekerberos \u542f\u7528\u540e\u6709\u6548"},{"location":"writer/hbase20xsqlwriter/#jdbcurl","title":"jdbcUrl","text":"

    queryServerAddress \u662f\u6ee1\u8db3 Phoenix \u94fe\u63a5\u7684\u5730\u5740\uff0c\u5177\u4f53\u683c\u5f0f\u548c\u8981\u6c42\u53ef\u4ee5\u53c2\u8003\u5b98\u65b9\u6587\u6863 \uff0c\u5176 jdbc \u8fde\u63a5\u4e32\u683c\u5f0f\u5982\u4e0b\uff1a

    jdbc:phoenix [ :<zookeeper quorum> [ :<port number> [ :<root node> [ :<principal> [ :<keytab file> ] ] ] ] ] \n

    \u5982\u679c\u4f60\u5e0c\u671b\u901a\u8fc7\u8fde\u63a5 Phoenix Query Server (a.k.a PQS) \uff0c\u5219 JDBC \u8fde\u63a5\u4e32\u5982\u4e0b\uff1a

    jdbc:phoenix:thin:url=<scheme>://<server-hostname>:<port>[;option=value...]\n

    \u66f4\u8be6\u7ec6\u7684\u63cf\u8ff0\uff0c\u53ef\u4ee5\u53c2\u8003\u5b98\u65b9\u6587\u6863

    "},{"location":"writer/hbase20xsqlwriter/#nullmode","title":"nullMode","text":"

    skip \u8868\u793a\u4e0d\u5411hbase\u5199\u8fd9\u5217\uff1bempty\uff1a\u5199\u5165 HConstants.EMPTY_BYTE_ARRAY\uff0c\u5373new byte [0]

    \u6ce8\u610f\uff1a\u542f\u7528kerberos\u8ba4\u8bc1\u540e\uff0c\u7a0b\u5e8f\u9700\u8981\u77e5\u9053hbase-site.xml \u6240\u5728\u7684\u8def\u5f84\uff0c\u4e00\u79cd\u529e\u6cd5\u662f\u8fd0\u884c\u6267\u884c\u5728\u73af\u5883\u53d8\u91cf CLASSPATH \u4e2d\u589e\u52a0\u8be5\u6587\u4ef6\u7684\u6240\u5728\u8def\u5f84\u3002

    \u53e6\u5916\u4e00\u4e2a\u89e3\u51b3\u529e\u6cd5\u662f\u5c06 hbase-site.xml \u6587\u4ef6\u62f7\u8d1d\u5230\u63d2\u4ef6\u7684 libs \u76ee\u5f55\u91cc\u3002

    "},{"location":"writer/hdfswriter/","title":"HDFS Writer","text":"

    HDFS Writer \u63d0\u4f9b\u5411 HDFS \u6587\u4ef6\u7cfb\u7edf\u6307\u5b9a\u8def\u5f84\u4e2d\u5199\u5165 TextFile \uff0c ORCFile , Parquet \u7b49\u683c\u5f0f\u6587\u4ef6\u7684\u80fd\u529b\uff0c \u6587\u4ef6\u5185\u5bb9\u53ef\u4e0e hive \u4e2d\u8868\u5173\u8054\u3002

    "},{"location":"writer/hdfswriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 00:00:00\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        },\n        \"writer\": {\n          \"name\": \"hdfswriter\",\n          \"parameter\": {\n            \"defaultFS\": \"hdfs://xxx:port\",\n            \"fileType\": \"orc\",\n            \"path\": \"/user/hive/warehouse/writerorc.db/orcfull\",\n            \"fileName\": \"xxxx\",\n            \"column\": [\n              {\n                \"name\": \"col1\",\n                \"type\": \"string\"\n              },\n              {\n                \"name\": \"col2\",\n                \"type\": \"int\"\n              },\n              {\n                \"name\": \"col3\",\n                \"type\": \"string\"\n              },\n              {\n                \"name\": \"col4\",\n                \"type\": \"boolean\"\n              },\n              {\n                \"name\": \"col5\",\n                \"type\": \"string\"\n              }\n            ],\n            \"writeMode\": \"overwrite\",\n            \"fieldDelimiter\": \"\\u0001\",\n            \"compress\": \"SNAPPY\"\n          }\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/hdfswriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e path \u662f string \u65e0 \u8981\u8bfb\u53d6\u7684\u6587\u4ef6\u8def\u5f84 defaultFS \u662f string \u65e0 \u8be6\u8ff0\u89c1\u4e0b fileType \u662f string \u65e0 \u6587\u4ef6\u7684\u7c7b\u578b\uff0c\u8be6\u8ff0\u89c1\u4e0b fileName \u662f string \u65e0 \u8981\u5199\u5165\u7684\u6587\u4ef6\u540d\uff0c\u7528\u4e8e\u5f53\u4f5c\u524d\u7f00 column \u662f list<map> \u65e0 \u5199\u5165\u7684\u5b57\u6bb5\u5217\u8868 writeMode \u662f string \u65e0 \u5199\u5165\u6a21\u5f0f\uff0c\u8be6\u8ff0\u89c1\u4e0b skipTrash \u5426 boolean false \u662f\u5426\u8df3\u8fc7\u5783\u573e\u56de\u6536\u7ad9\uff0c\u548c writeMode \u914d\u7f6e\u76f8\u5173\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0 fieldDelimiter \u5426 string , \u6587\u672c\u6587\u4ef6\u7684\u5b57\u6bb5\u5206\u9694\u7b26\uff0c\u4e8c\u8fdb\u5236\u6587\u4ef6\u4e0d\u9700\u8981\u6307\u5b9a\u8be5\u9879 encoding \u5426 string utf-8 \u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e\uff0c \u76ee\u524d\u4ec5\u652f\u6301 utf-8 nullFormat \u5426 string \u65e0 \u5b9a\u4e49\u8868\u793a\u4e3a\u7a7a\u7684\u5b57\u7b26\uff0c\u4f8b\u5982\u5982\u679c\u7528\u6237\u914d\u7f6e: \"\\\\N\" \uff0c\u90a3\u4e48\u5982\u679c\u6e90\u5934\u6570\u636e\u662f \"\\N\" \uff0c\u89c6\u4f5c null \u5b57\u6bb5 haveKerberos \u5426 boolean false \u662f\u5426\u542f\u7528 Kerberos \u8ba4\u8bc1\uff0c\u5982\u679c\u542f\u7528\uff0c\u5219\u9700\u8981\u540c\u65f6\u914d\u7f6e\u4ee5\u4e0b\u4e24\u9879 kerberosKeytabFilePath \u5426 string \u65e0 \u7528\u4e8e Kerberos \u8ba4\u8bc1\u7684\u51ed\u8bc1\u6587\u4ef6\u8def\u5f84, \u6bd4\u5982 /your/path/addax.service.keytab kerberosPrincipal \u5426 string \u65e0 \u7528\u4e8e Kerberos \u8ba4\u8bc1\u7684\u51ed\u8bc1\u4e3b\u4f53, \u6bd4\u5982 addax/node1@WGZHAO.COM compress \u5426 string \u65e0 \u6587\u4ef6\u7684\u538b\u7f29\u683c\u5f0f\uff0c\u8be6\u89c1\u4e0b\u6587 hadoopConfig \u5426 map \u65e0 \u91cc\u53ef\u4ee5\u914d\u7f6e\u4e0e Hadoop \u76f8\u5173\u7684\u4e00\u4e9b\u9ad8\u7ea7\u53c2\u6570\uff0c\u6bd4\u5982HA\u7684\u914d\u7f6e preShell \u5426 list \u65e0 \u5199\u5165\u6570\u636e\u524d\u6267\u884c\u7684shell\u547d\u4ee4\uff0c\u6bd4\u5982 hive -e \"truncate table test.hello\" postShell \u5426 list \u65e0 \u5199\u5165\u6570\u636e\u540e\u6267\u884c\u7684shell\u547d\u4ee4\uff0c\u6bd4\u5982 hive -e \"select count(1) from test.hello\" ignoreError \u5426 boolean false \u662f\u5426\u5ffd\u7565preShell, postShell \u547d\u4ee4\u7684\u9519\u8bef"},{"location":"writer/hdfswriter/#path","title":"path","text":"

    \u5b58\u50a8\u5230 Hadoop hdfs\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0cHdfsWriter \u4f1a\u6839\u636e\u5e76\u53d1\u914d\u7f6e\u5728 Path \u76ee\u5f55\u4e0b\u5199\u5165\u591a\u4e2a\u6587\u4ef6\u3002\u4e3a\u4e0ehive\u8868\u5173\u8054\uff0c\u8bf7\u586b\u5199hive\u8868\u5728hdfs\u4e0a\u7684\u5b58\u50a8\u8def\u5f84\u3002 \u4f8b\uff1aHive\u4e0a\u8bbe\u7f6e\u7684\u6570\u636e\u4ed3\u5e93\u7684\u5b58\u50a8\u8def\u5f84\u4e3a\uff1a/user/hive/warehouse/ \uff0c\u5df2\u5efa\u7acb\u6570\u636e\u5e93\uff1atest\uff0c\u8868\uff1ahello\uff1b \u5219\u5bf9\u5e94\u7684\u5b58\u50a8\u8def\u5f84\u4e3a\uff1a/user/hive/warehouse/test.db/hello (\u5982\u679c\u5efa\u8868\u65f6\u6307\u5b9a\u4e86location \u5c5e\u6027\uff0c\u5219\u4f9d\u636e\u8be5\u5c5e\u6027\u7684\u8def\u5f84)

    "},{"location":"writer/hdfswriter/#defaultfs","title":"defaultFS","text":"

    Hadoop hdfs \u6587\u4ef6\u7cfb\u7edf namenode \u8282\u70b9\u5730\u5740\u3002\u683c\u5f0f\uff1ahdfs://ip:port \uff1b\u4f8b\u5982\uff1ahdfs://127.0.0.1:9000 , \u5982\u679c\u542f\u7528\u4e86HA\uff0c\u5219\u4e3a servicename \u6a21\u5f0f\uff0c\u6bd4\u5982 hdfs://sandbox

    "},{"location":"writer/hdfswriter/#filetype","title":"fileType","text":"

    \u63cf\u8ff0\uff1a\u6587\u4ef6\u7684\u7c7b\u578b\uff0c\u76ee\u524d\u53ea\u652f\u6301\u7528\u6237\u914d\u7f6e\u4e3a

    "},{"location":"writer/hdfswriter/#column","title":"column","text":"

    \u5199\u5165\u6570\u636e\u7684\u5b57\u6bb5\uff0c\u4e0d\u652f\u6301\u5bf9\u90e8\u5206\u5217\u5199\u5165\u3002\u4e3a\u4e0ehive\u4e2d\u8868\u5173\u8054\uff0c\u9700\u8981\u6307\u5b9a\u8868\u4e2d\u6240\u6709\u5b57\u6bb5\u540d\u548c\u5b57\u6bb5\u7c7b\u578b\uff0c \u5176\u4e2d\uff1aname \u6307\u5b9a\u5b57\u6bb5\u540d\uff0ctype \u6307\u5b9a\u5b57\u6bb5\u7c7b\u578b\u3002

    \u7528\u6237\u53ef\u4ee5\u6307\u5b9a column \u5b57\u6bb5\u4fe1\u606f\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"column\": [\n    {\n      \"name\": \"userName\",\n      \"type\": \"string\"\n    },\n    {\n      \"name\": \"age\",\n      \"type\": \"long\"\n    },\n    {\n      \"name\": \"salary\",\n      \"type\": \"decimal(8,2)\"\n    }\n  ]\n}\n

    \u5bf9\u4e8e\u6570\u636e\u7c7b\u578b\u662f decimal \u7c7b\u578b\u7684\uff0c\u9700\u8981\u6ce8\u610f\uff1a

    1. \u5982\u679c\u6ca1\u6709\u6307\u5b9a\u7cbe\u5ea6\u548c\u5c0f\u6570\u4f4d\uff0c\u5219\u4f7f\u7528\u9ed8\u8ba4\u7684 decimal(38,10) \u8868\u793a
    2. \u5982\u679c\u4ec5\u6307\u5b9a\u4e86\u7cbe\u5ea6\u4f46\u672a\u6307\u5b9a\u5c0f\u6570\u4f4d\uff0c\u5219\u5c0f\u6570\u4f4d\u75280\u8868\u793a\uff0c\u5373 decimal(p,0)
    3. \u5982\u679c\u90fd\u6307\u5b9a\uff0c\u5219\u4f7f\u7528\u6307\u5b9a\u7684\u89c4\u683c\uff0c\u5373 decimal(p,s)
    "},{"location":"writer/hdfswriter/#writemode","title":"writeMode","text":"

    \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff1a

    "},{"location":"writer/hdfswriter/#skiptrash","title":"skipTrash","text":"

    \u5f53 writeMode \u4e3a overwrite \u6a21\u5f0f\u65f6\uff0c\u5f53\u524d\u8981\u5220\u9664\u7684\u6587\u4ef6\u6216\u6587\u4ef6\u5939\u662f\u5426\u8fdb\u5165\u56de\u6536\u7ad9\uff0c\u9ed8\u8ba4\u4e3a\u8fdb\u56de\u6536\u7ad9\uff0c\u4ec5\u5f53\u914d\u7f6e\u4e3a true \u65f6\u4e3a\u76f4\u63a5\u5220\u9664\u3002

    \u8be5\u529f\u80fd\u7684\u5b9e\u73b0\u65b9\u5f0f\u4e3a\u83b7\u53d6 Hadoop HDFS \u7684 fs.trash.interval \u53c2\u6570\uff0c\u5982\u679c\u8be5\u53c2\u6570\u6ca1\u6709\u8bbe\u7f6e\uff0c\u6216\u8bbe\u7f6e\u4e3a0\u65f6\uff0c\u4f1a\u5728\u5220\u9664\u65f6\uff0c\u8bbe\u7f6e\u8be5\u53c2\u6570\u4e3a 10080 \uff0c\u8868\u793a 7 \u5929\u3002

    \u8fd9\u6837\uff0c\u8fdb\u5165\u56de\u6536\u7ad9\u7684\u6587\u4ef6\u4f1a\u4fdd\u75597\u5929\u3002

    \u4fee\u6539\u5220\u9664\u7684\u9ed8\u8ba4\u884c\u4e3a\u662f\u4e3a\u4e86\u7ed9\u56e0\u4e3a\u9519\u8bef\u7684\u91c7\u96c6\u800c\u5bfc\u81f4\u5220\u9664\u7684\u6570\u636e\u6709\u633d\u56de\u7684\u673a\u4f1a\u3002

    "},{"location":"writer/hdfswriter/#compress","title":"compress","text":"

    \u5f53 fileType\uff08\u6587\u4ef6\u7c7b\u578b\uff09\u4e3a csv \u4e0b\u7684\u6587\u4ef6\u538b\u7f29\u65b9\u5f0f\uff0c\u76ee\u524d\u4ec5\u652f\u6301 gzip\u3001bz2\u3001zip\u3001lzo\u3001lzo_deflate\u3001hadoop-snappy\u3001framing-snappy \u538b\u7f29\uff1b \u503c\u5f97\u6ce8\u610f\u7684\u662f\uff0clzo \u5b58\u5728\u4e24\u79cd\u538b\u7f29\u683c\u5f0f\uff1alzo \u548c lzo_deflate\uff0c\u7528\u6237\u5728\u914d\u7f6e\u7684\u65f6\u5019\u9700\u8981\u7559\u5fc3\uff0c\u4e0d\u8981\u914d\u9519\u4e86\uff1b

    \u53e6\u5916\uff0c\u7531\u4e8e snappy \u76ee\u524d\u6ca1\u6709\u7edf\u4e00\u7684 stream format\uff0caddax \u76ee\u524d\u53ea\u652f\u6301\u6700\u4e3b\u6d41\u7684\u4e24\u79cd\uff1ahadoop-snappy\uff08hadoop \u4e0a\u7684 snappy stream format\uff09 \u548c framing-snappy\uff08google \u5efa\u8bae\u7684 snappy stream format\uff09;

    "},{"location":"writer/hdfswriter/#hadoopconfig","title":"hadoopConfig","text":"

    hadoopConfig \u91cc\u53ef\u4ee5\u914d\u7f6e\u4e0e Hadoop \u76f8\u5173\u7684\u4e00\u4e9b\u9ad8\u7ea7\u53c2\u6570\uff0c\u6bd4\u5982HA\u7684\u914d\u7f6e

    {\n  \"hadoopConfig\": {\n    \"dfs.nameservices\": \"cluster\",\n    \"dfs.ha.namenodes.cluster\": \"nn1,nn2\",\n    \"dfs.namenode.rpc-address.cluster.nn1\": \"node1.example.com:8020\",\n    \"dfs.namenode.rpc-address.cluster.nn2\": \"node2.example.com:8020\",\n    \"dfs.client.failover.proxy.provider.cluster\": \"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\"\n  }\n}\n

    \u8fd9\u91cc\u7684 cluster \u8868\u793a HDFS \u914d\u7f6e\u6210HA\u65f6\u7684\u540d\u5b57\uff0c\u4e5f\u662f defaultFS \u914d\u7f6e\u9879\u4e2d\u7684\u540d\u5b57 \u5982\u679c\u5b9e\u9645\u73af\u5883\u4e2d\u7684\u540d\u5b57\u4e0d\u662f cluster \uff0c\u5219\u4e0a\u8ff0\u914d\u7f6e\u4e2d\u6240\u6709\u5199\u6709 cluster \u90fd\u9700\u8981\u66ff\u6362

    "},{"location":"writer/hdfswriter/#preshell-postshell","title":"preShell \u4e0e postShell","text":"

    \u5f15\u5165 preShell \u4e0e postShell \u7684\u76ee\u7684\u662f\u4e3a\u4e86\u5728\u5199\u5165\u6570\u636e\u524d\u540e\u6267\u884c\u4e00\u4e9b\u989d\u5916\u7684\u64cd\u4f5c\uff0c\u6bd4\u5982\u5728\u5199\u5165\u6570\u636e\u524d\u6e05\u7a7a\u8868\uff0c\u5199\u5165\u6570\u636e\u540e\u67e5\u8be2\u8868\u7684\u884c\u6570\u7b49\u3002\u4e00\u4e2a\u5178\u578b\u7684\u751f\u4ea7\u73af\u5883\u573a\u666f\u65f6\uff0c\u91c7\u96c6\u7684\u6570\u636e\u6309\u65e5\u5206\u533a\u4fdd\u5b58\u5728 HDFS \u4e0a\uff0c \u91c7\u96c6\u4e4b\u524d\u9700\u8981\u521b\u5efa\u5206\u533a\uff0c\u8fd9\u6837\u5c31\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e preShell \u6765\u5b9e\u73b0\uff0c\u6bd4\u5982 hive -e \"alter table test.hello add partition(dt='${logdate}')\"

    "},{"location":"writer/hdfswriter/#ignoreerror","title":"ignoreError","text":"

    \u8be5\u914d\u7f6e\u9879\u7528\u4e8e\u63a7\u5236\u662f\u5426\u5ffd\u7565 preShell \u548c postShell \u547d\u4ee4\u7684\u9519\u8bef\uff0c\u5982\u679c\u914d\u7f6e\u4e3a true\uff0c\u5219\u5728\u6267\u884c preShell \u548c postShell \u547d\u4ee4\u65f6\uff0c\u5982\u679c\u547d\u4ee4\u6267\u884c\u5931\u8d25\uff0c\u4e0d\u4f1a\u5bfc\u81f4\u4efb\u52a1\u5931\u8d25\uff0c\u800c\u662f\u4f1a\u6253\u5370\u9519\u8bef\u65e5\u5fd7\uff0c\u7ee7\u7eed\u6267\u884c\u4efb\u52a1\u3002 \u5426\u5219\uff0c\u5982\u679c\u914d\u7f6e\u4e3a false\uff0c\u5219\u5728\u6267\u884c preShell \u548c postShell \u547d\u4ee4\u65f6\uff0c\u5982\u679c\u547d\u4ee4\u6267\u884c\u5931\u8d25\uff0c\u4f1a\u5bfc\u81f4\u4efb\u52a1\u5931\u8d25\u3002

    "},{"location":"writer/hdfswriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b HIVE \u6570\u636e\u7c7b\u578b Long TINYINT,SMALLINT,INT,INTEGER,BIGINT Double FLOAT,DOUBLE,DECIMAL String STRING,VARCHAR,CHAR Boolean BOOLEAN Date DATE,TIMESTAMP Bytes BINARY"},{"location":"writer/hdfswriter/#_4","title":"\u529f\u80fd\u4e0e\u9650\u5236","text":"
    1. \u76ee\u524d\u4e0d\u652f\u6301\uff1abinary\u3001arrays\u3001maps\u3001structs\u3001union \u7c7b\u578b
    "},{"location":"writer/influxdb2writer/","title":"InfluxDB2 Writer","text":"

    InfluxDB2 Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5c06\u6570\u636e\u5199\u5165 InfluxDB 2.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u6570\u636e\u5e93\u7684\u529f\u80fd\u3002

    \u6ce8\u610f\uff0c\u5982\u679c\u4f60\u7684 InfluxDB \u662f 1.8 \u53ca\u4ee5\u4e0b\u7248\u672c\uff0c\u5219\u5e94\u8be5\u4f7f\u7528 InfluxDBWriter \u63d2\u4ef6

    "},{"location":"writer/influxdb2writer/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u793a\u4f8b\u7528\u6765\u6f14\u793a\u8be5\u63d2\u4ef6\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\u5e76\u5199\u5165\u5230\u6307\u5b9a\u8868

    "},{"location":"writer/influxdb2writer/#job","title":"\u521b\u5efa job \u6587\u4ef6","text":"

    \u521b\u5efa job/stream2influx2.json \u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    job/stream2influx2.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"incr\": \"2021-10-17 22:40:00,1,s\",\n              \"type\": \"date\"\n            },\n            {\n              \"random\": \"1,1000\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"1,10\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"1000,50000\",\n              \"type\": \"double\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"influxdb2writer\",\n        \"parameter\": {\n          \"connection\": {\n            \"endpoint\": \"http://localhost:8086\",\n            \"org\": \"com.wgzhao\",\n            \"bucket\": \"test\",\n            \"table\": \"addax_tbl\"\n          },\n          \"token\": \"YOUR_SECURE_TOKEN\",\n          \"tag\": [\n            {\n              \"location\": \"east\"\n            },\n            {\n              \"lat\": \"23.123445\"\n            }\n          ],\n          \"interval\": \"ms\",\n          \"column\": [\n            \"c_long\",\n            \"c_string\",\n            \"c_double\"\n          ],\n          \"batchSize\": 1024\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/influxdb2writer/#_2","title":"\u8fd0\u884c","text":"

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/stream2influx2.json\n
    "},{"location":"writer/influxdb2writer/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 InfluxDB \u8fde\u63a5\u4e32 table \u662f string \u65e0 \u8981\u5199\u5165\u7684\u8868\uff08\u6307\u6807\uff09 org \u662f string \u65e0 \u6307\u5b9a InfluxDB \u7684 org \u540d\u79f0 bucket \u662f string \u65e0 \u6307\u5b9a InfluxDB \u7684 bucket \u540d\u79f0 token \u662f string \u65e0 \u8bbf\u95ee\u6570\u636e\u5e93\u7684 token column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408 tag \u5426 list<map> \u65e0 \u8981\u6307\u5b9a\u7684 tag interval \u5426 string ms \u6307\u5b9a\u65f6\u95f4\u95f4\u9694\uff0c\u53ef\u4ee5\u6307\u5b9a s,ms,us, ns batchSize \u5426 int 1024 \u6279\u91cf\u5199\u5165\u7684\u5927\u5c0f"},{"location":"writer/influxdb2writer/#column","title":"column","text":"

    InfluxDB \u4f5c\u4e3a\u65f6\u5e8f\u6570\u636e\u5e93\uff0c\u9700\u8981\u6bcf\u6761\u8bb0\u5f55\u90fd\u6709\u65f6\u95f4\u6233\u5b57\u6bb5\uff0c\u56e0\u6b64\u4f1a\u628a\u6bcf\u6761\u7684\u8bb0\u5f55\u7684\u7b2c\u4e00\u4e2a\u5b57\u6bb5\u5f53\u4f5c\u65f6\u95f4\u6233\u6765\u5904\u7406\u3002 column \u53ea\u9700\u8981\u6307\u5b9a\u9664\u4e86\u7b2c\u4e00\u4e2a\u5b57\u6bb5\u5916\u7684\u5176\u4ed6\u5b57\u6bb5\u3002 \u6bd4\u5982\u793a\u4f8b\u4e2d\uff0cstreamreader \u8bbe\u7f6e\u4e864\u4e2a\u5b57\u6bb5\uff0c\u4f46\u5728 influxdb2writer \u4e2d\u7684 column \u53ea\u6307\u5b9a\u4e86\u4e09\u4e2a\u5b57\u6bb5\uff0c\u5c31\u662f\u56e0\u4e3a\u7b2c\u4e00\u4e2a\u5b57\u6bb5\u5df2\u7ecf\u9ed8\u8ba4\u4f5c\u4e3a\u65f6\u95f4\u6233\u4e86\u3002

    "},{"location":"writer/influxdb2writer/#tag","title":"tag","text":"

    \u7528\u4e8e\u6307\u5b9a\u6307\u6807\uff08\u8fd9\u91cc\u5f53\u4f5c\u8868\uff09\u7684 \u6807\u7b7e\uff0c\u6bcf\u4e2a tag \u4f7f\u7528 map \u65b9\u5f0f\u6307\u5b9a\uff0c\u6bd4\u5982\u793a\u4f8b\u4e2d\uff1a

    {\n  \"tag\": [\n    {\n      \"location\": \"east\"\n    },\n    {\n      \"lat\": 23.123445\n    }\n  ]\n}\n

    map\u4e2d\u7684 key \u8868\u793a\u6807\u7b7e\u7684\u540d\u79f0\uff0cvalue \u8868\u793a\u6807\u7b7e\u503c

    "},{"location":"writer/influxdb2writer/#interval","title":"interval","text":"

    \u8bbe\u7f6e\u65f6\u95f4\u6233\u7684\u95f4\u9694\u9891\u7387\uff0c\u8be5\u5b57\u6bb5\u7684\u5b9a\u4e49\u6765\u6e90\u4e8e influxdb-client-java \u4e2d\u7684 WritePrecision.java \u5176\u5b57\u7b26\u4e32\u8868\u8fbe\u7684\u542b\u4e49\u5206\u522b\u4e3a\uff1a

    "},{"location":"writer/influxdb2writer/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u5f53\u524d\u652f\u6301 InfluxDB 2.0 \u7684\u57fa\u672c\u7c7b\u578b

    "},{"location":"writer/influxdbwriter/","title":"InfluxDB Writer","text":"

    InfluxDB Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5c06\u6570\u636e\u5199\u5165 InfluxDB \u8bfb\u53d6\u6570\u636e\u7684\u529f\u80fd\u3002 \u5e95\u5c42\u5b9e\u73b0\u4e0a\uff0c\u662f\u901a\u8fc7\u8c03\u7528 InfluQL \u8bed\u8a00\u63a5\u53e3\uff0c\u6784\u5efa\u63d2\u5165\u8bed\u53e5\uff0c\u7136\u540e\u8fdb\u884c\u6570\u636e\u63d2\u5165\u3002

    "},{"location":"writer/influxdbwriter/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u793a\u4f8b\u7528\u6765\u6f14\u793a\u8be5\u63d2\u4ef6\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\u5e76\u5199\u5165\u5230\u6307\u5b9a\u8868

    "},{"location":"writer/influxdbwriter/#_2","title":"\u521b\u5efa\u9700\u8981\u7684\u5e93","text":"

    \u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u6765\u521b\u5efa\u9700\u8981\u5199\u5165\u7684\u5e93

    # create database\ninflux --execute \"CREATE DATABASE addax\"\n
    "},{"location":"writer/influxdbwriter/#job","title":"\u521b\u5efa job \u6587\u4ef6","text":"

    \u521b\u5efa job/stream2influxdb.json \u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    job/stream2influxdb.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"2001-01-01 00:00:00, 2016-07-07 23:59:59\",\n              \"type\": \"date\"\n            },\n            {\n              \"random\": \"1,1000\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"1,10\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"1000,50000\",\n              \"type\": \"double\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"influxdbwriter\",\n        \"parameter\": {\n          \"connection\": {\n            \"endpoint\": \"http://localhost:8086\",\n            \"database\": \"addax\",\n            \"table\": \"addax_tbl\"\n          },\n          \"connTimeout\": 15,\n          \"readTimeout\": 20,\n          \"writeTimeout\": 20,\n          \"username\": \"influx\",\n          \"password\": \"influx123\",\n          \"column\": [\n            {\n              \"name\": \"time\",\n              \"type\": \"timestamp\"\n            },\n            {\n              \"name\": \"user_id\",\n              \"type\": \"int\"\n            },\n            {\n              \"name\": \"user_name\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"salary\",\n              \"type\": \"double\"\n            }\n          ],\n          \"preSql\": [\n            \"delete from addax_tbl\"\n          ],\n          \"batchSize\": 1024,\n          \"retentionPolicy\": {\n            \"name\": \"one_day_only\",\n            \"duration\": \"1d\",\n            \"replication\": 1\n          }\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/influxdbwriter/#_3","title":"\u8fd0\u884c","text":"

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/stream2influxdb.json\n
    "},{"location":"writer/influxdbwriter/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 InfluxDB \u8fde\u63a5\u4e32 username \u662f string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801 database \u662f string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7684\u6570\u636e\u5e93 table \u662f string \u65e0 \u8981\u5199\u5165\u7684\u8868\uff08\u6307\u6807\uff09 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408 connTimeout \u5426 int 15 \u8bbe\u7f6e\u8fde\u63a5\u8d85\u65f6\u503c\uff0c\u5355\u4f4d\u4e3a\u79d2 readTimeout \u5426 int 20 \u8bbe\u7f6e\u8bfb\u53d6\u8d85\u65f6\u503c\uff0c\u5355\u4f4d\u4e3a\u79d2 writeTimeout \u5426 int 20 \u8bbe\u7f6e\u5199\u5165\u8d85\u65f6\u503c\uff0c\u5355\u4f4d\u4e3a\u79d2 preSql \u5426 list \u65e0 \u63d2\u5165\u6570\u636e\u524d\u6267\u884c\u7684SQL\u8bed\u53e5 postSql \u5426 list \u65e0 \u6570\u636e\u63d2\u5165\u5b8c\u6bd5\u540e\u9700\u8981\u6267\u884c\u7684\u8bed\u53e5 retentionPolicy \u5426 map \u65e0 \u8bbe\u7f6e\u6570\u636e\u5e93\u7684 Retention Policy \u7b56\u7565"},{"location":"writer/influxdbwriter/#column","title":"column","text":"

    InfluxDB \u4f5c\u4e3a\u65f6\u5e8f\u6570\u636e\u5e93\uff0c\u9700\u8981\u6bcf\u6761\u8bb0\u5f55\u90fd\u6709\u65f6\u95f4\u6233\u5b57\u6bb5\uff0c\u56e0\u6b64\u8fd9\u91cc\u4f1a\u628a column \u914d\u7f6e\u7684\u7b2c\u4e00\u4e2a\u5b57\u6bb5\u9ed8\u8ba4\u5f53\u4f5c\u65f6\u95f4\u6233

    "},{"location":"writer/influxdbwriter/#retentionpolicy","title":"retentionPolicy","text":"

    \u8bbe\u5b9a\u6570\u636e\u5e93\u7684 Retention Policy \u7b56\u7565\uff0c\u4f9d\u636e\u7ed9\u5b9a\u7684\u914d\u7f6e\uff0c\u5728\u6307\u5b9a\u6570\u636e\u5e93\u4e0a\u521b\u5efa\u4e00\u6761 Retention Policy \u4fe1\u606f\u3002 \u6709\u5173 Retention Policy \u66f4\u8be6\u7ec6\u7684\u4fe1\u606f\uff0c\u53ef\u4ee5\u53c2\u8003 [\u5b98\u65b9\u6587\u6863][2]

    "},{"location":"writer/influxdbwriter/#_5","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u5f53\u524d\u652f\u6301 InfluxDB \u7684\u57fa\u672c\u7c7b\u578b

    "},{"location":"writer/influxdbwriter/#_6","title":"\u9650\u5236","text":"
    1. \u5f53\u524d\u63d2\u4ef6\u4ec5\u652f\u6301 1.x \u7248\u672c\uff0c2.0 \u53ca\u4ee5\u4e0a\u5e76\u4e0d\u652f\u6301
    "},{"location":"writer/kafkawriter/","title":"Kafka Writer","text":"

    Kafka Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5c06\u6570\u636e\u4ee5 json \u683c\u5f0f\u5199\u5165 Kafka \u7684\u529f\u80fd\u3002

    "},{"location":"writer/kafkawriter/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u6f14\u793a\u4e86\u5982\u4f55\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\u5e76\u5199\u5165\u5230 kafka \u7684\u6307\u5b9a topic \u4e2d\u3002

    "},{"location":"writer/kafkawriter/#_2","title":"\u521b\u5efa\u4efb\u52a1\u6587\u4ef6","text":"

    \u9996\u5148\u521b\u5efa\u4e00\u4e2a\u4efb\u52a1\u6587\u4ef6 stream2kafka.json , \u5185\u5bb9\u5982\u4e0b\uff1a

    {\n  \"job\": {\n    \"setting\": {\n        \"speed\": {\n            \"channel\": 1\n        }\n    },\n    \"content\": [\n      {\n        \"reader\": {\n            \"name\": \"streamreader\",\n            \"parameter\": {\n              \"column\": [\n                    {\"random\": \"10,1000\", \"type\": \"long\"},\n                    {\"value\": \"1.1.1.1\", \"type\": \"string\"},\n                    {\"value\": 19890604.0, \"type\": \"double\"},\n                    {\"value\": 19890604, \"type\": \"long\"},\n                    {\"value\": 19890604, \"type\": \"long\"},\n                    {\"value\": \"hello world\", \"type\": \"string\"},\n                    {\"value\": \"long text\", \"type\": \"string\"},\n                    {\"value\": \"41.12,-71.34\", \"type\": \"string\"},\n                    {\"value\": \"2017-05-25 11:22:33\", \"type\": \"string\"}\n                    ],\n            \"sliceRecordCount\": 100\n            }\n        },\n        \"writer\": {\n          \"name\": \"kafkawriter\",\n          \"parameter\": {\n            \"brokerList\": \"localhost:9092\",\n            \"topic\": \"test-1\",\n            \"partitions\": 0,\n            \"batchSize\": 1000,\n            \"column\": [\"col1\", \"col2\",\"col3\",\"col4\",\"col5\", \"col6\", \"col7\", \"col8\", \"col9\"]\n          }\n        }\n      }\n    ]\n  }\n}\n
    "},{"location":"writer/kafkawriter/#_3","title":"\u8fd0\u884c","text":"

    \u6267\u884c bin/addax.sh stream2kafka.json \u547d\u4ee4\uff0c\u83b7\u5f97\u7c7b\u4f3c\u4e0b\u9762\u7684\u8f93\u51fa\uff1a

    2022-02-26 21:59:22.975 [        main] INFO  VMInfo               - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl\n2022-02-26 21:59:22.985 [        main] INFO  Engine               - \n{\n    \"content\":{\n        \"reader\":{\n            \"parameter\":{\n                \"column\":[\n                    {\n                        \"random\":\"10,1000\",\n                        \"type\":\"long\"\n                    },\n                    {\n                        \"type\":\"string\",\n                        \"value\":\"1.1.1.1\"\n                    },\n                    {\n                        \"type\":\"double\",\n                        \"value\":19890604.0\n                    },\n                    {\n                        \"type\":\"long\",\n                        \"value\":19890604\n                    },\n                    {\n                        \"type\":\"long\",\n                        \"value\":19890604\n                    },\n                    {\n                        \"type\":\"string\",\n                        \"value\":\"hello world\"\n                    },\n                    {\n                        \"type\":\"string\",\n                        \"value\":\"long text\"\n                    },\n                    {\n                        \"type\":\"string\",\n                        \"value\":\"41.12,-71.34\"\n                    },\n                    {\n                        \"type\":\"string\",\n                        \"value\":\"2017-05-25 11:22:33\"\n                    }\n                ],\n                \"sliceRecordCount\":100\n            },\n            \"name\":\"streamreader\"\n        },\n        \"writer\":{\n            \"parameter\":{\n                \"partitions\":0,\n                \"column\":[\n                    \"col1\",\n                    \"col2\",\n                    \"col3\",\n                    \"col4\",\n                    \"col5\",\n                    \"col6\",\n                    \"col7\",\n                    \"col8\",\n                    \"col9\"\n                ],\n                \"topic\":\"test-1\",\n                \"batchSize\":1000,\n                \"brokerList\":\"localhost:9092\"\n            },\n            \"name\":\"kafkawriter\"\n        }\n    },\n    \"setting\":{\n        \"speed\":{\n            \"channel\":1\n        }\n    }\n}\n\n2022-02-26 21:59:23.002 [        main] INFO  PerfTrace            - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2022-02-26 21:59:23.003 [        main] INFO  JobContainer         - Addax jobContainer starts job.\n2022-02-26 21:59:23.004 [        main] INFO  JobContainer         - Set jobId = 0\n2022-02-26 21:59:23.017 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do prepare work .\n2022-02-26 21:59:23.017 [       job-0] INFO  JobContainer         - Addax Writer.Job [kafkawriter] do prepare work .\n2022-02-26 21:59:23.017 [       job-0] INFO  JobContainer         - Job set Channel-Number to 1 channel(s).\n2022-02-26 21:59:23.018 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] splits to [1] tasks.\n2022-02-26 21:59:23.019 [       job-0] INFO  JobContainer         - Addax Writer.Job [kafkawriter] splits to [1] tasks.\n2022-02-26 21:59:23.039 [       job-0] INFO  JobContainer         - Scheduler starts [1] taskGroups.\n2022-02-26 21:59:23.047 [ taskGroup-0] INFO  TaskGroupContainer   - taskGroupId=[0] start [1] channels for [1] tasks.\n2022-02-26 21:59:23.050 [ taskGroup-0] INFO  Channel              - Channel set byte_speed_limit to -1, No bps activated.\n2022-02-26 21:59:23.050 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\n2022-02-26 21:59:23.082 [0-0-0-writer] INFO  ProducerConfig       - ProducerConfig values: \n    acks = 1\n    batch.size = 1000\n    bootstrap.servers = [localhost:9092]\n    buffer.memory = 33554432\n    client.id = addax-kafka-writer\n    compression.type = none\n    connections.max.idle.ms = 540000\n    enable.idempotence = false\n    interceptor.classes = []\n    key.serializer = class org.apache.kafka.common.serialization.StringSerializer\n    linger.ms = 0\n    max.block.ms = 60000\n    max.in.flight.requests.per.connection = 5\n    max.request.size = 1048576\n    metadata.max.age.ms = 300000\n    metric.reporters = []\n    metrics.num.samples = 2\n    metrics.recording.level = INFO\n    metrics.sample.window.ms = 30000\n    partitioner.class = class org.apache.kafka.clients.producer.internals.DefaultPartitioner\n    receive.buffer.bytes = 32768\n    reconnect.backoff.max.ms = 1000\n    reconnect.backoff.ms = 50\n    request.timeout.ms = 30000\n    retries = 0\n    retry.backoff.ms = 100\n    sasl.client.callback.handler.class = null\n    sasl.jaas.config = null\n    sasl.kerberos.kinit.cmd = /usr/bin/kinit\n    sasl.kerberos.min.time.before.relogin = 60000\n    sasl.kerberos.service.name = null\n    sasl.kerberos.ticket.renew.jitter = 0.05\n    sasl.kerberos.ticket.renew.window.factor = 0.8\n    sasl.login.callback.handler.class = null\n    sasl.login.class = null\n    sasl.login.refresh.buffer.seconds = 300\n    sasl.login.refresh.min.period.seconds = 60\n    sasl.login.refresh.window.factor = 0.8\n    sasl.login.refresh.window.jitter = 0.05\n    sasl.mechanism = GSSAPI\n    security.protocol = PLAINTEXT\n    send.buffer.bytes = 131072\n    ssl.cipher.suites = null\n    ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]\n    ssl.endpoint.identification.algorithm = https\n    ssl.key.password = null\n    ssl.keymanager.algorithm = SunX509\n    ssl.keystore.location = null\n    ssl.keystore.password = null\n    ssl.keystore.type = JKS\n    ssl.protocol = TLS\n    ssl.provider = null\n    ssl.secure.random.implementation = null\n    ssl.trustmanager.algorithm = PKIX\n    ssl.truststore.location = null\n    ssl.truststore.password = null\n    ssl.truststore.type = JKS\n    transaction.timeout.ms = 60000\n    transactional.id = null\n    value.serializer = class org.apache.kafka.common.serialization.StringSerializer\n\n2022-02-26 21:59:23.412 [0-0-0-writer] INFO  AppInfoParser        - Kafka version : 2.0.0\n2022-02-26 21:59:23.413 [0-0-0-writer] INFO  AppInfoParser        - Kafka commitId : 3402a8361b734732\n2022-02-26 21:59:23.534 [kafka-producer-network-thread | addax-kafka-writer] INFO  Metadata             - Cluster ID: xPAQZFNDTp6y63nZO4LACA\n2022-02-26 21:59:26.061 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2022-02-26 21:59:26.062 [       job-0] INFO  JobContainer         - Addax Writer.Job [kafkawriter] do post work.\n2022-02-26 21:59:26.062 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do post work.\n2022-02-26 21:59:26.063 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2022-02-26 21:59:26.064 [       job-0] INFO  StandAloneJobContainerCommunicator - Total 100 records, 9200 bytes | Speed 2.99KB/s, 33 records/s | Error 0 records, 0 bytes |  All Task WaitWriterTime 0.000s |  All Task WaitReaderTime 0.000s | Percentage 100.00%\n2022-02-26 21:59:26.065 [       job-0] INFO  JobContainer         - \n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2022-02-26 21:59:23\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2022-02-26 21:59:26\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :            2.99KB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :             33rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                 100\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n

    \u6211\u4eec\u4f7f\u7528 kafka \u81ea\u5e26\u7684 kafka-console-consumer.sh \u5c1d\u8bd5\u8bfb\u53d6\u6570\u636e\uff0c\u8f93\u51fa\u5982\u4e0b\uff1a

    $ bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test-1 --from-beginning\n\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":916}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":572}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":88}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":33}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":697}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":381}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":304}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":103}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":967}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":147}\n
    "},{"location":"writer/kafkawriter/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 brokerList \u662f string \u65e0 \u8fde\u63a5 kafka \u670d\u52a1\u7684 broker \u914d\u7f6e\uff0c\u591a\u4e2a broker\u4e4b\u95f4\u7528\u9017\u53f7(,)\u5206\u9694 topic \u662f string \u65e0 \u8981\u5199\u5165\u7684 topic batchSize \u5426 int 1204 \u8bbe\u7f6e Kafka \u7684 batch.size \u53c2\u6570 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u4e0d\u5141\u8bb8\u4e3a * properties \u5426 map \u65e0 \u9700\u8981\u8bbe\u7f6e\u7684\u5176\u4ed6 kafka \u8fde\u63a5\u53c2\u6570"},{"location":"writer/kafkawriter/#_5","title":"\u9650\u5236","text":"
    1. \u4ec5\u652f\u6301 Kafka 1.0 \u53ca\u4ee5\u4e0a\u7248\u672c\uff0c\u4f4e\u4e8e\u8be5\u7248\u672c\u7684\u65e0\u6cd5\u786e\u5b9a\u662f\u5426\u80fd\u5199\u5165
    2. \u5f53\u524d\u4e0d\u652f\u6301\u542f\u7528\u4e86 kerberos \u8ba4\u8bc1\u7684 kafka \u670d\u52a1
    "},{"location":"writer/kuduwriter/","title":"Kudu Writer","text":"

    Kudu Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5c06\u6570\u636e\u5199\u5165\u5230 kudu \u7684\u80fd\u529b\uff0c\u5f53\u524d\u662f\u901a\u8fc7\u8c03\u7528\u539f\u751fRPC\u63a5\u53e3\u6765\u5b9e\u73b0\u7684\u3002 \u540e\u671f\u5e0c\u671b\u901a\u8fc7 impala \u63a5\u53e3\u5b9e\u73b0\uff0c\u4ece\u800c\u589e\u52a0\u66f4\u591a\u7684\u529f\u80fd\u3002

    "},{"location":"writer/kuduwriter/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u793a\u4f8b\u6f14\u793a\u4e86\u5982\u4f55\u4ece\u5185\u5b58\u8bfb\u53d6\u6837\u4f8b\u6570\u636e\u5e76\u5199\u5165\u5230 kudu \u8868\u4e2d\u7684\u3002

    "},{"location":"writer/kuduwriter/#_2","title":"\u8868\u7ed3\u6784","text":"

    \u6211\u4eec\u7528 trino \u5de5\u5177\u8fde\u63a5\u5230 kudu \u670d\u52a1\uff0c\u7136\u540e\u901a\u8fc7\u4e0b\u9762\u7684 SQL \u8bed\u53e5\u521b\u5efa\u8868

    CREATE TABLE kudu.default.users (\n  user_id int WITH (primary_key = true),\n  user_name varchar,\n  salary double\n) WITH (\n  partition_by_hash_columns = ARRAY['user_id'],\n  partition_by_hash_buckets = 2\n);\n
    "},{"location":"writer/kuduwriter/#job","title":"job \u914d\u7f6e\u6587\u4ef6","text":"

    \u521b\u5efa job/stream2kudu.json \u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    job/stream2kudu.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"1,1000\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"1,10\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"1000,50000\",\n              \"type\": \"double\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"kuduwriter\",\n        \"parameter\": {\n          \"masterAddress\": \"127.0.0.1:7051,127.0.0.1:7151,127.0.0.1:7251\",\n          \"timeout\": 60,\n          \"table\": \"users\",\n          \"writeMode\": \"upsert\",\n          \"column\": [ \"user_id\", \"user_name\", \"salary\"],\n          \"batchSize\": 1024,\n          \"bufferSize\": 2048,\n          \"skipFail\": false,\n          \"encoding\": \"UTF-8\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/kuduwriter/#_3","title":"\u8fd0\u884c","text":"

    \u6267\u884c\u4e0b\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/stream2kudu.json\n
    "},{"location":"writer/kuduwriter/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 masterAddress \u662f string \u65e0 Kudu Master\u96c6\u7fa4RPC\u5730\u5740,\u591a\u4e2a\u5730\u5740\u7528\u9017\u53f7(,)\u5206\u9694 table \u662f string \u65e0 kudu \u8868\u540d writeMode \u5426 string upsert \u8868\u6570\u636e\u5199\u5165\u6a21\u5f0f\uff0c\u652f\u6301 upsert, insert \u4e24\u8005 timeout \u5426 int 100 \u5199\u5165\u6570\u636e\u8d85\u65f6\u65f6\u95f4(\u79d2), 0 \u8868\u793a\u4e0d\u53d7\u9650\u5236 column \u662f list \u65e0 \u8981\u5199\u5165\u7684\u8868\u5b57\u6bb5\uff0c\u914d\u7f6e\u65b9\u5f0f\u89c1\u4e0a\u793a\u4f8b skipFail \u5426 boolean false \u662f\u5426\u8df3\u8fc7\u63d2\u5165\u5931\u8d25\u7684\u8bb0\u5f55\uff0c\u5982\u679c\u8bbe\u7f6e\u4e3atrue\uff0c\u5219\u63d2\u4ef6\u4e0d\u4f1a\u628a\u63d2\u5165\u5931\u8d25\u7684\u5f53\u4f5c\u5f02\u5e38"},{"location":"writer/kuduwriter/#column","title":"column","text":"

    column \u53ef\u4ee5\u76f4\u63a5\u6307\u5b9a\u8981\u5199\u5165\u7684\u5217\uff0c\u5982\u540c\u4e0a\u8ff0\u4f8b\u5b50\uff0c\u4e5f\u53ef\u4ee5\u8bbe\u7f6e [\"*\"] \u6765\u8868\u793a\u5199\u5165\u6240\u6709\u5217\u3002

    "},{"location":"writer/mongodbwriter/","title":"MongoDB Writer","text":"

    MongoDB Writer \u63d2\u4ef6\u7528\u4e8e\u5411 MongoDB \u5199\u5165\u6570\u636e\u3002

    "},{"location":"writer/mongodbwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u8be5\u793a\u4f8b\u5c06\u6d41\u5f0f\u6570\u636e\u5199\u5165\u5230 MongoDB \u8868\u4e2d

    job/stream2mongo.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"unique_id\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"sid\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"user_id\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"auction_id\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"content_type\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"pool_type\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"a1 a2 a3\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"c1 c2 c3\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"2020-09-06\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"tag1 tag2 tag3\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"property\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 1984,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": 1900,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": 75,\n              \"type\": \"long\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"mongodbwriter\",\n        \"parameter\": {\n          \"connection\": {\n            \"address\": [\n              \"127.0.0.1:27017\"\n            ],\n            \"database\": \"my_database\",\n            \"collection\": \"addax_writer\",\n            \"authDb\": \"my_database\"\n          },\n          \"username\": \"my_user\",\n          \"password\": \"password123\",\n          \"column\": [\n            {\n              \"name\": \"unique_id\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"sid\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"user_id\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"auction_id\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"content_type\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"pool_type\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"frontcat_id\",\n              \"type\": \"Array\",\n              \"splitter\": \" \"\n            },\n            {\n              \"name\": \"categoryid\",\n              \"type\": \"Array\",\n              \"splitter\": \" \"\n            },\n            {\n              \"name\": \"gmt_create\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"taglist\",\n              \"type\": \"Array\",\n              \"splitter\": \" \"\n            },\n            {\n              \"name\": \"property\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"scorea\",\n              \"type\": \"int\"\n            },\n            {\n              \"name\": \"scoreb\",\n              \"type\": \"int\"\n            },\n            {\n              \"name\": \"scorec\",\n              \"type\": \"int\"\n            }\n          ],\n          \"writeMode\": \"insert\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/mongodbwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 address \u662f list \u65e0 MongoDB \u7684\u6570\u636e\u5730\u5740\u4fe1\u606f username \u5426 string \u65e0 MongoDB \u7684\u7528\u6237\u540d password \u5426 string \u65e0 MongoDB \u7684\u5bc6\u7801 collection \u662f string \u65e0 MongoDB \u7684\u96c6\u5408\u540d column \u662f list<map> \u65e0 MongoDB \u7684\u6587\u6863\u5217\u540d splitter \u5426 string \u65e0 \u7279\u6b8a\u5206\u9694\u7b26\uff0c\u8be6\u89c1\u4e0b\u6587 writeMode \u5426 string insert \u6307\u5b9a\u4e86\u4f20\u8f93\u6570\u636e\u65f6\u66f4\u65b0\u7684\u4fe1\u606f,\u652f\u6301 insert\uff0c update \u4e24\u79cd batchSize \u5426 int 2048 \u6307\u5b9a\u6279\u6b21\u8f93\u5165\u7684\u6570\u91cf isUpsert \u5426 boolean \u65e0 \u5f53\u8bbe\u7f6e\u4e3a true \u65f6\uff0c\u8868\u793a\u9488\u5bf9\u76f8\u540c\u7684 upsertKey \u505a\u66f4\u65b0\u64cd\u4f5c upsertKey \u5426 string \u65e0 upsertKey \u6307\u5b9a\u4e86\u6ca1\u884c\u8bb0\u5f55\u7684\u4e1a\u52a1\u4e3b\u952e\u3002\u7528\u6765\u505a\u66f4\u65b0\u65f6\u4f7f\u7528"},{"location":"writer/mongodbwriter/#column","title":"column","text":"

    column \u6307\u5b9a mongo collection \u7684\u5b57\u6bb5\u4ee5\u53ca\u7c7b\u578b\uff0c\u5982\u679c\u662f\u6570\u7ec4\u7c7b\u578b\uff0c\u8fd8\u9700\u8981\u6307\u5b9a\u63a5\u6536\u5230\u7684\u6570\u636e\u6309\u7167\u4ec0\u4e48\u5206\u5272\uff0c\u4e00\u4e2a column \u5b57\u6bb5\u81f3\u5c11\u9700\u8981\u6307\u5b9a name \u4ee5\u53ca type\uff0c\u6bd4\u5982

    {\n  \"column\": [\n    {\n      \"name\": \"user_id\",\n      \"type\": \"string\"\n    }\n  ]\n}\n

    \u5982\u679c\u662f\u6570\u7ec4\u7c7b\u578b\uff0c\u5219\u9700\u8981\u914d\u7f6e splitter \u6765\u544a\u77e5\u5206\u9694\u7b26\uff0c\u7c7b\u4f3c\u5982\u4e0b\uff1a

    {\n  \"column\": {\n    \"name\": \"taglist\",\n    \"type\": \"Array\",\n    \"splitter\": \" \"\n  }\n}\n
    "},{"location":"writer/mongodbwriter/#splitter","title":"splitter","text":"

    \u5f53\u4e14\u4ec5\u5f53\u8981\u5904\u7406\u7684\u5b57\u7b26\u4e32\u8981\u7528\u5206\u9694\u7b26\u5206\u9694\u4e3a\u5b57\u7b26\u6570\u7ec4\u65f6\uff0c\u624d\u4f7f\u7528\u8fd9\u4e2a\u53c2\u6570\uff0c\u901a\u8fc7\u8fd9\u4e2a\u53c2\u6570\u6307\u5b9a\u7684\u5206\u9694\u7b26\uff0c\u5c06\u5b57\u7b26\u4e32\u5206\u9694\u5b58\u50a8\u5230 MongoDB \u7684\u6570\u7ec4\u4e2d

    "},{"location":"writer/mongodbwriter/#writemode","title":"writeMode","text":"

    \u4e0d\u914d\u7f6e\u7684\u60c5\u51b5\u4e0b\uff0c\u9ed8\u8ba4\u91c7\u53d6\u76f4\u63a5\u63d2\u5165\u8bb0\u5f55\u7684\u65b9\u5f0f\uff0c\u5982\u679c\u5e0c\u671b\u5b9e\u73b0\u63d2\u5165\u66f4\u65b0\uff08\u5373\u8bb0\u5f55\u5b58\u5728\u5219\u66f4\u65b0\u5426\u5219\u63d2\u5165\uff09\uff0c\u53ef\u4ee5\u6307\u5b9a\u4e3a update \u6a21\u5f0f\uff0c\u8be5\u6a21\u5f0f\u4e0b\uff0c\u5fc5\u987b\u540c\u65f6\u66f4\u65b0\u7684\u5b57\u6bb5\u662f\u54ea\u4e2a\uff0c\u6bd4\u5982\uff1a

    {\n  \"writeMode\": \"update(unique_id)\"\n}\n

    \u4e0a\u8ff0\u914d\u7f6e\u8868\u793a\u4f9d\u636e\u5b57\u6bb5 unique_id \u6765\u51b3\u5b9a\u5f53\u524d\u8bb0\u5f55\u662f\u63d2\u5165\u8fd8\u662f\u66f4\u65b0\uff0c\u5f53\u524d\u6682\u4e0d\u652f\u6301\u6307\u5b9a\u591a\u4e2a\u5b57\u6bb5

    "},{"location":"writer/mongodbwriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b MongoDB \u6570\u636e\u7c7b\u578b Long int, Long Double double String string, array Date date Boolean boolean Bytes bytes"},{"location":"writer/mysqlwriter/","title":"MySQL Writer","text":"

    MySQL Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 MySQL \u76ee\u7684\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/mysqlwriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684 MySQL \u8868\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    create table test.addax_tbl\n(\n  col1 varchar(20) ,\n  col2 int(4),\n  col3 datetime,\n  col4 boolean,\n  col5 binary\n) default charset utf8;\n

    \u8fd9\u91cc\u4f7f\u7528\u4e00\u4efd\u4ece\u5185\u5b58\u4ea7\u751f\u5230 MySQL \u5bfc\u5165\u7684\u6570\u636e\u3002

    job/stream2mysql.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"mysqlwriter\",\n        \"parameter\": {\n          \"writeMode\": \"insert\",\n          \"username\": \"root\",\n          \"password\": \"\",\n          \"column\": [\n            \"*\"\n          ],\n          \"session\": [\n            \"set session sql_mode='ANSI'\"\n          ],\n          \"preSql\": [\n            \"delete from @table\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:mysql://127.0.0.1:3306/test?useSSL=false\",\n            \"table\": [\n              \"addax_tbl\"\n            ],\n            \"driver\": \"com.mysql.jdbc.Driver\"\n          }\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2mysql.json

    "},{"location":"writer/mysqlwriter/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/stream2mysql.json\n
    "},{"location":"writer/mysqlwriter/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\uff0c\u5e76\u4e14\u589e\u52a0\u4e86\u4e00\u4e9b MySQL \u7279\u6709\u7684\u914d\u7f6e\u9879\u3002

    \u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 writeMode \u662f string insert \u6570\u636e\u5199\u5165\u8868\u7684\u65b9\u5f0f\uff0c\u8be6\u89c1\u4e0b\u6587 batchSize \u5426 int 1024 \u5b9a\u4e49\u4e86\u63d2\u4ef6\u548c\u6570\u636e\u5e93\u670d\u52a1\u5668\u7aef\u6bcf\u6b21\u6279\u91cf\u6570\u636e\u83b7\u53d6\u6761\u6570"},{"location":"writer/mysqlwriter/#driver","title":"driver","text":"

    \u5f53\u524d\u91c7\u7528\u7684 MySQL JDBC \u9a71\u52a8\u4e3a 8.0 \u4ee5\u4e0a\u7248\u672c\uff0c\u9a71\u52a8\u7c7b\u540d\u4f7f\u7528\u7684 com.mysql.cj.jdbc.Driver\uff0c\u800c\u4e0d\u662f com.mysql.jdbc.Driver\u3002 \u5982\u679c\u4f60\u9700\u8981\u91c7\u96c6\u7684 MySQL \u670d\u52a1\u4f4e\u4e8e 5.6\uff0c\u9700\u8981\u4f7f\u7528\u5230 Connector/J 5.1 \u9a71\u52a8\uff0c\u5219\u53ef\u4ee5\u91c7\u53d6\u4e0b\u9762\u7684\u6b65\u9aa4\uff1a

    1. \u66ff\u6362\u63d2\u4ef6\u5185\u7f6e\u7684\u9a71\u52a8 rm -f plugin/writer/mysqlwriter/libs/mysql-connector-java-*.jar

    2. \u62f7\u8d1d\u8001\u7684\u9a71\u52a8\u5230\u63d2\u4ef6\u76ee\u5f55 cp mysql-connector-java-5.1.48.jar plugin/writer/mysqlwriter/libs/

    3. \u6307\u5b9a\u9a71\u52a8\u7c7b\u540d\u79f0 \u5728\u4f60\u7684 json \u6587\u4ef6\u7c7b\uff0c\u914d\u7f6e \"driver\": \"com.mysql.jdbc.Driver\"

    "},{"location":"writer/mysqlwriter/#writemode","title":"writeMode","text":""},{"location":"writer/oraclewriter/","title":"Oracle Writer","text":"

    Oracle Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 Oracle \u76ee\u7684\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/oraclewriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u8fd9\u91cc\u4f7f\u7528\u4e00\u4efd\u4ece\u5185\u5b58\u4ea7\u751f\u5230 Oracle \u5bfc\u5165\u7684\u6570\u636e\u3002

    job/stream2oracle.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"oraclewriter\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"root\",\n          \"column\": [\n            \"id\",\n            \"name\"\n          ],\n          \"preSql\": [\n            \"delete from test\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:oracle:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]\",\n            \"table\": [\n              \"test\"\n            ]\n          }\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/oraclewriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\uff0c\u5e76\u5728\u6b64\u57fa\u7840\u4e0a\u589e\u52a0\u4e86\u4e00\u4e9b OracleWriter \u7279\u6709\u7684\u914d\u7f6e\u9879\u3002

    \u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u9ed8\u8ba4\u503c \u63cf\u8ff0 writeMode \u5426 insert \u5199\u5165\u65b9\u5f0f\uff0c\u652f\u6301 insert\uff0c update\uff0c\u8be6\u89c1\u4e0b\u6587"},{"location":"writer/oraclewriter/#writemode","title":"writeMode","text":"

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c \u91c7\u53d6 insert into \u8bed\u6cd5\u5199\u5165 Oracle \u8868\uff0c\u5982\u679c\u4f60\u5e0c\u671b\u91c7\u53d6\u4e3b\u952e\u5b58\u5728\u65f6\u66f4\u65b0\uff0c\u4e0d\u5b58\u5728\u5219\u5199\u5165\u7684\u65b9\u5f0f\uff0c\u4e5f\u5c31\u662f Oracle \u7684 merge into \u8bed\u6cd5\uff0c \u53ef\u4ee5\u4f7f\u7528 update \u6a21\u5f0f\u3002\u5047\u5b9a\u8868\u7684\u4e3b\u952e\u4e3a id ,\u5219 writeMode \u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff1a

    \"writeMode\": \"update(id)\"\n

    \u5982\u679c\u662f\u8054\u5408\u552f\u4e00\u7d22\u5f15\uff0c\u5219\u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff1a

    \"writeMode\": \"update(col1, col2)\"\n
    "},{"location":"writer/paimonwriter/","title":"Paimon Writer","text":"

    Paimon Writer \u63d0\u4f9b\u5411 \u5df2\u6709\u7684paimon\u8868\u5199\u5165\u6570\u636e\u7684\u80fd\u529b\u3002

    "},{"location":"writer/paimonwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0\n      }\n    },\n    \"content\": [\n      {\n        \"reader\": {\n          \"name\": \"rdbmsreader\",\n          \"parameter\": {\n            \"username\": \"root\",\n            \"password\": \"root\",\n            \"column\": [\n              \"*\"\n            ],\n            \"connection\": [\n              {\n                \"querySql\": [\n                  \"select 1+0 id ,'test1' as name\"\n                ],\n                \"jdbcUrl\": [\"jdbc:mysql://localhost:3306/ruoyi_vue_camunda?allowPublicKeyRetrieval=true\",]\n              }\n            ],\n            \"fetchSize\": 1024\n          }\n        },\n        \"writer\": {\n          \"name\": \"paimonwriter\",\n          \"parameter\": {\n            \"dbName\": \"test\",\n            \"tableName\": \"test2\",\n            \"writeMode\": \"truncate\",\n            \"paimonConfig\": {\n              \"warehouse\": \"file:///g:/paimon\",\n              \"metastore\": \"filesystem\"\n            }\n          }\n        }\n      }\n    ]\n  }\n}\n
    "},{"location":"writer/paimonwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e dbName \u662f string \u65e0 \u8981\u5199\u5165\u7684paimon\u6570\u636e\u5e93\u540d tableName \u662f string \u65e0 \u8981\u5199\u5165\u7684paimon\u8868\u540d writeMode \u662f string \u65e0 \u5199\u5165\u6a21\u5f0f\uff0c\u8be6\u8ff0\u89c1\u4e0b paimonConfig \u662f json {} \u91cc\u53ef\u4ee5\u914d\u7f6e\u4e0e Paimon catalog\u548cHadoop \u76f8\u5173\u7684\u4e00\u4e9b\u9ad8\u7ea7\u53c2\u6570\uff0c\u6bd4\u5982HA\u7684\u914d\u7f6e"},{"location":"writer/paimonwriter/#writemode","title":"writeMode","text":"

    \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff1a

    "},{"location":"writer/paimonwriter/#paimonconfig","title":"paimonConfig","text":"

    paimonConfig \u91cc\u53ef\u4ee5\u914d\u7f6e\u4e0e Paimon catalog\u548cHadoop \u76f8\u5173\u7684\u4e00\u4e9b\u9ad8\u7ea7\u53c2\u6570\uff0c\u6bd4\u5982HA\u7684\u914d\u7f6e

    \u672c\u5730\u76ee\u5f55\u521b\u5efapaimon\u8868

    pom.xml

    <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\"\n         xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n         xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <modelVersion>4.0.0</modelVersion>\n\n    <groupId>com.test</groupId>\n    <artifactId>paimon-java-api-test</artifactId>\n    <version>1.0-SNAPSHOT</version>\n\n    <properties>\n        <maven.compiler.source>8</maven.compiler.source>\n        <maven.compiler.target>8</maven.compiler.target>\n        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>\n        <hadoop.version>3.2.4</hadoop.version>\n        <woodstox.version>7.0.0</woodstox.version>\n    </properties>\n<dependencies>\n    <dependency>\n        <groupId>org.apache.paimon</groupId>\n        <artifactId>paimon-bundle</artifactId>\n        <version>1.0.0</version>\n    </dependency>\n\n\n    <dependency>\n        <groupId>org.apache.hadoop</groupId>\n        <artifactId>hadoop-common</artifactId>\n        <version>${hadoop.version}</version>\n        <exclusions>\n            <exclusion>\n                <groupId>com.fasterxml.jackson.core</groupId>\n                <artifactId>jackson-databind</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.codehaus.jackson</groupId>\n                <artifactId>jackson-core-asl</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.codehaus.jackson</groupId>\n                <artifactId>jackson-mapper-asl</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>com.fasterxml.woodstox</groupId>\n                <artifactId>woodstox-core</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>commons-codec</groupId>\n                <artifactId>commons-codec</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>commons-net</groupId>\n                <artifactId>commons-net</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>io.netty</groupId>\n                <artifactId>netty</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>log4j</groupId>\n                <artifactId>log4j</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>net.minidev</groupId>\n                <artifactId>json-smart</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.codehaus.jettison</groupId>\n                <artifactId>jettison</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.eclipse.jetty</groupId>\n                <artifactId>jetty-server</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.xerial.snappy</groupId>\n                <artifactId>snappy-java</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.apache.zookeeper</groupId>\n                <artifactId>zookeeper</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.eclipse.jetty</groupId>\n                <artifactId>jetty-util</artifactId>\n            </exclusion>\n        </exclusions>\n    </dependency>\n\n    <dependency>\n        <groupId>org.apache.hadoop</groupId>\n        <artifactId>hadoop-aws</artifactId>\n        <version>${hadoop.version}</version>\n        <exclusions>\n            <exclusion>\n                <groupId>com.fasterxml.jackson.core</groupId>\n                <artifactId>jackson-databind</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.codehaus.jackson</groupId>\n                <artifactId>jackson-core-asl</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.codehaus.jackson</groupId>\n                <artifactId>jackson-mapper-asl</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>com.fasterxml.woodstox</groupId>\n                <artifactId>woodstox-core</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>commons-codec</groupId>\n                <artifactId>commons-codec</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>commons-net</groupId>\n                <artifactId>commons-net</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>io.netty</groupId>\n                <artifactId>netty</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>log4j</groupId>\n                <artifactId>log4j</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>net.minidev</groupId>\n                <artifactId>json-smart</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.codehaus.jettison</groupId>\n                <artifactId>jettison</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.eclipse.jetty</groupId>\n                <artifactId>jetty-server</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.xerial.snappy</groupId>\n                <artifactId>snappy-java</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.apache.zookeeper</groupId>\n                <artifactId>zookeeper</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.eclipse.jetty</groupId>\n                <artifactId>jetty-util</artifactId>\n            </exclusion>\n        </exclusions>\n    </dependency>\n\n    <dependency>\n        <groupId>org.apache.hadoop</groupId>\n        <artifactId>hadoop-mapreduce-client-core</artifactId>\n        <version>${hadoop.version}</version>\n        <exclusions>\n            <exclusion>\n                <groupId>com.fasterxml.jackson.core</groupId>\n                <artifactId>jackson-databind</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>commons-codec</groupId>\n                <artifactId>commons-codec</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>io.netty</groupId>\n                <artifactId>netty</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.eclipse.jetty</groupId>\n                <artifactId>jetty-util</artifactId>\n            </exclusion>\n        </exclusions>\n    </dependency>\n\n\n    <dependency>\n        <groupId>com.fasterxml.woodstox</groupId>\n        <artifactId>woodstox-core</artifactId>\n        <version>${woodstox.version}</version>\n    </dependency>\n</dependencies>\n</project>\n
    import org.apache.paimon.catalog.Catalog;\nimport org.apache.paimon.catalog.CatalogContext;\nimport org.apache.paimon.catalog.CatalogFactory;\nimport org.apache.paimon.catalog.Identifier;\nimport org.apache.paimon.fs.Path;\nimport org.apache.paimon.schema.Schema;\nimport org.apache.paimon.types.DataTypes;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\npublic class CreatePaimonTable {\n\n    public static Catalog createFilesystemCatalog() {\n        CatalogContext context = CatalogContext.create(new Path(\"file:///g:/paimon\"));\n        return CatalogFactory.createCatalog(context);\n    }\n    /* \u5982\u679c\u662fminio\u5219\u4f8b\u5b50\u5982\u4e0b\n\n     public static Catalog createFilesystemCatalog() {\n        Options options = new Options();\n        options.set(\"warehouse\", \"s3a://pvc-91d1e2cd-4d25-45c9-8613-6c4f7bf0a4cc/paimon\");\n        Configuration hadoopConf = new Configuration();\n        hadoopConf.set(\"fs.s3a.endpoint\", \"http://localhost:9000\");\n        hadoopConf.set(\"fs.s3a.access.key\", \"gy0dX5lALP176g6c9fYf\");\n        hadoopConf.set(\"fs.s3a.secret.key\", \"ReuUrCzzu5wKWAegtswoHIWV389BYl9AB1ZQbiKr\");\n        hadoopConf.set(\"fs.s3a.connection.ssl.enabled\", \"false\");\n        hadoopConf.set(\"fs.s3a.path.style.access\", \"true\");\n        hadoopConf.set(\"fs.s3a.impl\",\"org.apache.hadoop.fs.s3a.S3AFileSystem\");\n        CatalogContext context = CatalogContext.create(options,hadoopConf);\n\n\n        return CatalogFactory.createCatalog(context);\n    }\n     * \n     * \n     * */\n\n    public static void main(String[] args) {\n        Schema.Builder schemaBuilder = Schema.newBuilder();\n        schemaBuilder.primaryKey(\"id\");\n        schemaBuilder.column(\"id\", DataTypes.INT());\n        schemaBuilder.column(\"name\", DataTypes.STRING());\n        Map<String, String> options = new HashMap<>();\n        options.put(\"bucket\", \"1\");//\u7531\u4e8epaimon java api \u9650\u5236\u9700\u8981bucket>0\n        options.put(\"bucket-key\", \"id\");\n        options.put(\"file.format\", \"orc\");\n        options.put(\"file.compression\", \"lz4\");\n        options.put(\"lookup.cache-spill-compression\", \"lz4\");\n        options.put(\"spill-compression\", \"LZ4\");\n        options.put(\"orc.compress\", \"lz4\");\n        options.put(\"manifest.format\", \"orc\");\n\n        schemaBuilder.options(options);\n        Schema schema = schemaBuilder.build();\n\n        Identifier identifier = Identifier.create(\"test\", \"test2\");\n        try {\n            Catalog catalog = CreatePaimonTable.createFilesystemCatalog();\n            catalog.createDatabase(\"test\",true);\n            catalog.createTable(identifier, schema, true);\n        } catch (Catalog.TableAlreadyExistException e) {\n            e.printStackTrace();\n        } catch (Catalog.DatabaseNotExistException e) {\n            e.printStackTrace();\n        } catch (Catalog.DatabaseAlreadyExistException e) {\n            throw new RuntimeException(e);\n        }\n\n\n    }\n}\n

    Spark \u6216\u8005 flink \u73af\u5883\u521b\u5efa\u8868

    CREATE TABLE if not exists test.test2(id int ,name string)  tblproperties (\n    'primary-key' = 'id',\n    'bucket' = '1',\n    'bucket-key' = 'id'\n    'file.format'='orc',\n    'file.compression'='lz4',\n    'lookup.cache-spill-compression'='lz4',\n    'spill-compression'='LZ4',\n    'orc.compress'='lz4',\n    'manifest.format'='orc'\n)\n

    \u672c\u5730\u6587\u4ef6\u4f8b\u5b50

    {\n                    \"name\": \"paimonwriter\",\n                    \"parameter\": {\n                        \"dbName\": \"test\",\n                        \"tableName\": \"test2\",\n                        \"writeMode\": \"truncate\",\n                        \"paimonConfig\": {\n                           \"warehouse\": \"file:///g:/paimon\",\n                           \"metastore\": \"filesystem\"\n                         }\n                    }\n}\n

    s3 \u6216\u8005 minio catalog\u4f8b\u5b50 

    {\n    \"job\": {\n        \"setting\": {\n            \"speed\": {\n                \"channel\": 3\n            },\n            \"errorLimit\": {\n                \"record\": 0,\n                \"percentage\": 0\n            }\n        },\n        \"content\": [\n            {\n                \"reader\": {\n                    \"name\": \"rdbmsreader\",\n                    \"parameter\": {\n                        \"username\": \"root\",\n                        \"password\": \"root\",\n                        \"column\": [\n                            \"*\"\n                        ],\n                        \"connection\": [\n                            {\n                                \"querySql\": [\n                                    \"select 1+0 id ,'test1' as name\"\n                                ],\n                                \"jdbcUrl\": [\n                                    \"jdbc:mysql://localhost:3306/ruoyi_vue_camunda?allowPublicKeyRetrieval=true\"\n                                ]\n                            }\n                        ],\n                        \"fetchSize\": 1024\n                    }\n                },\n                \"writer\": {\n                    \"name\": \"paimonwriter\",\n                    \"parameter\": {\n                        \"dbName\": \"test\",\n                        \"tableName\": \"test2\",\n                        \"writeMode\": \"truncate\",\n                        \"paimonConfig\": {\n                            \"warehouse\": \"s3a://pvc-91d1e2cd-4d25-45c9-8613-6c4f7bf0a4cc/paimon\",\n                            \"metastore\": \"filesystem\",\n                            \"fs.s3a.endpoint\": \"http://localhost:9000\",\n                            \"fs.s3a.access.key\": \"gy0dX5lALP176g6c9fYf\",\n                            \"fs.s3a.secret.key\": \"ReuUrCzzu5wKWAegtswoHIWV389BYl9AB1ZQbiKr\",\n                            \"fs.s3a.connection.ssl.enabled\": \"false\",\n                            \"fs.s3a.path.style.access\": \"true\",\n                            \"fs.s3a.impl\": \"org.apache.hadoop.fs.s3a.S3AFileSystem\"\n                        }\n                    }\n                }\n            }\n        ]\n    }\n}\n

    hdfs catalog\u4f8b\u5b50

    {\n  \"paimonConfig\": {\n    \"warehouse\": \"hdfs://nameservice1/user/hive/paimon\",\n    \"metastore\": \"filesystem\",\n    \"fs.defaultFS\":\"hdfs://nameservice1\",\n    \"hadoop.security.authentication\" : \"kerberos\",\n    \"hadoop.kerberos.principal\" : \"hive/_HOST@XXXX.COM\",\n    \"hadoop.kerberos.keytab\" : \"/tmp/hive@XXXX.COM.keytab\",\n    \"ha.zookeeper.quorum\" : \"test-pr-nn1:2181,test-pr-nn2:2181,test-pr-nn3:2181\",\n    \"dfs.nameservices\" : \"nameservice1\",\n    \"dfs.namenode.rpc-address.nameservice1.namenode371\" : \"test-pr-nn2:8020\",\n    \"dfs.namenode.rpc-address.nameservice1.namenode265\": \"test-pr-nn1:8020\",\n    \"dfs.namenode.keytab.file\" : \"/tmp/hdfs@XXXX.COM.keytab\",\n    \"dfs.namenode.keytab.enabled\" : \"true\",\n    \"dfs.namenode.kerberos.principal\" : \"hdfs/_HOST@XXXX.COM\",\n    \"dfs.namenode.kerberos.internal.spnego.principal\" : \"HTTP/_HOST@XXXX.COM\",\n    \"dfs.ha.namenodes.nameservice1\" : \"namenode265,namenode371\",\n    \"dfs.datanode.keytab.file\" : \"/tmp/hdfs@XXXX.COM.keytab\",\n    \"dfs.datanode.keytab.enabled\" : \"true\",\n    \"dfs.datanode.kerberos.principal\" : \"hdfs/_HOST@XXXX.COM\",\n    \"dfs.client.use.datanode.hostname\" : \"false\",\n    \"dfs.client.failover.proxy.provider.nameservice1\" : \"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\",\n    \"dfs.balancer.keytab.file\" : \"/tmp/hdfs@XXXX.COM.keytab\",\n    \"dfs.balancer.keytab.enabled\" : \"true\",\n    \"dfs.balancer.kerberos.principal\" : \"hdfs/_HOST@XXXX.COM\"\n  }\n}\n
    "},{"location":"writer/paimonwriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b Paimon \u6570\u636e\u7c7b\u578b Integer TINYINT,SMALLINT,INT,INTEGER Long BIGINT Double FLOAT,DOUBLE,DECIMAL String STRING,VARCHAR,CHAR Boolean BOOLEAN Date DATE,TIMESTAMP Bytes BINARY"},{"location":"writer/postgresqlwriter/","title":"Postgresql Writer","text":"

    Postgresql Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 PostgreSQL \u6570\u636e\u5e93\u5e93\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/postgresqlwriter/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u6f14\u793a\u4ece postgresql \u6307\u5b9a\u7684\u8868\u8bfb\u53d6\u6570\u636e\uff0c\u5e76\u63d2\u5165\u5230\u5177\u6709\u76f8\u540c\u8868\u7ed3\u6784\u7684\u53e6\u5916\u4e00\u5f20\u8868\u4e2d\uff0c\u7528\u6765\u6d4b\u8bd5\u8be5\u63d2\u4ef6\u6240\u652f\u6301\u7684\u6570\u636e\u7c7b\u578b\u3002

    "},{"location":"writer/postgresqlwriter/#_2","title":"\u8868\u7ed3\u6784\u4fe1\u606f","text":"

    \u5047\u5b9a\u5efa\u8868\u8bed\u53e5\u4ee5\u53ca\u8f93\u5165\u63d2\u5165\u8bed\u53e5\u5982\u4e0b\uff1a

    create table if not exists addax_tbl\n(\n    c_bigint bigint,\n    c_bit bit(3),\n    c_bool boolean,\n    c_byte bytea,\n    c_char char(10),\n    c_varchar varchar(20),\n    c_date date,\n    c_double float8,\n    c_int integer,\n    c_json json,\n    c_number decimal(8, 3),\n    c_real real,\n    c_small smallint,\n    c_text text,\n    c_ts timestamp,\n    c_uuid uuid,\n    c_xml xml,\n    c_money money,\n    c_inet inet,\n    c_cidr cidr,\n    c_macaddr macaddr\n    );\n\ninsert into addax_tbl\nvalues (999988887777,\n        b'101',\n        TRUE,\n        '\\xDEADBEEF',\n        'hello',\n        'hello, world',\n        '2021-01-04',\n        999888.9972,\n        9876542,\n        '{\"bar\": \"baz\", \"balance\": 7.77, \"active\": false}'::json,\n        12345.123,\n        123.123,\n        126,\n        'this is a long text ',\n        '2020-01-04 12:13:14',\n        'A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11'::uuid,\n        '<foo>bar</foo>'::xml,\n        '52093.89'::money,\n        '192.168.1.1'::inet,\n        '192.168.1/24'::cidr,\n        '08002b:010203'::macaddr);\n

    \u521b\u5efa\u9700\u8981\u63d2\u5165\u7684\u8868\u7684\u8bed\u53e5\u5982\u4e0b:

    create table addax_tbl1 as select * from  addax_tbl where 1=2;\n
    "},{"location":"writer/postgresqlwriter/#_3","title":"\u4efb\u52a1\u914d\u7f6e","text":"

    \u4ee5\u4e0b\u662f\u914d\u7f6e\u6587\u4ef6

    job/pg2pg.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"postgresqlreader\",\n        \"parameter\": {\n          \"username\": \"pgtest\",\n          \"password\": \"pgtest\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"addax_tbl\"\n            ],\n            \"jdbcUrl\": \"jdbc:postgresql://localhost:5432/pgtest\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"postgresqlwriter\",\n        \"parameter\": {\n          \"column\": [\n            \"*\"\n          ],\n          \"preSql\": [\n            \"truncate table @table\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:postgresql://127.0.0.1:5432/pgtest\",\n            \"table\": [\n              \"addax_tbl1\"\n            ]\n          },\n          \"username\": \"pgtest\",\n          \"password\": \"pgtest\",\n          \"writeMode\": \"insert\"\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/pg2pg.json

    "},{"location":"writer/postgresqlwriter/#_4","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/pg2pg.json\n
    "},{"location":"writer/postgresqlwriter/#_5","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/postgresqlwriter/#writemode","title":"writeMode","text":"

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c \u91c7\u53d6 insert into \u8bed\u6cd5\u5199\u5165 postgresql \u8868\uff0c\u5982\u679c\u4f60\u5e0c\u671b\u91c7\u53d6\u4e3b\u952e\u5b58\u5728\u65f6\u66f4\u65b0\uff0c\u4e0d\u5b58\u5728\u5219\u5199\u5165\u7684\u65b9\u5f0f\uff0c \u53ef\u4ee5\u4f7f\u7528 update \u6a21\u5f0f\u3002\u5047\u5b9a\u8868\u7684\u4e3b\u952e\u4e3a id ,\u5219 writeMode \u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff1a

    \"writeMode\": \"update(id)\"\n

    \u5982\u679c\u662f\u8054\u5408\u552f\u4e00\u7d22\u5f15\uff0c\u5219\u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff1a

    \"writeMode\": \"update(col1, col2)\"\n

    \u6ce8\uff1a update \u6a21\u5f0f\u5728 3.1.6 \u7248\u672c\u9996\u6b21\u589e\u52a0\uff0c\u4e4b\u524d\u7248\u672c\u5e76\u4e0d\u652f\u6301\u3002

    "},{"location":"writer/postgresqlwriter/#_6","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u76ee\u524d PostgresqlWriter \u652f\u6301\u5927\u90e8\u5206 PostgreSQL \u7c7b\u578b\uff0c\u4f46\u4e5f\u5b58\u5728\u90e8\u5206\u6ca1\u6709\u652f\u6301\u7684\u60c5\u51b5\uff0c\u8bf7\u6ce8\u610f\u68c0\u67e5\u4f60\u7684\u7c7b\u578b\u3002

    \u4e0b\u9762\u5217\u51fa PostgresqlWriter \u9488\u5bf9 PostgreSQL \u7c7b\u578b\u8f6c\u6362\u5217\u8868:

    Addax \u5185\u90e8\u7c7b\u578b PostgreSQL \u6570\u636e\u7c7b\u578b Long bigint, bigserial, integer, smallint, serial Double double precision, money, numeric, real String varchar, char, text, bit, inet,cidr,macaddr,uuid,xml,json Date date, time, timestamp Boolean bool Bytes bytea"},{"location":"writer/postgresqlwriter/#_7","title":"\u5df2\u77e5\u9650\u5236","text":"

    \u9664\u4ee5\u4e0a\u5217\u51fa\u7684\u6570\u636e\u7c7b\u578b\u5916\uff0c\u5176\u4ed6\u6570\u636e\u7c7b\u578b\u7406\u8bba\u4e0a\u5747\u4e3a\u8f6c\u4e3a\u5b57\u7b26\u4e32\u7c7b\u578b\uff0c\u4f46\u4e0d\u786e\u4fdd\u51c6\u786e\u6027

    "},{"location":"writer/rdbmswriter/","title":"RDBMS Writer","text":"

    RDBMS Writer \u63d2\u4ef6\u652f\u6301\u4ece\u4f20\u7edf RDBMS \u8bfb\u53d6\u6570\u636e\u3002\u8fd9\u662f\u4e00\u4e2a\u901a\u7528\u5173\u7cfb\u6570\u636e\u5e93\u8bfb\u53d6\u63d2\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u6ce8\u518c\u6570\u636e\u5e93\u9a71\u52a8\u7b49\u65b9\u5f0f\u652f\u6301\u66f4\u591a\u5173\u7cfb\u6570\u636e\u5e93\u8bfb\u53d6\u3002

    \u540c\u65f6 RDBMS Writer \u53c8\u662f\u5176\u4ed6\u5173\u7cfb\u578b\u6570\u636e\u5e93\u8bfb\u53d6\u63d2\u4ef6\u7684\u7684\u57fa\u7840\u7c7b\u3002\u4ee5\u4e0b\u8bfb\u53d6\u63d2\u4ef6\u5747\u4f9d\u8d56\u8be5\u63d2\u4ef6

    \u6ce8\u610f, \u5982\u679c\u5df2\u7ecf\u63d0\u4f9b\u4e86\u4e13\u95e8\u7684\u6570\u636e\u5e93\u5199\u5165\u63d2\u4ef6\u7684\uff0c\u63a8\u8350\u4f7f\u7528\u4e13\u7528\u63d2\u4ef6\uff0c\u5982\u679c\u4f60\u9700\u8981\u5199\u5165\u7684\u6570\u636e\u5e93\u6ca1\u6709\u4e13\u95e8\u63d2\u4ef6\uff0c\u5219\u8003\u8651\u4f7f\u7528\u8be5\u901a\u7528\u63d2\u4ef6\u3002 \u5728\u4f7f\u7528\u4e4b\u524d\uff0c\u8fd8\u9700\u8981\u6267\u884c\u4ee5\u4e0b\u64cd\u4f5c\u624d\u53ef\u4ee5\u6b63\u5e38\u8fd0\u884c\uff0c\u5426\u5219\u8fd0\u884c\u4f1a\u51fa\u73b0\u5f02\u5e38\u3002

    "},{"location":"writer/rdbmswriter/#_1","title":"\u914d\u7f6e\u9a71\u52a8","text":"

    \u5047\u5b9a\u4f60\u9700\u8981\u5199\u5165 IBM DB2 \u7684\u6570\u636e\uff0c\u56e0\u4e3a\u6ca1\u6709\u63d0\u4f9b\u4e13\u95e8\u7684\u8bfb\u53d6\u63d2\u4ef6\uff0c\u6240\u4ee5\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528\u8be5\u63d2\u4ef6\u6765\u5b9e\u73b0\uff0c\u5728\u4f7f\u7528\u4e4b\u524d\uff0c\u9700\u8981\u6267\u884c\u4e0b\u9762\u4e24\u4e2a\u64cd\u4f5c\uff1a

    1. \u4e0b\u8f7d\u5bf9\u5e94\u7684 JDBC \u9a71\u52a8\uff0c\u5e76\u62f7\u8d1d\u5230 plugin/writer/rdbmswriter/libs \u76ee\u5f55
    2. \u4fee\u6539\u4efb\u52a1\u914d\u7f6e\u6587\u4ef6\uff0c\u627e\u5230 driver \u4e00\u9879\uff0c\u586b\u5199\u6b63\u786e\u7684 JDBC \u9a71\u52a8\u540d\uff0c\u6bd4\u5982 DB2 \u7684\u9a71\u52a8\u540d\u4e3a com.ibm.db2.jcc.DB2Driver

    \u4ee5\u4e0b\u5217\u51fa\u5e38\u89c1\u7684\u6570\u636e\u5e93\u4ee5\u53ca\u5bf9\u5e94\u7684\u9a71\u52a8\u540d\u79f0

    "},{"location":"writer/rdbmswriter/#_2","title":"\u914d\u7f6e\u8bf4\u660e","text":"

    \u914d\u7f6e\u4e00\u4e2a\u5199\u5165RDBMS\u7684\u4f5c\u4e1a\u3002

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"rdbmswriter\",\n        \"parameter\": {\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:dm://ip:port/database\",\n            \"table\": [\n              \"table\"\n            ]\n          },\n          \"username\": \"username\",\n          \"password\": \"password\",\n          \"driver\": \"dm.jdbc.driver.DmDriver\",\n          \"column\": [\n            \"*\"\n          ],\n          \"preSql\": [\n            \"delete from XXX;\"\n          ]\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/rdbmswriter/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 jdbcUrl \u662f string \u65e0 \u5bf9\u7aef\u6570\u636e\u5e93\u7684JDBC\u8fde\u63a5\u4fe1\u606f\uff0cjdbcUrl\u6309\u7167RDBMS\u5b98\u65b9\u89c4\u8303\uff0c\u5e76\u53ef\u4ee5\u586b\u5199\u8fde\u63a5\u9644\u4ef6\u63a7\u5236\u4fe1\u606f \uff5c driver \u662f string \u65e0 \u81ea\u5b9a\u4e49\u9a71\u52a8\u7c7b\u540d\uff0c\u89e3\u51b3\u517c\u5bb9\u6027\u95ee\u9898\uff0c\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0 username \u662f string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801 table \u662f list \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d,\u4f7f\u7528JSON\u6570\u636e\u683c\u5f0f\uff0c\u5f53\u914d\u7f6e\u4e3a\u591a\u5f20\u8868\u65f6\uff0c\u7528\u6237\u81ea\u5df1\u9700\u4fdd\u8bc1\u591a\u5f20\u8868\u662f\u540c\u4e00\u8868\u7ed3\u6784 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1\u540e preSql \u5426 list \u65e0 \u6267\u884c\u6570\u636e\u540c\u6b65\u4efb\u52a1\u4e4b\u524d\u7387\u5148\u6267\u884c\u7684sql\u8bed\u53e5\uff0c\u76ee\u524d\u53ea\u5141\u8bb8\u6267\u884c\u4e00\u6761SQL\u8bed\u53e5\uff0c\u4f8b\u5982\u6e05\u9664\u65e7\u6570\u636e,\u6d89\u53ca\u5230\u7684\u8868\u53ef\u7528 @table\u8868\u793a postSql \u5426 list \u65e0 \u6267\u884c\u6570\u636e\u540c\u6b65\u4efb\u52a1\u4e4b\u540e\u6267\u884c\u7684sql\u8bed\u53e5\uff0c\u76ee\u524d\u53ea\u5141\u8bb8\u6267\u884c\u4e00\u6761SQL\u8bed\u53e5\uff0c\u4f8b\u5982\u52a0\u4e0a\u67d0\u4e00\u4e2a\u65f6\u95f4\u6233 batchSize \u5426 int 1024 \u5b9a\u4e49\u4e86\u63d2\u4ef6\u548c\u6570\u636e\u5e93\u670d\u52a1\u5668\u7aef\u6bcf\u6b21\u6279\u91cf\u6570\u636e\u83b7\u53d6\u6761\u6570\uff0c\u8c03\u9ad8\u8be5\u503c\u53ef\u80fd\u5bfc\u81f4 Addax \u51fa\u73b0OOM\u6216\u8005\u76ee\u6807\u6570\u636e\u5e93\u4e8b\u52a1\u63d0\u4ea4\u5931\u8d25\u5bfc\u81f4\u6302\u8d77 session \u662f\u5426 list \u65e0 \u9488\u5bf9\u672c\u5730\u8fde\u63a5,\u4fee\u6539\u4f1a\u8bdd\u914d\u7f6e,\u8be6\u89c1\u4e0b\u6587"},{"location":"writer/rdbmswriter/#column","title":"column","text":"

    \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u4f7f\u7528JSON\u7684\u6570\u7ec4\u63cf\u8ff0\u5b57\u6bb5\u4fe1\u606f\u3002\u7528\u6237\u4f7f\u7528 * \u4ee3\u8868\u9ed8\u8ba4\u4f7f\u7528\u6240\u6709\u5217\u914d\u7f6e\uff0c\u4f8b\u5982 [\"*\"]\u3002

    \u652f\u6301\u5217\u88c1\u526a\uff0c\u5373\u5217\u53ef\u4ee5\u6311\u9009\u90e8\u5206\u5217\u8fdb\u884c\u5bfc\u51fa\u3002

    \u652f\u6301\u5217\u6362\u5e8f\uff0c\u5373\u5217\u53ef\u4ee5\u4e0d\u6309\u7167\u8868schema\u4fe1\u606f\u8fdb\u884c\u5bfc\u51fa\u3002

    \u652f\u6301\u5e38\u91cf\u914d\u7f6e\uff0c\u7528\u6237\u9700\u8981\u6309\u7167JSON\u683c\u5f0f:

    [\"id\", \"`table`\", \"1\", \"'bazhen.csy'\", \"null\", \"to_char(a + 1)\", \"2.3\" , \"true\"]

    Column\u5fc5\u987b\u663e\u793a\u586b\u5199\uff0c\u4e0d\u5141\u8bb8\u4e3a\u7a7a\uff01

    "},{"location":"writer/rdbmswriter/#jdbcurl","title":"jdbcUrl","text":"

    jdbcUrl \u914d\u7f6e\u9664\u4e86\u914d\u7f6e\u5fc5\u8981\u7684\u4fe1\u606f\u5916\uff0c\u6211\u4eec\u8fd8\u53ef\u4ee5\u5728\u589e\u52a0\u6bcf\u79cd\u7279\u5b9a\u9a71\u52a8\u7684\u7279\u5b9a\u914d\u7f6e\u5c5e\u6027\uff0c\u8fd9\u91cc\u7279\u522b\u63d0\u5230\u6211\u4eec\u53ef\u4ee5\u5229\u7528\u914d\u7f6e\u5c5e\u6027\u5bf9\u4ee3\u7406\u7684\u652f\u6301\u4ece\u800c\u5b9e\u73b0\u901a\u8fc7\u4ee3\u7406\u8bbf\u95ee\u6570\u636e\u5e93\u7684\u529f\u80fd\u3002 \u6bd4\u5982\u5bf9\u4e8e PrestoSQL \u6570\u636e\u5e93\u7684 JDBC \u9a71\u52a8\u800c\u8a00\uff0c\u652f\u6301 socksProxy \u53c2\u6570\uff0c\u6bd4\u5982\u4e00\u4e2a\u53ef\u80fd\u7684 jdbcUrl \u4e3a

    jdbc:presto://127.0.0.1:8080/hive?socksProxy=192.168.1.101:1081

    \u5927\u90e8\u5206\u5173\u7cfb\u578b\u6570\u636e\u5e93\u7684 JDBC \u9a71\u52a8\u652f\u6301 socksProxyHost,socksProxyPort \u53c2\u6570\u6765\u652f\u6301\u4ee3\u7406\u8bbf\u95ee\u3002\u4e5f\u6709\u4e00\u4e9b\u7279\u522b\u7684\u60c5\u51b5\u3002

    \u4ee5\u4e0b\u662f\u5404\u7c7b\u6570\u636e\u5e93 JDBC \u9a71\u52a8\u6240\u652f\u6301\u7684\u4ee3\u7406\u7c7b\u578b\u4ee5\u53ca\u914d\u7f6e\u65b9\u5f0f

    \u6570\u636e\u5e93 \u4ee3\u7406\u7c7b\u578b \u4ee3\u7406\u914d\u7f6e \u4f8b\u5b50 MySQL socks socksProxyHost,socksProxyPort socksProxyHost=192.168.1.101&socksProxyPort=1081 Presto socks socksProxy socksProxy=192.168.1.101:1081 Presto http httpProxy httpProxy=192.168.1.101:3128"},{"location":"writer/rdbmswriter/#driver","title":"driver","text":"

    \u5927\u90e8\u5206\u60c5\u51b5\u4e0b\uff0c\u4e00\u4e2a\u6570\u636e\u5e93\u7684JDBC\u9a71\u52a8\u662f\u56fa\u5b9a\u7684\uff0c\u4f46\u6709\u4e9b\u56e0\u4e3a\u7248\u672c\u7684\u4e0d\u540c\uff0c\u6240\u5efa\u8bae\u7684\u9a71\u52a8\u7c7b\u540d\u4e0d\u540c\uff0c\u6bd4\u5982 MySQL\u3002 \u65b0\u7684 MySQL JDBC \u9a71\u52a8\u7c7b\u578b\u63a8\u8350\u4f7f\u7528 com.mysql.cj.jdbc.Driver \u800c\u4e0d\u662f\u4ee5\u524d\u7684 com.mysql.jdbc.Drver\u3002 \u5982\u679c\u60f3\u8981\u4f7f\u7528\u5c31\u7684\u9a71\u52a8\u540d\u79f0\uff0c\u5219\u53ef\u4ee5\u914d\u7f6e driver \u914d\u7f6e\u9879\u3002

    "},{"location":"writer/rdbmswriter/#session","title":"session","text":"

    \u63cf\u8ff0\uff1a\u8bbe\u7f6e\u6570\u636e\u5e93\u8fde\u63a5\u65f6\u7684session\u4fe1\u606f\uff0c\u6bd4\u5982\u9488\u5bf9 Oracle \u6570\u636e\u5e93\uff0c\u53ef\u4ee5\u8bbe\u7f6e\u5982\u4e0b\uff1a

    {\n  \"session\": [\n    \"alter session set nls_date_format = 'dd.mm.yyyy hh24:mi:ss';\",\n    \"alter session set NLS_LANG = 'AMERICAN';\"\n  ]\n}\n
    "},{"location":"writer/rediswriter/","title":"Redis Writer","text":"

    Redis Writer \u63d0\u4f9b\u4e86\u8fd8\u539f Redis dump \u547d\u4ee4\u7684\u80fd\u529b\uff0c\u5e76\u5199\u5165\u5230\u76ee\u6807 Redis\u3002\u652f\u6301 redis cluster \u96c6\u7fa4\u3001proxy\u3001\u4ee5\u53ca\u5355\u673a

    "},{"location":"writer/rediswriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"redisreader\",\n        \"parameter\": {\n          \"connection\": [\n            {\n              \"uri\": \"tcp://127.0.0.1:7003\"\n            }\n          ]\n        }\n      },\n      \"writer\": {\n        \"name\": \"rediswriter\",\n        \"parameter\": {\n          \"connection\": {\n            \"uri\": \"tcp://127.0.0.1:6379\",\n            \"auth\": \"123456\"\n          },\n          \"redisCluster\": false,\n          \"flushDB\": false\n        }\n      }\n    },\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    }\n  }\n}\n
    "},{"location":"writer/rediswriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 uri \u662f string \u5426 redis\u94fe\u63a5 redisCluster \u5426 boolean false \u662f\u5426\u4e3aredis cluster\u96c6\u7fa4,\u5982\u679c\u662f proxy \u6216\u5355\u673a\u5ffd\u7565\u8be5\u9879 flushDB \u5426 boolean false \u8fc1\u79fb\u524d\u662f\u5426\u6e05\u7a7a\u76ee\u6807 Redis batchSize \u5426 string 1000 \u6bcf\u6b21\u6279\u91cf\u5904\u7406\u6570\u91cf\u3002\u5982\u679ckey\u8fc7\u5927/\u5c0f,\u53ef\u4ee5\u76f8\u5e94\u7684\u8c03\u6574 timeout \u5426 string 60000 \u6bcf\u6b21\u6267\u884c\u6700\u5927\u8d85\u65f6\u65f6\u95f4, \u5355\u4f4d\u6beb\u79d2(ms)"},{"location":"writer/s3writer/","title":"S3 Writer","text":"

    S3 Writer \u63d2\u4ef6\u7528\u4e8e\u5c06\u6570\u636e\u5199\u5165 Amazon AWS S3 \u5b58\u50a8\uff0c\u4ee5\u53ca\u517c\u5bb9 S3 \u534f\u8bae\u7684\u5b58\u50a8\uff0c\u6bd4\u5982 MinIO\u3002

    \u5728\u5b9e\u73b0\u4e0a\uff0c\u672c\u63d2\u4ef6\u57fa\u4e8e S3 \u5b98\u65b9\u7684 SDK 2.0 \u7f16\u5199\u3002

    "},{"location":"writer/s3writer/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u4e0b\u9762\u7684\u914d\u7f6e\u7528\u4e8e\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\uff0c\u5e76\u5199\u5165\u5230\u6307\u5b9a\u7684 S3 bucket \u4e0a\u3002

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 11:22:33\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"s3writer\",\n        \"parameter\": {\n          \"endpoint\": \"https://s3.amazonaws.com\",\n          \"accessId\": \"xxxxxxxxxxxx\",\n          \"accessKey\": \"xxxxxxxxxxxxxxxxxxxxxxx\",\n          \"bucket\": \"test\",\n          \"object\": \"upload.csv\",\n          \"region\": \"ap-northeast-1\",\n          \"encoding\": \"\",\n          \"fieldDelimiter\": \",\",\n          \"writeMode\": \"truncate\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/s3writer/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 S3 Server\u7684 EndPoint\u5730\u5740\uff0c\u4f8b\u5982 s3.xx.amazonaws.com region \u662f string \u65e0 S3 Server\u7684 Region \u5730\u5740\uff0c\u4f8b\u5982 ap-southeast-1 accessId \u662f string \u65e0 \u8bbf\u95ee ID accessKey \u662f string \u65e0 \u8bbf\u95ee Key bucket \u662f string \u65e0 \u8981\u5199\u5165\u7684 bucket object \u662f string \u65e0 \u8981\u5199\u5165\u7684 object\uff0c\u6ce8\u610f\u4e8b\u9879\u89c1\u4e0b fieldDelimiter \u5426 char ',' \u5b57\u6bb5\u7684\u5206\u9694\u7b26 nullFormat \u5426 char \\N \u5f53\u503c\u4e3a\u7a7a\u65f6\uff0c\u7528\u4ec0\u4e48\u5b57\u7b26\u8868\u793a header \u5426 list \u65e0 \u5199\u5165\u6587\u4ef6\u5934\u4fe1\u606f\uff0c\u6bd4\u5982 [\"id\",\"title\",\"url\"] maxFileSize \u5426 int 100000 \u5355\u4e2a object \u7684\u5927\u5c0f\uff0c\u5355\u4f4d\u4e3a MB encoding \u5426 string utf-8 \u6587\u4ef6\u7f16\u7801\u683c\u5f0f writeMode \u5426 string append \u5199\u5165\u6a21\u5f0f\uff0c\u8be6\u89c1 hdfswriter \u4e2d\u76f8\u5173\u63cf\u8ff0"},{"location":"writer/s3writer/#object","title":"object","text":"

    \u4e0a\u8ff0\u914d\u7f6e\u4e2d\u7684 object \u914d\u7f6e\u7684\u867d\u7136\u662f upload.csv \u6587\u4ef6\uff0c\u5b9e\u9645\u4e0a\u5728 S3 \u5199\u5165\u7684\u6587\u4ef6\u540d\u4f1a\u5728\u6307\u5b9a\u7684\u6587\u4ef6\u540d\u540e\u9762\u52a0\u4e0a uuid \u540e\u7f00\uff0c \u7c7b\u4f3c upload.csv_c0d2ca7df0444933a6f18ea76718b569\u3002 \u8fd9\u662f\u7528\u4e8e\u5728\u591a\u901a\u9053\u5199\u5165\u7684\u60c5\u51b5\u4e0b\uff0c\u786e\u4fdd\u6587\u4ef6\u540d\u4e0d\u4f1a\u91cd\u540d\u3002

    "},{"location":"writer/s3writer/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b S3 \u6570\u636e\u7c7b\u578b Long int, tinyint, smallint, mediumint, int, bigint Double float, double, decimal String varchar, char, tinytext, text, mediumtext, longtext, year,xml Date date, datetime, timestamp, time Boolean bit, bool Bytes tinyblob, mediumblob, blob, longblob, varbinary"},{"location":"writer/s3writer/#_4","title":"\u9650\u5236\u8bf4\u660e","text":"
    1. \u4ec5\u652f\u6301\u5199\u5165\u6587\u672c\u6587\u4ef6\u683c\u5f0f
    2. \u6682\u4e0d\u652f\u6301\u538b\u7f29
    "},{"location":"writer/sqlitewriter/","title":"SQLite Writer","text":"

    SQLite Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 SQLite \u6570\u636e\u5e93\u7684\u529f\u80fd\u3002

    "},{"location":"writer/sqlitewriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684\u8868\u5982\u4e0b\uff1a

    create table addax_tbl\n(\n    col1 varchar(20) ,\n    col2 int(4),\n    col3 datetime,\n    col4 boolean,\n    col5 binary\n);\n

    \u8fd9\u91cc\u4f7f\u7528\u4e00\u4efd\u4ece\u5185\u5b58\u4ea7\u751f\u5230 SQLite \u7684\u6570\u636e\u3002

    job/stream2sqlite.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"sqlitewriter\",\n        \"parameter\": {\n          \"writeMode\": \"insert\",\n          \"column\": [\n            \"*\"\n          ],\n          \"preSql\": [\n            \"delete from @table\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:sqlite://tmp/writer.sqlite3\",\n            \"table\": [\n              \"addax_tbl\"\n            ]\n          }\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2sqlite.json

    "},{"location":"writer/sqlitewriter/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/stream2sqlite.json\n
    "},{"location":"writer/sqlitewriter/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002\u56e0\u4e3a SQLite \u8fde\u63a5\u65e0\u9700\u8d26\u53f7\u5bc6\u7801\uff0c\u56e0\u6b64\u5176\u4ed6\u6570\u636e\u5e93\u5199\u5165\u63d2\u4ef6\u9700\u8981\u914d\u7f6e\u7684 username, password \u5728\u8fd9\u91cc\u4e0d\u9700\u8981\u3002

    "},{"location":"writer/sqlitewriter/#writemode","title":"writeMode","text":""},{"location":"writer/sqlitewriter/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b SQLite \u6570\u636e\u7c7b\u578b Long integer Double real String varchar Date datetime Boolean bool Bytes blob, binary"},{"location":"writer/sqlserverwriter/","title":"SQLServer Writer","text":"

    SQLServer Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 SQL Server \u5e93\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/sqlserverwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u8fd9\u91cc\u4f7f\u7528\u4e00\u4efd\u4ece\u5185\u5b58\u4ea7\u751f\u5230 SQL Server \u5bfc\u5165\u7684\u6570\u636e\u3002

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {},\n      \"writer\": {\n        \"name\": \"sqlserverwriter\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"root\",\n          \"column\": [\n            \"db_id\",\n            \"db_type\",\n            \"db_ip\",\n            \"db_port\",\n            \"db_role\",\n            \"db_name\",\n            \"db_username\",\n            \"db_password\",\n            \"db_modify_time\",\n            \"db_modify_user\",\n            \"db_description\",\n            \"db_tddl_info\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"db_info_for_writer\"\n            ],\n            \"jdbcUrl\": \"jdbc:sqlserver://[HOST_NAME]:PORT;DatabaseName=[DATABASE_NAME]\"\n          },\n          \"preSql\": [\n            \"delete from @table where db_id = -1;\"\n          ],\n          \"postSql\": [\n            \"update @table set db_modify_time = now() where db_id = 1;\"\n          ]\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/sqlserverwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/sqlserverwriter/#writemode","title":"writeMode","text":"

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c \u91c7\u53d6 insert into \u8bed\u6cd5\u5199\u5165 SQL Server \u8868\uff0c\u5982\u679c\u4f60\u5e0c\u671b\u91c7\u53d6\u4e3b\u952e\u5b58\u5728\u65f6\u66f4\u65b0\uff0c\u4e0d\u5b58\u5728\u5219\u5199\u5165\u7684\u65b9\u5f0f\uff0c\u4e5f\u5c31\u662f SQL Server \u7684 MERGE INTO \u8bed\u6cd5, \u53ef\u4ee5\u4f7f\u7528 update \u6a21\u5f0f\u3002 \u5047\u5b9a\u8868\u7684\u4e3b\u952e\u4e3a id ,\u5219 writeMode \u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff1a

    {\n  \"writeMode\": \"update(id)\"\n}\n

    \u5982\u679c\u662f\u8054\u5408\u552f\u4e00\u7d22\u5f15\uff0c\u5219\u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff1a

    {\n  \"writeMode\": \"update(col1, col2)\"\n}\n
    "},{"location":"writer/starrockswriter/","title":"StarRocks Writer","text":"

    StarRocks Writer \u63d2\u4ef6\u7528\u4e8e\u5411 Starrocks \u6570\u636e\u5e93\u4ee5\u6d41\u5f0f\u65b9\u5f0f\u5199\u5165\u6570\u636e\u3002 \u5176\u5b9e\u73b0\u4e0a\u662f\u901a\u8fc7\u8bbf\u95ee Doris http \u8fde\u63a5(8030) \uff0c\u7136\u540e\u901a\u8fc7 stream load \u52a0\u8f7d\u6570\u636e\u5230\u6570\u636e\u4e2d\uff0c\u76f8\u6bd4 insert into \u65b9\u5f0f\u6548\u7387\u8981\u9ad8\u4e0d\u5c11\uff0c\u4e5f\u662f\u5b98\u65b9\u63a8\u8350\u7684\u751f\u4ea7\u73af\u5883\u4e0b\u7684\u6570\u636e\u52a0\u8f7d\u65b9\u5f0f\u3002

    StarRocks \u662f\u4e00\u4e2a\u517c\u5bb9 MySQL \u534f\u8bae\u7684\u6570\u636e\u5e93\u540e\u7aef\uff0c\u56e0\u6b64 Doris \u8bfb\u53d6\u53ef\u4ee5\u4f7f\u7528 MySQLReader \u8fdb\u884c\u8bbf\u95ee\u3002

    "},{"location":"writer/starrockswriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684\u8868\u7684\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    CREATE DATABASE example_db;\nCREATE TABLE example_db.table1\n(\n    siteid   INT         DEFAULT '10',\n    citycode SMALLINT,\n    username VARCHAR(32) DEFAULT '',\n    pv       BIGINT SUM DEFAULT '0'\n) AGGREGATE KEY(siteid, citycode, username)\nDISTRIBUTED BY HASH(siteid) BUCKETS 10\nPROPERTIES(\"replication_num\" = \"1\");\n

    \u4e0b\u9762\u914d\u7f6e\u4e00\u4e2a\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\uff0c\u7136\u540e\u5199\u5165\u5230 doris \u8868\u7684\u914d\u7f6e\u6587\u4ef6

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2\n      }\n    },\n    \"content\": {\n      \"writer\": {\n        \"name\": \"starrockswriter\",\n        \"parameter\": {\n          \"username\": \"test\",\n          \"password\": \"123456\",\n          \"database\": \"example_db\",\n          \"table\": \"table1\",\n          \"jdbcUrl\": \"jdbc:mysql://172.28.17.100:9030/\",\n          \"loadUrl\": [\"172.28.17.100:8030\", \"172.28.17.100:8030\"],\n          \"loadProps\": {\n               \"column_separator\": \"\\\\x01\",\n               \"row_delimiter\": \"\\\\x02\"\n           },\n          \"column\": [\"siteid\", \"citycode\", \"username\", \"pv\"]\n        }\n      },\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"1,500\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"1,127\",\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"this is a text\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"5,200\",\n              \"type\": \"long\"\n            }\n          ],\n          \"sliceRecordCount\": 100\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2starrocks.json

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4

    bin/addax.sh job/stream2starrocks.json\n
    "},{"location":"writer/starrockswriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\uff0c\u5e76\u589e\u52a0\u4e86\u4e0b\u9762\u7684\u914d\u7f6e\u9879\u3002

    \u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 loadUrl \u662f string \u65e0 StarRocks FE \u7684\u5730\u5740\u7528\u4e8eStreamLoad1\uff0c\u53ef\u4ee5\u4e3a\u591a\u4e2afe\u5730\u5740\uff0cfe_ip:fe_http_port maxBatchRows \u5426 int 500000 \u5355\u6b21StreamLoad\u5bfc\u5165\u7684\u6700\u5927\u884c\u6570 maxBatchSize \u5426 int 104857600 \u5355\u6b21StreamLoad\u5bfc\u5165\u7684\u6700\u5927\u5b57\u8282\u6570 flushInterval \u5426 int 300000 \u4e0a\u4e00\u6b21StreamLoad\u7ed3\u675f\u81f3\u4e0b\u4e00\u6b21\u5f00\u59cb\u7684\u65f6\u95f4\u95f4\u9694(\u5355\u4f4d\uff1ams) loadProps \u5426 map streamLoad \u7684\u8bf7\u6c42\u53c2\u6570\uff0c\u8be6\u60c5\u53c2\u7167StreamLoad\u4ecb\u7ecd\u9875\u9762"},{"location":"writer/starrockswriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u9ed8\u8ba4\u4f20\u5165\u7684\u6570\u636e\u5747\u4f1a\u88ab\u8f6c\u4e3a\u5b57\u7b26\u4e32\uff0c\u5e76\u4ee5\\t\u4f5c\u4e3a\u5217\u5206\u9694\u7b26\uff0c\\n\u4f5c\u4e3a\u884c\u5206\u9694\u7b26\uff0c\u7ec4\u6210csv\u6587\u4ef6\u8fdb\u884cStreamLoad\u5bfc\u5165\u64cd\u4f5c\u3002 \u5982\u9700\u66f4\u6539\u5217\u5206\u9694\u7b26\uff0c \u5219\u6b63\u786e\u914d\u7f6e loadProps \u5373\u53ef\uff1a 

    \"loadProps\": {\n    \"column_separator\": \"\\\\x01\",\n    \"row_delimiter\": \"\\\\x02\"\n}\n

    \u5982\u9700\u66f4\u6539\u5bfc\u5165\u683c\u5f0f\u4e3ajson\uff0c \u5219\u6b63\u786e\u914d\u7f6e loadProps \u5373\u53ef\uff1a 

    \"loadProps\": {\n    \"format\": \"json\",\n    \"strip_outer_array\": true\n}\n

    "},{"location":"writer/streamwriter/","title":"Stream Writer","text":"

    Stream Writer \u662f\u4e00\u4e2a\u5c06\u6570\u636e\u5199\u5165\u5185\u5b58\u7684\u63d2\u4ef6\uff0c\u4e00\u822c\u7528\u6765\u5c06\u83b7\u53d6\u5230\u7684\u6570\u636e\u5199\u5230\u7ec8\u7aef\uff0c\u7528\u6765\u8c03\u8bd5\u8bfb\u53d6\u63d2\u4ef6\u7684\u6570\u636e\u5904\u7406\u60c5\u51b5\u3002

    \u4e00\u4e2a\u5178\u578b\u7684 Stream Writer \u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"name\": \"streamwriter\",\n  \"parameter\": {\n    \"encoding\": \"UTF-8\",\n    \"print\": true,\n    \"nullFormat\": \"NULL\"\n  }\n}\n

    \u4e0a\u8ff0\u914d\u7f6e\u4f1a\u5c06\u83b7\u53d6\u7684\u6570\u636e\u76f4\u63a5\u6253\u5370\u5230\u7ec8\u7aef\u3002 \u5176\u4e2d nullFormat \u7528\u6765\u6307\u5b9a\u5f53\u503c\u4e3a\u7a7a\u65f6\uff0c\u5982\u4f55\u5728\u7ec8\u7aef\u8868\u793a\uff0c\u9ed8\u8ba4\u662f\u5b57\u7b26\u4e32 NULL, \u5982\u679c\u4e0d\u60f3\u6253\u5370\u7a7a\u503c\uff0c\u53ef\u4ee5\u5c06\u5176\u8bbe\u7f6e\u4e3a \"\"\u3002

    \u8be5\u63d2\u4ef6\u4e5f\u652f\u6301\u5c06\u6570\u636e\u5199\u5165\u5230\u6587\u4ef6\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"name\": \"streamwriter\",\n  \"parameter\": {\n    \"encoding\": \"UTF-8\",\n    \"path\": \"/tmp/out\",\n    \"fileName\": \"out.txt\",\n    \"fieldDelimiter\": \",\",\n    \"recordNumBeforeSleep\": \"100\",\n    \"sleepTime\": \"5\"\n  }\n}\n

    \u4e0a\u8ff0\u914d\u7f6e\u4e2d:

    \u4e0a\u8ff0\u914d\u7f6e\u7684\u542b\u4e49\u662f\u5c06\u6570\u636e\u5199\u5165\u5230 /tmp/out/out.txt \u6587\u4ef6\uff0c \u6bcf\u83b7\u53d6100\u6761\u8bb0\u5f55\u540e\uff0c\u4f11\u77205\u79d2\u3002

    "},{"location":"writer/sybasewriter/","title":"Sybase Writer","text":"

    Sybase Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 Sybase \u5e93\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/sybasewriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u6211\u4eec\u53ef\u4ee5\u7528 Docker \u5bb9\u5668\u6765\u542f\u52a8\u4e00\u4e2a Sybase \u6570\u636e\u5e93

    docker run -tid --rm  -h dksybase --name sybase  -p 5000:5000  ifnazar/sybase_15_7 bash /sybase/start\n

    \u7136\u540e\u521b\u5efa\u4e00\u5f20\u5982\u4e0b\u8868

    create table addax_writer \n(\n    id int,\n    name varchar(255),\n    salary float(2),\n    created_at datetime,\n    updated_at datetime\n);\n

    \u518d\u4f7f\u7528\u4e0b\u9762\u7684\u4efb\u52a1\u914d\u7f6e\u6587\u4ef6

    {\n  \"job\": {\n    \"content\": {\n      \"writer\": {\n        \"name\": \"sybasewriter\",\n        \"parameter\": {\n          \"column\": [\n            \"id\",\n            \"name\",\n            \"salary\",\n            \"created_at\",\n            \"updated_at\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:sybase:Tds:127.0.0.1:5000/master\",\n            \"table\": [\n              \"dbo.addax_writer\"\n            ]\n          },\n          \"username\": \"sa\",\n          \"password\": \"password\"\n        }\n      },\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"100,1000\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"10,100\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"10,1000\",\n              \"type\": \"double\"\n            },\n            {\n              \"incr\": \"2022-01-01 13:00:00,2,d\",\n              \"type\": \"date\"\n            },\n            {\n              \"incr\": \"2023-01-01 13:00:00,2,d\",\n              \"type\": \"date\"\n            }\n          ],\n          \"sliceRecordCount\": 100\n        }\n      }\n    },\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    }\n  }\n}\n
    "},{"location":"writer/sybasewriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/tdenginewriter/","title":"TDengine Writer","text":"

    TDengine Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5c06\u6570\u636e\u5199\u5165 TDengine \u6570\u636e\u5e93\u7cfb\u7edf\u3002\u5728\u5e95\u5c42\u5b9e\u73b0\u4e0a\uff0cTDengine Writer \u901a\u8fc7JDBC JNI \u9a71\u52a8\u8fde\u63a5\u8fdc\u7a0b TDengine \u6570\u636e\u5e93\uff0c \u5e76\u6267\u884c\u76f8\u5e94\u7684sql\u8bed\u53e5\u5c06\u6570\u636e\u6279\u91cf\u5199\u5165 TDengine \u5e93\u4e2d\u3002

    "},{"location":"writer/tdenginewriter/#_1","title":"\u524d\u7f6e\u6761\u4ef6","text":"

    \u8003\u8651\u5230\u6027\u80fd\u95ee\u9898\uff0c\u8be5\u63d2\u4ef6\u4f7f\u7528\u4e86 TDengine \u7684 JDBC-JNI \u9a71\u52a8\uff0c \u8be5\u9a71\u52a8\u76f4\u63a5\u8c03\u7528\u5ba2\u6237\u7aef API\uff08libtaos.so \u6216 taos.dll\uff09\u5c06\u5199\u5165\u548c\u67e5\u8be2\u8bf7\u6c42\u53d1\u9001\u5230 taosd \u5b9e\u4f8b\u3002 \u56e0\u6b64\u5728\u4f7f\u7528\u4e4b\u524d\u9700\u8981\u914d\u7f6e\u597d\u52a8\u6001\u5e93\u94fe\u63a5\u6587\u4ef6\u3002

    \u9996\u5148\u5c06 plugin/writer/tdenginewriter/libs/libtaos.so.2.0.16.0 \u62f7\u8d1d\u5230 /usr/lib64 \u76ee\u5f55\uff0c\u7136\u540e\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u521b\u5efa\u8f6f\u94fe\u63a5

    ln -sf /usr/lib64/libtaos.so.2.0.16.0 /usr/lib64/libtaos.so.1\nln -sf /usr/lib64/libtaos.so.1 /usr/lib64/libtaos.so\n
    "},{"location":"writer/tdenginewriter/#_2","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684\u8868\u5982\u4e0b\uff1a

    create table test.addax_test (\n    ts timestamp,\n    name nchar(100),\n    file_size int,\n    file_date timestamp,\n    flag_open bool,\n    memo nchar(100)\n);\n

    \u4ee5\u4e0b\u662f\u914d\u7f6e\u6587\u4ef6

    job/stream2tdengine.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"2017-08-01 00:01:02,2020-01-01 12:13:14\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"tdenginewriter\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"taosdata\",\n          \"column\": [\n            \"ts\",\n            \"name\",\n            \"file_size\",\n            \"file_date\",\n            \"flag_open\",\n            \"memo\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:TAOS://127.0.0.1:6030/test\",\n            \"table\": [\n              \"addax_test\"\n            ]\n          }\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2tdengine.json

    "},{"location":"writer/tdenginewriter/#_3","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/tdengine2stream.json\n

    \u547d\u4ee4\u8f93\u51fa\u7c7b\u4f3c\u5982\u4e0b\uff1a

    2021-02-20 15:52:07.691 [main] INFO  VMInfo - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl\n2021-02-20 15:52:07.748 [main] INFO  Engine -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"column\":[\n                        {\n                            \"random\":\"2017-08-01 00:01:02,2020-01-01 12:13:14\",\n                            \"type\":\"date\"\n                        },\n                        {\n                            \"type\":\"string\",\n                            \"value\":\"Addax\"\n                        },\n                        {\n                            \"type\":\"long\",\n                            \"value\":19880808\n                        },\n                        {\n                            \"type\":\"date\",\n                            \"value\":\"1988-08-08 08:08:08\"\n                        },\n                        {\n                            \"type\":\"bool\",\n                            \"value\":true\n                        },\n                        {\n                            \"type\":\"bytes\",\n                            \"value\":\"test\"\n                        }\n                    ],\n                    \"sliceRecordCount\":1000\n                },\n                \"name\":\"streamreader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"password\":\"*****\",\n                    \"column\":[\n                        \"ts\",\n                        \"name\",\n                        \"file_size\",\n                        \"file_date\",\n                        \"flag_open\",\n                        \"memo\"\n                    ],\n                    \"connection\":[\n                        {\n                            \"jdbcUrl\":\"jdbc:TAOS://127.0.0.1:6030/test\",\n                            \"table\":[\n                                \"addax_test\"\n                            ]\n                        }\n                    ],\n                    \"username\":\"root\",\n                    \"preSql\":[]\n                },\n                \"name\":\"tdenginewriter\"\n            }\n    },\n    \"setting\":{\n        \"speed\":{\n            \"bytes\":-1,\n            \"channel\":1\n        }\n    }\n}\n\n2021-02-20 15:52:07.786 [main] INFO  PerfTrace - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-02-20 15:52:07.787 [main] INFO  JobContainer - Addax jobContainer starts job.\n2021-02-20 15:52:07.789 [main] INFO  JobContainer - Set jobId = 0\njava.library.path:/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib\n2021-02-20 15:52:08.048 [job-0] INFO  OriginalConfPretreatmentUtil - table:[addax_test] all columns:[ts,name,file_size,file_date,flag_open,memo].\n2021-02-20 15:52:08.056 [job-0] INFO  OriginalConfPretreatmentUtil - Write data [\nINSERT INTO %s (ts,name,file_size,file_date,flag_open,memo) VALUES(?,?,?,?,?,?)\n], which jdbcUrl like:[jdbc:TAOS://127.0.0.1:6030/test]\n\n2021-02-20 15:52:11.158 [job-0] INFO  JobContainer -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-02-20 15:52:07\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-02-20 15:52:11\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :           11.07KB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :            333rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                1000\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"writer/tdenginewriter/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\uff0c\u5e76\u4e14\u589e\u52a0\u4e86\u4e00\u4e9b TDengine \u7279\u6709\u7684\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/tdenginewriter/#jdbc-restful","title":"\u4f7f\u7528 JDBC-RESTful \u63a5\u53e3","text":"

    \u5982\u679c\u4e0d\u60f3\u4f9d\u8d56\u672c\u5730\u5e93\uff0c\u6216\u8005\u6ca1\u6709\u6743\u9650\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528 JDBC-RESTful \u63a5\u53e3\u6765\u5199\u5165\u8868\uff0c\u76f8\u6bd4 JDBC-JNI \u800c\u8a00\uff0c\u914d\u7f6e\u533a\u522b\u662f\uff1a

    \u6240\u4ee5\u4e0a\u8ff0\u914d\u7f6e\u4e2d\u7684 connection \u5e94\u8be5\u4fee\u6539\u4e3a\u5982\u4e0b\uff1a

    {\n  \"connection\": [\n    {\n      \"jdbcUrl\": \"jdbc:TAOS-RS://127.0.0.1:6041/test\",\n      \"table\": [\n        \"addax_test\"\n      ],\n      \"driver\": \"com.taosdata.jdbc.rs.RestfulDriver\"\n    }\n  ]\n}\n
    "},{"location":"writer/tdenginewriter/#_5","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u76ee\u524d TDenginereader \u652f\u6301 TDengine \u6240\u6709\u7c7b\u578b\uff0c\u5177\u4f53\u5982\u4e0b

    Addax \u5185\u90e8\u7c7b\u578b TDengine \u6570\u636e\u7c7b\u578b Long SMALLINT, TINYINT, INT, BIGINT, TIMESTAMP Double FLOAT, DOUBLE String BINARY, NCHAR Boolean BOOL"},{"location":"writer/tdenginewriter/#_6","title":"\u5f53\u524d\u652f\u6301\u7248\u672c","text":"

    TDengine 2.0.16

    "},{"location":"writer/tdenginewriter/#_7","title":"\u6ce8\u610f\u4e8b\u9879","text":""},{"location":"writer/txtfilewriter/","title":"TxtFile Writer","text":"

    TxtFile Writer \u63d0\u4f9b\u4e86\u5411\u672c\u5730\u6587\u4ef6\u5199\u5165\u7c7b CSV \u683c\u5f0f\u7684\u4e00\u4e2a\u6216\u8005\u591a\u4e2a\u8868\u6587\u4ef6\u3002

    "},{"location":"writer/txtfilewriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"txtfilereader\",\n        \"parameter\": {\n          \"path\": [\n            \"/tmp/data\"\n          ],\n          \"encoding\": \"UTF-8\",\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"long\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"boolean\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"double\"\n            },\n            {\n              \"index\": 3,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 4,\n              \"type\": \"date\",\n              \"format\": \"yyyy.MM.dd\"\n            }\n          ],\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"txtfilewriter\",\n        \"parameter\": {\n          \"path\": \"/tmp/result\",\n          \"fileName\": \"luohw\",\n          \"writeMode\": \"truncate\",\n          \"dateFormat\": \"yyyy-MM-dd\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/txtfilewriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f string \u65e0 \u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u5199\u5165 Path \u76ee\u5f55\u4e0b\u5c5e\u591a\u4e2a\u6587\u4ef6 fileName \u662f string \u65e0 \u5199\u5165\u7684\u6587\u4ef6\u540d\uff0c\u8be5\u6587\u4ef6\u540d\u4f1a\u6dfb\u52a0\u968f\u673a\u7684\u540e\u7f00\u4f5c\u4e3a\u6bcf\u4e2a\u7ebf\u7a0b\u5199\u5165\u5b9e\u9645\u6587\u4ef6\u540d writeMode \u662f string \u65e0 \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff0c\u8be6\u89c1\u4e0b\u6587 fieldDelimiter \u662f string , \u63cf\u8ff0\uff1a\u8bfb\u53d6\u7684\u5b57\u6bb5\u5206\u9694\u7b26 compress \u5426 string \u65e0 \u6587\u672c\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u538b\u7f29\u7c7b\u578b\u4e3a zip\u3001lzo\u3001lzop\u3001tgz\u3001bzip2 encoding \u5426 string utf-8 \u8bfb\u53d6\u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e nullFormat \u5426 string \\N \u5b9a\u4e49\u54ea\u4e9b\u5b57\u7b26\u4e32\u53ef\u4ee5\u8868\u793a\u4e3a null dateFormat \u5426 string \u65e0 \u65e5\u671f\u7c7b\u578b\u7684\u6570\u636e\u5e8f\u5217\u5316\u5230\u6587\u4ef6\u4e2d\u65f6\u7684\u683c\u5f0f\uff0c\u4f8b\u5982 \"yyyy-MM-dd\" fileFormat \u5426 string text \u6587\u4ef6\u5199\u51fa\u7684\u683c\u5f0f\uff0c\u8be6\u89c1\u4e0b\u6587 table \u662f string \u65e0 sql \u6a21\u5f0f\u65f6\u9700\u8981\u6307\u5b9a\u8868\u540d\uff0c column \u5426 list \u65e0 sql \u6a21\u5f0f\u65f6\u53ef\u9009\u6307\u5b9a\u5217\u540d\uff0c extendedInsert \u5426 boolean true sql \u6a21\u5f0f\u65f6\u662f\u5426\u4f7f\u7528\u6279\u91cf\u63d2\u5165\u8bed\u6cd5\uff0c\u8be6\u89c1\u4e0b\u6587 batchSize \u5426 int 2048 sql \u6a21\u5f0f\u65f6\u6279\u91cf\u63d2\u5165\u8bed\u6cd5\u7684\u6279\u6b21\u5927\u5c0f\uff0c\u8be6\u89c1\u4e0b\u6587 header \u5426 list \u65e0 text \u5199\u51fa\u65f6\u7684\u8868\u5934\uff0c\u793a\u4f8b ['id', 'name', 'age']"},{"location":"writer/txtfilewriter/#writemode","title":"writeMode","text":"

    \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff1a

    "},{"location":"writer/txtfilewriter/#fileformat","title":"fileFormat","text":"

    \u6587\u4ef6\u5199\u51fa\u7684\u683c\u5f0f\uff0c\u5305\u62ec csv \u548c text \u548c 4.1.3 \u7248\u672c\u5f15\u5165\u7684 sql \u4e09\u79cd\uff0ccsv \u662f\u4e25\u683c\u7684 csv \u683c\u5f0f\uff0c\u5982\u679c\u5f85\u5199\u6570\u636e\u5305\u62ec\u5217\u5206\u9694\u7b26\uff0c\u5219\u4f1a\u6309\u7167 csv \u7684\u8f6c\u4e49\u8bed\u6cd5\u8f6c\u4e49\uff0c\u8f6c\u4e49\u7b26\u53f7\u4e3a\u53cc\u5f15\u53f7 \"\uff1b text \u683c\u5f0f\u662f\u7528\u5217\u5206\u9694\u7b26\u7b80\u5355\u5206\u5272\u5f85\u5199\u6570\u636e\uff0c\u5bf9\u4e8e\u5f85\u5199\u6570\u636e\u5305\u62ec\u5217\u5206\u9694\u7b26\u60c5\u51b5\u4e0b\u4e0d\u505a\u8f6c\u4e49\u3002 sql \u683c\u5f0f\u8868\u793a\u5c06\u6570\u636e\u4ee5 SQL \u8bed\u53e5 (INSERT INTO ... VALUES) \u7684\u65b9\u5f0f\u5199\u5165\u5230\u6587\u4ef6

    "},{"location":"writer/txtfilewriter/#table","title":"table","text":"

    \u4ec5\u5728 sql \u6587\u4ef6\u683c\u5f0f\u4e0b\u9700\u8981\uff0c\u7528\u6765\u6307\u5b9a\u5199\u5165\u7684\u8868\u540d

    "},{"location":"writer/txtfilewriter/#column","title":"column","text":"

    \u5728 sql \u6587\u4ef6\u683c\u5f0f\u4e0b\uff0c\u53ef\u4ee5\u6307\u5b9a\u5199\u5165\u7684\u5217\u540d\uff0c\u5982\u679c\u6307\u5b9a\uff0c\u5219 sql \u8bed\u53e5\u7c7b\u4f3c INSERT INTO table (col1, col2, col3) VALUES (val1, val2, val3)\uff0c\u6a21\u5f0f \u5426\u5219\u4e3a INSERT INTO table VALUES (val1, val2, val3)\u3002\u6a21\u5f0f

    "},{"location":"writer/txtfilewriter/#extendedinsert","title":"extendedInsert","text":"

    \u662f\u5426\u542f\u7528\u6279\u91cf\u63d2\u5165\u8bed\u6cd5\uff0c\u5982\u679c\u542f\u7528\uff0c\u5219\u4f1a\u5c06 batchSize \u4e2a\u6570\u636e\u4e00\u6b21\u6027\u5199\u5165\u5230\u6587\u4ef6\u4e2d\uff0c\u5426\u5219\u6bcf\u4e2a\u6570\u636e\u4e00\u884c\u3002\u8be5\u53c2\u6570\u501f\u9274\u4e86 mysqldump \u5de5\u5177\u7684 extended-insert \u53c2\u6570\u8bed\u6cd5

    "},{"location":"writer/txtfilewriter/#batchsize","title":"batchSize","text":"

    \u6279\u91cf\u63d2\u5165\u8bed\u6cd5\u7684\u6279\u6b21\u5927\u5c0f\uff0c\u5982\u679c extendedInsert \u4e3a true\uff0c\u5219\u6bcf batchSize \u4e2a\u6570\u636e\u4e00\u6b21\u6027\u5199\u5165\u5230\u6587\u4ef6\u4e2d\uff0c\u5426\u5219\u6bcf\u4e2a\u6570\u636e\u4e00\u884c\u3002

    "},{"location":"writer/txtfilewriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b \u672c\u5730\u6587\u4ef6 \u6570\u636e\u7c7b\u578b Long Long Double Double string string Boolean Boolean Date Date"}]} \ No newline at end of file +{"config":{"lang":["en"],"separator":"[\\s\\u200b\\u3000\\-\u3001\u3002\uff0c\uff0e\uff1f\uff01\uff1b]+","pipeline":["stemmer"]},"docs":[{"location":"","title":"Addax \u4ecb\u7ecd","text":""},{"location":"#_1","title":"\u6982\u89c8","text":"

    Addax \u662f\u4e00\u4e2a\u5f02\u6784\u6570\u636e\u6e90\u79bb\u7ebf\u540c\u6b65\u5de5\u5177\uff0c\u6700\u521d\u6765\u6e90\u4e8e\u963f\u91cc\u7684 DataX \uff0c\u81f4\u529b\u4e8e\u5b9e\u73b0\u5305\u62ec\u5173\u7cfb\u578b\u6570\u636e\u5e93(MySQL\u3001Oracle \u7b49)\u3001HDFS\u3001Hive\u3001HBase\u3001FTP \u7b49\u5404\u79cd\u5f02\u6784\u6570\u636e\u6e90\u4e4b\u95f4\u7a33\u5b9a\u9ad8\u6548\u7684\u6570\u636e\u540c\u6b65\u529f\u80fd\u3002

    \u4e3a\u4e86\u89e3\u51b3\u5f02\u6784\u6570\u636e\u6e90\u540c\u6b65\u95ee\u9898\uff0cAddax \u5c06\u590d\u6742\u7684\u7f51\u72b6\u7684\u540c\u6b65\u94fe\u8def\u53d8\u6210\u4e86\u661f\u578b\u6570\u636e\u94fe\u8def\uff0cAddax \u4f5c\u4e3a\u4e2d\u95f4\u4f20\u8f93\u8f7d\u4f53\u8d1f\u8d23\u8fde\u63a5\u5404\u79cd\u6570\u636e\u6e90\u3002\u5f53\u9700\u8981\u63a5\u5165\u4e00\u4e2a\u65b0\u7684\u6570\u636e\u6e90\u7684\u65f6\u5019\uff0c\u53ea\u9700\u8981\u5c06\u6b64\u6570\u636e\u6e90\u5bf9\u63a5\u5230 Addax\uff0c\u4fbf\u80fd\u8ddf\u5df2\u6709\u7684\u6570\u636e\u6e90\u505a\u5230\u65e0\u7f1d\u6570\u636e\u540c\u6b65\u3002

    "},{"location":"#_2","title":"\u6846\u67b6\u8bbe\u8ba1","text":"
    graph LR\nMySQL\nsubgraph Addax\n    direction LR\n    subgraph reader[\"Reader Plugin\"]\n        mr[\"MySQLReader\"]\n    end\n    subgraph writer[\"Writer Plugin\"]\n    hw[\"HDFSWriter\"]\n    end\n    Framework\n    mr --> Framework --> writer\nend\n\nMySQL ==> Addax ==> HDFS\n

    Addax \u672c\u8eab\u4f5c\u4e3a\u79bb\u7ebf\u6570\u636e\u540c\u6b65\u6846\u67b6\uff0c\u91c7\u7528 Framework + plugin \u67b6\u6784\u6784\u5efa\u3002\u5c06\u6570\u636e\u6e90\u8bfb\u53d6\u548c\u5199\u5165\u62bd\u8c61\u6210\u4e3a Reader/Writer \u63d2\u4ef6\uff0c\u7eb3\u5165\u5230\u6574\u4e2a\u540c\u6b65\u6846\u67b6\u4e2d\u3002

    Addax Framework \u63d0\u4f9b\u4e86\u7b80\u5355\u7684\u63a5\u53e3\u4e0e\u63d2\u4ef6\u4ea4\u4e92\uff0c\u63d0\u4f9b\u7b80\u5355\u7684\u63d2\u4ef6\u63a5\u5165\u673a\u5236\uff0c\u53ea\u9700\u8981\u4efb\u610f\u52a0\u4e0a\u4e00\u79cd\u63d2\u4ef6\uff0c\u5c31\u80fd\u65e0\u7f1d\u5bf9\u63a5\u5176\u4ed6\u6570\u636e\u6e90\u3002

    "},{"location":"#_3","title":"\u6838\u5fc3\u67b6\u6784","text":"

    \u672c\u5c0f\u8282\u6309\u4e00\u4e2a Addax \u4f5c\u4e1a\u751f\u547d\u5468\u671f\u7684\u65f6\u5e8f\u56fe\uff0c\u4ece\u6574\u4f53\u67b6\u6784\u8bbe\u8ba1\u975e\u5e38\u7b80\u8981\u8bf4\u660e\u5404\u4e2a\u6a21\u5757\u76f8\u4e92\u5173\u7cfb\u3002

    graph TB\nsubgraph Job\nend\nsubgraph task\n  direction TB\n  t1[\"Task\"]\n  t2[\"Task\"]\n  t3[\"Task\"]\n  t4[\"Task\"]\n  t5[\"Task\"]\n  t6[\"Task\"]\nend\nsubgraph taskgroup[\" \"]\n    direction TB\n  subgraph tg1[\"TaskGroup\"]\n    subgraph tg1_Task[\"Task\"]\n      tg1_r[\"Reader\"]\n      tg1_c[\"Channel\"]\n      tg1_w[\"Writer\"]\n    end\n    t7[\"Task\"]\n    t8[\"Task\"]\n  end\n\n  subgraph tg2[\"TaskGroup\"]\n    subgraph tg2_Task[\"Task\"]\n      direction LR\n      tg2_r[\"Reader\"]\n      tg2_c[\"Channel\"]\n      tg2_w[\"Writer\"]\n    end\n    t9[\"Task\"]\n    t10[\"Task\"]\n  end\n\n  subgraph tg3[\"TaskGroup\"]\n    direction LR\n    subgraph tg3_Task[\"Task\"]\n      tg3_r[\"Reader\"]\n      tg3_c[\"Channel\"]\n      tg3_w[\"Writer\"]\n    end\n    t11[\"Task\"]\n    t12[\"Task\"]\n  end\nend\n\nJob == split ==> task\ntask == Schedule ==> taskgroup
    "},{"location":"#_4","title":"\u6838\u5fc3\u6a21\u5757\u4ecb\u7ecd","text":"
    1. Addax \u5b8c\u6210\u5355\u4e2a\u6570\u636e\u540c\u6b65\u7684\u4f5c\u4e1a\uff0c\u6211\u4eec\u79f0\u4e4b\u4e3a Job\uff0cAddax \u63a5\u6536\u5230\u4e00\u4e2a Job \u4e4b\u540e\uff0c\u5c06\u542f\u52a8\u4e00\u4e2a\u8fdb\u7a0b\u6765\u5b8c\u6210\u6574\u4e2a\u4f5c\u4e1a\u540c\u6b65\u8fc7\u7a0b\u3002Addax Job \u6a21\u5757\u662f\u5355\u4e2a\u4f5c\u4e1a\u7684\u4e2d\u67a2\u7ba1\u7406\u8282\u70b9\uff0c\u627f\u62c5\u4e86\u6570\u636e\u6e05\u7406\u3001\u5b50\u4efb\u52a1\u5207\u5206(\u5c06\u5355\u4e00\u4f5c\u4e1a\u8ba1\u7b97\u8f6c\u5316\u4e3a\u591a\u4e2a\u5b50 Task)\u3001TaskGroup \u7ba1\u7406\u7b49\u529f\u80fd\u3002
    2. Addax Job \u542f\u52a8\u540e\uff0c\u4f1a\u6839\u636e\u4e0d\u540c\u7684\u6e90\u7aef\u5207\u5206\u7b56\u7565\uff0c\u5c06 Job \u5207\u5206\u6210\u591a\u4e2a\u5c0f\u7684 Task (\u5b50\u4efb\u52a1)\uff0c\u4ee5\u4fbf\u4e8e\u5e76\u53d1\u6267\u884c\u3002Task \u4fbf\u662f Addax \u4f5c\u4e1a\u7684\u6700\u5c0f\u5355\u5143\uff0c\u6bcf\u4e00\u4e2a Task \u90fd\u4f1a\u8d1f\u8d23\u4e00\u90e8\u5206\u6570\u636e\u7684\u540c\u6b65\u5de5\u4f5c\u3002
    3. \u5207\u5206\u591a\u4e2a Task \u4e4b\u540e\uff0cAddax Job \u4f1a\u8c03\u7528 Scheduler \u6a21\u5757\uff0c\u6839\u636e\u914d\u7f6e\u7684\u5e76\u53d1\u6570\u636e\u91cf\uff0c\u5c06\u62c6\u5206\u6210\u7684 Task \u91cd\u65b0\u7ec4\u5408\uff0c\u7ec4\u88c5\u6210 TaskGroup(\u4efb\u52a1\u7ec4)\u3002\u6bcf\u4e00\u4e2a TaskGroup \u8d1f\u8d23\u4ee5\u4e00\u5b9a\u7684\u5e76\u53d1\u8fd0\u884c\u5b8c\u6bd5\u5206\u914d\u597d\u7684\u6240\u6709 Task\uff0c\u9ed8\u8ba4\u5355\u4e2a\u4efb\u52a1\u7ec4\u7684\u5e76\u53d1\u6570\u91cf\u4e3a 5\u3002
    4. \u6bcf\u4e00\u4e2a Task \u90fd\u7531 TaskGroup \u8d1f\u8d23\u542f\u52a8\uff0cTask \u542f\u52a8\u540e\uff0c\u4f1a\u56fa\u5b9a\u542f\u52a8 Reader\u2014>Channel\u2014>Writer \u7684\u7ebf\u7a0b\u6765\u5b8c\u6210\u4efb\u52a1\u540c\u6b65\u5de5\u4f5c\u3002
    5. Addax \u4f5c\u4e1a\u8fd0\u884c\u8d77\u6765\u4e4b\u540e\uff0c Job \u76d1\u63a7\u5e76\u7b49\u5f85\u591a\u4e2a TaskGroup \u6a21\u5757\u4efb\u52a1\u5b8c\u6210\uff0c\u7b49\u5f85\u6240\u6709 TaskGroup \u4efb\u52a1\u5b8c\u6210\u540e Job \u6210\u529f\u9000\u51fa\u3002\u5426\u5219\uff0c\u5f02\u5e38\u9000\u51fa\uff0c\u8fdb\u7a0b\u9000\u51fa\u503c\u975e 0
    "},{"location":"#_5","title":"\u8c03\u5ea6\u6d41\u7a0b","text":"

    \u4e3e\u4f8b\u6765\u8bf4\uff0c\u7528\u6237\u63d0\u4ea4\u4e86\u4e00\u4e2a\u4f5c\u4e1a\uff0c\u5e76\u4e14\u914d\u7f6e\u4e86 20 \u4e2a\u5e76\u53d1\uff0c\u76ee\u7684\u662f\u5c06\u4e00\u4e2a 100 \u5f20\u5206\u8868\u7684 MySQL \u6570\u636e\u540c\u6b65\u5230 Oracle \u91cc\u9762\u3002 \u8c03\u5ea6\u51b3\u7b56\u601d\u8def\u662f\uff1a

    1. Addax Job \u6839\u636e\u5206\u5e93\u5206\u8868\u5207\u5206\u6210\u4e86 100 \u4e2a Task\u3002
    2. \u6839\u636e 20 \u4e2a\u5e76\u53d1\uff0c\u8ba1\u7b97\u5171\u9700\u8981\u5206\u914d 20/5 = 4 \u4e2a TaskGroup\u3002
    3. 4 \u4e2a TaskGroup \u5e73\u5206\u5207\u5206\u597d\u7684 100 \u4e2a Task\uff0c\u6bcf\u4e00\u4e2a TaskGroup \u8d1f\u8d23\u4ee5 5 \u4e2a\u5e76\u53d1\u5171\u8ba1\u8fd0\u884c 25 \u4e2a Task\u3002
    "},{"location":"#_6","title":"\u6838\u5fc3\u4f18\u52bf","text":""},{"location":"#_7","title":"\u53ef\u9760\u7684\u6570\u636e\u8d28\u91cf\u76d1\u63a7","text":"

    \u652f\u6301\u6240\u6709\u7684\u5f3a\u6570\u636e\u7c7b\u578b\uff0c\u6bcf\u4e00\u79cd\u63d2\u4ef6\u90fd\u6709\u81ea\u5df1\u7684\u6570\u636e\u7c7b\u578b\u8f6c\u6362\u7b56\u7565\uff0c\u8ba9\u6570\u636e\u53ef\u4ee5\u5b8c\u6574\u65e0\u635f\u7684\u4f20\u8f93\u5230\u76ee\u7684\u7aef\u3002

    \u8fd0\u884c\u8fc7\u7a0b\u4e2d\u53ef\u4ee5\u5c06\u4f5c\u4e1a\u672c\u8eab\u72b6\u6001\u3001\u6570\u636e\u6d41\u91cf\u3001\u6570\u636e\u901f\u5ea6\u3001\u6267\u884c\u8fdb\u5ea6\u7b49\u4fe1\u606f\u8fdb\u884c\u5168\u9762\u7684\u5c55\u793a\uff0c\u8ba9\u7528\u6237\u53ef\u4ee5\u5b9e\u65f6\u4e86\u89e3\u4f5c\u4e1a\u72b6\u6001\u3002\u5e76\u53ef\u5728\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u667a\u80fd\u5224\u65ad\u6e90\u7aef\u548c\u76ee\u7684\u7aef\u7684\u901f\u5ea6\u5bf9\u6bd4\u60c5\u51b5\uff0c\u7ed9\u4e88\u7528\u6237\u66f4\u591a\u6027\u80fd\u6392\u67e5\u4fe1\u606f\u3002

    \u5728\u5927\u91cf\u6570\u636e\u7684\u4f20\u8f93\u8fc7\u7a0b\u4e2d\uff0c\u5fc5\u5b9a\u4f1a\u7531\u4e8e\u5404\u79cd\u539f\u56e0\u5bfc\u81f4\u5f88\u591a\u6570\u636e\u4f20\u8f93\u62a5\u9519(\u6bd4\u5982\u7c7b\u578b\u8f6c\u6362\u9519\u8bef)\uff0c\u8fd9\u79cd\u6570\u636e Addax \u8ba4\u4e3a\u5c31\u662f\u810f\u6570\u636e\u3002Addax \u76ee\u524d\u53ef\u4ee5\u5b9e\u73b0\u810f\u6570\u636e\u7cbe\u786e\u8fc7\u6ee4\u3001\u8bc6\u522b\u3001\u91c7\u96c6\u3001\u5c55\u793a\uff0c\u4e3a\u7528\u6237\u63d0\u4f9b\u591a\u79cd\u7684\u810f\u6570\u636e\u5904\u7406\u6a21\u5f0f\uff0c\u8ba9\u7528\u6237\u51c6\u786e\u628a\u63a7\u6570\u636e\u8d28\u91cf\u5173\uff01

    "},{"location":"#_8","title":"\u4e30\u5bcc\u7684\u6570\u636e\u8f6c\u6362\u529f\u80fd","text":"

    \u4f5c\u4e3a\u4e00\u4e2a\u670d\u52a1\u4e8e\u5927\u6570\u636e\u7684 ETL \u5de5\u5177\uff0c\u9664\u4e86\u63d0\u4f9b\u6570\u636e\u5feb\u7167\u642c\u8fc1\u529f\u80fd\u4e4b\u5916\uff0c\u8fd8\u63d0\u4f9b\u4e86\u4e30\u5bcc\u6570\u636e\u8f6c\u6362\u7684\u529f\u80fd\uff0c\u8ba9\u6570\u636e\u5728\u4f20\u8f93\u8fc7\u7a0b\u4e2d\u53ef\u4ee5\u8f7b\u677e\u5b8c\u6210\u6570\u636e\u8131\u654f\uff0c\u8865\u5168\uff0c\u8fc7\u6ee4\u7b49\u6570\u636e\u8f6c\u6362\u529f\u80fd\uff0c\u53e6\u5916\u8fd8\u63d0\u4f9b\u4e86\u81ea\u52a8 groovy \u51fd\u6570\uff0c\u8ba9\u7528\u6237\u81ea\u5b9a\u4e49\u8f6c\u6362\u51fd\u6570\u3002\u8be6\u60c5\u8bf7\u770b transformer \u8be6\u7ec6\u4ecb\u7ecd\u3002

    "},{"location":"#_9","title":"\u7cbe\u51c6\u7684\u901f\u5ea6\u63a7\u5236","text":"

    \u63d0\u4f9b\u4e86\u5305\u62ec\u901a\u9053(\u5e76\u53d1)\u3001\u8bb0\u5f55\u6d41\u3001\u5b57\u8282\u6d41\u4e09\u79cd\u6d41\u63a7\u6a21\u5f0f\uff0c\u53ef\u4ee5\u968f\u610f\u63a7\u5236\u4f60\u7684\u4f5c\u4e1a\u901f\u5ea6\uff0c\u8ba9\u4f60\u7684\u4f5c\u4e1a\u5728\u5e93\u53ef\u4ee5\u627f\u53d7\u7684\u8303\u56f4\u5185\u8fbe\u5230\u6700\u4f73\u7684\u540c\u6b65\u901f\u5ea6\u3002

    {\n  \"speed\": {\n    \"channel\": 5,\n    \"byte\": 1048576,\n    \"record\": 10000\n  }\n}\n
    "},{"location":"#_10","title":"\u5f3a\u52b2\u5730\u540c\u6b65\u6027\u80fd","text":"

    \u6bcf\u4e00\u79cd\u8bfb\u63d2\u4ef6\u90fd\u6709\u4e00\u79cd\u6216\u591a\u79cd\u5207\u5206\u7b56\u7565\uff0c\u90fd\u80fd\u5c06\u4f5c\u4e1a\u5408\u7406\u5207\u5206\u6210\u591a\u4e2a Task \u5e76\u884c\u6267\u884c\uff0c\u5355\u673a\u591a\u7ebf\u7a0b\u6267\u884c\u6a21\u578b\u53ef\u4ee5\u8ba9\u901f\u5ea6\u968f\u5e76\u53d1\u6210\u7ebf\u6027\u589e\u957f\u3002 \u5728\u6e90\u7aef\u548c\u76ee\u7684\u7aef\u6027\u80fd\u90fd\u8db3\u591f\u7684\u60c5\u51b5\u4e0b\uff0c\u5355\u4e2a\u4f5c\u4e1a\u4e00\u5b9a\u53ef\u4ee5\u6253\u6ee1\u7f51\u5361\u3002

    "},{"location":"#_11","title":"\u5065\u58ee\u7684\u5bb9\u9519\u673a\u5236","text":"

    \u4f5c\u4e1a\u662f\u6781\u6613\u53d7\u5916\u90e8\u56e0\u7d20\u7684\u5e72\u6270\uff0c\u7f51\u7edc\u95ea\u65ad\u3001\u6570\u636e\u6e90\u4e0d\u7a33\u5b9a\u7b49\u56e0\u7d20\u5f88\u5bb9\u6613\u8ba9\u540c\u6b65\u5230\u4e00\u534a\u7684\u4f5c\u4e1a\u62a5\u9519\u505c\u6b62\u3002\u56e0\u6b64\u7a33\u5b9a\u6027\u662f Addax \u7684\u57fa\u672c\u8981\u6c42\uff0c\u5728 Addax \u7684\u8bbe\u8ba1\u4e2d\uff0c\u91cd\u70b9\u5b8c\u5584\u4e86\u6846\u67b6\u548c\u63d2\u4ef6\u7684\u7a33\u5b9a\u6027\u3002 \u76ee\u524d Addax \u53ef\u4ee5\u505a\u5230\u7ebf\u7a0b\u7ea7\u522b\u3001\u4f5c\u4e1a\u7ea7\u522b\u591a\u5c42\u6b21\u5c40\u90e8/\u5168\u5c40\u7684\u91cd\u8bd5\uff0c\u4fdd\u8bc1\u7528\u6237\u7684\u4f5c\u4e1a\u7a33\u5b9a\u8fd0\u884c\u3002

    "},{"location":"debug/","title":"\u5982\u4f55\u8c03\u8bd5\u9879\u76ee","text":"

    \u5982\u679c\u4f60\u60f3\u5728\u672c\u5730\u4fee\u6539\u6216\u8005\u8c03\u8bd5\u4ee3\u7801\uff0c\u53ef\u4ee5\u53c2\u8003\u4ee5\u4e0b\u65b9\u5f0f\uff0c\u4ee5\u4e0b\u7684\u64cd\u4f5c\u5747\u4ee5 Intellij IDEA \u5f00\u53d1\u5de5\u5177\u4e3a\u4f8b\u3002

    \u8c03\u8bd5\u53ef\u4ee5\u5206\u6210\u4e24\u65b9\u65b9\u5f0f\uff0c\u4e00\u79cd\u662f\u4ee3\u7801\u548c\u4e8c\u8fdb\u5236\u5305\u90fd\u5728\u672c\u5730\uff0c\u6211\u4eec\u79f0\u4e4b\u4e3a\u672c\u5730\u8c03\u8bd5\uff1b\u53e6\u5916\u4e00\u79cd\u6e90\u4ee3\u7801\u5728\u672c\u5730\uff0c\u4f46\u4e8c\u8fdb\u5236\u7a0b\u5e8f\u5df2\u7ecf\u90e8\u7f72\u5728\u8fdc\u7a0b\uff0c\u8fd9\u79cd\u60c5\u51b5\u4e0b\u6211\u4eec\u79f0\u4e4b\u4e3a\u8fdc\u7a0b\u8c03\u8bd5\u3002

    \u4e0b\u9762\u5206\u522b\u63cf\u8ff0

    "},{"location":"debug/#_2","title":"\u672c\u5730\u8c03\u8bd5","text":""},{"location":"debug/#_3","title":"\u4e00\u4e9b\u8bbe\u5b9a","text":"

    \u6211\u4eec\u5047\u5b9a\u672c\u5730\u90e8\u7f72\u7684 Addax \u5728 /opt/app/addax/4.0.3 \u6587\u4ef6\u5939\u4e0b\u3002\u5176 job \u76ee\u5f55\u4e0b\u6709\u8fd9\u6837\u7684\u4e00\u4e2a job.json \u914d\u7f6e\u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    job/job.json 
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 00:00:00\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u4e0a\u8ff0 job \u6587\u4ef6\u8fd0\u884c\u6ca1\u6709\u7b26\u5408\u6211\u4eec\u7684\u9884\u671f\uff0c\u731c\u6d4b\u662f streamreader \u8fd9\u4e2a\u63d2\u4ef6\u7684 parseMixupFunctions \u51fd\u6570\u6709\u95ee\u9898\uff0c\u6211\u60f3\u8c03\u8bd5\u770b\u5177\u4f53\u95ee\u9898\u5728\u54ea\u91cc\u3002

    "},{"location":"debug/#idea","title":"\u914d\u7f6e IDEA","text":"

    \u6253\u5f00 IDEA\u5de5\u5177\uff0c\u5e76\u6253\u5f00 addax \u9879\u76ee\u6e90\u4ee3\u7801\uff0c\u6253\u5f00 plugin/reader/streamreader/StreamReader.java \u6587\u4ef6\uff0c\u627e\u5230 parseMixupFunctions \u51fd\u6570\uff0c\u5e76\u5728\u51fd\u6570\u7533\u660e\u5904\u7684\u70b9\u51fb\u5de6\u4fa7\u8fb9\u7f18\u5904\u589e\u52a0\u65ad\u70b9\u3002\u5982\u4e0b\u56fe\u6240\u793a\uff1a

    \u70b9\u51fb IDEA \u7684 Run->Edit Configurations... \u83dc\u5355\uff0c\u5728\u5f39\u51fa\u7684 Run/Debug Configurations \u7a97\u53e3\uff0c\u70b9\u51fb\u5de6\u4e0a\u89d2\u7684 + \u6309\u94ae\uff0c\u7136\u540e\u9009\u62e9 \u9009\u62e9 Application \uff0c\u5728\u53f3\u4fa7\u914d\u7f6e\u6846\u4e2d\uff0c\u586b\u5199\u76f8\u5173\u4fe1\u606f\u5982\u4e0b\uff1a

    \u5176\u4ed6\u4fdd\u6301\u4e0d\u53d8\uff0c\u70b9\u51fb Apply \u6309\u94ae\u3002\u5f97\u5230\u7c7b\u4f3c\u4e0b\u56fe\u914d\u7f6e\u5185\u5bb9

    \u70b9\u51fb OK \u6309\u94ae\u4fdd\u5b58\u4e0a\u8ff0\u914d\u7f6e\uff0c\u56de\u5230 IDEA \u4e3b\u7a97\u53e3\uff0c\u5728\u7a97\u53e3\u83dc\u5355\u680f\u6709\u7eff\u8272\ud83d\udd28\u7684\u53f3\u4fa7\uff0c\u5e94\u8be5\u53ef\u4ee5\u770b\u5230\u521a\u624d\u914d\u7f6e\u7684\u63cf\u8ff0\u6587\u4ef6\uff0c\u7c7b\u4f3c\u4e0b\u56fe\uff1a

    \u70b9\u51fb\u4e0a\u8ff0\u622a\u56fe\u4e2d\u7684\u7eff\u8272 DEBUG \u5c0f\u866b\u6309\u94ae\uff0c\u8fdb\u5165\u8c03\u8bd5\uff0c\u5f97\u5230\u7c7b\u4f3c\u4e0b\u56fe\u7684\u8c03\u8bd5\u7a97\u53e3\uff1a

    "},{"location":"debug/#_4","title":"\u8fdc\u7a0b\u8c03\u8bd5","text":""},{"location":"debug/#_5","title":"\u4e00\u4e9b\u5047\u5b9a","text":"

    \u5047\u5b9a\u7a0b\u5e8f\u90e8\u7f72\u5728\u8fdc\u7a0b\u670d\u52a1\u5668\u4e0a\uff0c\u9700\u8981\u76f4\u63a5\u8c03\u8bd5\u8fdc\u7a0b\u670d\u52a1\u5668\u4e0a\u8fd0\u884c\u7684\u7a0b\u5e8f\uff0c\u5047\u5b9a\u8fdc\u7a0b\u670d\u52a1\u5668IP\u5730\u5740\u4e3a 192.168.1.100\uff0c\bAddax \u90e8\u7f72\u5728 /opt/addax/4.0.3 \u76ee\u5f55\u4e0b\uff0c\u5176 job \u6587\u4ef6\u5939\u4e0b\uff0c\u4e5f\u6709\u4e00\u4e2a\u548c\u672c\u5730\u8c03\u8bd5\u4e2d\u63cf\u8ff0\u7684 job.json \u6587\u4ef6\u3002 \u540c\u6837\u7684\uff0c\u4e0a\u8ff0 job \u6587\u4ef6\u8fd0\u884c\u6ca1\u6709\u7b26\u5408\u6211\u4eec\u7684\u9884\u671f\uff0c\u731c\u6d4b\u662f streamreader \u8fd9\u4e2a\u63d2\u4ef6\u7684 parseMixupFunctions \u51fd\u6570\u6709\u95ee\u9898\uff0c\u6211\u60f3\u8c03\u8bd5\u770b\u5177\u4f53\u95ee\u9898\u5728\u54ea\u91cc\u3002

    \u6ce8\u610f\uff1a\u8fdc\u7a0b\u8c03\u8bd5\u9700\u8981\u5728\u670d\u52a1\u5668\u6253\u5f00 9999 \u7aef\u53e3\uff0c\u56e0\u6b64\u8981\u4fdd\u8bc1\u670d\u52a1\u5668\u4e0a\u7684 9999 \u7aef\u53e3\u6ca1\u6709\u88ab\u5360\u7528\uff0c\u5982\u679c\u88ab\u5360\u7528\uff0c\u5219\u9700\u8981\u4fee\u6539\u6b64\u7aef\u53e3\u3002

    \u4fee\u6539\u65b9\u5f0f\u5982\u4e0b\uff1a

    1. \u6253\u5f00 bin/addax.sh \u811a\u672c
    2. \u5b9a\u4f4d\u5230\u5927\u7ea624\u884c\u5904\uff0c\u627e\u5230 address=0.0.0.0:9999 \u5b57\u7b26\u4e32
    3. \u5c06 9999 \u4fee\u6539\u6210\u5176\u4ed6\u672a\u88ab\u5360\u7528\u7684\u7aef\u53e3
    4. \u4fdd\u5b58\u9000\u51fa
    "},{"location":"debug/#idea_1","title":"\u914d\u7f6e IDEA","text":"

    \u6253\u5f00 IDEA\u5de5\u5177\uff0c\u5e76\u6253\u5f00 addax \u9879\u76ee\u6e90\u4ee3\u7801\uff0c\u6253\u5f00 plugin/reader/streamreader/StreamReader.java \u6587\u4ef6\uff0c\u627e\u5230 parseMixupFunctions \u51fd\u6570\uff0c\u5e76\u5728\u51fd\u6570\u7533\u660e\u5904\u7684\u70b9\u51fb\u5de6\u4fa7\u8fb9\u7f18\u5904\u589e\u52a0\u65ad\u70b9\u3002\u5982\u4e0b\u56fe\u6240\u793a\uff1a

    \u70b9\u51fb IDEA \u7684 Run->Edit Configurations... \u83dc\u5355\uff0c\u5728\u5f39\u51fa\u7684 Run/Debug Configurations \u7a97\u53e3\uff0c\u70b9\u51fb\u5de6\u4e0a\u89d2\u7684 + \u6309\u94ae\uff0c\u7136\u540e\u9009\u62e9 \u9009\u62e9 Remove JVM Debug \uff0c\u5728\u53f3\u4fa7\u914d\u7f6e\u6846\u4e2d\uff0c\u586b\u5199\u76f8\u5173\u4fe1\u606f\u5982\u4e0b\uff1a

    \u5176\u4ed6\u4fdd\u6301\u4e0d\u53d8\uff0c\u70b9\u51fb Apply \u6309\u94ae\uff0c\u5f97\u5230\u5982\u4e0b\u914d\u7f6e\u4fe1\u606f\uff1a

    \u70b9\u51fb OK \u6309\u94ae\u4fdd\u5b58\u5e76\u8fd4\u56de\u5230 IDEA \u4e3b\u7a97\u53e3

    \u786e\u4fdd\u5728\u7a97\u53e3\u5de5\u5177\u680f\u6709\u7eff\u8272\ud83d\udd28\u7684\u53f3\u4fa7\u9009\u62e9\u7684\u662f\u4e0a\u8ff0\u586b\u5199 Name \u7684\u63cf\u8ff0\u914d\u7f6e\uff0c\u5426\u5219\u5728\u4e0b\u62c9\u6846\u4e2d\u9009\u62e9\u521a\u624d\u7684\u914d\u7f6e\u3002

    "},{"location":"debug/#_6","title":"\u8fd0\u884c\u8c03\u8bd5","text":"

    \u8fd0\u884c\u8fdc\u7a0b\u8c03\u8bd5\u5206\u6210\u4e24\u4e2a\u6b65\u9aa4\uff0c\u4e00\u662f\u542f\u52a8\u7a0b\u5e8f\uff0c\u4e8c\u662f\u8c03\u8bd5\u5de5\u5177\u8054\u63a5\u5230\u8fd0\u884c\u7684\u7a0b\u5e8f\u4e0a\u3002

    \u5728\u8fdc\u7a0b\u670d\u52a1\u5668\u4e0a\u8fd0\u884c\u5982\u4e0b\u547d\u4ee4\uff1a

    bin/addax.sh -d job/job.json

    \u5982\u679c\u8fd0\u884c\u6b63\u5e38\uff0c\u4f1a\u5f97\u5230\u5982\u4e0b\u4fe1\u606f\uff1a

    bin/addax.sh -d job/job.json\nListening for transport dt_socket at address: 9999\n

    \u8868\u793a\u7a0b\u5e8f\u76d1\u542c\u5728 9999 \u7aef\u53e3\u4e0a\uff0c\u7b49\u5f85\u8054\u63a5\u3002

    \u8fd4\u56de IDEA \u7a97\u53e3\uff0c\u70b9\u51fb\u5de5\u5177\u680f\u4e0a\u7eff\u8272 DEBUG \u5c0f\u866b\u6309\u94ae\uff0c\u5f00\u59cb\u8c03\u8bd5\uff0c\u5982\u679c\u8fd0\u884c\u6b63\u5e38\uff0c\u4f1a\u5f97\u5230\u7c7b\u4f3c\u4e0b\u56fe\u7684\u8c03\u8bd5\u7a97\u53e3\uff1a

    "},{"location":"encrypt_password/","title":"\u52a0\u5bc6\u914d\u7f6e\u6587\u4ef6\u7684\u4e2d\u5bc6\u7801","text":"

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u5bc6\u7801\u662f\u660e\u6587\u7684\uff0c\u8fd9\u5e26\u6765\u4e86\u4e00\u5b9a\u7684\u5b89\u5168\u9690\u60a3\uff0c\u4ece 4.0.9 \u7248\u672c\u8d77\uff0c\u6211\u4eec\u589e\u52a0\u4e86\u5bf9\u914d\u7f6e\u6587\u4ef6\u7684\u7684\u5bc6\u7801\u52a0\u5bc6\u529f\u80fd\u3002 \u540c\u65f6\u6211\u4eec\u63d0\u4f9b\u4e86\u4e00\u4e2a shell \u811a\u672c encrypt_password.sh \u6765\u5e2e\u52a9\u4f60\u52a0\u5bc6\u914d\u7f6e\u6587\u4ef6\u7684\u5bc6\u7801\u3002

    \u5047\u5b9a\u4f60\u7684\u539f\u59cb\u5bc6\u7801\u662f 123456\uff0c\u4f60\u5e0c\u671b\u5728\u914d\u7f6e\u6587\u4ef6\u4e2d\u4f7f\u7528\u52a0\u5bc6\u5bc6\u7801\u914d\u7f6e\u3002 \u9996\u5148\u6267\u884c\u4e0b\u9762\u7684\u6307\u4ee4\uff1a

    $ bin/encrypt_password.sh 123456\nThe encrypt string is : '${enc:tFd05jnm1mSq+PEK9t/Rgg==}', you can paste it to json file.\n

    \u4e0a\u8ff0\u7ed3\u679c\u4e2d\u7684 tFd05jnm1mSq+PEK9t/Rgg== \u4e3a 123456 \u7684\u5bc6\u6587\u3002 ${enc: \u5f00\u5934\u662f\u4e3a\u4e86\u8ba9 addax \u77e5\u9053\u8fd9\u662f\u4e00\u4e2a\u52a0\u5bc6\u5bc6\u6587\u3002

    \u4f60\u5c06\u4e0a\u8ff0\u5b57\u7b26\u4e32 ${enc:tFd05jnm1mSq+PEK9t/Rgg==} \u66ff\u6362\u4f60\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u8bbe\u7f6e\u5bc6\u7801\u4e3a 123456 \u7684\u5730\u65b9\u5373\u53ef\u3002

    "},{"location":"plugin_development/","title":"\u63d2\u4ef6\u5f00\u53d1","text":"

    \u672c\u6307\u5357\u4e3b\u8981\u9762\u5411\u90a3\u4e9b\u9700\u8981\u5f00\u53d1\u7b26\u5408\u81ea\u5df1\u9700\u6c42\u7684 Addax \u63d2\u4ef6\u5f00\u53d1\u4eba\u5458\u3002

    "},{"location":"plugin_development/#addax","title":"Addax \u6d41\u7a0b","text":"

    Addax \u8fd0\u884c\u4e00\u4e2a\u4efb\u52a1\u7684\u5927\u81f4\u6d41\u7a0b\u5982\u4e0b\uff1a

    \u542f\u52a8\u6b65\u9aa4\u4e3a\uff1a

    1. \u89e3\u6790\u914d\u7f6e\uff0c\u5305\u62ec job.json\u3001core.json\u3001plugin.json \u4e09\u4e2a\u914d\u7f6e
    2. \u8bbe\u7f6e jobId \u5230 configuration \u5f53\u4e2d
    3. \u542f\u52a8 Engine\uff0c\u901a\u8fc7 Engine.start() \u8fdb\u5165\u542f\u52a8\u7a0b\u5e8f
    4. \u8bbe\u7f6e RUNTIME_MODE \u5230 configuration \u5f53\u4e2d
    5. \u901a\u8fc7 JobContainer \u7684 start() \u65b9\u6cd5\u542f\u52a8
    6. \u4f9d\u6b21\u6267\u884c job \u7684 preHandler()\u3001init()\u3001prepare()\u3001split()\u3001schedule()\u3001post()\u3001postHandle() \u7b49\u65b9\u6cd5\u3002
    7. init() \u65b9\u6cd5\u6d89\u53ca\u5230\u6839\u636e configuration \u6765\u521d\u59cb\u5316 reader \u548c writer \u63d2\u4ef6\uff0c\u8fd9\u91cc\u6d89\u53ca\u5230 jar \u5305\u70ed\u52a0\u8f7d\u4ee5\u53ca\u8c03\u7528\u63d2\u4ef6 init() \u64cd\u4f5c\u65b9\u6cd5\uff0c\u540c\u65f6\u8bbe\u7f6e reader \u548c writer \u7684 configuration \u4fe1\u606f
    8. prepare() \u65b9\u6cd5\u6d89\u53ca\u5230\u521d\u59cb\u5316 reader \u548c writer \u63d2\u4ef6\u7684\u521d\u59cb\u5316\uff0c\u901a\u8fc7\u8c03\u7528\u63d2\u4ef6\u7684 prepare() \u65b9\u6cd5\u5b9e\u73b0\uff0c\u6bcf\u4e2a\u63d2\u4ef6\u90fd\u6709\u81ea\u5df1\u7684 jarLoader\uff0c\u901a\u8fc7\u96c6\u6210 URLClassloader \u5b9e\u73b0\u800c\u6765
    9. split() \u65b9\u6cd5\u901a\u8fc7 adjustChannelNumber() \u65b9\u6cd5\u8c03\u6574 channel \u4e2a\u6570\uff0c\u540c\u65f6\u6267\u884c reader \u548c writer \u6700\u7ec6\u7c92\u5ea6\u7684\u5207\u5206\uff0c\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0cwriter \u7684\u5207\u5206\u7ed3\u679c\u8981\u53c2\u7167 reader \u7684\u5207\u5206\u7ed3\u679c\uff0c\u8fbe\u5230\u5207\u5206\u540e\u6570\u76ee\u76f8\u7b49\uff0c\u624d\u80fd\u6ee1\u8db3 1\uff1a1 \u7684\u901a\u9053\u6a21\u578b
    10. channel \u7684\u8ba1\u6570\u4e3b\u8981\u662f\u6839\u636e byte \u548c record \u7684\u9650\u901f\u6765\u5b9e\u73b0\u7684\uff0c\u5728 split() \u7684\u51fd\u6570\u4e2d\u7b2c\u4e00\u6b65\u5c31\u662f\u8ba1\u7b97 channel \u7684\u5927\u5c0f
    11. split() \u65b9\u6cd5 reader \u63d2\u4ef6\u4f1a\u6839\u636e channel \u7684\u503c\u8fdb\u884c\u62c6\u5206\uff0c\u4f46\u662f\u6709\u4e9b reader \u63d2\u4ef6\u53ef\u80fd\u4e0d\u4f1a\u53c2\u8003 channel \u7684\u503c\uff0cwriter \u63d2\u4ef6\u4f1a\u5b8c\u5168\u6839\u636e reader \u7684\u63d2\u4ef6 1:1 \u8fdb\u884c\u8fd4\u56de
    12. split() \u65b9\u6cd5\u5185\u90e8\u7684 mergeReaderAndWriterTaskConfigs() \u8d1f\u8d23\u5408\u5e76 reader\u3001writer\u3001\u4ee5\u53ca transformer \u4e09\u8005\u5173\u7cfb\uff0c\u751f\u6210 task \u7684\u914d\u7f6e\uff0c\u5e76\u4e14\u91cd\u5199 job.content \u7684\u914d\u7f6e
    13. schedule() \u65b9\u6cd5\u6839\u636e split() \u62c6\u5206\u751f\u6210\u7684 task \u914d\u7f6e\u5206\u914d\u751f\u6210 taskGroup \u5bf9\u8c61\uff0c\u6839\u636e task \u7684\u6570\u91cf\u548c\u5355\u4e2a taskGroup \u652f\u6301\u7684 task \u6570\u91cf\u8fdb\u884c\u914d\u7f6e\uff0c\u4e24\u8005\u76f8\u9664\u5c31\u53ef\u4ee5\u5f97\u51fa taskGroup \u7684\u6570\u91cf
    14. schedule() \u5185\u90e8\u901a\u8fc7 AbstractScheduler \u7684 schedule() \u6267\u884c\uff0c\u7ee7\u7eed\u6267\u884cstartAllTaskGroup() \u65b9\u6cd5\u521b\u5efa\u6240\u6709\u7684 TaskGroupContainer \u7ec4\u7ec7\u76f8\u5173\u7684 task\uff0cTaskGroupContainerRunner \u8d1f\u8d23\u8fd0\u884c TaskGroupContainer \u6267\u884c\u5206\u914d\u7684 task\u3002scheduler \u7684\u5177\u4f53\u5b9e\u73b0\u7c7b\u4e3a ProcessInnerScheduler\u3002
    15. taskGroupContainerExecutorService \u542f\u52a8\u56fa\u5b9a\u7684\u7ebf\u7a0b\u6c60\u7528\u4ee5\u6267\u884c TaskGroupContainerRunner \u5bf9\u8c61\uff0cTaskGroupContainerRunner \u7684 run() \u65b9\u6cd5\u8c03\u7528 taskGroupContainer. start() \u65b9\u6cd5\uff0c\u9488\u5bf9\u6bcf\u4e2a channel \u521b\u5efa\u4e00\u4e2a TaskExecutor\uff0c\u901a\u8fc7 taskExecutor.doStart() \u542f\u52a8\u4efb\u52a1\u3002
    "},{"location":"plugin_development/#_2","title":"\u63d2\u4ef6\u673a\u5236","text":"

    Addax \u4e3a\u4e86\u5e94\u5bf9\u4e0d\u540c\u6570\u636e\u6e90\u7684\u5dee\u5f02\u3001\u540c\u65f6\u63d0\u4f9b\u4e00\u81f4\u5730\u540c\u6b65\u539f\u8bed\u548c\u6269\u5c55\u80fd\u529b\uff0c\u91c7\u7528\u4e86 \u6846\u67b6 + \u63d2\u4ef6 \u7684\u6a21\u5f0f\uff1a

    \u4f5c\u4e3a\u63d2\u4ef6\u5f00\u53d1\u4eba\u5458\uff0c\u5219\u9700\u8981\u5173\u6ce8\u4e24\u4e2a\u95ee\u9898\uff1a

    1. \u6570\u636e\u6e90\u672c\u8eab\u7684\u8bfb\u5199\u6570\u636e\u6b63\u786e\u6027\u3002
    2. \u5982\u4f55\u4e0e\u6846\u67b6\u6c9f\u901a\u3001\u5408\u7406\u6b63\u786e\u5730\u4f7f\u7528\u6846\u67b6\u3002
    "},{"location":"plugin_development/#_3","title":"\u63d2\u4ef6\u89c6\u89d2\u770b\u6846\u67b6","text":""},{"location":"plugin_development/#_4","title":"\u903b\u8f91\u6267\u884c\u6a21\u578b","text":"

    \u63d2\u4ef6\u5f00\u53d1\u8005\u4e0d\u7528\u5173\u5fc3\u592a\u591a\uff0c\u57fa\u672c\u53ea\u9700\u8981\u5173\u6ce8\u7279\u5b9a\u7cfb\u7edf\u8bfb\u548c\u5199\uff0c\u4ee5\u53ca\u81ea\u5df1\u7684\u4ee3\u7801\u5728\u903b\u8f91\u4e0a\u662f\u600e\u6837\u88ab\u6267\u884c\u7684\uff0c\u54ea\u4e00\u4e2a\u65b9\u6cd5\u662f\u5728\u4ec0\u4e48\u65f6\u5019\u88ab\u8c03\u7528\u7684\u3002\u5728\u6b64\u4e4b\u524d\uff0c\u9700\u8981\u660e\u786e\u4ee5\u4e0b\u6982\u5ff5\uff1a

    \u7b80\u800c\u8a00\u4e4b\uff0c Job\u62c6\u5206\u6210Task\uff0c\u5206\u522b\u5728\u6846\u67b6\u63d0\u4f9b\u7684\u5bb9\u5668\u4e2d\u6267\u884c\uff0c\u63d2\u4ef6\u53ea\u9700\u8981\u5b9e\u73b0 Job \u548c Task \u4e24\u90e8\u5206\u903b\u8f91\u3002

    "},{"location":"plugin_development/#_5","title":"\u7f16\u7a0b\u63a5\u53e3","text":"

    \u90a3\u4e48\uff0cJob \u548c Task \u7684\u903b\u8f91\u5e94\u662f\u600e\u4e48\u5bf9\u5e94\u5230\u5177\u4f53\u7684\u4ee3\u7801\u4e2d\u7684\uff1f

    \u9996\u5148\uff0c\u63d2\u4ef6\u7684\u5165\u53e3\u7c7b\u5fc5\u987b\u6269\u5c55 Reader \u6216 Writer \u62bd\u8c61\u7c7b\uff0c\u5e76\u4e14\u5b9e\u73b0\u5206\u522b\u5b9e\u73b0 Job \u548c Task \u4e24\u4e2a\u5185\u90e8\u62bd\u8c61\u7c7b\uff0cJob \u548c Task \u7684\u5b9e\u73b0\u5fc5\u987b\u662f \u5185\u90e8\u7c7b \u7684\u5f62\u5f0f\uff0c\u539f\u56e0\u89c1 \u52a0\u8f7d\u539f\u7406 \u4e00\u8282\u3002\u4ee5 Reader \u4e3a\u4f8b\uff1a

    public class SomeReader\n        extends Reader\n{\n    public static class Job\n            extends Reader.Job\n    {\n        @Override\n        public void init()\n        {\n        }\n\n        @Override\n        public void prepare()\n        {\n        }\n\n        @Override\n        public List<Configuration> split(int adviceNumber)\n        {\n            return null;\n        }\n\n        @Override\n        public void post()\n        {\n        }\n\n        @Override\n        public void destroy()\n        {\n        }\n    }\n\n    public static class Task\n            extends Reader.Task\n    {\n\n        @Override\n        public void init()\n        {\n        }\n\n        @Override\n        public void prepare()\n        {\n        }\n\n        @Override\n        public void startRead(RecordSender recordSender)\n        {\n        }\n\n        @Override\n        public void post()\n        {\n        }\n\n        @Override\n        public void destroy()\n        {\n        }\n    }\n}\n

    Job \u63a5\u53e3\u529f\u80fd\u5982\u4e0b\uff1a

    Task \u63a5\u53e3\u529f\u80fd\u5982\u4e0b\uff1a

    \u9700\u8981\u6ce8\u610f\u7684\u662f\uff1a

    \u6846\u67b6\u6309\u7167\u5982\u4e0b\u7684\u987a\u5e8f\u6267\u884c Job \u548c Task \u7684\u63a5\u53e3\uff1a

    stateDiagram-v2\ndirection TB\nInit:::job --> Prepare:::job\nPrepare --> Split:::job\nSplit --> Schedule:::fw\nstate Schedule {\n    direction LR\n    init\\nprepare\\nstartRead\\npost\\ndestroy1 --> init\\nprepare\\nstartRead\\npost\\ndestroy : Channel\n}\nSchedule --> Post:::job\n\nclassDef job fill:yellow\nclassDef fw fill:#c6fac4\nclassDef ctask fill:blue

    \u4e0a\u56fe\u4e2d\uff0c\u9ec4\u8272\u8868\u793a Job \u90e8\u5206\u7684\u6267\u884c\u9636\u6bb5\uff0c\u7070\u8272\u8868\u793a Task \u90e8\u5206\u7684\u6267\u884c\u9636\u6bb5\uff0c\u7eff\u8272\u8868\u793a\u6846\u67b6\u6267\u884c\u9636\u6bb5\u3002

    \u76f8\u5173\u7c7b\u5173\u7cfb\u5982\u4e0b\uff1a

    %%{init: {\"theme\": \"neutral\"}}%%\nclassDiagram\n    class Pluginable {\n    + init()\n    + destroy()\n    + others()\n    }\n    class AbstractPlugin {\n        + prepare()\n        + post()\n        + others()\n    }\n    class AbstractJobPlugin {\n        + getJobPluginCollector(): JobPluginCollector\n        + setJobPluginCollector(JobPluginCollector)\n    }\n\n    class AbstractTaskPlugin {\n        + getTaskPluginCollector(): TaskPluginCollector\n        + setTaskPluginCollector(TaskPluginCollector)\n    }\n    class Reader_Job {\n        + split(init): List<<Configuration>>\n    }\n\n    class Writer_Job {\n        + split(init): List<<Configuration>>\n    }\n\n    class Reader_Task {\n        + startRead(RecordSender)\n    }\n\n    class Writer_Task {\n        + startWrite(RecordReceiver)\n    }\n\n    AbstractJobPlugin <|-- Reader_Job\n    AbstractJobPlugin <|-- Writer_Job\n\n    AbstractTaskPlugin <|-- Reader_Task\n    AbstractTaskPlugin <|-- Writer_Task\n\n    AbstractPlugin <|-- AbstractJobPlugin\n    AbstractPlugin <|-- AbstractTaskPlugin\n\n    Pluginable <|-- AbstractPlugin\n
    "},{"location":"plugin_development/#_6","title":"\u63d2\u4ef6\u5b9a\u4e49","text":"

    \u5728\u6bcf\u4e2a\u63d2\u4ef6\u7684\u9879\u76ee\u4e2d\uff0c\u90fd\u6709\u4e00\u4e2aplugin.json\u6587\u4ef6\uff0c\u8fd9\u4e2a\u6587\u4ef6\u5b9a\u4e49\u4e86\u63d2\u4ef6\u7684\u76f8\u5173\u4fe1\u606f\uff0c\u5305\u62ec\u5165\u53e3\u7c7b\u3002\u4f8b\u5982\uff1a

    {\n  \"name\": \"mysqlwriter\",\n  \"class\": \"com.wgzhao.addax.plugin.writer.mysqlwriter.MysqlWriter\",\n  \"description\": \"Use Jdbc connect to database, execute insert sql.\",\n  \"developer\": \"wgzhao\"\n}\n
    "},{"location":"plugin_development/#_7","title":"\u6253\u5305\u53d1\u5e03","text":"

    Addax \u4f7f\u7528 assembly \u6253\u5305\uff0c\u6253\u5305\u547d\u4ee4\u5982\u4e0b\uff1a

    mvn clean package\nmvn package assembly:single\n

    Addax \u63d2\u4ef6\u9700\u8981\u9075\u5faa\u7edf\u4e00\u7684\u76ee\u5f55\u7ed3\u6784\uff1a

    ${ADDAX_HOME}\n\u251c\u2500\u2500 bin\n\u2502     \u251c\u2500\u2500 addax.sh\n\u251c\u2500\u2500 conf\n\u2502     \u251c\u2500\u2500 core.json\n\u2502     \u2514\u2500\u2500 logback.xml\n\u251c\u2500\u2500 job\n\u251c\u2500\u2500 lib\n\u2502     \u251c\u2500\u2500 addax-common-<version>.jar\n\u2502     \u251c\u2500\u2500 addax-core-<version>.jar\n\u2502     \u251c\u2500\u2500 addax-rdbms-<version>.jar\n\u2502     \u251c\u2500\u2500 addax-storage-<version>.jar\n\u251c\u2500\u2500 log\n\u251c\u2500\u2500 plugin\n\u2502     \u251c\u2500\u2500 reader\n\u2502     \u2502     \u251c\u2500\u2500 cassandrareader\n\u2502     \u2502     \u2502     \u251c\u2500\u2500 cassandrareader-<version>.jar\n\u2502     \u2502     \u2502     \u251c\u2500\u2500 libs\n\u2502     \u2502     \u2502     \u2502     \u251c\u2500\u2500 <symbol link to shared folder>\n\u2502     \u2502     \u2502     \u251c\u2500\u2500 plugin.json\n\u2502     \u2502     \u2502     \u2514\u2500\u2500 plugin_job_template.json\n\u2502     \u2514\u2500\u2500 writer\n\u2502         \u251c\u2500\u2500 cassandrawriter\n\u2502         \u2502     \u251c\u2500\u2500 cassandrawriter-<version>.jar\n\u2502         \u2502     \u251c\u2500\u2500 libs\n\u2502         \u2502     \u2502     \u251c\u2500\u2500 <symbol link to shared folder>\n\u2502         \u2502     \u251c\u2500\u2500 plugin.json\n\u2502         \u2502     \u2514\u2500\u2500 plugin_job_template.json\n\u251c\u2500\u2500 shared\n

    \u63d2\u4ef6\u76ee\u5f55\u5206\u4e3a reader \u548c writer \u5b50\u76ee\u5f55\uff0c\u8bfb\u5199\u63d2\u4ef6\u5206\u522b\u5b58\u653e\u3002\u63d2\u4ef6\u76ee\u5f55\u89c4\u8303\u5982\u4e0b\uff1a

    \u5c3d\u7ba1\u6846\u67b6\u52a0\u8f7d\u63d2\u4ef6\u65f6\uff0c\u4f1a\u628a ${PLUGIN_HOME} \u4e0b\u6240\u6709\u7684 jar \u5305\u6dfb\u52a0\u5230 classpath \u73af\u5883\u53d8\u91cf\u4e2d\uff0c\u4f46\u8fd8\u662f\u63a8\u8350\u4f9d\u8d56\u5e93\u7684 jar \u548c\u63d2\u4ef6\u672c\u8eab\u7684 jar \u5206\u5f00\u5b58\u653e\u3002

    \u7279\u522b\u63d0\u9192

    \u63d2\u4ef6\u7684\u76ee\u5f55\u540d\u5b57\u5fc5\u987b\u548c plugin.json \u4e2d\u5b9a\u4e49\u7684\u63d2\u4ef6\u540d\u79f0\u4e00\u81f4\u3002

    "},{"location":"plugin_development/#_8","title":"\u914d\u7f6e\u6587\u4ef6","text":"

    Addax \u4f7f\u7528 json \u4f5c\u4e3a\u914d\u7f6e\u6587\u4ef6\u7684\u683c\u5f0f\u3002\u4e00\u4e2a\u5178\u578b\u7684 Addax \u4efb\u52a1\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"postgresqlreader\",\n        \"parameter\": {\n          \"username\": \"pgtest\",\n          \"password\": \"pgtest\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"addax_tbl\"\n            ],\n            \"jdbcUrl\": \"jdbc:postgresql://localhost:5432/pgtest\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"postgresqlwriter\",\n        \"parameter\": {\n          \"column\": [\n            \"*\"\n          ],\n          \"preSql\": [\n            \"truncate table @table\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:postgresql://127.0.0.1:5432/pgtest\",\n            \"table\": [\n              \"addax_tbl1\"\n            ]\n          },\n          \"username\": \"pgtest\",\n          \"password\": \"pgtest\",\n          \"writeMode\": \"insert\"\n        }\n      }\n    }\n  }\n}\n

    Addax \u6846\u67b6\u6709 core.json \u914d\u7f6e\u6587\u4ef6\uff0c\u6307\u5b9a\u4e86\u6846\u67b6\u7684\u9ed8\u8ba4\u884c\u4e3a\u3002\u4efb\u52a1\u7684\u914d\u7f6e\u91cc\u5934\u53ef\u4ee5\u6307\u5b9a\u6846\u67b6\u4e2d\u5df2\u7ecf\u5b58\u5728\u7684\u914d\u7f6e\u9879\uff0c\u800c\u4e14\u5177\u6709\u66f4\u9ad8\u7684\u4f18\u5148\u7ea7\uff0c\u4f1a\u8986\u76d6 core.json \u4e2d\u7684\u9ed8\u8ba4\u503c\u3002

    \u914d\u7f6e\u4e2djob.content.reader.parameter \u7684 value \u90e8\u5206\u4f1a\u4f20\u7ed9 Reader.Job\uff1bjob.content.writer.parameter \u7684 value \u90e8\u5206\u4f1a\u4f20\u7ed9Writer.Job \uff0c Reader.Job \u548c Writer.Job \u53ef\u4ee5\u901a\u8fc7 super.getPluginJobConf() \u6765\u83b7\u53d6\u3002

    "},{"location":"plugin_development/#_9","title":"\u5982\u4f55\u8bbe\u8ba1\u914d\u7f6e\u53c2\u6570","text":"

    \u914d\u7f6e\u6587\u4ef6\u7684\u8bbe\u8ba1\u662f\u63d2\u4ef6\u5f00\u53d1\u7684\u7b2c\u4e00\u6b65\uff01

    \u4efb\u52a1\u914d\u7f6e\u4e2d reader \u548c writer \u4e0b parameter \u90e8\u5206\u662f\u63d2\u4ef6\u7684\u914d\u7f6e\u53c2\u6570\uff0c\u63d2\u4ef6\u7684\u914d\u7f6e\u53c2\u6570\u5e94\u5f53\u9075\u5faa\u4ee5\u4e0b\u539f\u5219\uff1a

    {\n  \"connection\": [\n    {\n      \"table\": [\"table_1\", \"table_2\"],\n      \"jdbcUrl\": [\n        \"jdbc:mysql://127.0.0.1:3306/database_1\",\n        \"jdbc:mysql://127.0.0.2:3306/database_1_slave\"\n      ]\n    },\n    {\n      \"table\": [\"table_3\", \"table_4\"],\n      \"jdbcUrl\": [\n        \"jdbc:mysql://127.0.0.3:3306/database_2\",\n        \"jdbc:mysql://127.0.0.4:3306/database_2_slave\"\n      ]\n    }\n  ]\n}\n
    "},{"location":"plugin_development/#configuration","title":"\u5982\u4f55\u4f7f\u7528 Configuration \u7c7b","text":"

    \u4e3a\u4e86\u7b80\u5316\u5bf9 json \u7684\u64cd\u4f5c\uff0cAddax \u63d0\u4f9b\u4e86\u7b80\u5355\u7684 DSL \u914d\u5408 Configuration \u7c7b\u4f7f\u7528\u3002

    Configuration \u63d0\u4f9b\u4e86\u5e38\u89c1\u7684 get, \u5e26\u7c7b\u578bget\uff0c\u5e26\u9ed8\u8ba4\u503cget\uff0cset \u7b49\u8bfb\u5199\u914d\u7f6e\u9879\u7684\u64cd\u4f5c\uff0c\u4ee5\u53ca clone, toJSON \u7b49\u65b9\u6cd5\u3002\u914d\u7f6e\u9879\u8bfb\u5199\u64cd\u4f5c\u90fd\u9700\u8981\u4f20\u5165\u4e00\u4e2a path \u505a\u4e3a\u53c2\u6570\uff0c \u8fd9\u4e2a path \u5c31\u662f Addax \u5b9a\u4e49\u7684 DSL\u3002\u8bed\u6cd5\u6709\u4e24\u6761\uff1a

    1. \u5b50 map \u7528 .key \u8868\u793a\uff0cpath \u7684\u7b2c\u4e00\u4e2a\u70b9\u7701\u7565\u3002
    2. \u6570\u7ec4\u5143\u7d20\u7528 [index] \u8868\u793a\u3002

    \u6bd4\u5982\u64cd\u4f5c\u5982\u4e0b json\uff1a

    {\n  \"a\": {\n    \"b\": {\n      \"c\": 2\n    },\n    \"f\": [\n      1,\n      2,\n      {\n        \"g\": true,\n        \"h\": false\n      },\n      4\n    ]\n  },\n  \"x\": 4\n}\n

    \u6bd4\u5982\u8c03\u7528 configuration.get(path) \u65b9\u6cd5\uff0c\u5f53 path \u4e3a\u5982\u4e0b\u503c\u7684\u65f6\u5019\u5f97\u5230\u7684\u7ed3\u679c\u4e3a\uff1a

    \u6ce8\u610f\uff0c\u56e0\u4e3a\u63d2\u4ef6\u770b\u5230\u7684\u914d\u7f6e\u53ea\u662f\u6574\u4e2a\u914d\u7f6e\u7684\u4e00\u90e8\u5206\u3002\u4f7f\u7528 Configuration \u5bf9\u8c61\u65f6\uff0c\u9700\u8981\u6ce8\u610f\u5f53\u524d\u7684\u6839\u8def\u5f84\u662f\u4ec0\u4e48\u3002

    \u66f4\u591a Configuration \u7684\u64cd\u4f5c\u8bf7\u53c2\u8003 Configuration.java \u3002

    "},{"location":"plugin_development/#_10","title":"\u63d2\u4ef6\u6570\u636e\u4f20\u8f93","text":"

    \u8ddf\u4e00\u822c\u7684 \u751f\u4ea7\u8005-\u6d88\u8d39\u8005 \u6a21\u5f0f\u4e00\u6837\uff0cReader \u63d2\u4ef6\u548c Writer \u63d2\u4ef6\u4e4b\u95f4\u4e5f\u662f\u901a\u8fc7 channel \u6765\u5b9e\u73b0\u6570\u636e\u7684\u4f20\u8f93\u7684\u3002channel \u53ef\u4ee5\u662f\u5185\u5b58\u7684\uff0c\u4e5f\u53ef\u80fd\u662f\u6301\u4e45\u5316\u7684\uff0c\u63d2\u4ef6\u4e0d\u5fc5\u5173\u5fc3\u3002 \u63d2\u4ef6\u901a\u8fc7 RecordSender \u5f80 channel \u5199\u5165\u6570\u636e\uff0c\u901a\u8fc7 RecordReceiver \u4ece channel \u8bfb\u53d6\u6570\u636e\u3002

    channel \u4e2d\u7684\u4e00\u6761\u6570\u636e\u4e3a\u4e00\u4e2a Record \u7684\u5bf9\u8c61\uff0cRecord \u4e2d\u53ef\u4ee5\u653e\u591a\u4e2a Column \u5bf9\u8c61\uff0c\u8fd9\u53ef\u4ee5\u7b80\u5355\u7406\u89e3\u4e3a\u6570\u636e\u5e93\u4e2d\u7684\u8bb0\u5f55\u548c\u5217\u3002

    Record \u6709\u5982\u4e0b\u65b9\u6cd5\uff1a

    public interface Record\n{\n    // \u52a0\u5165\u4e00\u4e2a\u5217\uff0c\u653e\u5728\u6700\u540e\u7684\u4f4d\u7f6e\n    void addColumn(Column column);\n\n    // \u5728\u6307\u5b9a\u4e0b\u6807\u5904\u653e\u7f6e\u4e00\u4e2a\u5217\n    void setColumn(int i, final Column column);\n\n    // \u83b7\u53d6\u4e00\u4e2a\u5217\n    Column getColumn(int i);\n\n    // \u8f6c\u6362\u4e3ajson String\n    String toString();\n\n    // \u83b7\u53d6\u603b\u5217\u6570\n    int getColumnNumber();\n\n    // \u8ba1\u7b97\u6574\u6761\u8bb0\u5f55\u5728\u5185\u5b58\u4e2d\u5360\u7528\u7684\u5b57\u8282\u6570\n    int getByteSize();\n}\n

    \u56e0\u4e3a Record \u662f\u4e00\u4e2a\u63a5\u53e3\uff0cReader \u63d2\u4ef6\u9996\u5148\u8c03\u7528 RecordSender.createRecord() \u521b\u5efa\u4e00\u4e2a Record \u5b9e\u4f8b\uff0c\u7136\u540e\u628a Column \u4e00\u4e2a\u4e2a\u6dfb\u52a0\u5230 Record \u4e2d\u3002

    Writer \u63d2\u4ef6\u8c03\u7528 RecordReceiver.getFromReader() \u65b9\u6cd5\u83b7\u53d6 Record\uff0c\u7136\u540e\u628a Column \u904d\u5386\u51fa\u6765\uff0c\u5199\u5165\u76ee\u6807\u5b58\u50a8\u4e2d\u3002\u5f53 Reader \u5c1a\u672a\u9000\u51fa\uff0c\u4f20\u8f93\u8fd8\u5728\u8fdb\u884c\u65f6\uff0c\u5982\u679c\u6682\u65f6\u6ca1\u6709\u6570\u636e RecordReceiver.getFromReader() \u65b9\u6cd5\u4f1a\u963b\u585e\u76f4\u5230\u6709\u6570\u636e\u3002 \u5982\u679c\u4f20\u8f93\u5df2\u7ecf\u7ed3\u675f\uff0c\u4f1a\u8fd4\u56denull\uff0cWriter \u63d2\u4ef6\u53ef\u4ee5\u636e\u6b64\u5224\u65ad\u662f\u5426\u7ed3\u675f startWrite \u65b9\u6cd5\u3002

    "},{"location":"plugin_development/#_11","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u4e3a\u4e86\u89c4\u8303\u6e90\u7aef\u548c\u76ee\u7684\u7aef\u7c7b\u578b\u8f6c\u6362\u64cd\u4f5c\uff0c\u4fdd\u8bc1\u6570\u636e\u4e0d\u5931\u771f\uff0cAddax \u652f\u6301\u516d\u79cd\u5185\u90e8\u6570\u636e\u7c7b\u578b\uff1a

    \u5bf9\u5e94\u5730\uff0c\u6709 DateColumn\u3001LongColumn\u3001DoubleColumn\u3001BytesColumn\u3001StringColumn \u3001BoolColumn \u548c TimestampColumn \u4e03\u79cd Column \u7684\u5b9e\u73b0\u3002

    Column \u9664\u4e86\u63d0\u4f9b\u6570\u636e\u76f8\u5173\u7684\u65b9\u6cd5\u5916\uff0c\u8fd8\u63d0\u4f9b\u4e00\u7cfb\u5217\u4ee5 as \u5f00\u5934\u7684\u6570\u636e\u7c7b\u578b\u8f6c\u6362\u8f6c\u6362\u65b9\u6cd5\u3002

    %%{init: {\"theme\": \"neutral\"}}%%\nclassDiagram\ndirection TB\nclass Column {\n    <<interface>>\n    - rawData: Object\n    - type: Type\n    + getRawData(): Object\n    + getType(): Type\n    + getByteSize(): init\n    + asLong(): Long\n    + asDouble(): Doule\n    + asString(): String\n    + asDate(): Date\n    + asBytes(): Bytes\n    + asBigDecimal(): BigDecimal\n    + asBoolean(): Boolean\n}\nColumn <|-- Stringcolumn\nColumn <|-- Doublecolumn\nColumn <|-- Longcolumn\nColumn <|-- Datecolumn\nColumn <|-- Boolcolumn\nColumn <|-- Bytescolumn

    Addax \u7684\u5185\u90e8\u7c7b\u578b\u5728\u5b9e\u73b0\u4e0a\u4f1a\u9009\u7528\u4e0d\u540c\u7684 java \u7c7b\u578b\uff1a

    \u5185\u90e8\u7c7b\u578b \u5b9e\u73b0\u7c7b\u578b \u5907\u6ce8 Date java.util.Date Timestamp java.sql.Timestamp \u53ef\u4ee5\u7cbe\u786e\u5230\u7eb3\u79d2 Long java.math.BigInteger \u4f7f\u7528\u65e0\u9650\u7cbe\u5ea6\u7684\u5927\u6574\u6570\uff0c\u4fdd\u8bc1\u4e0d\u5931\u771f Double java.lang.String \u7528 String \u8868\u793a\uff0c\u4fdd\u8bc1\u4e0d\u5931\u771f Bytes byte[] String java.lang.String Bool java.lang.Boolean

    \u7c7b\u578b\u4e4b\u95f4\u76f8\u4e92\u8f6c\u6362\u7684\u5173\u7cfb\u5982\u4e0b\uff1a

    from/to Date Long Double Bytes String Bool Date - \u4f7f\u7528\u6beb\u79d2\u65f6\u95f4\u6233 \u4e0d\u652f\u6301 \u4e0d\u652f\u6301 \u6309\u914d\u7f6e\u683c\u5f0f\u8f6c\u6362 \u4e0d\u652f\u6301 Long \u4f5c\u4e3a\u6beb\u79d2\u65f6\u95f4\u6233\u6784\u9020 Date - BigDecimal.doubleValue() \u4e0d\u652f\u6301 BigInteger.toString() 0 \u4e3a false\uff0c\u5176\u4ed6\u4e3a true Double \u4e0d\u652f\u6301 BigDecimal.longValue() - \u4e0d\u652f\u6301 \u76f4\u63a5\u8fd4\u56de\u5185\u90e8 String \u4e0d\u652f\u6301 Bytes \u4e0d\u652f\u6301 \u4e0d\u652f\u6301 \u4e0d\u652f\u6301 - \u6309utf-8\u7f16\u7801\u8f6c\u6362\u4e3a byte[] \u4e0d\u652f\u6301 String \u6309\u914d\u7f6e\u7684 \u683c\u5f0f\u89e3\u6790 BigDecimal.longValueBigDecimal.doubleValue1 \u6309 utf-8 \u7f16\u78012\u8f6c\u6362\u4e3a byte[] - \"true\"\u4e3atrue, \"false\"\u4e3afalse\uff0c\u4e0d\u533a\u5206\u5927\u5c0f\u5199\u3002\u5176\u4ed6\u4e0d\u652f\u6301 Bool \u4e0d\u652f\u6301 true\u4e3a1L\uff0c\u5426\u52190L \u4e0d\u652f\u6301 true\u4e3a1.0\uff0c\u5426\u52190.0 \u4e0d\u652f\u6301 -"},{"location":"plugin_development/#_12","title":"\u810f\u6570\u636e\u5904\u7406","text":""},{"location":"plugin_development/#_13","title":"\u4ec0\u4e48\u662f\u810f\u6570\u636e","text":"

    \u76ee\u524d\u4e3b\u8981\u6709\u4e09\u7c7b\u810f\u6570\u636e\uff1a

    1. Reader \u8bfb\u5230\u4e0d\u652f\u6301\u7684\u7c7b\u578b\u3001\u4e0d\u5408\u6cd5\u7684\u503c\u3002
    2. \u4e0d\u652f\u6301\u7684\u7c7b\u578b\u8f6c\u6362\uff0c\u6bd4\u5982\uff1aBytes \u8f6c\u6362\u4e3a Date\u3002
    3. \u5199\u5165\u76ee\u6807\u7aef\u5931\u8d25\uff0c\u6bd4\u5982\uff1a\u5199 MySQL \u6574\u578b\u957f\u5ea6\u8d85\u957f\u3002
    "},{"location":"plugin_development/#_14","title":"\u5982\u4f55\u5904\u7406\u810f\u6570\u636e","text":"

    \u5728 Reader.Task \u548c Writer.Task \u4e2d\uff0c\u901a\u8fc7 AbstractTaskPlugin.getPluginCollector() \u53ef\u4ee5\u62ff\u5230\u4e00\u4e2a TaskPluginCollector\uff0c\u5b83\u63d0\u4f9b\u4e86\u4e00\u7cfb\u5217 collectDirtyRecord \u7684\u65b9\u6cd5\u3002 \u5f53\u810f\u6570\u636e\u51fa\u73b0\u65f6\uff0c\u53ea\u9700\u8981\u8c03\u7528\u5408\u9002\u7684 collectDirtyRecord \u65b9\u6cd5\uff0c\u628a\u88ab\u8ba4\u4e3a\u662f\u810f\u6570\u636e\u7684 Record \u4f20\u5165\u5373\u53ef\u3002

    \u7528\u6237\u53ef\u4ee5\u5728\u4efb\u52a1\u7684\u914d\u7f6e\u4e2d\u6307\u5b9a\u810f\u6570\u636e\u9650\u5236\u6761\u6570\u6216\u8005\u767e\u5206\u6bd4\u9650\u5236\uff0c\u5f53\u810f\u6570\u636e\u8d85\u51fa\u9650\u5236\u65f6\uff0c\u6846\u67b6\u4f1a\u7ed3\u675f\u540c\u6b65\u4efb\u52a1\uff0c\u9000\u51fa\u3002\u63d2\u4ef6\u9700\u8981\u4fdd\u8bc1\u810f\u6570\u636e\u90fd\u88ab\u6536\u96c6\u5230\uff0c\u5176\u4ed6\u5de5\u4f5c\u4ea4\u7ed9\u6846\u67b6\u5c31\u597d\u3002

    "},{"location":"plugin_development/#_15","title":"\u52a0\u8f7d\u539f\u7406","text":"
    1. \u6846\u67b6\u626b\u63cf plugin/reader \u548c plugin/writer\u76ee\u5f55\uff0c\u52a0\u8f7d\u6bcf\u4e2a\u63d2\u4ef6\u7684 plugin.json \u6587\u4ef6\u3002
    2. \u4ee5 plugin.json \u6587\u4ef6\u4e2d name \u4e3a key\uff0c\u7d22\u5f15\u6240\u6709\u7684\u63d2\u4ef6\u914d\u7f6e\u3002\u5982\u679c\u53d1\u73b0\u91cd\u540d\u7684\u63d2\u4ef6\uff0c\u6846\u67b6\u4f1a\u5f02\u5e38\u9000\u51fa\u3002
    3. \u7528\u6237\u5728\u63d2\u4ef6\u4e2d\u5728 reader/writer \u914d\u7f6e\u7684 name \u5b57\u6bb5\u6307\u5b9a\u63d2\u4ef6\u540d\u5b57\u3002\u6846\u67b6\u6839\u636e\u63d2\u4ef6\u7684\u7c7b\u578b\uff08reader/writer\uff09\u548c\u63d2\u4ef6\u540d\u79f0\u53bb\u63d2\u4ef6\u7684\u8def\u5f84\u4e0b\u626b\u63cf\u6240\u6709\u7684 jar\uff0c\u52a0\u5165 classpath\u3002
    4. \u6839\u636e\u63d2\u4ef6\u914d\u7f6e\u4e2d\u5b9a\u4e49\u7684\u5165\u53e3\u7c7b\uff0c\u6846\u67b6\u901a\u8fc7\u53cd\u5c04\u5b9e\u4f8b\u5316\u5bf9\u5e94\u7684 Job \u548c Task \u5bf9\u8c61\u3002

    1. \u5904\u7406 NaN, Infinity, -Infinity \u7b49\u6570\u503c\u00a0\u21a9

    2. \u9664\u975e\u53e6\u6709\u6307\u5b9a\u7f16\u7801\u683c\u5f0f\u00a0\u21a9

    "},{"location":"quickstart/","title":"\u5feb\u901f\u4f7f\u7528","text":""},{"location":"quickstart/#addax","title":"\u5b89\u88c5 Addax","text":""},{"location":"quickstart/#docker","title":"\u4f7f\u7528 Docker \u955c\u50cf","text":"

    \u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528 Docker \u955c\u50cf\uff0c\u53ea\u9700\u8981\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u5373\u53ef

    docker run -it --rm quay.io/wgzhao/addax:latest /opt/addax/bin/addax.sh /opt/addax/job/job.json\n
    "},{"location":"quickstart/#_2","title":"\u4e00\u952e\u5b89\u88c5","text":"

    \u5982\u679c\u4f60\u4e0d\u60f3\u7f16\u8bd1\uff0c\u4f60\u53ef\u4ee5\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\uff0c\u4e00\u952e\u5b89\u88c5\uff08\u5f53\u524d\u4ec5\u652f\u6301 Linux \u548c macOS \uff09

    /bin/bash -c \"$(curl -fsSL https://raw.githubusercontent.com/wgzhao/Addax/master/install.sh)\"\n

    \u5982\u679c\u662f macOS \uff0c\u9ed8\u8ba4\u5b89\u88c5\u5728 /usr/local/addax \u76ee\u5f55\u4e0b\uff0c \u5982\u679c\u662f Linux\uff0c \u5219\u5b89\u88c5\u5728 /opt/addax \u76ee\u5f55\u4e0b

    "},{"location":"quickstart/#_3","title":"\u6e90\u4ee3\u7801\u7f16\u8bd1\u5b89\u88c5","text":"

    \u4f60\u53ef\u4ee5\u9009\u62e9\u4ece\u6e90\u4ee3\u7801\u7f16\u8bd1\u5b89\u88c5\uff0c\u57fa\u672c\u64cd\u4f5c\u5982\u4e0b\uff1a

    git clone https://github.com/wgzhao/addax.git\ncd addax\nmvn clean package\nmvn package assembly:single\ncd target/addax/addax-<version>\n
    "},{"location":"quickstart/#_4","title":"\u5f00\u59cb\u7b2c\u4e00\u4e2a\u91c7\u96c6\u4efb\u52a1","text":"

    \u8981\u4f7f\u7528 Addax \u8fdb\u884c\u6570\u636e\u91c7\u96c6\uff0c\u53ea\u9700\u8981\u7f16\u5199\u4e00\u4e2a\u4efb\u52a1\u91c7\u96c6\u6587\u4ef6\uff0c\u8be5\u6587\u4ef6\u4e3a JSON \u683c\u5f0f\uff0c\u4ee5\u4e0b\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u914d\u7f6e\u6587\u4ef6\uff0c\u8be5\u4efb\u52a1\u7684\u76ee\u7684\u662f\u4ece\u5185\u5b58\u8bfb\u53d6\u8bfb\u53d6\u6307\u5b9a\u5185\u5bb9\u7684\u6570\u636e\uff0c\u5e76\u5c06\u5176\u6253\u5370\u51fa\u6765\uff0c\u6587\u4ef6\u4fdd\u5b58\u5728 job/test.json \u4e2d

    job/test.json 
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 00:00:00\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u6587\u4ef6\u4fdd\u5b58\u4e3a job/test.json

    \u7136\u540e\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\uff1a

    bin/addax.sh job/test.json\n

    \u5982\u679c\u6ca1\u6709\u62a5\u9519\uff0c\u5e94\u8be5\u4f1a\u6709\u7c7b\u4f3c\u8fd9\u6837\u7684\u8f93\u51fa

      ___      _     _\n / _ \\    | |   | |\n/ /_\\ \\ __| | __| | __ ___  __\n|  _  |/ _` |/ _` |/ _` \\ \\/ /\n| | | | (_| | (_| | (_| |>  <\n\\_| |_/\\__,_|\\__,_|\\__,_/_/\\_\\\n\n:: Addax version ::    (v4.0.3-SNAPSHOT)\n\n2021-08-23 13:45:17.199 [        main] INFO  VMInfo               - VMInfo# operatingSystem class => com.sun.management.internal.OperatingSystemImpl\n2021-08-23 13:45:17.223 [        main] INFO  Engine               -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"column\":[\n                        {\n                            \"type\":\"string\",\n                            \"value\":\"addax\"\n                        },\n                        {\n                            \"type\":\"long\",\n                            \"value\":19890604\n                        },\n                        {\n                            \"type\":\"date\",\n                            \"value\":\"1989-06-04 00:00:00\"\n                        },\n                        {\n                            \"type\":\"bool\",\n                            \"value\":true\n                        }\n                    ],\n                    \"sliceRecordCount\":10\n                },\n                \"name\":\"streamreader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":true\n                },\n                \"name\":\"streamwriter\"\n            }\n        }\n}\n\n2021-08-23 13:45:17.238 [        main] INFO  PerfTrace            - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-08-23 13:45:17.239 [        main] INFO  JobContainer         - Addax jobContainer starts job.\n2021-08-23 13:45:17.240 [        main] INFO  JobContainer         - Set jobId = 0\n2021-08-23 13:45:17.250 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do prepare work .\n2021-08-23 13:45:17.250 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do prepare work .\n2021-08-23 13:45:17.251 [       job-0] INFO  JobContainer         - Job set Channel-Number to 1 channels.\n2021-08-23 13:45:17.251 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] splits to [1] tasks.\n2021-08-23 13:45:17.252 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] splits to [1] tasks.\n2021-08-23 13:45:17.276 [       job-0] INFO  JobContainer         - Scheduler starts [1] taskGroups.\n2021-08-23 13:45:17.282 [ taskGroup-0] INFO  TaskGroupContainer   - taskGroupId=[0] start [1] channels for [1] tasks.\n2021-08-23 13:45:17.287 [ taskGroup-0] INFO  Channel              - Channel set byte_speed_limit to -1, No bps activated.\n2021-08-23 13:45:17.288 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\naddax   19890604    1989-06-04 00:00:00 true\n2021-08-23 13:45:20.295 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2021-08-23 13:45:20.296 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do post work.\n2021-08-23 13:45:20.297 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do post work.\n2021-08-23 13:45:20.302 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2021-08-23 13:45:20.305 [       job-0] INFO  StandAloneJobContainerCommunicator - Total 10 records, 220 bytes | Speed 73B/s, 3 records/s | Error 0 records, 0 bytes |  All Task WaitWriterTime 0.000s |  All Task WaitReaderTime 0.011s | Percentage 100.00%\n2021-08-23 13:45:20.307 [       job-0] INFO  JobContainer         -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-08-23 13:45:17\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-08-23 13:45:20\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :               73B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :              3rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                  10\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"setupJob/","title":"\u4efb\u52a1\u914d\u7f6e","text":"

    \u4e00\u4e2a\u91c7\u96c6\u4efb\u52a1\u5c31\u662f\u4e00\u4e2a JSON \u683c\u5f0f\u914d\u7f6e\u6587\u4ef6\uff0c\u8be5\u914d\u7f6e\u6587\u4ef6\u7684\u6a21\u677f\u5982\u4e0b\uff1a

    {\n  \"job\": {\n    \"settings\": {},\n    \"content\": {\n      \"reader\": {},\n      \"writer\": {},\n      \"transformer\": []\n    }\n  }\n}\n

    \u4efb\u52a1\u914d\u7f6e\u7531 key \u4e3a job \u7684\u5b57\u5178\u7ec4\u6210\uff0c\u5176\u5b57\u5178\u5143\u7d20\u7531\u4e09\u90e8\u5206\u7ec4\u6210\uff1a

    "},{"location":"setupJob/#reader","title":"reader \u914d\u7f6e\u9879","text":"

    reader \u914d\u7f6e\u9879\u4f9d\u636e\u4e0d\u540c\u7684 reader \u63d2\u4ef6\u800c\u6709\u4e9b\u5fae\u4e0d\u540c\uff0c\u4f46\u5927\u90e8\u5206\u7684\u914d\u7f6e\u57fa\u672c\u76f8\u540c\uff0c\u7279\u522b\u662f\u9488\u5bf9\u5173\u7cfb\u578b\u6570\u636e\u5e93\u800c\u8a00\uff0c\u5176\u57fa\u672c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"name\": \"mysqlreader\",\n  \"parameter\": {\n    \"username\": \"\",\n    \"password\": \"\",\n    \"column\": [],\n    \"autoPk\": false,\n    \"splitPk\": \"\",\n    \"connection\": [\n      {\n        \"jdbcUrl\": [],\n        \"table\": []\n      }\n    ],\n    \"where\": \"\"\n  }\n}\n

    \u5176\u4e2d name \u662f\u63d2\u4ef6\u7684\u540d\u79f0\uff0c\u6bcf\u4e2a\u63d2\u4ef6\u7684\u540d\u79f0\u90fd\u662f\u552f\u4e00\u7684\uff0c\u6bcf\u4e2a\u63d2\u4ef6\u66f4\u8be6\u7ec6\u7684\u914d\u7f6e\u53ef\u4ee5\u53c2\u8003\u8bfb\u53d6\u63d2\u4ef6\u7ae0\u8282\u7684\u5404\u63d2\u4ef6\u5185\u5bb9

    "},{"location":"setupJob/#writer","title":"writer \u914d\u7f6e\u9879","text":"

    writer \u914d\u7f6e\u9879\u548c reader \u914d\u7f6e\u9879\u5dee\u4e0d\u591a\uff0c\u5176\u57fa\u672c\u6a21\u677f\u5982\u4e0b\uff1a

    {\n  \"name\": \"mysqlwriter\",\n  \"parameter\": {\n    \"username\": \"\",\n    \"password\": \"\",\n    \"writeMode\": \"\",\n    \"column\": [],\n    \"session\": [],\n    \"preSql\": [],\n    \"postSql\": [],\n    \"connection\": [\n      {\n        \"jdbcUrl\": \"\",\n        \"table\": []\n      }\n    ]\n  }\n}\n

    \u540c\u6837\u7684\uff0c\u8fd9\u91cc\u7684 name \u4e5f\u662f\u552f\u4e00\u7684\uff0c\u6bcf\u4e2a\u63d2\u4ef6\u66f4\u8be6\u7ec6\u7684\u914d\u7f6e\u53ef\u4ee5\u53c2\u8003\u5199\u5165\u63d2\u4ef6\u7ae0\u8282\u7684\u5404\u63d2\u4ef6\u5185\u5bb9

    "},{"location":"setupJob/#settings","title":"settings \u914d\u7f6e\u9879","text":"

    settings \u53ef\u914d\u7f6e\u7684\u5185\u5bb9\u5982\u4e0b\uff1a

    {\n  \"speed\": {\n    \"byte\": -1,\n    \"record\": 100,\n    \"channel\": 1\n  },\n  \"errorLimit\": {\n    \"record\": 0,\n    \"percentage\": 0.02\n  }\n}\n

    \u89e3\u91ca\u5982\u4e0b\uff1a

    "},{"location":"setupJob/#speed","title":"speed","text":"

    \u987e\u540d\u601d\u4e49\uff0c\u8fd9\u91cc\u662f\u7528\u6765\u505a\u6d41\u63a7\u7684\u914d\u7f6e\u9879\uff0c\u53ef\u4ee5\u4ece\u7f51\u7edc\u4f20\u8f93\u901f\u5ea6\uff0c\u6bcf\u79d2\u7684\u8bb0\u5f55\u6570\u4ee5\u53ca\u7ebf\u7a0b\u6570\u4e0a\u505a\u63a7\u5236\uff0c\u5206\u522b\u63cf\u8ff0\u5982\u4e0b\uff1a

    "},{"location":"setupJob/#speedbyte","title":"speed.byte","text":"

    \u8bbe\u7f6e\u6bcf\u79d2\u53ef\u83b7\u53d6\u7684\u5b57\u8282\u6570(Bps)\uff0c\u4e00\u822c\u662f\u4e3a\u4e86\u9632\u6b62\u6267\u884c\u4efb\u52a1\u65f6\u5c06\u6574\u4e2a\u5e26\u5bbd\u8dd1\u6ee1\uff0c\u4ece\u800c\u5f71\u54cd\u5230\u5176\u4ed6\u670d\u52a1\u3002\u5982\u679c\u4e0d\u505a\u9650\u5236\uff0c\u53ef\u8bbe\u7f6e\u4e3a -1

    "},{"location":"setupJob/#speedrecord","title":"speed.record","text":"

    \u8bbe\u7f6e\u8bb0\u5f55\u6bcf\u79d2\u53ef\u83b7\u53d6\u7684\u6700\u5927\u8bb0\u5f55\u6761\u6570\uff0c\u8be5\u53c2\u6570\u9700\u8981\u548c speed.byte \u914d\u5408\u4f7f\u7528

    "},{"location":"setupJob/#speedchannel","title":"speed.channel","text":"

    \u8bbe\u7f6e\u901a\u9053\u6570\uff0c\u8be5\u901a\u9053\u8def\u786e\u5b9a\u4e86\u603b\u7684 Task \u7ebf\u7a0b\u6570\uff0c\u5047\u5b9a\u8bbe\u5b9a speed.channel \u4e3a 13\uff0c \u5219\u4e00\u5171\u6709 13 \u4e2a Task. \u7136\u540e\u6839\u636e conf/core.json \u914d\u7f6e\u4e2d\u7684 taskGroup.channel \u914d\u7f6e\u6765\u786e\u5b9a\u8981\u521b\u5efa\u7684 taskGroup \u6570\u91cf\u3002\u5373 taskGroup = speed.channel / taskGroup.channel.

    "},{"location":"setupJob/#errorlimit","title":"errorLimit","text":"

    errorLimit \u7528\u6765\u914d\u7f6e\u5728\u6570\u636e\u5199\u5165\u62a5\u9519\u65f6\u7684\u884c\u4e3a\uff0c\u5177\u4f53\u5982\u4e0b

    "},{"location":"setupJob/#errorlimitrecord","title":"errorLimit.record","text":"

    \u5141\u8bb8\u9519\u8bef\u7684\u8bb0\u5f55\u6761\u6570\uff0c\u5982\u679c\u8d85\u8fc7\u8fd9\u4e2a\u6570\uff0c\u5219\u8ba4\u4e3a\u672c\u6b21\u4efb\u52a1\u5931\u8d25\uff0c\u5426\u5219\u8ba4\u4e3a\u6210\u529f

    "},{"location":"setupJob/#errorlimitpercentage","title":"errorLimit.percentage","text":"

    \u5141\u8bb8\u9519\u8bef\u8bb0\u5f55\u7684\u6bd4\u7387\uff0c\u8d85\u8fc7\u8fd9\u4e2a\u6bd4\u7387\uff0c\u5219\u8ba4\u4e3a\u672c\u6b21\u4efb\u52a1\u5931\u8d25\uff0c\u5426\u5219\u8ba4\u4e3a\u6210\u529f

    \u6ce8\u610f\uff0c\u4e0a\u8ff0\u53c2\u6570\u5728 conf/core.json \u914d\u7f6e\u6587\u4ef6\u5747\u6709\u9ed8\u8ba4\u914d\u7f6e\uff0c\u7528\u6765\u63a7\u5236\u5168\u5c40\u7684\u8bbe\u7f6e\u3002

    "},{"location":"statsreport/","title":"\u4efb\u52a1\u7ed3\u679c\u4e0a\u62a5","text":""},{"location":"statsreport/#_2","title":"\u5feb\u901f\u4ecb\u7ecd","text":"

    \u4e3b\u8981\u7528\u4e8e\u5c06\u5b9a\u65f6\u4efb\u52a1\u7684\u7ed3\u679c\u4e0a\u62a5\u7ed9\u6307\u5b9a\u670d\u52a1\u5668\uff0cAddax \u4f7f\u7528 HTTP \u534f\u8bae\u4ee5 POST \u65b9\u5f0f\u628a\u4e0a\u62a5\u6570\u636e\u4ee5 JSON \u683c\u5f0f\u53d1\u9001\u7ed9\u6307\u5b9a\u7684\u670d\u52a1\u5668\u3002

    \u53d1\u9001\u7684\u6570\u636e\u6837\u4f8b\u5982\u4e0b\uff1a

    {\n  \"jobName\": \"test\",\n  \"startTimeStamp\": 1587971621,\n  \"endTimeStamp\": 1587971621,\n  \"totalCosts\": 10,\n  \"byteSpeedPerSecond\": 33,\n  \"recordSpeedPerSecond\": 1,\n  \"totalReadRecords\": 6,\n  \"totalErrorRecords\": 0,\n  \"jobContent\": {\n    ...\n  }\n}\n
    \u4e0a\u8ff0\u53c2\u6570\u8bf4\u660e\u5982\u4e0b\uff1a

    \u53c2\u6570 \u63cf\u8ff0 \u5fc5\u9009 \u9ed8\u8ba4\u503c jobName \u4efb\u52a1\u540d \u662f jobName startTimeStamp \u4efb\u52a1\u6267\u884c\u7684\u5f00\u59cb\u65f6\u95f4 \u662f \u65e0 endTimeStamp \u4efb\u52a1\u6267\u884c\u7684\u7ed3\u675f\u65f6\u95f4 \u662f \u65e0 totalCosts \u4efb\u52a1\u603b\u8ba1\u8017\u65f6 \u662f \u65e0 byteSpeedPerSecond \u4efb\u52a1\u5e73\u5747\u6d41\u91cf \u662f \u65e0 recordSpeedPerSecond \u8bb0\u5f55\u5199\u5165\u901f\u5ea6 \u662f \u65e0 totalReadRecords \u8bfb\u51fa\u8bb0\u5f55\u603b\u6570 \u662f 0 totalErrorRecords \u8bfb\u5199\u5931\u8d25\u603b\u6570 \u662f 0 jobContent \u672c\u6b21\u4efb\u52a1\u7684json\u6587\u4ef6 \u662f \u65e0

    jobName \u7684\u8bbe\u7f6e\u89c4\u5219\u5982\u4e0b:

    1. \u5728\u547d\u4ee4\u884c\u901a\u8fc7\u4f20\u9012 -P\"-DjobName=xxxx\" \u65b9\u5f0f\u6307\u5b9a,\u5426\u5219
    2. \u914d\u7f6e\u6587\u4ef6\u7684 writer.parameters.path \u503c\u6309 / \u5206\u5272\u540e\u53d6\u7b2c2\uff0c3\u5217\u7528\u70b9(.)\u62fc\u63a5\u800c\u6210\uff0c\u5176\u542b\u4e49\u662f\u4e3a\u5e93\u540d\u53ca\u8868\u540d,\u5426\u5219
    3. \u5426\u5219\u8bbe\u7f6e\u4e3a jobName
    "},{"location":"transformer/","title":"\u6570\u636e\u8f6c\u6362","text":""},{"location":"transformer/#transformer","title":"Transformer \u5b9a\u4e49","text":"

    \u5728\u6570\u636e\u540c\u6b65\u3001\u4f20\u8f93\u8fc7\u7a0b\u4e2d\uff0c\u5b58\u5728\u7528\u6237\u5bf9\u4e8e\u6570\u636e\u4f20\u8f93\u8fdb\u884c\u7279\u6b8a\u5b9a\u5236\u5316\u7684\u9700\u6c42\u573a\u666f\uff0c\u5305\u62ec\u88c1\u526a\u5217\u3001\u8f6c\u6362\u5217\u7b49\u5de5\u4f5c\uff0c\u53ef\u4ee5\u501f\u52a9ETL\u7684T\u8fc7\u7a0b\u5b9e\u73b0(Transformer)\u3002Addax\u5305\u542b\u4e86\u5b8c\u6210\u7684E(Extract)\u3001T(Transformer)\u3001L(Load)\u652f\u6301\u3002

    "},{"location":"transformer/#_2","title":"\u8fd0\u884c\u6a21\u578b","text":"
    graph LR\nsource((\"source\"))\nsubgraph fr[\"Addax Framework\"]\n    direction LR\n    Reader ==> Transformer ==>Writer\nend\ntarget((\"target\"))\nsource ==> fr ==> target
    "},{"location":"transformer/#udf","title":"UDF \u51fd\u6570","text":""},{"location":"transformer/#dx_substr","title":"dx_substr","text":"

    dx_substr(idx, pos, length) -> str

    \u53c2\u6570

    \u8fd4\u56de\uff1a \u4ece\u5b57\u7b26\u4e32\u7684\u6307\u5b9a\u4f4d\u7f6e\uff08\u5305\u542b\uff09\u622a\u53d6\u6307\u5b9a\u957f\u5ea6\u7684\u5b57\u7b26\u4e32\u3002\u5982\u679c\u5f00\u59cb\u4f4d\u7f6e\u975e\u6cd5\u629b\u51fa\u5f02\u5e38\u3002\u5982\u679c\u5b57\u6bb5\u4e3a\u7a7a\u503c\uff0c\u76f4\u63a5\u8fd4\u56de\uff08\u5373\u4e0d\u53c2\u4e0e\u672ctransformer\uff09

    "},{"location":"transformer/#dx_pad","title":"dx_pad","text":"

    dx_pad(idx, flag, length, chr)

    \u53c2\u6570

    \u8fd4\u56de\uff1a \u5982\u679c\u6e90\u5b57\u7b26\u4e32\u957f\u5ea6\u5c0f\u4e8e\u76ee\u6807\u5b57\u6bb5\u957f\u5ea6\uff0c\u6309\u7167\u4f4d\u7f6e\u6dfb\u52a0pad\u5b57\u7b26\u540e\u8fd4\u56de\u3002\u5982\u679c\u957f\u4e8e\uff0c\u76f4\u63a5\u622a\u65ad\uff08\u90fd\u622a\u53f3\u8fb9\uff09\u3002\u5982\u679c\u5b57\u6bb5\u4e3a\u7a7a\u503c\uff0c\u8f6c\u6362\u4e3a\u7a7a\u5b57\u7b26\u4e32\u8fdb\u884cpad\uff0c\u5373\u6700\u540e\u7684\u5b57\u7b26\u4e32\u5168\u662f\u9700\u8981pad\u7684\u5b57\u7b26

    \u4e3e\u4f8b\uff1a

    "},{"location":"transformer/#dx_replace","title":"dx_replace","text":"

    dx_replace(idx, pos, length, str) -> str

    \u53c2\u6570

    \u8fd4\u56de\uff1a \u4ece\u5b57\u7b26\u4e32\u7684\u6307\u5b9a\u4f4d\u7f6e\uff08\u5305\u542b\uff09\u66ff\u6362\u6307\u5b9a\u957f\u5ea6\u7684\u5b57\u7b26\u4e32\u3002\u5982\u679c\u5f00\u59cb\u4f4d\u7f6e\u975e\u6cd5\u629b\u51fa\u5f02\u5e38\u3002\u5982\u679c\u5b57\u6bb5\u4e3a\u7a7a\u503c\uff0c\u76f4\u63a5\u8fd4\u56de\uff08\u5373\u4e0d\u53c2\u4e0e\u672ctransformer\uff09

    \u4e3e\u4f8b\uff1a

    "},{"location":"transformer/#dx_filter","title":"dx_filter","text":"

    dx_filter(idx, operator, expr) -> str

    \u53c2\u6570\uff1a

    \u8fd4\u56de\uff1a

    \u4e3e\u4f8b

    \u5173\u8054filter\u6682\u4e0d\u652f\u6301\uff0c\u5373\u591a\u4e2a\u5b57\u6bb5\u7684\u8054\u5408\u5224\u65ad\uff0c\u51fd\u53c2\u592a\u8fc7\u590d\u6742\uff0c\u7528\u6237\u96be\u4ee5\u4f7f\u7528\u3002

    "},{"location":"transformer/#dx_groovy","title":"dx_groovy","text":"

    dx_groovy(code, package) -> record

    \u53c2\u6570

    \u8fd4\u56de

    Record \u6570\u636e\u7c7b\u578b

    \u6ce8\u610f\uff1a

    \u4e3e\u4f8b:

    groovy \u5b9e\u73b0\u7684 subStr

    String code=\"Column column = record.getColumn(1);\\n\"+\n        \" String oriValue = column.asString();\\n\"+\n        \" String newValue = oriValue.substring(0, 3);\\n\"+\n        \" record.setColumn(1, new StringColumn(newValue));\\n\"+\n        \" return record;\";\n        dx_groovy(record);\n

    groovy \u5b9e\u73b0\u7684Replace

    String code2=\"Column column = record.getColumn(1);\\n\"+\n        \" String oriValue = column.asString();\\n\"+\n        \" String newValue = \\\"****\\\" + oriValue.substring(3, oriValue.length());\\n\"+\n        \" record.setColumn(1, new StringColumn(newValue));\\n\"+\n        \" return record;\";\n

    groovy \u5b9e\u73b0\u7684Pad

    String code3=\"Column column = record.getColumn(1);\\n\"+\n        \" String oriValue = column.asString();\\n\"+\n        \" String padString = \\\"12345\\\";\\n\"+\n        \" String finalPad = \\\"\\\";\\n\"+\n        \" int NeedLength = 8 - oriValue.length();\\n\"+\n        \"        while (NeedLength > 0) {\\n\"+\n        \"\\n\"+\n        \"            if (NeedLength >= padString.length()) {\\n\"+\n        \"                finalPad += padString;\\n\"+\n        \"                NeedLength -= padString.length();\\n\"+\n        \"            } else {\\n\"+\n        \"                finalPad += padString.substring(0, NeedLength);\\n\"+\n        \"                NeedLength = 0;\\n\"+\n        \"            }\\n\"+\n        \"        }\\n\"+\n        \" String newValue= finalPad + oriValue;\\n\"+\n        \" record.setColumn(1, new StringColumn(newValue));\\n\"+\n        \" return record;\";\n

    \u4ece 4.1.2 \u7248\u672c\u5f00\u59cb\uff0c dx_groovy \u652f\u6301\u4ece\u5916\u90e8\u6587\u4ef6\u52a0\u8f7d groovy \u4ee3\u7801\uff0c\u8bfb\u53d6\u6587\u4ef6\u7684\u76f8\u5bf9\u8def\u5f84\u4e3a $ADDAX_HOME \u53d8\u91cf\u6240\u5728\u7684\u76ee\u5f55\uff0c\u4e5f\u5c31\u662f Addax \u7684\u5b89\u88c5\u76ee\u5f55\u3002

    \u4ee5\u5b9e\u73b0 subStr \u4e3a\u4f8b\uff0c\u6211\u4eec\u53ef\u4ee5\u521b\u5efa job/substr.groovy \u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    Column column = record.getColumn(1)\nString oriValue = column.asString()\nString newValue = oriValue.substring(0, 3)\nrecord.setColumn(1, new StringColumn(newValue))\nreturn record\n

    \u7136\u540e\u5728 job \u6587\u4ef6\u4e2d\u8fd9\u6837\u53bb\u5b9a\u4e49\uff1a

    {\n  \"transformer\": [\n    {\n      \"name\": \"dx_groovy\",\n      \"parameter\": {\n        \"codeFile\": \"job/substr.groovy\"\n      }\n    }\n  ]\n}\n

    \u6587\u4ef6\u4e5f\u53ef\u4ee5\u4f7f\u7528\u7edd\u5bf9\u8def\u5f84\u6765\u6307\u5b9a\u3002

    "},{"location":"transformer/#job","title":"Job\u5b9a\u4e49","text":"

    \u672c\u4f8b\u4e2d\uff0c\u914d\u7f6e4\u4e2aUDF\u3002

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"My name is xxxx\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"password is Passw0rd\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 00:00:00\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            },\n            {\n              \"random\": \"0,10\",\n              \"type\": \"long\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true,\n          \"encoding\": \"UTF-8\"\n        }\n      },\n      \"transformer\": [\n        {\n          \"name\": \"dx_replace\",\n          \"parameter\": {\n            \"columnIndex\": 0,\n            \"paras\": [\n              \"11\",\n              \"6\",\n              \"wgzhao\"\n            ]\n          }\n        },\n        {\n          \"name\": \"dx_substr\",\n          \"parameter\": {\n            \"columnIndex\": 1,\n            \"paras\": [\n              \"0\",\n              \"12\"\n            ]\n          }\n        },\n        {\n          \"name\": \"dx_map\",\n          \"parameter\": {\n            \"columnIndex\": 2,\n            \"paras\": [\n              \"^\",\n              \"2\"\n            ]\n          }\n        },\n        {\n          \"name\": \"dx_filter\",\n          \"parameter\": {\n            \"columnIndex\": 6,\n            \"paras\": [\n              \"<\",\n              \"5\"\n            ]\n          }\n        }\n      ]\n    }\n  }\n}\n
    "},{"location":"transformer/#_3","title":"\u81ea\u5b9a\u4e49\u51fd\u6570","text":"

    \u5982\u679c\u81ea\u5e26\u7684\u51fd\u6570\u4e0d\u6ee1\u8db3\u6570\u636e\u8f6c\u6362\u8981\u6c42\uff0c\u6211\u4eec\u53ef\u4ee5\u5728 transformer \u7f16\u5199\u6ee1\u8db3 groovy \u89c4\u8303\u8981\u6c42\u7684\u4ee3\u7801\uff0c\u4e0b\u9762\u7ed9\u51fa\u4e00\u4e2a\u5b8c\u6574\u7684\u4f8b\u5b50

    \n

    \u4e0a\u8ff0 transformer \u4ee3\u7801\u9488\u5bf9\u6bcf\u6761\u8bb0\u5f55\u7684\u524d\u9762\u4e24\u4e2a\u5b57\u6bb5\u505a\u4e86\u4fee\u6539\uff0c\u5bf9\u7b2c\u4e00\u4e2a\u5b57\u6bb5\u7684\u5b57\u7b26\u4e32\uff0c\u5728\u5b57\u7b26\u4e32\u524d\u9762\u589e\u52a0 Header_ \u5b57\u7b26\uff1b \u7b2c\u4e8c\u4e2a\u6574\u6570\u5b57\u6bb5\u503c\u8fdb\u884c\u500d\u589e\u5904\u7406\u3002\u6700\u540e\u6267\u884c\u7684\u7ed3\u679c\u5982\u4e0b\uff1a

    $ bin/addax.sh job/transformer_demo.json \n\n  ___      _     _            \n / _ \\    | |   | |           \n/ /_\\ \\ __| | __| | __ ___  __\n|  _  |/ _` |/ _` |/ _` \\ \\/ /\n| | | | (_| | (_| | (_| |>  < \n\\_| |_/\\__,_|\\__,_|\\__,_/_/\\_\\\n\n:: Addax version ::    (v4.0.2-SNAPSHOT)\n\n2021-08-04 15:45:56.421 [        main] INFO  VMInfo               - VMInfo# operatingSystem class => com.sun.management.internal.OperatingSystemImpl\n2021-08-04 15:45:56.443 [        main] INFO  Engine               - \n\n.....\n\n2021-08-04 15:45:56.458 [        main] INFO  PerfTrace            - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-08-04 15:45:56.459 [        main] INFO  JobContainer         - Addax jobContainer starts job.\n2021-08-04 15:45:56.460 [        main] INFO  JobContainer         - Set jobId = 0\n2021-08-04 15:45:56.470 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do prepare work .\n2021-08-04 15:45:56.471 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do prepare work .\n2021-08-04 15:45:56.471 [       job-0] INFO  JobContainer         - Job set Channel-Number to 1 channels.\n2021-08-04 15:45:56.472 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] splits to [1] tasks.\n2021-08-04 15:45:56.472 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] splits to [1] tasks.\n2021-08-04 15:45:56.498 [       job-0] INFO  JobContainer         - Scheduler starts [1] taskGroups.\n2021-08-04 15:45:56.505 [ taskGroup-0] INFO  TaskGroupContainer   - taskGroupId=[0] start [1] channels for [1] tasks.\n2021-08-04 15:45:56.517 [ taskGroup-0] INFO  Channel              - Channel set byte_speed_limit to -1, No bps activated.\n2021-08-04 15:45:56.517 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\n2021-08-04 15:45:56.520 [ taskGroup-0] INFO  TransformerUtil      -  user config transformers [[dx_groovy]], loading...\n2021-08-04 15:45:56.531 [ taskGroup-0] INFO  TransformerUtil      -  1 of transformer init success. name=dx_groovy, isNative=true parameter = \n  {\"code\":\"record.setColumn(0, new StringColumn('Header_' + record.getColumn(0).asString()));record.setColumn(1, new LongColumn(record.getColumn(1).asLong() * 2));return record;\"}\n\nHeader_Addax    2       1989-06-04 00:00:01     true    test\nHeader_Addax    4       1989-06-03 00:00:01     true    test\nHeader_Addax    6       1989-06-02 00:00:01     true    test\nHeader_Addax    8       1989-06-01 00:00:01     true    test\nHeader_Addax    10      1989-05-31 00:00:01     true    test\nHeader_Addax    12      1989-05-30 00:00:01     true    test\nHeader_Addax    14      1989-05-29 00:00:01     true    test\nHeader_Addax    16      1989-05-28 00:00:01     true    test\nHeader_Addax    18      1989-05-27 00:00:01     true    test\nHeader_Addax    20      1989-05-26 00:00:01     true    test\n\n2021-08-04 15:45:59.515 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2021-08-04 15:45:59.517 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do post work.\n2021-08-04 15:45:59.518 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do post work.\n2021-08-04 15:45:59.521 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2021-08-04 15:45:59.524 [       job-0] INFO  StandAloneJobContainerCommunicator - Total 10 records, 330 bytes | Speed 110B/s, 3 records/s | Error 0 records, 0 bytes |  \n  All Task WaitWriterTime 0.000s |  All Task WaitReaderTime 0.000s | Transformer Success 10 records | Transformer Error 0 records | Transformer Filter 0 records \n  | Transformer usedTime 0.383s | Percentage 100.00%\n2021-08-04 15:45:59.527 [       job-0] INFO  JobContainer         - \n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-08-04 15:45:56\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-08-04 15:45:59\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :              110B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :              3rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                  10\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n\n2021-08-04 15:45:59.528 [       job-0] INFO  JobContainer         - \nTransformer\u6210\u529f\u8bb0\u5f55\u603b\u6570         :                  10\nTransformer\u5931\u8d25\u8bb0\u5f55\u603b\u6570         :                   0\nTransformer\u8fc7\u6ee4\u8bb0\u5f55\u603b\u6570         :                   0\n
    "},{"location":"transformer/#_4","title":"\u8ba1\u91cf\u548c\u810f\u6570\u636e","text":"

    Transform\u8fc7\u7a0b\u6d89\u53ca\u5230\u6570\u636e\u7684\u8f6c\u6362\uff0c\u53ef\u80fd\u9020\u6210\u6570\u636e\u7684\u589e\u52a0\u6216\u51cf\u5c11\uff0c\u56e0\u6b64\u66f4\u52a0\u9700\u8981\u7cbe\u786e\u5ea6\u91cf\uff0c\u5305\u62ec\uff1a

    \u6d89\u53ca\u5230\u8fd0\u884c\u8fc7\u7a0b\u7684\u8ba1\u91cf\u6570\u636e\u5c55\u73b0\u5b9a\u4e49\u5982\u4e0b\uff1a

    Total 1000000 records, 22000000 bytes | Transform 100000 records(in), 10000 records(out) | Speed 2.10MB/s, 100000 records/s | Error 0 records, 0 bytes | Percentage 100.00%\n

    \u6ce8\u610f\uff0c\u8fd9\u91cc\u4e3b\u8981\u8bb0\u5f55\u8f6c\u6362\u7684\u8f93\u5165\u8f93\u51fa\uff0c\u9700\u8981\u68c0\u6d4b\u6570\u636e\u8f93\u5165\u8f93\u51fa\u7684\u8bb0\u5f55\u6570\u91cf\u53d8\u5316\u3002

    \u6d89\u53ca\u5230\u6700\u7ec8\u4f5c\u4e1a\u7684\u8ba1\u91cf\u6570\u636e\u5c55\u73b0\u5b9a\u4e49\u5982\u4e0b\uff1a

    \u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2015-03-10 17:34:21\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2015-03-10 17:34:31\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                 10s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :            2.10MB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :         100000rec/s\n\u8f6c\u6362\u8f93\u5165\u603b\u6570                    :             1000000\n\u8f6c\u6362\u8f93\u51fa\u603b\u6570                    :             1000000\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :             1000000\n\u540c\u6b65\u5931\u8d25\u603b\u6570                    :                   0\n

    \u6ce8\u610f\uff0c\u8fd9\u91cc\u4e3b\u8981\u8bb0\u5f55\u8f6c\u6362\u7684\u8f93\u5165\u8f93\u51fa\uff0c\u9700\u8981\u68c0\u6d4b\u6570\u636e\u8f93\u5165\u8f93\u51fa\u7684\u8bb0\u5f55\u6570\u91cf\u53d8\u5316\u3002

    "},{"location":"reader/accessreader/","title":"Access Reader","text":"

    AccessReader \u5b9e\u73b0\u4e86\u4ece Access \u6570\u636e\u5e93\u4e0a\u8bfb\u53d6\u6570\u636e\u7684\u80fd\u529b\uff0c\u4ed6\u57fa\u4e8e Addax RDBMS Reader \u5b9e\u73b0\u3002

    "},{"location":"reader/accessreader/#_1","title":"\u793a\u4f8b","text":"

    \u6211\u4eec\u4e0b\u8f7d\u7528\u4e8e\u6d4b\u8bd5\u7528\u7684 AcessThemeDemo.zip \u6587\u4ef6\uff0c\u89e3\u538b\u540e\u5f97\u5230 AccessThemeDemo.mdb \u6587\u4ef6\uff0c\u8be5\u6587\u4ef6\u4e2d\u5305\u542b\u4e86\u4e00\u4e2a tbl_Users \u8868\uff0c\u6211\u4eec\u5c06\u8be5\u8868\u7684\u6570\u636e\u540c\u6b65\u5230\u7ec8\u7aef\u4e0a\u3002

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/access2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\":\n      {\n        \"reader\": {\n          \"name\": \"accessreader\",\n          \"parameter\": {\n            \"username\": \"root\",\n            \"password\": \"\",\n            \"column\": [\n              \"*\"\n            ],\n            \"connection\": {\n              \"table\": [\n                \"tbl_Users\"\n              ],\n              \"jdbcUrl\": \"jdbc:ucanaccess:///Users/wgzhao/Downloads/AccessThemeDemo.mdb\"\n            },\n            \"where\": \"\"\n          }\n        },\n        \"writer\": {\n          \"name\": \"streamwriter\",\n          \"parameter\": {\n            \"encoding\": \"utf-8\",\n            \"print\": true\n          }\n        }\n      }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/access2stream.json

    "},{"location":"reader/accessreader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/access2stream.json\n
    "},{"location":"reader/accessreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    AccessReader \u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/cassandrareader/","title":"Cassandra Reader","text":"

    CassandraReader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Cassandra \u8bfb\u53d6\u6570\u636e\u7684\u80fd\u529b\u3002

    "},{"location":"reader/cassandrareader/#_1","title":"\u914d\u7f6e","text":"

    \u4e0b\u9762\u662f\u914d\u7f6e\u4e00\u4e2a\u4ece Cassandra \u8bfb\u53d6\u6570\u636e\u5230\u7ec8\u7aef\u7684\u4f8b\u5b50

    job/cassandra2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"cassandrareader\",\n        \"parameter\": {\n          \"host\": \"localhost\",\n          \"port\": 9042,\n          \"useSSL\": false,\n          \"keyspace\": \"test\",\n          \"table\": \"addax_src\",\n          \"column\": [\n            \"textCol\",\n            \"blobCol\",\n            \"writetime(blobCol)\",\n            \"boolCol\",\n            \"smallintCol\",\n            \"tinyintCol\",\n            \"intCol\",\n            \"bigintCol\",\n            \"varintCol\",\n            \"floatCol\",\n            \"doubleCol\",\n            \"decimalCol\",\n            \"dateCol\",\n            \"timeCol\",\n            \"timeStampCol\",\n            \"uuidCol\",\n            \"inetCol\",\n            \"durationCol\",\n            \"listCol\",\n            \"mapCol\",\n            \"setCol\",\n            \"tupleCol\",\n            \"udtCol\"\n          ]\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/cassandrareader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 host \u662f list \u65e0 \u8fde\u63a5\u7684\u57df\u540d\u6216 IP\uff0c\u591a\u4e2a\u8282\u70b9\u4e4b\u95f4\u7528\u9017\u53f7\u5206\u9694 port \u662f int 9042 \u7aef\u53e3 username \u5426 string \u65e0 \u7528\u6237\u540d password \u5426 string \u65e0 \u5bc6\u7801 useSSL \u5426 boolean false \u662f\u5426\u4f7f\u7528SSL\u8fde\u63a5 keyspace \u662f string \u65e0 \u9700\u8981\u540c\u6b65\u7684\u8868\u6240\u5728\u7684 keyspace table \u662f string \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u96c6\u5408,\u5176\u4e2d\u7684\u5143\u7d20\u53ef\u4ee5\u6307\u5b9a\u5217\u7684\u540d\u79f0\u6216 writetime(column_name)\uff0c\u540e\u4e00\u79cd\u5f62\u5f0f\u4f1a\u8bfb\u53d6column_name\u5217\u7684\u65f6\u95f4\u6233\u800c\u4e0d\u662f\u6570\u636e where \u5426 string \u65e0 \u6570\u636e\u7b5b\u9009\u6761\u4ef6\u7684 cql \u8868\u8fbe\u5f0f allowFiltering \u5426 boolean \u65e0 \u662f\u5426\u5728\u670d\u52a1\u7aef\u8fc7\u6ee4\u6570\u636e\uff0c\u8be6\u7ec6\u63cf\u8ff0\u53c2\u8003\u5b98\u65b9\u6587\u6863\u7684\u76f8\u5173\u63cf\u8ff0 consistencyLevel \u5426 string LOCAL_QUORUM \u6570\u636e\u4e00\u81f4\u6027\u7ea7\u522b, \u53ef\u9009 ONE, QUORUM, LOCAL_QUORUM, EACH_QUORUM, ALL, ANY, TWO, THREE, LOCAL_ONE"},{"location":"reader/cassandrareader/#_3","title":"\u652f\u6301\u7684\u6570\u636e\u7c7b\u578b","text":"

    \u76ee\u524d\u652f\u6301\u9664 counter \u548c Custom \u7c7b\u578b\u4e4b\u5916\u7684\u6240\u6709\u7c7b\u578b\u3002

    \u4e0b\u9762\u5217\u51fa\u7c7b\u578b\u8f6c\u6362\u5217\u8868:

    Addax \u5185\u90e8\u7c7b\u578b Cassandra \u6570\u636e\u7c7b\u578b Long int, tinyint, smallint,varint,bigint,time,counter Double float, double, decimal String ascii,varchar, text,uuid,timeuuid,duration,list,map,set,tuple,udt,inet Date date, timestamp Boolean bool Bytes blob"},{"location":"reader/clickhousereader/","title":"ClickHouse Reader","text":"

    ClickHouseReader \u63d2\u4ef6\u652f\u6301\u4ece ClickHouse\u6570\u636e\u5e93\u8bfb\u53d6\u6570\u636e\u3002

    "},{"location":"reader/clickhousereader/#_1","title":"\u793a\u4f8b","text":""},{"location":"reader/clickhousereader/#_2","title":"\u8868\u7ed3\u6784\u53ca\u6570\u636e\u4fe1\u606f","text":"

    \u5047\u5b9a\u9700\u8981\u7684\u8bfb\u53d6\u7684\u8868\u7684\u7ed3\u6784\u4ee5\u53ca\u6570\u636e\u5982\u4e0b\uff1a

    CREATE TABLE ck_addax (\n    c_int8 Int8,\n    c_int16 Int16,\n    c_int32 Int32,\n    c_int64 Int64,\n    c_uint8 UInt8,\n    c_uint16 UInt16,\n    c_uint32 UInt32,\n    c_uint64 UInt64,\n    c_float32 Float32,\n    c_float64 Float64,\n    c_decimal Decimal(38,10),\n    c_string String,\n    c_fixstr FixedString(36),\n    c_uuid UUID,\n    c_date Date,\n    c_datetime DateTime('Asia/Chongqing'),\n    c_datetime64 DateTime64(3, 'Asia/Chongqing'),\n    c_enum Enum('hello' = 1, 'world'=2)\n) ENGINE = MergeTree() ORDER BY (c_int8, c_int16) SETTINGS index_granularity = 8192;\n\ninsert into ck_addax values(\n    127,\n    -32768,\n    2147483647,\n    -9223372036854775808,\n    255,\n    65535,\n    4294967295,\n    18446744073709551615,\n    0.999999999999,\n    0.99999999999999999,\n    1234567891234567891234567891.1234567891,\n    'Hello String',\n    '2c:16:db:a3:3a:4f',\n    '5F042A36-5B0C-4F71-ADFD-4DF4FCA1B863',\n    '2021-01-01',\n    '2021-01-01 11:22:33',\n    '2021-01-01 10:33:23.123',\n    'hello'\n);\n
    "},{"location":"reader/clickhousereader/#json","title":"\u914d\u7f6e json \u6587\u4ef6","text":"

    \u4e0b\u9762\u7684\u914d\u7f6e\u6587\u4ef6\u8868\u793a\u4ece ClickHouse \u6570\u636e\u5e93\u8bfb\u53d6\u6307\u5b9a\u7684\u8868\u6570\u636e\u5e76\u6253\u5370\u5230\u7ec8\u7aef

    job/clickhouse2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"clickhousereader\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"root\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"ck_addax\"\n            ],\n            \"jdbcUrl\": \"jdbc:clickhouse://127.0.0.1:8123/default\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/clickhouse2stream.json

    "},{"location":"reader/clickhousereader/#_3","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/clickhouse2stream.json\n

    \u5176\u8f93\u51fa\u4fe1\u606f\u5982\u4e0b\uff08\u5220\u9664\u4e86\u975e\u5173\u952e\u4fe1\u606f)

    021-01-06 14:39:35.742 [main] INFO  VMInfo - VMInfo# operatingSystem class => com.sun.management.internal.OperatingSystemImpl\n\n2021-01-06 14:39:35.767 [main] INFO  Engine -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"column\":[\n                        \"*\"\n                    ],\n                    \"connection\":[\n                        {\n                            \"jdbcUrl\":[\n                                \"jdbc:clickhouse://127.0.0.1:8123/\"\n                            ],\n                            \"table\":[\n                                \"ck_addax\"\n                            ]\n                        }\n                    ],\n                    \"username\":\"default\"\n                },\n                \"name\":\"clickhousereader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":true\n                },\n                \"name\":\"streamwriter\"\n            }\n    },\n    \"setting\":{\n        \"errorLimit\":{\n            \"record\":0,\n            \"percentage\":0.02\n        },\n        \"speed\":{\n            \"channel\":3\n        }\n    }\n}\n\n127 -32768  2147483647  -9223372036854775808    255 65535   4294967295  18446744073709551615    1   1   1234567891234567891234567891.1234567891Hello String 2c:16:db:a3:3a:4f   \n5f042a36-5b0c-4f71-adfd-4df4fca1b863    2021-01-01  2021-01-01 00:00:00 2021-01-01 00:00:00 hello\n\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-01-06 14:39:35\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-01-06 14:39:39\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :               77B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :              0rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                   1\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"reader/clickhousereader/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u53c2\u6570\u3002

    "},{"location":"reader/clickhousereader/#_5","title":"\u652f\u6301\u7684\u6570\u636e\u7c7b\u578b","text":"Addax \u5185\u90e8\u7c7b\u578b ClickHouse \u6570\u636e\u7c7b\u578b Long Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, Enum8, Enum16 Double Float32, Float64, Decimal String String, FixedString(N) Date Date, DateTime, DateTime64 Boolean UInt8 Bytes String"},{"location":"reader/clickhousereader/#_6","title":"\u9650\u5236","text":"

    \u9664\u4e0a\u8ff0\u7f57\u5217\u5b57\u6bb5\u7c7b\u578b\u5916\uff0c\u5176\u4ed6\u7c7b\u578b\u5747\u4e0d\u652f\u6301\uff0c\u5982 Array\u3001Nested \u7b49

    "},{"location":"reader/databendreader/","title":"Databend Reader","text":"

    DatabendReader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Databend \u8bfb\u53d6\u6570\u636e

    \u6ce8\u610f\uff0cdatabender \u6709\u517c\u5bb9 MySQL \u5ba2\u6237\u7aef\u7684\u534f\u8bae\u5b9e\u73b0\uff0c\u56e0\u6b64\u4f60\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528 MySQL Reader \u6765\u8bfb\u53d6 Databend \u6570\u636e\u3002

    "},{"location":"reader/databendreader/#_1","title":"\u793a\u4f8b","text":"

    \u6211\u4eec\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u65b9\u5f0f\u542f\u52a8 Databend \u6570\u636e\u5e93

    docker run  -tid  --rm  -p 8000:8000 \\\n   -e QUERY_DEFAULT_USER=databend \\\n   -e QUERY_DEFAULT_PASSWORD=databend \\\n   datafuselabs/databend\n

    \u7136\u540e\u521b\u5efa\u4e00\u5f20\u9700\u8981\u8bfb\u53d6\u7684\u8868

    (\n    id int,\n    name varchar(255),\n    salary float,\n    created_at datetime,\n    updated_at datetime\n);\n

    \u5e76\u586b\u5145\u5fc5\u8981\u7684\u6570\u636e

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/databend2stream.json 
    {\n  \"job\": {\n    \"content\": [\n      {\n        \"reader\": {\n          \"name\": \"databendreader\",\n          \"parameter\": {\n            \"column\": [\n              \"*\"\n            ],\n            \"connection\": {\n              \"jdbcUrl\": \"jdbc:databend://127.0.0.1:8000/default\",\n              \"table\": [\n                \"addax_reader\"\n              ]\n            },\n            \"username\": \"databend\",\n            \"password\": \"databend\"\n          }\n        },\n        \"writer\": {\n          \"name\": \"streamwriter\",\n          \"parameter\": {\n            \"print\": \"true\"\n          }\n        }\n      }\n    ],\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/databend2stream.json

    "},{"location":"reader/databendreader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/databend2stream.json\n
    "},{"location":"reader/databendreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u53c2\u6570\u3002

    "},{"location":"reader/databendreader/#_4","title":"\u9650\u5236","text":"

    \u6682\u65e0

    "},{"location":"reader/datareader/","title":"Data Reader","text":"

    DataReader \u63d2\u4ef6\u662f\u4e13\u95e8\u63d0\u4f9b\u7528\u4e8e\u5f00\u53d1\u548c\u6d4b\u8bd5\u73af\u5883\u4e2d\uff0c\u751f\u4ea7\u6ee1\u8db3\u4e00\u5b9a\u89c4\u5219\u8981\u6c42\u7684\u6570\u636e\u7684\u63d2\u4ef6\u3002

    \u5728\u5b9e\u9645\u5f00\u53d1\u548c\u6d4b\u8bd5\u4e2d\uff0c\u6211\u4eec\u9700\u8981\u6309\u7167\u4e00\u5b9a\u7684\u4e1a\u52a1\u89c4\u5219\u6765\u751f\u4ea7\u6d4b\u8bd5\u6570\u636e\uff0c\u800c\u4e0d\u4ec5\u4ec5\u662f\u968f\u673a\u5185\u5bb9\uff0c\u6bd4\u5982\u8eab\u4efd\u8bc1\u53f7\u7801\uff0c\u94f6\u884c\u8d26\u53f7\uff0c\u80a1\u7968\u4ee3\u7801\u7b49\u3002

    "},{"location":"reader/datareader/#_1","title":"\u4e3a\u4ec0\u4e48\u8981\u91cd\u590d\u53d1\u660e\u8f6e\u5b50","text":"

    \u8bda\u7136\uff0c\u7f51\u7edc\u4e0a\u6709\u76f8\u5f53\u591a\u7684\u4e13\u95e8\u7684\u6570\u636e\u751f\u4ea7\u5de5\u5177\uff0c\u5176\u4e2d\u4e0d\u4e4f\u529f\u80fd\u5f3a\u5927\u3001\u6027\u80fd\u4e5f\u5f3a\u608d\u3002 \u4f46\u8fd9\u4e9b\u5de5\u5177\u5927\u90e8\u5206\u662f\u8003\u8651\u5230\u4e86\u6570\u636e\u751f\u6210\u8fd9\u4e00\u6bb5\uff0c\u800c\u5ffd\u7565\u4e86\u6570\u636e\u5199\u5165\u5230\u76ee\u6807\u7aef\u7684\u95ee\u9898\uff0c\u6216\u8005\u8bf4\u6709\u4e9b\u8003\u8651\u5230\u4e86\uff0c\u4f46\u4ec5\u4ec5\u53ea\u8003\u8651\u4e86\u4e00\u79cd\u6216\u6709\u9650\u7684\u51e0\u79cd\u6570\u636e\u5e93\u3002

    \u6070\u597d Addax \u5de5\u5177\u80fd\u591f\u63d0\u4f9b\u8db3\u591f\u591a\u7684\u76ee\u6807\u7aef\u5199\u5165\u80fd\u529b\uff0c\u52a0\u4e0a\u4e4b\u524d\u7684\u5df2\u6709\u7684 Stream Reader \u5df2\u7ecf\u7b97\u662f\u4e00\u4e2a\u7b80\u5355\u7248\u7684\u6570\u636e\u751f\u6210\u5de5\u5177\uff0c\u56e0\u6b64\u5728\u6b64\u529f\u80fd\u4e0a \u589e\u52a0\u4e00\u4e9b\u7279\u5b9a\u89c4\u5219\uff0c\u518d\u5229\u7528\u5199\u5165\u7aef\u591a\u6837\u6027\u7684\u80fd\u529b\uff0c\u81ea\u7136\u5c31\u6210\u4e3a\u4e86\u4e00\u4e2a\u8f83\u597d\u7684\u6570\u636e\u751f\u6210\u5de5\u5177\u3002

    "},{"location":"reader/datareader/#_2","title":"\u914d\u7f6e\u793a\u4f8b","text":"

    \u8fd9\u91cc\u6211\u628a\u76ee\u524d\u63d2\u4ef6\u652f\u6301\u7684\u89c4\u5219\u5168\u90e8\u5217\u4e3e\u5230\u4e0b\u9762\u7684\u4f8b\u5b50\u4e2d

    datareader2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"datareader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"1,100,\",\n              \"rule\": \"random\",\n              \"type\": \"double\"\n            },\n            {\n              \"value\": \"DataX\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"1\",\n              \"rule\": \"incr\",\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989/06/04 00:00:01,-1\",\n              \"rule\": \"incr\",\n              \"type\": \"date\",\n              \"dateFormat\": \"yyyy/MM/dd hh:mm:ss\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            },\n            {\n              \"rule\": \"address\"\n            },\n            {\n              \"rule\": \"bank\"\n            },\n            {\n              \"rule\": \"company\"\n            },\n            {\n              \"rule\": \"creditCard\"\n            },\n            {\n              \"rule\": \"debitCard\"\n            },\n            {\n              \"rule\": \"idCard\"\n            },\n            {\n              \"rule\": \"lat\"\n            },\n            {\n              \"rule\": \"lng\"\n            },\n            {\n              \"rule\": \"name\"\n            },\n            {\n              \"rule\": \"job\"\n            },\n            {\n              \"rule\": \"phone\"\n            },\n            {\n              \"rule\": \"stockCode\"\n            },\n            {\n              \"rule\": \"stockAccount\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true,\n          \"encoding\": \"UTF-8\"\n        }\n      }\n    }\n  }\n}\n

    \u4fdd\u5b58\u4e0a\u8ff0\u5185\u5bb9\u5230 job/datareader2stream.json

    \u7136\u540e\u6267\u884c\u8be5\u4efb\u52a1\uff0c\u5176\u8f93\u51fa\u7ed3\u679c\u7c7b\u4f3c\u5982\u4e0b\uff1a

    $ bin/addax.sh job/datareader2stream.json\n\n  ___      _     _\n / _ \\    | |   | |\n/ /_\\ \\ __| | __| | __ ___  __\n|  _  |/ _` |/ _` |/ _` \\ \\/ /\n| | | | (_| | (_| | (_| |>  <\n\\_| |_/\\__,_|\\__,_|\\__,_/_/\\_\\\n\n:: Addax version ::    (v4.0.2-SNAPSHOT)\n\n2021-08-13 17:02:00.888 [        main] INFO  VMInfo               - VMInfo# operatingSystem class => com.sun.management.internal.OperatingSystemImpl\n2021-08-13 17:02:00.910 [        main] INFO  Engine               -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"column\":[\n                        {\n                            \"rule\":\"random\",\n                            \"type\":\"double\",\n                            \"scale\": \"2\",\n                            \"value\":\"1,100,\"\n                        },\n                        {\n                            \"type\":\"string\",\n                            \"value\":\"DataX\"\n                        },\n                        {\n                            \"rule\":\"incr\",\n                            \"type\":\"long\",\n                            \"value\":\"1\"\n                        },\n                        {\n                            \"dateFormat\":\"yyyy/MM/dd hh:mm:ss\",\n                            \"rule\":\"incr\",\n                            \"type\":\"date\",\n                            \"value\":\"1989/06/04 00:00:01,-1\"\n                        },\n                        {\n                            \"type\":\"bytes\",\n                            \"value\":\"test\"\n                        },\n                        {\n                            \"rule\":\"address\"\n                        },\n                        {\n                            \"rule\":\"bank\"\n                        },\n                        {\n                            \"rule\":\"company\"\n                        },\n                        {\n                            \"rule\":\"creditCard\"\n                        },\n                        {\n                            \"rule\":\"debitCard\"\n                        },\n                        {\n                            \"rule\":\"idCard\"\n                        },\n                        {\n                            \"rule\":\"lat\"\n                        },\n                        {\n                            \"rule\":\"lng\"\n                        },\n                        {\n                            \"rule\":\"name\"\n                        },\n                        {\n                            \"rule\":\"job\"\n                        },\n                        {\n                            \"rule\":\"phone\"\n                        },\n                        {\n                            \"rule\":\"stockCode\"\n                        },\n                        {\n                            \"rule\":\"stockAccount\"\n                        }\n                    ],\n                    \"sliceRecordCount\":10\n                },\n                \"name\":\"datareader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":true,\n                    \"encoding\":\"UTF-8\"\n                },\n                \"name\":\"streamwriter\"\n            }\n    },\n    \"setting\":{\n        \"errorLimit\":{\n            \"record\":0,\n            \"percentage\":0.02\n        },\n        \"speed\":{\n            \"byte\":-1,\n            \"channel\":1\n        }\n    }\n}\n\n2021-08-13 17:02:00.937 [        main] INFO  PerfTrace            - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-08-13 17:02:00.938 [        main] INFO  JobContainer         - Addax jobContainer starts job.\n2021-08-13 17:02:00.940 [        main] INFO  JobContainer         - Set jobId = 0\n2021-08-13 17:02:00.976 [       job-0] INFO  JobContainer         - Addax Reader.Job [datareader] do prepare work .\n2021-08-13 17:02:00.977 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do prepare work .\n2021-08-13 17:02:00.978 [       job-0] INFO  JobContainer         - Job set Channel-Number to 1 channels.\n2021-08-13 17:02:00.979 [       job-0] INFO  JobContainer         - Addax Reader.Job [datareader] splits to [1] tasks.\n2021-08-13 17:02:00.980 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] splits to [1] tasks.\n2021-08-13 17:02:01.002 [       job-0] INFO  JobContainer         - Scheduler starts [1] taskGroups.\n2021-08-13 17:02:01.009 [ taskGroup-0] INFO  TaskGroupContainer   - taskGroupId=[0] start [1] channels for [1] tasks.\n2021-08-13 17:02:01.017 [ taskGroup-0] INFO  Channel              - Channel set byte_speed_limit to -1, No bps activated.\n2021-08-13 17:02:01.017 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\n\n7.65    DataX   1   1989-06-04 00:00:01 test    \u5929\u6d25\u5e02\u5357\u4eac\u53bf\u957f\u5bff\u533a\u5149\u660e\u8def263\u53f7    \u4ea4\u901a\u94f6\u884c    \u6613\u52a8\u529b\u4fe1\u606f\u6709\u9650\u516c\u53f8   6227894836568607    6235712610856305437 450304194808316766  31.3732613  -125.3507716    \u9f9a\u519b  \u673a\u7535\u5de5\u7a0b\u5e08   13438631667 726929  8741848665\n18.58   DataX   2   1989-06-03 00:00:01 test    \u6c5f\u82cf\u7701\u592a\u539f\u5e02\u6d54\u9633\u533a\u4e1c\u5c71\u8def33\u53f7 \u4e2d\u56fd\u94f6\u884c    \u65f6\u7a7a\u76d2\u6570\u5b57\u4fe1\u606f\u6709\u9650\u516c\u53f8 4096666711928233    6217419359154239015 220301200008188547  48.6648764  104.8567048 \u5321\u98de  \u5316\u5986\u5e08 18093137306 006845  1815787371\n16.16   DataX   3   1989-06-02 00:00:01 test    \u53f0\u6e7e\u7701\u90af\u90f8\u5e02\u6e05\u6cb3\u533a\u4e07\u987a\u8def10\u53f7 \u5927\u540c\u5546\u884c    \u5f00\u53d1\u533a\u4e16\u521b\u79d1\u6280\u6709\u9650\u516c\u53f8 4096713966912225    6212977716107080594 150223196408276322  29.0134395  142.6426842 \u652f\u6ce2  \u5ba1\u6838\u5458 13013458079 020695  3545552026\n63.89   DataX   4   1989-06-01 00:00:01 test    \u4e0a\u6d77\u5e02\u8f9b\u96c6\u53bf\u516d\u679d\u7279\u533a\u7518\u56ed\u8def119\u53f7   \u4e2d\u56fd\u519c\u4e1a\u94f6\u884c  \u6cf0\u9e92\u9e9f\u4f20\u5a92\u6709\u9650\u516c\u53f8   6227893481508780    6215686558778997167 220822196208286838  -71.6484635 111.8181273 \u656c\u5764  \u623f\u5730\u4ea7\u5ba2\u670d   13384928291 174445  0799668655\n79.18   DataX   5   1989-05-31 00:00:01 test    \u9655\u897f\u7701\u5357\u4eac\u5e02\u671d\u9633\u533a\u5927\u80dc\u8def170\u53f7    \u5185\u8499\u53e4\u94f6\u884c   \u6656\u6765\u8ba1\u7b97\u673a\u4fe1\u606f\u6709\u9650\u516c\u53f8 6227535683896707    6217255315590053833 350600198508222018  -24.9783587 78.017024   \u848b\u6768  \u56fa\u5b9a\u8d44\u4ea7\u4f1a\u8ba1  18766298716 402188  9633759917\n14.97   DataX   6   1989-05-30 00:00:01 test    \u6d77\u5357\u7701\u957f\u6625\u53bf\u74a7\u5c71\u533a\u78a7\u6d77\u8857147\u53f7    \u534e\u590f\u94f6\u884c    \u6d59\u5927\u4e07\u670b\u79d1\u6280\u6709\u9650\u516c\u53f8  6224797475369912    6215680436662199846 220122199608190275  -3.5088667  -40.2634359 \u8fb9\u6768  \u7763\u5bfc/\u5de1\u5e97   13278765923 092780  2408887582\n45.49   DataX   7   1989-05-29 00:00:01 test    \u53f0\u6e7e\u7701\u6f5c\u6c5f\u53bf\u6881\u5e73\u533a\u4e03\u661f\u8857201\u53f7    \u664b\u57ce\u5546\u884c    \u5f00\u53d1\u533a\u4e16\u521b\u4fe1\u606f\u6709\u9650\u516c\u53f8 5257468530819766    6213336008535546044 141082197908244004  -72.9200596 120.6018163 \u6851\u660e  \u7cfb\u7edf\u5de5\u7a0b\u5e08   13853379719 175864  8303448618\n8.45    DataX   8   1989-05-28 00:00:01 test    \u6d77\u5357\u7701\u676d\u5dde\u53bf\u57ce\u5317\u533a\u5929\u5174\u8def11\u53f7 \u5927\u540c\u5546\u884c    \u4e07\u8fc5\u7535\u8111\u79d1\u6280\u6709\u9650\u516c\u53f8  6227639043120062    6270259717880740332 430405198908214042  -16.5115338 -39.336119  \u8983\u5065  \u4eba\u4e8b\u603b\u76d1    13950216061 687461  0216734574\n15.01   DataX   9   1989-05-27 00:00:01 test    \u4e91\u5357\u7701\u60e0\u5dde\u5e02\u548c\u5e73\u533a\u6d77\u9e25\u8857201\u53f7    \u5185\u8499\u53e4\u94f6\u884c   \u9ec4\u77f3\u91d1\u627f\u4fe1\u606f\u6709\u9650\u516c\u53f8  6200358843233005    6235730928871528500 130300195008312067  -61.646097  163.0882369 \u536b\u5efa\u534e \u7535\u8bdd\u91c7\u7f16    15292600492 001658  1045093445\n55.14   DataX   10  1989-05-26 00:00:01 test    \u8fbd\u5b81\u7701\u5170\u5dde\u5e02\u5f90\u6c47\u533a\u4e1c\u5c71\u8857176\u53f7    \u5eca\u574a\u94f6\u884c    \u521b\u6c47\u79d1\u6280\u6709\u9650\u516c\u53f8    6227605280751588    6270262330691012025 341822200908168063  77.2165746  139.5431377 \u6c60\u6d69  \u591a\u5a92\u4f53\u8bbe\u8ba1   18693948216 201678  0692522928\n\n2021-08-13 17:02:04.020 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2021-08-13 17:02:04.021 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do post work.\n2021-08-13 17:02:04.022 [       job-0] INFO  JobContainer         - Addax Reader.Job [datareader] do post work.\n2021-08-13 17:02:04.025 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2021-08-13 17:02:04.028 [       job-0] INFO  StandAloneJobContainerCommunicator - Total 10 records, 1817 bytes | Speed 605B/s, 3 records/s | Error 0 records, 0 bytes |  All Task WaitWriterTime 0.000s |  All Task WaitReaderTime 0.000s | Percentage 100.00%\n2021-08-13 17:02:04.030 [       job-0] INFO  JobContainer         -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-08-13 17:02:00\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-08-13 17:02:04\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :              605B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :              3rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                  10\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"reader/datareader/#_3","title":"\u914d\u7f6e\u8bf4\u660e","text":"

    column \u7684\u914d\u7f6e\u548c\u5176\u4ed6\u63d2\u4ef6\u7684\u914d\u7f6e\u7a0d\u6709\u4e0d\u540c\uff0c\u4e00\u4e2a\u5b57\u6bb5\u7531\u4ee5\u4e0b\u914d\u7f6e\u9879\u7ec4\u6210

    \u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u9ed8\u8ba4\u503c \u793a\u4f8b \u8bf4\u660e value \u5426 \u65e0 Addax \u6570\u636e\u503c\uff0c\u5728\u67d0\u4e9b\u60c5\u51b5\u4e0b\u4e3a\u5fc5\u9009\u9879 rule \u5426 constantidCard \u6570\u636e\u751f\u4ea7\u89c4\u5219\uff0c\u8be6\u7ec6\u4e0b\u9762\u7684\u63cf\u8ff0 type \u5426 stringdouble \u6570\u636e\u503c\u7c7b\u578b dateFormat \u5426 yyyy-MM-dd HH:mm:ssyyyy/MM/dd HH:mm:ss \u65e5\u671f\u683c\u5f0f\uff0c\u4ec5\u5728 type \u4e3a date \u65f6\u6709\u6548"},{"location":"reader/datareader/#rule","title":"rule \u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u7684\u5b57\u6bb5\u914d\u7f6e\u6838\u5fc3\u662f rule \u5b57\u6bb5\uff0c\u5b83\u7528\u6765\u6307\u793a\u5e94\u8be5\u751f\u6210\u4ec0\u4e48\u6837\u7684\u6570\u636e\uff0c\u5e76\u4f9d\u636e\u4e0d\u540c\u89c4\u5219\uff0c\u914d\u5408\u5176\u4ed6\u914d\u7f6e\u9009\u9879\u6765\u751f\u4ea7\u6ee1\u8db3\u671f\u671b\u7684\u6570\u636e\u3002 \u5f53\u524d rule \u7684\u914d\u7f6e\u5747\u4e3a\u5185\u7f6e\u652f\u6301\u7684\u89c4\u5219\uff0c\u6682\u4e0d\u652f\u6301\u81ea\u5b9a\u4e49\uff0c\u4ee5\u4e0b\u8be6\u7ec6\u8bf4\u660e

    "},{"location":"reader/datareader/#constant","title":"constant","text":"

    constant \u662f rule \u7684\u9ed8\u8ba4\u914d\u7f6e\uff0c\u8be5\u89c4\u5219\u610f\u5473\u7740\u8981\u751f\u6210\u7684\u6570\u636e\u503c\u7531 value \u914d\u7f6e\u9879\u51b3\u5b9a\uff0c\u5176\u4e0d\u505a\u4efb\u4f55\u53d8\u66f4\u3002\u6bd4\u5982

    {\n  \"value\": \"Addax\",\n  \"type\": \"string\",\n  \"rule\": \"constant\"\n}\n

    \u8868\u793a\u8be5\u5b57\u6bb5\u751f\u4ea7\u7684\u6570\u636e\u503c\u5747\u4e3a Addax

    "},{"location":"reader/datareader/#incr","title":"incr","text":"

    incr \u914d\u7f6e\u9879\u7684\u542b\u4e49\u548c streamreader \u63d2\u4ef6\u4e2d\u7684 incr \u542b\u4e49\u4e00\u81f4\uff0c\u8868\u793a\u8fd9\u662f\u4e00\u4e2a\u9012\u589e\u7684\u6570\u636e\u751f\u4ea7\u89c4\u5219\uff0c\u6bd4\u5982

    {\n  \"value\": \"1,2\",\n  \"rule\": \"incr\",\n  \"type\": \"long\"\n}\n

    \u8868\u793a\u8be5\u5b57\u6bb5\u7684\u6570\u636e\u662f\u4e00\u4e2a\u957f\u6574\u5f62\uff0c\u6570\u503c\u4ece 1 \u5f00\u59cb\uff0c\u6bcf\u6b21\u9012\u589e 2\uff0c\u4e5f\u5c31\u662f\u5f62\u6210 1 \u5f00\u59cb\uff0c\u6b65\u957f\u4e3a 2 \u7684\u9012\u589e\u6570\u5217\u3002

    \u8be5\u5b57\u6bb5\u66f4\u8be6\u7ec6\u7684\u914d\u7f6e\u89c4\u5219\u548c\u6ce8\u610f\u4e8b\u9879\uff0c\u53ef\u4ee5\u53c2\u8003 streamreader \u4e2d\u7684 incr \u8bf4\u660e\u3002

    "},{"location":"reader/datareader/#random","title":"random","text":"

    random \u914d\u7f6e\u9879\u7684\u542b\u4e49\u548c streamreader \u63d2\u4ef6\u4e2d\u7684 random \u542b\u4e49\u4e00\u81f4\uff0c\u8868\u793a\u8fd9\u662f\u4e00\u4e2a\u9012\u589e\u7684\u6570\u636e\u751f\u4ea7\u89c4\u5219\uff0c\u6bd4\u5982

    {\n  \"value\": \"1,10\",\n  \"rule\": \"random\",\n  \"type\": \"string\"\n}\n

    \u8868\u793a\u8be5\u5b57\u6bb5\u7684\u6570\u636e\u662f\u4e00\u4e2a\u957f\u5ea6\u4e3a 1 \u5230 10 \uff081 \u548c 10 \u90fd\u5305\u62ec\uff09\u968f\u673a\u5b57\u7b26\u4e32\u3002

    \u8be5\u5b57\u6bb5\u66f4\u8be6\u7ec6\u7684\u914d\u7f6e\u89c4\u5219\u548c\u6ce8\u610f\u4e8b\u9879\uff0c\u53ef\u4ee5\u53c2\u8003 streamreader \u4e2d\u7684 random \u8bf4\u660e\u3002

    \u89c4\u5219\u540d\u79f0 \u542b\u4e49 \u793a\u4f8b \u6570\u636e\u7c7b\u578b \u8bf4\u660e address \u968f\u673a\u751f\u6210\u4e00\u6761\u57fa\u672c\u6ee1\u8db3\u56fd\u5185\u5b9e\u9645\u60c5\u51b5\u7684\u5730\u5740\u4fe1\u606f \u8fbd\u5b81\u7701\u5170\u5dde\u5e02\u5f90\u6c47\u533a\u4e1c\u5c71\u8857176\u53f7 string bank \u968f\u673a\u751f\u6210\u4e00\u4e2a\u56fd\u5185\u94f6\u884c\u540d\u79f0 \u534e\u590f\u94f6\u884c string company \u968f\u673a\u751f\u6210\u4e00\u4e2a\u516c\u53f8\u7684\u540d\u79f0 \u4e07\u8fc5\u7535\u8111\u79d1\u6280\u6709\u9650\u516c\u53f8 string creditCard \u968f\u673a\u751f\u6210\u4e00\u4e2a\u4fe1\u7528\u5361\u5361\u53f7 430405198908214042 string 16 \u4f4d debitCard \u968f\u673a\u751f\u6210\u4e00\u4e2a\u50a8\u84c4\u5361\u5361\u53f7 6227894836568607 string 19 \u4f4d email \u968f\u673a\u751f\u6210\u4e00\u4e2a\u7535\u5b50\u90ae\u4ef6\u5730\u5740 ok2a@gmail.com string idCard \u968f\u673a\u751f\u6210\u4e00\u4e2a\u56fd\u5185\u8eab\u4efd\u8bc1\u53f7\u7801 350600198508222018 string 18 \u4f4d\uff0c\u8d1f\u8d23\u6821\u9a8c\u89c4\u5219\uff0c\u5934 6 \u4f4d\u7f16\u7801\u6ee1\u8db3\u884c\u653f\u533a\u5212\u8981\u6c42 lat \u968f\u673a\u751f\u6210\u7ef4\u5ea6\u6570\u636e 48.6648764 double \u56fa\u5b9a 7 \u4f4d\u5c0f\u6570 \uff0c\u4e5f\u53ef\u4ee5\u7528latitude \u8868\u793a lng \u968f\u673a\u751f\u6210\u7ecf\u5ea6\u6570\u636e 120.6018163 double \u56fa\u5b9a 7 \u4f4d\u5c0f\u6570\uff0c\u4e5f\u53ef\u4ee5\u4f7f\u7528longitude \u8868\u793a name \u968f\u673a\u751f\u6210\u4e00\u4e2a\u56fd\u5185\u540d\u5b57 \u6c60\u6d69 string \u6682\u6ca1\u8003\u8651\u59d3\u6c0f\u5728\u56fd\u5185\u7684\u5360\u6bd4\u5ea6 job \u968f\u673a\u751f\u6210\u4e00\u4e2a\u56fd\u5185\u5c97\u4f4d\u540d\u79f0 \u7cfb\u7edf\u5de5\u7a0b\u5e08 string \u6570\u636e\u6765\u6e90\u4e8e\u62db\u8058\u7f51\u7ad9 phone \u968f\u673a\u751f\u6210\u4e00\u4e2a\u56fd\u5185\u624b\u673a\u53f7\u7801 15292600492 string \u6682\u4e0d\u8003\u8651\u865a\u62df\u624b\u673a\u53f7 stockCode \u968f\u673a\u751f\u6210\u4e00\u4e2a 6 \u4f4d\u7684\u80a1\u7968\u4ee3\u7801 687461 string \u524d\u4e24\u4f4d\u6ee1\u8db3\u56fd\u5185\u80a1\u7968\u4ee3\u7801\u7f16\u53f7\u89c4\u8303 stockAccount \u968f\u673a\u751f\u6210\u4e00\u4e2a 10 \u4f4d\u7684\u80a1\u7968\u4ea4\u6613\u8d26\u6237 0692522928 string \u5b8c\u5168\u968f\u673a\uff0c\u4e0d\u6ee1\u8db3\u8d26\u6237\u89c4\u8303 uuid \u968f\u673a\u751f\u6210\u4e00\u4e2a UUID \u5b57\u7b26\u4e32 bc1cf125-929b-43b7-b324-d7c4cc5a75d2 string \u5b8c\u5168\u968f\u673a\uff0c\u4e0d\u6ee1\u8db3\u8d26\u6237\u89c4\u8303 zipCode \u968f\u673a\u751f\u4ea7\u4e00\u4e2a\u56fd\u5185\u90ae\u653f\u7f16\u53f7 411105 long \u4e0d\u5b8c\u5168\u6ee1\u8db3\u56fd\u5185\u90ae\u653f\u7f16\u53f7\u89c4\u8303

    \u6ce8\u610f\uff1a\u4e0a\u8ff0\u8868\u683c\u4e2d\u7684\u89c4\u5219\u8fd4\u56de\u7684\u6570\u636e\u7c7b\u578b\u662f\u56fa\u5b9a\u7684\uff0c\u4e14\u4e0d\u652f\u6301\u4fee\u6539\uff0c\u56e0\u6b64 type \u65e0\u9700\u914d\u7f6e\uff0c\u914d\u7f6e\u7684\u7c7b\u578b\u4e5f\u4f1a\u88ab\u5ffd\u7565\uff0c\u56e0\u4e3a\u6570\u636e\u751f\u6210\u6765\u81ea\u5185\u90e8\u89c4\u5219\uff0c\u6240\u4ee5 value \u4e5f\u65e0\u9700\u914d\u7f6e\uff0c\u914d\u7f6e\u7684\u5185\u5bb9\u4e5f\u4f1a\u88ab\u5ffd\u7565\u3002

    "},{"location":"reader/dbfreader/","title":"Dbf Reader","text":"

    DbfReader \u63d2\u4ef6\u652f\u6301\u8bfb\u53d6 DBF \u683c\u5f0f\u6587\u4ef6

    "},{"location":"reader/dbfreader/#_1","title":"\u914d\u7f6e\u8bf4\u660e","text":"

    \u4ee5\u4e0b\u662f\u8bfb\u53d6 DBF \u6587\u4ef6\u540e\u6253\u5370\u5230\u7ec8\u7aef\u7684\u914d\u7f6e\u6837\u4f8b

    jobs/dbf2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"dbfreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"long\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 3,\n              \"type\": \"boolean\"\n            },\n            {\n              \"index\": 4,\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"dbf\",\n              \"type\": \"string\"\n            }\n          ],\n          \"path\": [\n            \"/tmp/out\"\n          ],\n          \"encoding\": \"GBK\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": \"true\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/dbfreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    parameter \u914d\u7f6e\u9879\u652f\u6301\u4ee5\u4e0b\u914d\u7f6e

    \u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f \u65e0 DBF \u6587\u4ef6\u8def\u5f84\uff0c\u652f\u6301\u5199\u591a\u4e2a\u8def\u5f84\uff0c\u8be6\u7ec6\u60c5\u51b5\u89c1\u4e0b column \u662f \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u96c6\u5408, \u662f {type: value} \u6216 {type: index} \u7684\u96c6\u5408\uff0c\u8be6\u7ec6\u914d\u7f6e\u89c1\u4e0b encoding \u5426 GBK DBF \u6587\u4ef6\u7f16\u7801\uff0c\u6bd4\u5982 GBK, UTF-8 nullFormat \u5426 \\N \u5b9a\u4e49\u54ea\u4e2a\u5b57\u7b26\u4e32\u53ef\u4ee5\u8868\u793a\u4e3a null,"},{"location":"reader/dbfreader/#path","title":"path","text":"

    \u63cf\u8ff0\uff1a\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84\u3002

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0cAddax \u4f1a\u5c06\u4e00\u4e2a\u4f5c\u4e1a\u4e0b\u540c\u6b65\u7684\u6240\u6709 dbf File \u89c6\u4f5c\u540c\u4e00\u5f20\u6570\u636e\u8868\u3002\u7528\u6237\u5fc5\u987b\u81ea\u5df1\u4fdd\u8bc1\u6240\u6709\u7684 File \u80fd\u591f\u9002\u914d\u540c\u4e00\u5957 schema \u4fe1\u606f\u3002\u8bfb\u53d6\u6587\u4ef6\u7528\u6237\u5fc5\u987b\u4fdd\u8bc1\u4e3a\u7c7b dbf \u683c\u5f0f\uff0c\u5e76\u4e14\u63d0\u4f9b\u7ed9 Addax \u6743\u9650\u53ef\u8bfb\u3002

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0c\u5982\u679c Path \u6307\u5b9a\u7684\u8def\u5f84\u4e0b\u6ca1\u6709\u7b26\u5408\u5339\u914d\u7684\u6587\u4ef6\u62bd\u53d6\uff0cAddax \u5c06\u62a5\u9519\u3002

    "},{"location":"reader/dbfreader/#column","title":"column","text":"

    \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cname \u4e3a\u5b57\u6bb5\u540d,\u957f\u5ea6\u6700\u5927 8\uff0cvalue \u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece\u6e90\u5934\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636e value \u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u7528\u6237\u53ef\u4ee5\u5168\u90e8\u6309\u7167 String \u7c7b\u578b\u8bfb\u53d6\u6570\u636e\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"column\": [\"*\"]\n}\n

    \u7528\u6237\u53ef\u4ee5\u6307\u5b9a Column \u5b57\u6bb5\u4fe1\u606f\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    [\n  {\n    \"type\": \"long\",\n    \"index\": 0\n  },\n  {\n    \"type\": \"string\",\n    \"value\": \"addax\"\n  }\n]\n
    "},{"location":"reader/dbfreader/#_3","title":"\u652f\u6301\u7684\u6570\u636e\u7c7b\u578b","text":"

    \u672c\u5730\u6587\u4ef6\u672c\u8eab\u63d0\u4f9b\u6570\u636e\u7c7b\u578b\uff0c\u8be5\u7c7b\u578b\u662f Addax dbfFileReader \u5b9a\u4e49\uff1a

    Addax \u5185\u90e8\u7c7b\u578b \u672c\u5730\u6587\u4ef6 \u6570\u636e\u7c7b\u578b Long Long Double Double String String Boolean Boolean Date Date

    \u5176\u4e2d\uff1a

    "},{"location":"reader/elasticsearchreader/","title":"ElasticSearchReader","text":"

    ElasticSearchReader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Elasticsearch \u8bfb\u53d6\u7d22\u5f15\u7684\u529f\u80fd\uff0c \u5b83\u901a\u8fc7 Elasticsearch \u63d0\u4f9b\u7684 Rest API \uff08\u9ed8\u8ba4\u7aef\u53e39200\uff09\uff0c\u6267\u884c\u6307\u5b9a\u7684\u67e5\u8be2\u8bed\u53e5\u6279\u91cf\u83b7\u53d6\u6570\u636e

    "},{"location":"reader/elasticsearchreader/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u83b7\u53d6\u7684\u7d22\u5f15\u5185\u5bb9\u5982\u4e0b

    {\n  \"took\": 14,\n  \"timed_out\": false,\n  \"_shards\": {\n    \"total\": 1,\n    \"successful\": 1,\n    \"skipped\": 0,\n    \"failed\": 0\n  },\n  \"hits\": {\n    \"total\": 2,\n    \"max_score\": 1,\n    \"hits\": [\n      {\n        \"_index\": \"test-1\",\n        \"_type\": \"default\",\n        \"_id\": \"38\",\n        \"_score\": 1,\n        \"_source\": {\n          \"col_date\": \"2017-05-25T11:22:33.000+08:00\",\n          \"col_integer\": 19890604,\n          \"col_keyword\": \"hello world\",\n          \"col_ip\": \"1.1.1.1\",\n          \"col_text\": \"long text\",\n          \"col_double\": 19890604,\n          \"col_long\": 19890604,\n          \"col_geo_point\": \"41.12,-71.34\"\n        }\n      },\n      {\n        \"_index\": \"test-1\",\n        \"_type\": \"default\",\n        \"_id\": \"103\",\n        \"_score\": 1,\n        \"_source\": {\n          \"col_date\": \"2017-05-25T11:22:33.000+08:00\",\n          \"col_integer\": 19890604,\n          \"col_keyword\": \"hello world\",\n          \"col_ip\": \"1.1.1.1\",\n          \"col_text\": \"long text\",\n          \"col_double\": 19890604,\n          \"col_long\": 19890604,\n          \"col_geo_point\": \"41.12,-71.34\"\n        }\n      }\n    ]\n  }\n}\n

    \u914d\u7f6e\u4e00\u4e2a\u4ece Elasticsearch \u8bfb\u53d6\u6570\u636e\u5e76\u6253\u5370\u5230\u7ec8\u7aef\u7684\u4efb\u52a1

    job/es2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"elasticsearchreader\",\n        \"parameter\": {\n          \"endpoint\": \"http://127.0.0.1:9200\",\n          \"accessId\": \"\",\n          \"accesskey\": \"\",\n          \"index\": \"test-1\",\n          \"type\": \"default\",\n          \"searchType\": \"dfs_query_then_fetch\",\n          \"headers\": {},\n          \"scroll\": \"3m\",\n          \"search\": [\n            {\n              \"query\": {\n                \"match\": {\n                  \"col_ip\": \"1.1.1.1\"\n                }\n              },\n              \"aggregations\": {\n                \"top_10_states\": {\n                  \"terms\": {\n                    \"field\": \"col_date\",\n                    \"size\": 10\n                  }\n                }\n              }\n            }\n          ],\n          \"column\": [\n            \"col_ip\",\n            \"col_double\",\n            \"col_long\",\n            \"col_integer\",\n            \"col_keyword\",\n            \"col_text\",\n            \"col_geo_point\",\n            \"col_date\"\n          ]\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true,\n          \"encoding\": \"UTF-8\"\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u5185\u5bb9\u4fdd\u5b58\u4e3a job/es2stream.json

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91c7\u96c6

    bin/addax.sh job/es2stream.json\n

    \u5176\u8f93\u51fa\u7ed3\u679c\u7c7b\u4f3c\u5982\u4e0b\uff08\u8f93\u51fa\u8bb0\u5f55\u6570\u6709\u5220\u51cf)

    2021-02-19 13:38:15.860 [main] INFO  VMInfo - VMInfo# operatingSystem class => com.sun.management.internal.OperatingSystemImpl\n2021-02-19 13:38:15.895 [main] INFO  Engine -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"accessId\":\"\",\n                    \"headers\":{},\n                    \"endpoint\":\"http://127.0.0.1:9200\",\n                    \"search\":[\n                      {\n                        \"query\": {\n                          \"match\": {\n                            \"col_ip\": \"1.1.1.1\"\n                          }\n                        },\n                        \"aggregations\": {\n                          \"top_10_states\": {\n                            \"terms\": {\n                              \"field\": \"col_date\",\n                              \"size\": 10\n                            }\n                          }\n                        }\n                      }\n                    ],\n                    \"accesskey\":\"*****\",\n                    \"searchType\":\"dfs_query_then_fetch\",\n                    \"scroll\":\"3m\",\n                    \"column\":[\n                        \"col_ip\",\n                        \"col_double\",\n                        \"col_long\",\n                        \"col_integer\",\n                        \"col_keyword\",\n                        \"col_text\",\n                        \"col_geo_point\",\n                        \"col_date\"\n                    ],\n                    \"index\":\"test-1\",\n                    \"type\":\"default\"\n                },\n                \"name\":\"elasticsearchreader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":true,\n                    \"encoding\":\"UTF-8\"\n                },\n                \"name\":\"streamwriter\"\n            }\n        },\n    \"setting\":{\n        \"errorLimit\":{\n            \"record\":0,\n            \"percentage\":0.02\n        },\n        \"speed\":{\n            \"byte\":-1,\n            \"channel\":1\n        }\n    }\n}\n\n2021-02-19 13:38:15.934 [main] INFO  PerfTrace - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-02-19 13:38:15.934 [main] INFO  JobContainer - Addax jobContainer starts job.\n2021-02-19 13:38:15.937 [main] INFO  JobContainer - Set jobId = 0\n\n2017-05-25T11:22:33.000+08:00   19890604    hello world 1.1.1.1 long text   19890604    19890604    41.12,-71.34\n2017-05-25T11:22:33.000+08:00   19890604    hello world 1.1.1.1 long text   19890604    19890604    41.12,-71.34\n\n2021-02-19 13:38:19.845 [job-0] INFO  AbstractScheduler - Scheduler accomplished all tasks.\n2021-02-19 13:38:19.848 [job-0] INFO  JobContainer - Addax Writer.Job [streamwriter] do post work.\n2021-02-19 13:38:19.849 [job-0] INFO  JobContainer - Addax Reader.Job [elasticsearchreader] do post work.\n2021-02-19 13:38:19.855 [job-0] INFO  JobContainer - PerfTrace not enable!\n2021-02-19 13:38:19.858 [job-0] INFO  StandAloneJobContainerCommunicator - Total 95 records, 8740 bytes | Speed 2.84KB/s, 31 records/s | Error 0 records, 0 bytes |  All Task WaitWriterTime 0.000s |  All Task WaitReaderTime 0.103s | Percentage 100.00%\n2021-02-19 13:38:19.861 [job-0] INFO  JobContainer -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-02-19 13:38:15\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-02-19 13:38:19\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :            2.84KB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :             31rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                   2\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"reader/elasticsearchreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 ElasticSearch\u7684\u8fde\u63a5\u5730\u5740 accessId \u5426 string \"\" http auth\u4e2d\u7684user accessKey \u5426 string \"\" http auth\u4e2d\u7684password index \u662f string \u65e0 elasticsearch\u4e2d\u7684index\u540d type \u5426 string index\u540d elasticsearch\u4e2dindex\u7684type\u540d search \u662f list [] json\u683c\u5f0fapi\u641c\u7d22\u6570\u636e\u4f53 column \u662f list \u65e0 \u9700\u8981\u8bfb\u53d6\u7684\u5b57\u6bb5 timeout \u5426 int 60 \u5ba2\u6237\u7aef\u8d85\u65f6\u65f6\u95f4(\u5355\u4f4d\uff1a\u79d2) discovery \u5426 boolean false \u542f\u7528\u8282\u70b9\u53d1\u73b0(\u8f6e\u8be2)\u5e76\u5b9a\u671f\u66f4\u65b0\u5ba2\u6237\u673a\u4e2d\u7684\u670d\u52a1\u5668\u5217\u8868 compression \u5426 boolean true http\u8bf7\u6c42\uff0c\u5f00\u542f\u538b\u7f29 multiThread \u5426 boolean true http\u8bf7\u6c42\uff0c\u662f\u5426\u6709\u591a\u7ebf\u7a0b searchType \u5426 string dfs_query_then_fetch \u641c\u7d22\u7c7b\u578b headers \u5426 map {} http\u8bf7\u6c42\u5934 scroll \u5426 string \"\" \u6eda\u52a8\u5206\u9875\u914d\u7f6e"},{"location":"reader/elasticsearchreader/#search","title":"search","text":"

    search \u914d\u7f6e\u9879\u5141\u8bb8\u914d\u7f6e\u4e3a\u6ee1\u8db3 Elasticsearch API \u67e5\u8be2\u8981\u6c42\u7684\u5185\u5bb9\uff0c\u6bd4\u5982\u8fd9\u6837\uff1a

    {\n  \"query\": {\n    \"match\": {\n      \"message\": \"myProduct\"\n    }\n  },\n  \"aggregations\": {\n    \"top_10_states\": {\n      \"terms\": {\n        \"field\": \"state\",\n        \"size\": 10\n      }\n    }\n  }\n}\n
    "},{"location":"reader/elasticsearchreader/#searchtype","title":"searchType","text":"

    searchType \u76ee\u524d\u652f\u6301\u4ee5\u4e0b\u51e0\u79cd\uff1a

    "},{"location":"reader/excelreader/","title":"Excel Reader","text":"

    Excel Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Microsoft Excel \u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u7684\u80fd\u529b\u3002

    "},{"location":"reader/excelreader/#_1","title":"\u914d\u7f6e","text":""},{"location":"reader/excelreader/#_2","title":"\u83b7\u53d6\u6837\u4f8b\u6587\u4ef6","text":"

    \u4ece\u8fd9\u91cc\u4e0b\u8f7d\u7528\u4e8e\u6f14\u793a\u7684 Excel \u538b\u7f29\u6587\u4ef6\uff0c\u5e76\u89e3\u538b\u7f29\u653e\u7f6e\u5230 /tmp/in \u76ee\u5f55\u4e0b\u3002 \u4e09\u4e2a\u6587\u4ef6\u5939\u7684\u5185\u5bb9\u76f8\u540c\uff0c\u5176\u4e2d

    \u6587\u4ef6\u5185\u5bb9\uff0c\u5982\u4e0b\u8868\u6240\u793a\uff1a

    \u7f16\u53f7 \u6574\u6570\u7c7b\u578b \u6d6e\u70b9\u6570\u7c7b\u578b \u5b57\u7b26\u4e32\u7c7b\u578b \u65e5\u671f\u7c7b\u578b \u516c\u5f0f\u8ba1\u7b97 \u5355\u5143\u683c\u5f0f\u5316 1 11 1102.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/10 5544.17 \u00a51,102.23 2 12 1103.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/11 5552.17 \u00a51,103.23 3 13 1104.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/12 5560.17 \u00a51,104.23 4 14 1105.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/13 5568.17 \u00a51,105.23 5 15 1106.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/14 5576.17 \u00a51,106.23 6 16 1107.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/15 5584.17 \u00a51,107.23 7 17 1108.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/16 5592.17 \u00a51,108.23 8 18 1109.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/17 5600.17 \u00a51,109.23 9 19 1110.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/18 5608.17 \u00a51,110.23 10 20 1111.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/19 5616.17 \u00a51,111.23 11 21 1112.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/20 5624.17 \u00a51,112.23 12 22 1113.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/21 5632.17 \u00a51,113.23 13 23 1114.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/22 5640.17 \u00a51,114.23 14 24 1115.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/23 5648.17 \u00a51,115.23 15 25 1116.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/24 5656.17 \u00a51,116.23 16 26 1117.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/25 5664.17 \u00a51,117.23 17 27 1118.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/26 5672.17 \u00a51,118.23 18 28 1119.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/27 5680.17 \u00a51,119.23 19 29 1120.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/28 5688.17 \u00a51,120.23 20 30 1121.234 Addax \u52a0\u4e0a\u4e2d\u6587 2021/9/29 5696.17 \u00a51,121.23

    \u8868\u5934\u5927\u81f4\u8bf4\u660e\u4e86\u5355\u5143\u6570\u636e\u7684\u7279\u5f81

    "},{"location":"reader/excelreader/#job","title":"\u521b\u5efa job \u6587\u4ef6","text":"

    \u521b\u5efa\u5982\u4e0b json \u6587\u4ef6

    excel2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"excelreader\",\n        \"parameter\": {\n          \"path\": [\n            \"/tmp/in\"\n          ],\n          \"header\": true,\n          \"skipRows\": 0\n        }\n      },\n      \"writer\": {\n        \"parameter\": {\n          \"print\": true\n        },\n        \"name\": \"streamwriter\"\n      }\n    }\n  }\n}\n

    \u5c06\u8f93\u51fa\u5185\u5bb9\u5b58\u4fdd\u4e3a\u5230 job/excel2stream.json \u6587\u4ef6\u4e2d\uff0c\u6267\u884c\u91c7\u96c6\u547d\u4ee4\uff1a

    $ bin/addax.sh job/excel2stream.json\n

    \u5982\u679c\u6ca1\u6709\u62a5\u9519\uff0c\u5e94\u8be5\u5f97\u5230\u5982\u4e0b\u8f93\u51fa\uff1a

    \u70b9\u51fb\u5c55\u5f00 
     ___      _     _\n / _ \\    | |   | |\n/ /_\\ \\ __| | __| | __ ___  __\n|  _  |/ _` |/ _` |/ _` \\ \\/ /\n| | | | (_| | (_| | (_| |>  <\n\\_| |_/\\__,_|\\__,_|\\__,_/_/\\_\\\n\n:: Addax version ::    (v4.0.3)\n\n2021-09-09 14:43:42.579 [        main] INFO  VMInfo               - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl\n2021-09-09 14:43:42.621 [        main] INFO  Engine               -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"path\":[\n                        \"/tmp/in\"\n                    ],\n                    \"column\":[\n                        {\n                            \"name\":\"no\",\n                            \"type\":\"long\"\n                        },\n                        {\n                            \"name\":\"birth\",\n                            \"format\":\"yyyy-MM-dd HH:mm:ss\",\n                            \"type\":\"date\"\n                        },\n                        {\n                            \"name\":\"kk\",\n                            \"type\":\"string\"\n                        }\n                    ],\n                    \"header\":true,\n                    \"skipHeader\":true,\n                    \"encoding\":\"UTF-8\",\n                    \"fieldDelimiter\":\",\"\n                },\n                \"name\":\"excelreader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":true\n                },\n                \"name\":\"streamwriter\"\n            }\n        },\n    \"setting\":{\n        \"speed\":{\n            \"bytes\":-1,\n            \"channel\":2\n        }\n    }\n}\n\n2021-09-09 14:43:42.653 [        main] INFO  PerfTrace            - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-09-09 14:43:42.653 [        main] INFO  JobContainer         - Addax jobContainer starts job.\n2021-09-09 14:43:42.655 [        main] INFO  JobContainer         - Set jobId = 0\n2021-09-09 14:43:42.669 [       job-0] INFO  ExcelReader$Job      - add file [/tmp/in/demo_old.xls] as a candidate to be read.\n2021-09-09 14:43:42.669 [       job-0] INFO  ExcelReader$Job      - add file [/tmp/in/demo_gbk.xlsx] as a candidate to be read.\n2021-09-09 14:43:42.670 [       job-0] INFO  ExcelReader$Job      - add file [/tmp/in/demo.xlsx] as a candidate to be read.\n2021-09-09 14:43:42.670 [       job-0] INFO  ExcelReader$Job      - The number of files to read is: [3]\n2021-09-09 14:43:42.677 [       job-0] INFO  JobContainer         - Addax Reader.Job [excelreader] do prepare work .\n2021-09-09 14:43:42.678 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do prepare work .\n2021-09-09 14:43:42.679 [       job-0] INFO  JobContainer         - Job set Channel-Number to 2 channels.\n2021-09-09 14:43:42.681 [       job-0] INFO  JobContainer         - Addax Reader.Job [excelreader] splits to [3] tasks.\n2021-09-09 14:43:42.682 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] splits to [3] tasks.\n2021-09-09 14:43:42.727 [       job-0] INFO  JobContainer         - Scheduler starts [1] taskGroups.\n2021-09-09 14:43:42.736 [ taskGroup-0] INFO  TaskGroupContainer   - taskGroupId=[0] start [2] channels for [3] tasks.\n2021-09-09 14:43:42.741 [ taskGroup-0] INFO  Channel              - Channel set byte_speed_limit to -1, No bps activated.\n2021-09-09 14:43:42.742 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\n2021-09-09 14:43:42.755 [0-0-1-reader] INFO  ExcelReader$Task     - The first row is skipped as a table header\n2021-09-09 14:43:42.755 [0-0-1-reader] INFO  ExcelReader$Task     - begin read file /tmp/in/demo.xlsx\n2021-09-09 14:43:42.757 [0-0-0-reader] INFO  ExcelReader$Task     - The first row is skipped as a table header\n2021-09-09 14:43:42.758 [0-0-0-reader] INFO  ExcelReader$Task     - begin read file /tmp/in/demo_gbk.xlsx\n1   11  1102.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-10 00:00:00 5544.17 1102.234\n1   12  1103.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-11 00:00:00 5552.17 1103.234\n1   13  1104.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-12 00:00:00 5560.17 1104.234\n1   14  1105.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-13 00:00:00 5568.17 1105.234\n1   15  1106.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-14 00:00:00 5576.17 1106.234\n1   16  1107.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-15 00:00:00 5584.17 1107.234\n1   17  1108.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-16 00:00:00 5592.17 1108.234\n1   18  1109.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-17 00:00:00 5600.17 1109.234\n1   19  1110.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-18 00:00:00 5608.17 1110.234\n1   20  1111.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-19 00:00:00 5616.17 1111.234\n1   21  1112.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-20 00:00:00 5624.17 1112.234\n1   22  1113.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-21 00:00:00 5632.17 1113.234\n1   23  1114.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-22 00:00:00 5640.17 1114.234\n1   24  1115.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-23 00:00:00 5648.17 1115.234\n1   25  1116.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-24 00:00:00 5656.17 1116.234\n1   26  1117.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-25 00:00:00 5664.17 1117.234\n1   27  1118.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-26 00:00:00 5672.17 1118.234\n1   28  1119.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-27 00:00:00 5680.17 1119.234\n1   29  1120.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-28 00:00:00 5688.17 1120.234\n1   30  1121.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-29 00:00:00 5696.17 1121.234\n1   11  1102.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-10 00:00:00 5544.17 1102.234\n2   12  1103.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-11 00:00:00 5552.17 1103.234\n3   13  1104.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-12 00:00:00 5560.17 1104.234\n4   14  1105.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-13 00:00:00 5568.17 1105.234\n5   15  1106.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-14 00:00:00 5576.17 1106.234\n6   16  1107.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-15 00:00:00 5584.17 1107.234\n7   17  1108.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-16 00:00:00 5592.17 1108.234\n8   18  1109.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-17 00:00:00 5600.17 1109.234\n9   19  1110.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-18 00:00:00 5608.17 1110.234\n10  20  1111.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-19 00:00:00 5616.17 1111.234\n11  21  1112.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-20 00:00:00 5624.17 1112.234\n12  22  1113.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-21 00:00:00 5632.17 1113.234\n13  23  1114.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-22 00:00:00 5640.17 1114.234\n14  24  1115.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-23 00:00:00 5648.17 1115.234\n15  25  1116.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-24 00:00:00 5656.17 1116.234\n16  26  1117.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-25 00:00:00 5664.17 1117.234\n17  27  1118.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-26 00:00:00 5672.17 1118.234\n18  28  1119.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-27 00:00:00 5680.17 1119.234\n19  29  1120.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-28 00:00:00 5688.17 1120.234\n20  30  1121.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-29 00:00:00 5696.17 1121.234\n2021-09-09 14:43:43.894 [0-0-2-reader] INFO  ExcelReader$Task     - The first row is skipped as a table header\n2021-09-09 14:43:43.894 [0-0-2-reader] INFO  ExcelReader$Task     - begin read file /tmp/in/demo_old.xls\n1   11  1102.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-10 00:00:00 5544.17 1102.234\n2   12  1103.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-11 00:00:00 5552.17 1103.234\n3   13  1104.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-12 00:00:00 5560.17 1104.234\n4   14  1105.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-13 00:00:00 5568.17 1105.234\n5   15  1106.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-14 00:00:00 5576.17 1106.234\n6   16  1107.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-15 00:00:00 5584.17 1107.234\n7   17  1108.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-16 00:00:00 5592.17 1108.234\n8   18  1109.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-17 00:00:00 5600.17 1109.234\n9   19  1110.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-18 00:00:00 5608.17 1110.234\n10  20  1111.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-19 00:00:00 5616.17 1111.234\n11  21  1112.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-20 00:00:00 5624.17 1112.234\n12  22  1113.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-21 00:00:00 5632.17 1113.234\n13  23  1114.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-22 00:00:00 5640.17 1114.234\n14  24  1115.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-23 00:00:00 5648.17 1115.234\n15  25  1116.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-24 00:00:00 5656.17 1116.234\n16  26  1117.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-25 00:00:00 5664.17 1117.234\n17  27  1118.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-26 00:00:00 5672.17 1118.234\n18  28  1119.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-27 00:00:00 5680.17 1119.234\n19  29  1120.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-28 00:00:00 5688.17 1120.234\n20  30  1121.234    Addax\u52a0\u4e0a\u4e2d\u6587   2021-09-29 00:00:00 5696.17 1121.234\n2021-09-09 14:43:45.753 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2021-09-09 14:43:45.754 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do post work.\n2021-09-09 14:43:45.756 [       job-0] INFO  JobContainer         - Addax Reader.Job [excelreader] do post work.\n2021-09-09 14:43:45.761 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2021-09-09 14:43:45.762 [       job-0] INFO  StandAloneJobContainerCommunicator - Total 60 records, 3360 bytes | Speed 1.09KB/s, 20 records/s | Error 0 records, 0 bytes |  All Task WaitWriterTime 0.000s |  All Task WaitReaderTime 0.993s | Percentage 100.00%\n2021-09-09 14:43:45.764 [       job-0] INFO  JobContainer         -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-09-09 14:43:42\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-09-09 14:43:45\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :            1.09KB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :             20rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                  60\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"reader/excelreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f string/list \u65e0 \u6307\u5b9a\u8981\u8bfb\u53d6\u7684\u6587\u4ef6\u5939\uff0c\u53ef\u4ee5\u6307\u5b9a\u591a\u4e2a header \u5426 boolean false \u6587\u4ef6\u662f\u5426\u5305\u542b\u5934 skipRows \u5426 int 0 \u8981\u8df3\u8fc7\u524d\u591a\u5c11\u884c"},{"location":"reader/excelreader/#header","title":"header","text":"

    Excel \u6587\u4ef6\u662f\u5426\u5305\u542b\u5934\uff0c\u5982\u679c\u5305\u542b\uff0c\u5219\u8df3\u8fc7

    "},{"location":"reader/excelreader/#skiprows","title":"skipRows","text":"

    \u6307\u5b9a\u8981\u8df3\u8fc7\u7684\u884c\u6570\uff0c \u9ed8\u8ba4\u4e3a 0\uff0c\u8868\u793a\u4e0d\u8df3\u8fc7\u3002\u8fd9\u91cc\u8981\u6ce8\u610f\u7684\u662f\uff0c\u5047\u5b9a \u8bbe\u7f6e\u4e86 header \u4e3a true\uff0c\u540c\u65f6\u8bbe\u7f6e skipRows \u4e3a 2\u3002\u5219\u8868\u793a\u524d\u4e09\u884c\u90fd\u8df3\u8fc7\u3002 \u5982\u679c header \u4e3a false\uff0c \u5219\u8868\u793a\u8df3\u8fc7\u524d\u4e24\u884c\u3002

    "},{"location":"reader/excelreader/#_4","title":"\u652f\u6301\u7684\u6570\u636e\u7c7b\u578b","text":"

    Excel \u8bfb\u53d6\u529f\u80fd\u7684\u5b9e\u73b0\u4f9d\u8d56\u4e8e Apache POI \u9879\u76ee\uff0c\u8be5\u5b9e\u73b0\u5bf9\u5355\u5143\u683c\u7684\u6570\u636e\u7c7b\u578b\u5b9a\u4e49\u5f88\u5bbd\u6cdb\u3002 \u4ec5\u5b9a\u4e49\u4e86\u5e03\u5c14\u578b(Boolean)\uff0c\u6570\u503c\u578b (Double)\uff0c\u5b57\u7b26\u4e32\u578b(String) \u4e09\u79cd\u3002\u5176\u4e2d\u6570\u503c\u578b\u5305\u62ec\u6240\u6709\u6574\u6570\uff0c\u5c0f\u6570\u548c\u65e5\u671f\u3002 \u76ee\u524d\u5bf9\u4e8e\u6570\u503c\u7c7b\u578b\u505a\u4e86\u7b80\u5355\u533a\u5206

    1. \u4f7f\u7528\u5e93\u5de5\u5177\u7c7b\u63a2\u6d4b\u662f\u5426\u4e3a\u65e5\u671f\u7c7b\u578b\uff0c\u5982\u679c\u662f\uff0c\u5219\u5224\u65ad\u4e3a\u65e5\u671f\u7c7b\u578b
    2. \u5c06\u6570\u503c\u8f6c\u6362\u4e3a\u957f\u6574\u5f62\uff0c\u5e76\u548c\u539f\u503c\u6bd4\u8f83\uff0c\u5982\u679c\u5927\u5c0f\u76f8\u7b49\uff0c\u5219\u5224\u65ad\u4e3a\u957f\u6574\u578b(Long)
    3. \u5426\u5219\u5224\u65ad\u4e3a\u6d6e\u70b9\u578b\uff08Double\uff09
    "},{"location":"reader/excelreader/#_5","title":"\u9650\u5236","text":"
    1. \u5f53\u524d\u4ec5\u8bfb\u53d6\u6587\u4ef6\u7684\u7b2c\u4e00\u4e2a Sheet \u800c\u5ffd\u7565\u5176\u4ed6 Sheets
    2. \u6682\u4e0d\u652f\u6301\u6307\u5b9a\u5217\u8bfb\u53d6
    3. \u6682\u4e0d\u652f\u6301\u8df3\u8fc7\u5c3e\u90e8\u884c\u6570\uff08\u6bd4\u5982\u6709\u603b\u7ed3\u7684\u5c3e\u884c\u53ef\u80fd\u5e76\u4e0d\u7b26\u5408\u8981\u6c42\uff09
    4. \u6682\u4e0d\u5224\u65ad\u6bcf\u4e00\u884c\u7684\u5217\u6570\u662f\u5426\u76f8\u7b49\uff0c\u9700\u8981 Excel \u81ea\u884c\u4fdd\u8bc1
    5. \u4ec5\u4f1a\u8bfb\u53d6\u6307\u5b9a\u76ee\u5f55\u4e0b\u6587\u4ef6\u540e\u7f00\u4e3a xlsx \u6216 xls \u7684\u6587\u4ef6\uff0c\u5176\u4ed6\u540e\u7f00\u6587\u4ef6\u5c06\u4f1a\u5ffd\u7565\u5e76\u7ed9\u51fa\u544a\u8b66\u6d88\u606f
    "},{"location":"reader/ftpreader/","title":"Ftp Reader","text":"

    Ftp Reader \u63d0\u4f9b\u4e86\u8bfb\u53d6\u8fdc\u7a0b FTP/SFTP \u6587\u4ef6\u7cfb\u7edf\u6570\u636e\u5b58\u50a8\u7684\u80fd\u529b\u3002

    "},{"location":"reader/ftpreader/#_1","title":"\u529f\u80fd\u8bf4\u660e","text":""},{"location":"reader/ftpreader/#_2","title":"\u914d\u7f6e\u6837\u4f8b","text":"job/ftp2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"ftpreader\",\n        \"parameter\": {\n          \"protocol\": \"sftp\",\n          \"host\": \"127.0.0.1\",\n          \"port\": 22,\n          \"username\": \"xx\",\n          \"password\": \"xxx\",\n          \"path\": [\n            \"/var/ftp/test.txt\",\n            \"/var/tmp/*.txt\",\n            \"/public/ftp\",\n            \"/public/a??.txt\"\n          ],\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"long\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"boolean\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"double\"\n            },\n            {\n              \"index\": 3,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 4,\n              \"type\": \"date\",\n              \"format\": \"yyyy.MM.dd\"\n            }\n          ],\n          \"encoding\": \"UTF-8\",\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"ftpWriter\",\n        \"parameter\": {\n          \"path\": \"/var/ftp/FtpWriter/result\",\n          \"fileName\": \"shihf\",\n          \"writeMode\": \"truncate\",\n          \"format\": \"yyyy-MM-dd\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/ftpreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 protocol \u662f string \u65e0 \u670d\u52a1\u5668\u534f\u8bae\uff0c\u76ee\u524d\u652f\u6301\u4f20\u8f93\u534f\u8bae\u6709 ftp \u548c sftp host \u662f string \u65e0 \u670d\u52a1\u5668\u5730\u5740 port \u5426 int 22/21 \u82e5\u4f20\u8f93\u534f\u8bae\u662f sftp \u534f\u8bae\uff0c\u9ed8\u8ba4\u503c\u662f 22\uff1b\u82e5\u4f20\u8f93\u534f\u8bae\u662f\u6807\u51c6 ftp \u534f\u8bae\uff0c\u9ed8\u8ba4\u503c\u662f 21 timeout \u5426 int 60000 \u8fde\u63a5 ftp \u670d\u52a1\u5668\u8fde\u63a5\u8d85\u65f6\u65f6\u95f4\uff0c\u5355\u4f4d\u6beb\u79d2(ms) connectPattern \u5426 string PASV \u8fde\u63a5\u6a21\u5f0f\uff0c\u4ec5\u652f\u6301 PORT, PASV \u6a21\u5f0f\u3002\u8be5\u53c2\u6570\u4ec5\u5728 ftp \u534f\u8bae\u65f6\u4f7f\u7528 username \u662f string \u65e0 ftp \u670d\u52a1\u5668\u8bbf\u95ee\u7528\u6237\u540d password \u5426 string \u65e0 ftp \u670d\u52a1\u5668\u8bbf\u95ee\u5bc6\u7801 useKey \u5426 boolean false \u662f\u5426\u4f7f\u7528\u79c1\u94a5\u767b\u5f55\uff0c\u4ec5\u9488\u5bf9 sftp \u767b\u5f55\u6709\u6548 keyPath \u5426 string ~/.ssh/id_rsa \u79c1\u94a5\u5730\u5740 keyPass \u5426 string \u65e0 \u79c1\u94a5\u5bc6\u7801\uff0c\u82e5\u6ca1\u6709\u8bbe\u7f6e\u79c1\u94a5\u5bc6\u7801\uff0c\u5219\u65e0\u9700\u914d\u7f6e\u8be5\u9879 path \u662f list \u65e0 \u8fdc\u7a0b FTP \u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1\u4e0b column \u662f list<map> \u65e0 \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0c\u8be6\u89c1\u4e0b\u6587 fieldDelimiter \u662f string , \u63cf\u8ff0\uff1a\u8bfb\u53d6\u7684\u5b57\u6bb5\u5206\u9694\u7b26 compress \u5426 string \u65e0 \u6587\u672c\u538b\u7f29\u7c7b\u578b\uff0c\u9ed8\u8ba4\u4e0d\u586b\u5199\u610f\u5473\u7740\u6ca1\u6709\u538b\u7f29\u3002\u652f\u6301\u538b\u7f29\u7c7b\u578b\u4e3a zip\u3001gz\u3001bzip2 encoding \u5426 string utf-8 \u8bfb\u53d6\u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e skipHeader \u5426 boolean false \u7c7b CSV \u683c\u5f0f\u6587\u4ef6\u53ef\u80fd\u5b58\u5728\u8868\u5934\u4e3a\u6807\u9898\u60c5\u51b5\uff0c\u9700\u8981\u8df3\u8fc7\u3002\u9ed8\u8ba4\u4e0d\u8df3\u8fc7 nullFormat \u5426 char \\N \u5b9a\u4e49\u54ea\u4e9b\u5b57\u7b26\u4e32\u53ef\u4ee5\u8868\u793a\u4e3a null maxTraversalLevel \u5426 int 100 \u5141\u8bb8\u904d\u5386\u6587\u4ef6\u5939\u7684\u6700\u5927\u5c42\u6570 csvReaderConfig \u5426 map \u65e0 \u8bfb\u53d6 CSV \u7c7b\u578b\u6587\u4ef6\u53c2\u6570\u914d\u7f6e\uff0cMap \u7c7b\u578b\u3002\u4e0d\u914d\u7f6e\u5219\u4f7f\u7528\u9ed8\u8ba4\u503c,\u8be6\u89c1\u4e0b\u6587"},{"location":"reader/ftpreader/#path","title":"path","text":"

    \u8fdc\u7a0b FTP \u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84\uff0c\u793a\u4f8b\u914d\u7f6e\u4e2d\u6f14\u793a\u4e86\u5982\u4f55\u586b\u5199\u591a\u4e2a\u8def\u5f84\u3002

    {\n  \"path\": [\n    \"/var/ftp/test.txt\", // \u8bfb\u53d6 /var/ftp \u76ee\u5f55\u4e0b\u7684 test.txt \u6587\u4ef6\n    \"/var/tmp/*.txt\", // \u8bfb\u53d6 /var/tmp \u76ee\u5f55\u4e0b\u6240\u6709 txt \u6587\u4ef6\n    \"/public/ftp\", // \u8bfb\u53d6 /public/ftp \u76ee\u5f55\u4e0b\u6240\u6709\u6587\u4ef6, \u5982\u679c ftp \u662f\u6587\u4ef6\u7684\u8bdd\uff0c\u5219\u76f4\u63a5\u8bfb\u53d6\n    \"/public/a??.txt\" // \u8bfb\u53d6 /public \u76ee\u5f55\u4e0b\u6240\u6709 a \u5f00\u5934\uff0c\u540e\u9762\u8ddf\u4e24\u4e2a\u5b57\u7b26\uff0c\u6700\u540e\u662f txt \u7ed3\u5c3e\u7684\u6587\u4ef6\n  ]\n}\n

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0cAddax \u4f1a\u5c06\u4e00\u4e2a\u4f5c\u4e1a\u4e0b\u540c\u6b65\u7684\u6240\u6709 Text File \u89c6\u4f5c\u540c\u4e00\u5f20\u6570\u636e\u8868\u3002\u7528\u6237\u5fc5\u987b\u81ea\u5df1\u4fdd\u8bc1\u6240\u6709\u7684 File \u80fd\u591f\u9002\u914d\u540c\u4e00\u5957 schema \u4fe1\u606f\u3002\u8bfb\u53d6\u6587\u4ef6\u7528\u6237\u5fc5\u987b\u4fdd\u8bc1\u4e3a\u7c7b CSV \u683c\u5f0f\uff0c\u5e76\u4e14\u63d0\u4f9b\u7ed9 Addax \u6743\u9650\u53ef\u8bfb\u3002

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0c\u5982\u679c Path \u6307\u5b9a\u7684\u8def\u5f84\u4e0b\u6ca1\u6709\u7b26\u5408\u5339\u914d\u7684\u6587\u4ef6\u62bd\u53d6\uff0cAddax \u5c06\u62a5\u9519\u3002

    "},{"location":"reader/ftpreader/#column","title":"column","text":"

    \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cindex \u6307\u5b9a\u5f53\u524d\u5217\u6765\u81ea\u4e8e\u6587\u672c\u7b2c\u51e0\u5217(\u4ee5 0 \u5f00\u59cb)\uff0cvalue \u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece\u6e90\u5934\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636e value \u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u7528\u6237\u53ef\u4ee5\u5168\u90e8\u6309\u7167 String \u7c7b\u578b\u8bfb\u53d6\u6570\u636e\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"column\": [\"*\"]\n}\n

    \u7528\u6237\u53ef\u4ee5\u6307\u5b9a Column \u5b57\u6bb5\u4fe1\u606f\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    [\n  {\n    \"type\": \"long\",\n    \"index\": 0,\n    \"description\": \"\u4ece\u8fdc\u7a0bFTP\u6587\u4ef6\u6587\u672c\u7b2c\u4e00\u5217\u83b7\u53d6int\u5b57\u6bb5\"\n  },\n  {\n    \"type\": \"string\",\n    \"value\": \"addax\",\n    \"description\": \"\u4eceFtpReader\u5185\u90e8\u751f\u6210alibaba\u7684\u5b57\u7b26\u4e32\u5b57\u6bb5\u4f5c\u4e3a\u5f53\u524d\u5b57\u6bb5\"\n  }\n]\n

    \u5bf9\u4e8e\u7528\u6237\u6307\u5b9a Column \u4fe1\u606f\uff0ctype \u5fc5\u987b\u586b\u5199\uff0cindex/value \u5fc5\u987b\u9009\u62e9\u5176\u4e00\u3002

    "},{"location":"reader/ftpreader/#csvreaderconfig","title":"csvReaderConfig","text":"

    \u5e38\u89c1\u914d\u7f6e\uff1a

    {\n  \"csvReaderConfig\": {\n    \"safetySwitch\": false,\n    \"skipEmptyRecords\": false,\n    \"useTextQualifier\": false\n  }\n}\n

    \u6240\u6709\u914d\u7f6e\u9879\u53ca\u9ed8\u8ba4\u503c,\u914d\u7f6e\u65f6 csvReaderConfig \u7684 map \u4e2d\u8bf7 \u4e25\u683c\u6309\u7167\u4ee5\u4e0b\u5b57\u6bb5\u540d\u5b57\u8fdb\u884c\u914d\u7f6e\uff1a

    boolean caseSensitive = true;\nchar textQualifier = 34;\nboolean trimWhitespace = true;\nboolean useTextQualifier = true;//\u662f\u5426\u4f7f\u7528csv\u8f6c\u4e49\u5b57\u7b26\nchar delimiter = 44;//\u5206\u9694\u7b26\nchar recordDelimiter = 0;\nchar comment = 35;\nboolean useComments = false;\nint escapeMode = 1;\nboolean safetySwitch = true;//\u5355\u5217\u957f\u5ea6\u662f\u5426\u9650\u5236100000\u5b57\u7b26\nboolean skipEmptyRecords = true;//\u662f\u5426\u8df3\u8fc7\u7a7a\u884c\nboolean captureRawRecord = true;\n
    "},{"location":"reader/ftpreader/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u8fdc\u7a0b FTP \u6587\u4ef6\u672c\u8eab\u4e0d\u63d0\u4f9b\u6570\u636e\u7c7b\u578b\uff0c\u8be5\u7c7b\u578b\u662f Addax FtpReader \u5b9a\u4e49\uff1a

    Addax \u5185\u90e8\u7c7b\u578b \u8fdc\u7a0b FTP \u6587\u4ef6 \u6570\u636e\u7c7b\u578b Long Long Double Double String String Boolean Boolean Date Date

    \u5176\u4e2d\uff1a

    "},{"location":"reader/ftpreader/#_5","title":"\u9650\u5236","text":"
    1. \u5355\u4e2a File \u652f\u6301\u591a\u7ebf\u7a0b\u5e76\u53d1\u8bfb\u53d6\uff0c\u8fd9\u91cc\u6d89\u53ca\u5230\u5355\u4e2a File \u5185\u90e8\u5207\u5206\u7b97\u6cd5
    2. \u5355\u4e2a File \u5728\u538b\u7f29\u60c5\u51b5\u4e0b\uff0c\u4ece\u6280\u672f\u4e0a\u65e0\u6cd5\u652f\u6301\u591a\u7ebf\u7a0b\u5e76\u53d1\u8bfb\u53d6\u3002
    "},{"location":"reader/hanareader/","title":"HANA Reader","text":"

    HANA Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece SAP HANA \u8bfb\u53d6\u6570\u636e\u7684\u80fd\u529b

    "},{"location":"reader/hanareader/#_1","title":"\u793a\u4f8b","text":"

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/hanareader.json 
    {\n  \"job\": {\n    \"content\": [\n      {\n        \"reader\": {\n          \"name\": \"hanareader\",\n          \"parameter\": {\n            \"column\": [\n              \"*\"\n            ],\n            \"connection\": {\n              \"jdbcUrl\": \"jdbc:sap://wgzhao-pc:39017/system\",\n              \"table\": [\n                \"addax_tbl\"\n              ]\n            },\n            \"username\": \"system\",\n            \"password\": \"HXEHana1\"\n          }\n        },\n        \"writer\": {\n          \"name\": \"streamwriter\",\n          \"parameter\": {\n            \"print\": true\n          }\n        }\n      }\n    ],\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/hana2stream.json

    "},{"location":"reader/hanareader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/hana2stream.json\n
    "},{"location":"reader/hanareader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u53c2\u6570\u3002

    "},{"location":"reader/hbase11xreader/","title":"HBase11X Reader","text":"

    HBase11X Reader \u63d2\u4ef6\u652f\u6301\u4ece HBase 1.x \u7248\u672c\u8bfb\u53d6\u6570\u636e\uff0c \u5176\u5b9e\u73b0\u65b9\u5f0f\u4e3a \u901a\u8fc7 HBase \u7684 Java \u5ba2\u6237\u7aef\u8fde\u63a5\u8fdc\u7a0b HBase \u670d\u52a1\uff0c\u5e76\u901a\u8fc7 Scan \u65b9\u5f0f\u8bfb\u53d6\u4f60\u6307\u5b9a rowkey \u8303\u56f4\u5185\u7684\u6570\u636e\u3002

    "},{"location":"reader/hbase11xreader/#_1","title":"\u914d\u7f6e","text":""},{"location":"reader/hbase11xreader/#_2","title":"\u5efa\u8868\u4ee5\u53ca\u586b\u5145\u6570\u636e","text":"

    \u4ee5\u4e0b\u6f14\u793a\u57fa\u4e8e\u4e0b\u9762\u521b\u5efa\u7684\u8868\u4ee5\u53ca\u6570\u636e

    create 'users', 'address','info'\nput 'users', 'lisi', 'address:country', 'china'\nput 'users', 'lisi', 'address:province',    'beijing'\nput 'users', 'lisi', 'info:age',        27\nput 'users', 'lisi', 'info:birthday',   '1987-06-17'\nput 'users', 'lisi', 'info:company',    'baidu'\nput 'users', 'xiaoming', 'address:city',    'hangzhou'\nput 'users', 'xiaoming', 'address:country', 'china'\nput 'users', 'xiaoming', 'address:province',    'zhejiang'\nput 'users', 'xiaoming', 'info:age',        29\nput 'users', 'xiaoming', 'info:birthday',   '1987-06-17'\nput 'users', 'xiaoming', 'info:company',    'alibaba'\n
    "},{"location":"reader/hbase11xreader/#normal","title":"normal \u6a21\u5f0f","text":"

    \u628a HBase \u4e2d\u7684\u8868\uff0c\u5f53\u6210\u666e\u901a\u4e8c\u7ef4\u8868\uff08\u6a2a\u8868\uff09\u8fdb\u884c\u8bfb\u53d6,\u8bfb\u53d6\u6700\u65b0\u7248\u672c\u6570\u636e\u3002\u5982\uff1a

    hbase(main):017:0> scan 'users'\nROW           COLUMN+CELL\n lisi         column=address:city, timestamp=1457101972764, value=beijing\n lisi         column=address:country, timestamp=1457102773908, value=china\n lisi         column=address:province, timestamp=1457101972736, value=beijing\n lisi         column=info:age, timestamp=1457101972548, value=27\n lisi         column=info:birthday, timestamp=1457101972604, value=1987-06-17\n lisi         column=info:company, timestamp=1457101972653, value=baidu\n xiaoming     column=address:city, timestamp=1457082196082, value=hangzhou\n xiaoming     column=address:country, timestamp=1457082195729, value=china\n xiaoming     column=address:province, timestamp=1457082195773, value=zhejiang\n xiaoming     column=info:age, timestamp=1457082218735, value=29\n xiaoming     column=info:birthday, timestamp=1457082186830, value=1987-06-17\n xiaoming     column=info:company, timestamp=1457082189826, value=alibaba\n2 row(s) in 0.0580 seconds\n

    \u8bfb\u53d6\u540e\u6570\u636e

    rowKey addres:city address:country address:province info:age info:birthday info:company lisi beijing china beijing 27 1987-06-17 baidu xiaoming hangzhou china zhejiang 29 1987-06-17 alibaba"},{"location":"reader/hbase11xreader/#multiversionfixedcolumn","title":"multiVersionFixedColumn \u6a21\u5f0f","text":"

    \u628a HBase \u4e2d\u7684\u8868\uff0c\u5f53\u6210\u7ad6\u8868\u8fdb\u884c\u8bfb\u53d6\u3002\u8bfb\u51fa\u7684\u6bcf\u6761\u8bb0\u5f55\u4e00\u5b9a\u662f\u56db\u5217\u5f62\u5f0f\uff0c\u4f9d\u6b21\u4e3a\uff1arowKey\uff0cfamily:qualifier\uff0ctimestamp\uff0cvalue\u3002

    \u8bfb\u53d6\u65f6\u9700\u8981\u660e\u786e\u6307\u5b9a\u8981\u8bfb\u53d6\u7684\u5217\uff0c\u628a\u6bcf\u4e00\u4e2a cell \u4e2d\u7684\u503c\uff0c\u4f5c\u4e3a\u4e00\u6761\u8bb0\u5f55\uff08record\uff09\uff0c\u82e5\u6709\u591a\u4e2a\u7248\u672c\u5c31\u6709\u591a\u6761\u8bb0\u5f55\uff08record\uff09\u3002\u5982\uff1a

    hbase(main):018:0> scan 'users',{VERSIONS=>5}\nROW               COLUMN+CELL\n lisi             column=address:city, timestamp=1457101972764, value=beijing\n lisi             column=address:contry, timestamp=1457102773908, value=china\n lisi             column=address:province, timestamp=1457101972736, value=beijing\n lisi             column=info:age, timestamp=1457101972548, value=27\n lisi             column=info:birthday, timestamp=1457101972604, value=1987-06-17\n lisi             column=info:company, timestamp=1457101972653, value=baidu\n xiaoming         column=address:city, timestamp=1457082196082, value=hangzhou\n xiaoming         column=address:contry, timestamp=1457082195729, value=china\n xiaoming         column=address:province, timestamp=1457082195773, value=zhejiang\n xiaoming         column=info:age, timestamp=1457082218735, value=29\n xiaoming         column=info:age, timestamp=1457082178630, value=24\n xiaoming         column=info:birthday, timestamp=1457082186830, value=1987-06-17\n xiaoming         column=info:company, timestamp=1457082189826, value=alibaba\n2 row(s) in 0.0260 seconds\n

    \u8bfb\u53d6\u540e\u6570\u636e(4 \u5217)

    rowKey column:qualifier timestamp value lisi address:city 1457101972764 beijing lisi address:contry 1457102773908 china lisi address:province 1457101972736 beijing lisi info:age 1457101972548 27 lisi info:birthday 1457101972604 1987-06-17 lisi info:company 1457101972653 beijing xiaoming address:city 1457082196082 hangzhou xiaoming address:contry 1457082195729 china xiaoming address:province 1457082195773 zhejiang xiaoming info:age 1457082218735 29 xiaoming info:age 1457082178630 24 xiaoming info:birthday 1457082186830 1987-06-17 xiaoming info:company 1457082189826 alibaba"},{"location":"reader/hbase11xreader/#_3","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4ece HBase \u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a\uff0c\u5206\u522b\u4e3a\u6807\u51c6\u6a21\u5f0f\u548c\u591a\u7248\u672c\u6a21\u5f0f

    job/hbase11x2stream_normal.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"hbase11xreader\",\n        \"parameter\": {\n          \"hbaseConfig\": {\n            \"hbase.zookeeper.quorum\": \"xxxf\"\n          },\n          \"table\": \"users\",\n          \"encoding\": \"utf-8\",\n          \"mode\": \"normal\",\n          \"column\": [\n            {\n              \"name\": \"rowkey\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: age\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: birthday\",\n              \"type\": \"date\",\n              \"format\": \"yyyy-MM-dd\"\n            },\n            {\n              \"name\": \"info: company\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: country\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: province\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: city\",\n              \"type\": \"string\"\n            }\n          ],\n          \"range\": {\n            \"startRowkey\": \"\",\n            \"endRowkey\": \"\",\n            \"isBinaryRowkey\": true\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"txtfilewriter\",\n        \"parameter\": {\n          \"path\": \"/Users/shf/workplace/addax_test/hbase11xreader/result\",\n          \"fileName\": \"qiran\",\n          \"writeMode\": \"truncate\"\n        }\n      }\n    }\n  }\n}\n
    job/hbase11x2srtream_version.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"hbase11xreader\",\n        \"parameter\": {\n          \"hbaseConfig\": {\n            \"hbase.zookeeper.quorum\": \"127.0.0.1:2181\"\n          },\n          \"table\": \"users\",\n          \"encoding\": \"utf-8\",\n          \"mode\": \"multiVersionFixedColumn\",\n          \"maxVersion\": \"-1\",\n          \"column\": [\n            {\n              \"name\": \"rowkey\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: age\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: birthday\",\n              \"type\": \"date\",\n              \"format\": \"yyyy-MM-dd\"\n            },\n            {\n              \"name\": \"info: company\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: contry\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: province\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: city\",\n              \"type\": \"string\"\n            }\n          ],\n          \"range\": {\n            \"startRowkey\": \"\",\n            \"endRowkey\": \"\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"txtfilewriter\",\n        \"parameter\": {\n          \"path\": \"/Users/shf/workplace/addax_test/hbase11xreader/result\",\n          \"fileName\": \"qiran\",\n          \"writeMode\": \"truncate\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/hbase11xreader/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 hbaseConfig \u662f map \u65e0 \u8fde\u63a5 HBase \u96c6\u7fa4\u9700\u8981\u7684\u914d\u7f6e\u4fe1\u606f, hbase.zookeeper.quorum \u4e3a\u5fc5\u586b\u9879\uff0c\u5176\u4ed6 client \u7684\u914d\u7f6e\u4e3a\u53ef\u9009\u9879 mode \u662f string \u65e0 \u8bfb\u53d6 HBase \u7684\u6a21\u5f0f\uff0c\u53ef\u586b\u5199 normal \u6216 multiVersionFixedColumn table \u662f string \u65e0 \u8981\u8bfb\u53d6\u7684 hbase \u8868\u540d\uff08\u5927\u5c0f\u5199\u654f\u611f\uff09 encoding \u5426 string UTF-8 \u7f16\u7801\u65b9\u5f0f\uff0cUTF-8 \u6216\u662f GBK\uff0c\u7528\u4e8e\u5bf9\u4e8c\u8fdb\u5236\u5b58\u50a8\u7684 HBase byte[] \u8f6c\u4e3a String \u65f6\u7684\u7f16\u7801 column \u662f list<map> \u65e0 \u8981\u8bfb\u53d6\u7684 hbase \u5b57\u6bb5\uff0cnormal \u6a21\u5f0f\u4e0e multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b\u9879, \u8be6\u7ec6\u8bf4\u660e\u89c1\u4e0b\u6587 maxVersion \u662f string \u65e0 \u6307\u5b9a\u5728\u591a\u7248\u672c\u6a21\u5f0f\u4e0b\u8bfb\u53d6\u7684\u7248\u672c\u6570\uff0c-1 \u8868\u793a\u8bfb\u53d6\u6240\u6709\u7248\u672c, multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b range \u5426 string \u65e0 \u6307\u5b9a\u8bfb\u53d6\u7684rowkey \u8303\u56f4, \u8be6\u89c1\u4e0b\u6587 scanCacheSize \u5426 int 256 \u6bcf\u6b21\u4ece\u670d\u52a1\u5668\u7aef\u8bfb\u53d6\u7684\u884c\u6570 scanBatchSize \u5426 int 100 \u6bcf\u6b21\u4ece\u670d\u52a1\u5668\u7aef\u8bfb\u53d6\u7684\u5217\u6570"},{"location":"reader/hbase11xreader/#column","title":"column","text":"

    \u63cf\u8ff0\uff1a\u8981\u8bfb\u53d6\u7684 hbase \u5b57\u6bb5\uff0cnormal \u6a21\u5f0f\u4e0e multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b\u9879\u3002

    "},{"location":"reader/hbase11xreader/#normal_1","title":"normal \u6a21\u5f0f","text":"

    name \u6307\u5b9a\u8bfb\u53d6\u7684 hbase \u5217\uff0c\u9664\u4e86 rowkey \u5916\uff0c\u5fc5\u987b\u4e3a \u5217\u65cf:\u5217\u540d \u7684\u683c\u5f0f\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cformat\u6307\u5b9a\u65e5\u671f\u7c7b\u578b\u7684\u683c\u5f0f\uff0c value \u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece hbase \u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636e value \u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002\u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"column\": [\n    {\n      \"name\": \"rowkey\",\n      \"type\": \"string\"\n    },\n    {\n      \"value\": \"test\",\n      \"type\": \"string\"\n    }\n  ]\n}\n

    normal \u6a21\u5f0f\u4e0b\uff0c\u5bf9\u4e8e\u7528\u6237\u6307\u5b9a Column \u4fe1\u606f\uff0ctype \u5fc5\u987b\u586b\u5199\uff0cname/value \u5fc5\u987b\u9009\u62e9\u5176\u4e00\u3002

    "},{"location":"reader/hbase11xreader/#multiversionfixedcolumn_1","title":"multiVersionFixedColumn \u6a21\u5f0f","text":"

    name \u6307\u5b9a\u8bfb\u53d6\u7684 hbase \u5217\uff0c\u9664\u4e86 rowkey \u5916\uff0c\u5fc5\u987b\u4e3a \u5217\u65cf:\u5217\u540d \u7684\u683c\u5f0f\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cformat\u6307\u5b9a\u65e5\u671f\u7c7b\u578b\u7684\u683c\u5f0f \u3002 multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u4e0d\u652f\u6301\u5e38\u91cf\u5217\u3002\u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"column\": [\n    {\n      \"name\": \"rowkey\",\n      \"type\": \"string\"\n    },\n    {\n      \"name\": \"info: age\",\n      \"type\": \"string\"\n    }\n  ]\n}\n
    "},{"location":"reader/hbase11xreader/#range","title":"range","text":"

    \u6307\u5b9a\u8bfb\u53d6\u7684 rowkey \u8303\u56f4

    \u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"range\": {\n    \"startRowkey\": \"aaa\",\n    \"endRowkey\": \"ccc\",\n    \"isBinaryRowkey\": false\n  }\n}\n
    "},{"location":"reader/hbase11xreader/#_5","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u4e0b\u9762\u5217\u51fa\u652f\u6301\u7684\u8bfb\u53d6 HBase \u6570\u636e\u7c7b\u578b\uff0cHbaseReader \u9488\u5bf9 HBase \u7c7b\u578b\u8f6c\u6362\u5217\u8868:

    Addax \u5185\u90e8\u7c7b\u578b HBase \u6570\u636e\u7c7b\u578b Long int, short ,long Double float, double String string, binarystring Date date Boolean boolean

    \u8bf7\u6ce8\u610f:

    \u9664\u4e0a\u8ff0\u7f57\u5217\u5b57\u6bb5\u7c7b\u578b\u5916\uff0c\u5176\u4ed6\u7c7b\u578b\u5747\u4e0d\u652f\u6301

    "},{"location":"reader/hbase11xreader/#_6","title":"\u9650\u5236","text":"
    1. \u76ee\u524d\u4e0d\u652f\u6301\u52a8\u6001\u5217\u7684\u8bfb\u53d6\u3002\u8003\u8651\u7f51\u7edc\u4f20\u8f93\u6d41\u91cf\uff08\u652f\u6301\u52a8\u6001\u5217\uff0c\u9700\u8981\u5148\u5c06 hbase \u6240\u6709\u5217\u7684\u6570\u636e\u8bfb\u53d6\u51fa\u6765\uff0c\u518d\u6309\u89c4\u5219\u8fdb\u884c\u8fc7\u6ee4\uff09\uff0c\u73b0\u652f\u6301\u7684\u4e24\u79cd\u8bfb\u53d6\u6a21\u5f0f\u4e2d\u9700\u8981\u7528\u6237\u660e\u786e\u6307\u5b9a\u8981\u8bfb\u53d6\u7684\u5217\u3002
    2. \u5173\u4e8e\u540c\u6b65\u4f5c\u4e1a\u7684\u5207\u5206\uff1a\u76ee\u524d\u7684\u5207\u5206\u65b9\u5f0f\u662f\u6839\u636e\u7528\u6237 hbase \u8868\u6570\u636e\u7684 region \u5206\u5e03\u8fdb\u884c\u5207\u5206\u3002\u5373\uff1a\u5728\u7528\u6237\u586b\u5199\u7684 [startrowkey\uff0cendrowkey\uff3d \u8303\u56f4\u5185\uff0c\u4e00\u4e2a region \u4f1a\u5207\u5206\u6210\u4e00\u4e2a task\uff0c\u5355\u4e2a region \u4e0d\u8fdb\u884c\u5207\u5206\u3002
    3. multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u4e0d\u652f\u6301\u589e\u52a0\u5e38\u91cf\u5217
    "},{"location":"reader/hbase11xsqlreader/","title":"HBase11x SQL Reader","text":"

    HBase11x SQL Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Phoenix(HBase SQL)\u8bfb\u53d6\u6570\u636e, \u652f\u6301\u7684 HBase \u7248\u672c\u4e3a 1.x

    "},{"location":"reader/hbase11xsqlreader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4ecePhoenix\u540c\u6b65\u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a:

    {\n    \"job\": {\n        \"setting\": {\n            \"speed\": {\n                \"byte\":-1,\n              \"channel\": 1\n            }\n        },\n        \"content\": [ {\n                \"reader\": {\n                    \"name\": \"hbase11xsqlreader\",\n                    \"parameter\": {\n                        \"hbaseConfig\": {\n                            \"hbase.zookeeper.quorum\": \"node1,node2,node3\"\n                        },\n                        \"table\": \"US_POPULATION\",\n                        \"column\": [],\n                        \"where\": \"1=1\",\n                        \"querySql\": \"\"\n                    }\n                },\n                \"writer\": {\n                    \"name\": \"streamwriter\",\n                    \"parameter\": {\n                        \"print\":true,\n                        \"encoding\": \"UTF-8\"\n                    }\n                }\n            }\n        ]\n    }\n}\n
    "},{"location":"reader/hbase11xsqlreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 hbaseConfig \u662f map \u65e0 \u9700\u8981\u901a\u8fc7 Phoenix \u5ba2\u6237\u7aef\u53bb\u8fde\u63a5 hbase \u96c6\u7fa4\uff0c\u56e0\u6b64\u8fd9\u91cc\u9700\u8981\u586b\u5199\u5bf9\u5e94 hbase \u96c6\u7fa4\u7684 zkurl\u5730\u5740 table \u662f string \u65e0 \u6307\u5b9a Phoenix \u4e2d\u7684\u8868\u540d,\u5982\u679c\u6709 namespace\uff0c\u8be5\u503c\u8bbe\u7f6e\u4e3a namespace.tablename querySql \u5426 string \u65e0 \u4e0d\u662f\u76f4\u63a5\u67e5\u8be2\u8868\uff0c\u800c\u662f\u63d0\u4f9b\u5177\u4f53\u7684\u67e5\u8be2\u8bed\u53e5\uff0c\u5982\u679c\u8be5\u53c2\u6570\u548c table \u53c2\u6570\u540c\u65f6\u5b58\u5728\uff0c\u5219\u4f18\u5148\u4f7f\u7528\u8be5\u53c2\u6570 column \u662f list<map> \u65e0 \u586b\u5199\u9700\u8981\u4ecephoenix\u8868\u4e2d\u8bfb\u53d6\u7684\u5217\u540d\u96c6\u5408\uff0c\u4f7f\u7528JSON\u7684\u6570\u7ec4\u63cf\u8ff0\u5b57\u6bb5\u4fe1\u606f\uff0c\u7a7a\u503c\u6216 \"*\" \u8868\u793a\u8bfb\u53d6\u6240\u6709\u5217 where \u5426 string \u65e0 where \u6761\u4ef6"},{"location":"reader/hbase11xsqlreader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u76ee\u524d\u652f\u6301\u5927\u90e8\u5206 Phoenix\u7c7b\u578b\uff0c\u4f46\u4e5f\u5b58\u5728\u90e8\u5206\u4e2a\u522b\u7c7b\u578b\u6ca1\u6709\u652f\u6301\u7684\u60c5\u51b5\uff0c\u8bf7\u6ce8\u610f\u68c0\u67e5\u4f60\u7684\u7c7b\u578b\u3002

    \u4e0b\u9762\u5217\u51fa\u7c7b\u578b\u8f6c\u6362\u5217\u8868:

    Addax \u5185\u90e8\u7c7b\u578b Phoenix \u6570\u636e\u7c7b\u578b String CHAR, VARCHAR Bytes BINARY, VARBINARY Bool BOOLEAN Long INTEGER, TINYINT, SMALLINT, BIGINT Double FLOAT, DECIMAL, DOUBLE, Date DATE, TIME, TIMESTAMP"},{"location":"reader/hbase20xreader/","title":"HBase20 Reader","text":"

    HBase20 Reader \u63d2\u4ef6\u652f\u6301\u4ece HBase 2.x \u7248\u672c\u8bfb\u53d6\u6570\u636e\uff0c \u5176\u5b9e\u73b0\u65b9\u5f0f\u4e3a \u901a\u8fc7 HBase \u7684 Java \u5ba2\u6237\u7aef\u8fde\u63a5\u8fdc\u7a0b HBase \u670d\u52a1\uff0c\u5e76\u901a\u8fc7 Scan \u65b9\u5f0f\u8bfb\u53d6\u4f60\u6307\u5b9a rowkey \u8303\u56f4\u5185\u7684\u6570\u636e\u3002

    "},{"location":"reader/hbase20xreader/#_1","title":"\u914d\u7f6e","text":"

    \u4ee5\u4e0b\u6f14\u793a\u57fa\u4e8e\u4e0b\u9762\u521b\u5efa\u7684\u8868\u4ee5\u53ca\u6570\u636e

    create 'users', {NAME=>'address', VERSIONS=>100},{NAME=>'info',VERSIONS=>1000}\nput 'users', 'lisi', 'address:country', 'china1', 20200101\nput 'users', 'lisi', 'address:province',    'beijing1', 20200101\nput 'users', 'lisi', 'info:age',        27, 20200101\nput 'users', 'lisi', 'info:birthday',   '1987-06-17', 20200101\nput 'users', 'lisi', 'info:company',    'baidu1', 20200101\nput 'users', 'xiaoming', 'address:city',    'hangzhou1', 20200101\nput 'users', 'xiaoming', 'address:country', 'china1', 20200101\nput 'users', 'xiaoming', 'address:province',    'zhejiang1',20200101\nput 'users', 'xiaoming', 'info:age',        29, 20200101\nput 'users', 'xiaoming', 'info:birthday',   '1987-06-17',20200101\nput 'users', 'xiaoming', 'info:company',    'alibaba1', 20200101\nput 'users', 'lisi', 'address:country', 'china2', 20200102\nput 'users', 'lisi', 'address:province',    'beijing2', 20200102\nput 'users', 'lisi', 'info:age',        27, 20200102\nput 'users', 'lisi', 'info:birthday',   '1987-06-17', 20200102\nput 'users', 'lisi', 'info:company',    'baidu2', 20200102\nput 'users', 'xiaoming', 'address:city',    'hangzhou2', 20200102\nput 'users', 'xiaoming', 'address:country', 'china2', 20200102\nput 'users', 'xiaoming', 'address:province',    'zhejiang2', 20200102\nput 'users', 'xiaoming', 'info:age',        29, 20200102\nput 'users', 'xiaoming', 'info:birthday',   '1987-06-17', 20200102\nput 'users', 'xiaoming', 'info:company',    'alibaba2', 20200102\n
    "},{"location":"reader/hbase20xreader/#normal","title":"normal \u6a21\u5f0f","text":"

    \u628aHBase\u4e2d\u7684\u8868\uff0c\u5f53\u6210\u666e\u901a\u4e8c\u7ef4\u8868\uff08\u6a2a\u8868\uff09\u8fdb\u884c\u8bfb\u53d6,\u8bfb\u53d6\u6700\u65b0\u7248\u672c\u6570\u636e\u3002\u5982\uff1a

    hbase(main):017:0> scan 'users'\nROW           COLUMN+CELL\n lisi         column=address:city, timestamp=1457101972764, value=beijing\n lisi         column=address:country, timestamp=1457102773908, value=china\n lisi         column=address:province, timestamp=1457101972736, value=beijing\n lisi         column=info:age, timestamp=1457101972548, value=27\n lisi         column=info:birthday, timestamp=1457101972604, value=1987-06-17\n lisi         column=info:company, timestamp=1457101972653, value=baidu\n xiaoming     column=address:city, timestamp=1457082196082, value=hangzhou\n xiaoming     column=address:country, timestamp=1457082195729, value=china\n xiaoming     column=address:province, timestamp=1457082195773, value=zhejiang\n xiaoming     column=info:age, timestamp=1457082218735, value=29\n xiaoming     column=info:birthday, timestamp=1457082186830, value=1987-06-17\n xiaoming     column=info:company, timestamp=1457082189826, value=alibaba\n2 row(s) in 0.0580 seconds\n

    \u8bfb\u53d6\u540e\u6570\u636e

    rowKey addres:city address:country address:province info:age info:birthday info:company lisi beijing china beijing 27 1987-06-17 baidu xiaoming hangzhou china zhejiang 29 1987-06-17 alibaba"},{"location":"reader/hbase20xreader/#multiversionfixedcolumn","title":"multiVersionFixedColumn \u6a21\u5f0f","text":"

    \u628aHBase\u4e2d\u7684\u8868\uff0c\u5f53\u6210\u7ad6\u8868\u8fdb\u884c\u8bfb\u53d6\u3002\u8bfb\u51fa\u7684\u6bcf\u6761\u8bb0\u5f55\u4e00\u5b9a\u662f\u56db\u5217\u5f62\u5f0f\uff0c\u4f9d\u6b21\u4e3a\uff1arowKey\uff0cfamily:qualifier\uff0ctimestamp\uff0cvalue\u3002

    \u8bfb\u53d6\u65f6\u9700\u8981\u660e\u786e\u6307\u5b9a\u8981\u8bfb\u53d6\u7684\u5217\uff0c\u628a\u6bcf\u4e00\u4e2a cell \u4e2d\u7684\u503c\uff0c\u4f5c\u4e3a\u4e00\u6761\u8bb0\u5f55\uff08record\uff09\uff0c\u82e5\u6709\u591a\u4e2a\u7248\u672c\u5c31\u6709\u591a\u6761\u8bb0\u5f55\uff08record\uff09\u3002\u5982\uff1a

    hbase(main):018:0> scan 'users',{VERSIONS=>5}\nROW              COLUMN+CELL\n lisi            column=address:city, timestamp=1457101972764, value=beijing\n lisi            column=address:contry, timestamp=1457102773908, value=china\n lisi            column=address:province, timestamp=1457101972736, value=beijing\n lisi            column=info:age, timestamp=1457101972548, value=27\n lisi            column=info:birthday, timestamp=1457101972604, value=1987-06-17\n lisi            column=info:company, timestamp=1457101972653, value=baidu\n xiaoming        column=address:city, timestamp=1457082196082, value=hangzhou\n xiaoming        column=address:contry, timestamp=1457082195729, value=china\n xiaoming        column=address:province, timestamp=1457082195773, value=zhejiang\n xiaoming        column=info:age, timestamp=1457082218735, value=29\n xiaoming        column=info:age, timestamp=1457082178630, value=24\n xiaoming        column=info:birthday, timestamp=1457082186830, value=1987-06-17\n xiaoming        column=info:company, timestamp=1457082189826, value=alibaba\n2 row(s) in 0.0260 seconds\n

    \u8bfb\u53d6\u540e\u6570\u636e(4\u5217)

    rowKey column:qualifier timestamp value lisi address:city 1457101972764 beijing lisi address:contry 1457102773908 china lisi address:province 1457101972736 beijing lisi info:age 1457101972548 27 lisi info:birthday 1457101972604 1987-06-17 lisi info:company 1457101972653 beijing xiaoming address:city 1457082196082 hangzhou xiaoming address:contry 1457082195729 china xiaoming address:province 1457082195773 zhejiang xiaoming info:age 1457082218735 29 xiaoming info:age 1457082178630 24 xiaoming info:birthday 1457082186830 1987-06-17 xiaoming info:company 1457082189826 alibaba

    \u914d\u7f6e\u4e00\u4e2a\u4ece HBase \u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a\uff0c\u5206\u522b\u4e3a\u6807\u51c6\u6a21\u5f0f\u548c\u591a\u7248\u672c\u6a21\u5f0f

    hbase20x2stream_normal.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"hbase11xreader\",\n        \"parameter\": {\n          \"hbaseConfig\": {\n            \"hbase.zookeeper.quorum\": \"xxxf\"\n          },\n          \"table\": \"users\",\n          \"encoding\": \"utf-8\",\n          \"mode\": \"normal\",\n          \"column\": [\n            {\n              \"name\": \"rowkey\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: age\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: birthday\",\n              \"type\": \"date\",\n              \"format\": \"yyyy-MM-dd\"\n            },\n            {\n              \"name\": \"info: company\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: country\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: province\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: city\",\n              \"type\": \"string\"\n            }\n          ],\n          \"range\": {\n            \"startRowkey\": \"\",\n            \"endRowkey\": \"\",\n            \"isBinaryRowkey\": true\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"txtfilewriter\",\n        \"parameter\": {\n          \"path\": \"/Users/shf/workplace/addax_test/hbase11xreader/result\",\n          \"fileName\": \"qiran\",\n          \"writeMode\": \"truncate\"\n        }\n      }\n    }\n  }\n}\n
    hbase20x2stream_version.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"hbase11xreader\",\n        \"parameter\": {\n          \"hbaseConfig\": {\n            \"hbase.zookeeper.quorum\": \"xxx\"\n          },\n          \"table\": \"users\",\n          \"encoding\": \"utf-8\",\n          \"mode\": \"multiVersionFixedColumn\",\n          \"maxVersion\": \"-1\",\n          \"column\": [\n            {\n              \"name\": \"rowkey\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: age\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"info: birthday\",\n              \"type\": \"date\",\n              \"format\": \"yyyy-MM-dd\"\n            },\n            {\n              \"name\": \"info: company\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: contry\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: province\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"address: city\",\n              \"type\": \"string\"\n            }\n          ],\n          \"range\": {\n            \"startRowkey\": \"\",\n            \"endRowkey\": \"\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"txtfilewriter\",\n        \"parameter\": {\n          \"path\": \"/Users/shf/workplace/addax_test/hbase11xreader/result\",\n          \"fileName\": \"qiran\",\n          \"writeMode\": \"truncate\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/hbase20xreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 hbaseConfig \u662f map \u65e0 \u8fde\u63a5HBase\u96c6\u7fa4\u9700\u8981\u7684\u914d\u7f6e\u4fe1\u606f, hbase.zookeeper.quorum \u4e3a\u5fc5\u586b\u9879\uff0c\u5176\u4ed6\u4e3a\u53ef\u9009\u9879 mode \u662f string \u65e0 \u8bfb\u53d6hbase\u7684\u6a21\u5f0f\uff0c\u53ef\u586b\u5199 normal \u6216 multiVersionFixedColumn table \u662f string \u65e0 \u8981\u8bfb\u53d6\u7684 hbase \u8868\u540d\uff08\u5927\u5c0f\u5199\u654f\u611f\uff09 encoding \u5426 string UTF-8 \u7f16\u7801\u65b9\u5f0f\uff0cUTF-8 \u6216\u662f GBK\uff0c\u7528\u4e8e\u5bf9\u4e8c\u8fdb\u5236\u5b58\u50a8\u7684 HBase byte[] \u8f6c\u4e3a String \u65f6\u7684\u7f16\u7801 column \u662f list<map> \u65e0 \u8981\u8bfb\u53d6\u7684\u5b57\u6bb5\uff0cnormal \u6a21\u5f0f\u4e0e multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b\u9879, \u8be6\u7ec6\u8bf4\u660e\u89c1\u4e0b\u6587 maxVersion \u662f string \u65e0 \u6307\u5b9a\u5728\u591a\u7248\u672c\u6a21\u5f0f\u4e0b\u8bfb\u53d6\u7684\u7248\u672c\u6570\uff0c-1 \u8868\u793a\u8bfb\u53d6\u6240\u6709\u7248\u672c, multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b range \u5426 string \u65e0 \u6307\u5b9a\u8bfb\u53d6\u7684 rowkey \u8303\u56f4, \u8be6\u89c1\u4e0b\u6587 scanCacheSize \u5426 int 256 \u6bcf\u6b21\u4ece\u670d\u52a1\u5668\u7aef\u8bfb\u53d6\u7684\u884c\u6570 scanBatchSize \u5426 int 100 \u6bcf\u6b21\u4ece\u670d\u52a1\u5668\u7aef\u8bfb\u53d6\u7684\u5217\u6570"},{"location":"reader/hbase20xreader/#column","title":"column","text":"

    \u63cf\u8ff0\uff1a\u8981\u8bfb\u53d6\u7684\u5b57\u6bb5\uff0cnormal \u6a21\u5f0f\u4e0e multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b\u9879\u3002

    "},{"location":"reader/hbase20xreader/#normal_1","title":"normal \u6a21\u5f0f","text":"

    name \u6307\u5b9a\u8bfb\u53d6\u7684 hbase \u5217\uff0c\u9664\u4e86 rowkey \u5916\uff0c\u5fc5\u987b\u4e3a \u5217\u65cf:\u5217\u540d \u7684\u683c\u5f0f\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cformat\u6307\u5b9a\u65e5\u671f\u7c7b\u578b\u7684\u683c\u5f0f\uff0c value \u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece hbase \u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636e value \u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002\u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"column\": [\n    {\n      \"name\": \"rowkey\",\n      \"type\": \"string\"\n    },\n    {\n      \"value\": \"test\",\n      \"type\": \"string\"\n    }\n  ]\n}\n

    normal \u6a21\u5f0f\u4e0b\uff0c\u5bf9\u4e8e\u7528\u6237\u6307\u5b9aColumn\u4fe1\u606f\uff0ctype\u5fc5\u987b\u586b\u5199\uff0cname/value\u5fc5\u987b\u9009\u62e9\u5176\u4e00\u3002

    "},{"location":"reader/hbase20xreader/#multiversionfixedcolumn_1","title":"multiVersionFixedColumn \u6a21\u5f0f","text":"

    name \u6307\u5b9a\u8bfb\u53d6\u7684 hbase \u5217\uff0c\u9664\u4e86 rowkey \u5916\uff0c\u5fc5\u987b\u4e3a \u5217\u65cf:\u5217\u540d \u7684\u683c\u5f0f\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cformat\u6307\u5b9a\u65e5\u671f\u7c7b\u578b\u7684\u683c\u5f0f \u3002 multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u4e0d\u652f\u6301\u5e38\u91cf\u5217\u3002 \u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"mode\": \"multiVersionFixedColumn\",\n  \"maxVersion\": 3,\n  \"column\": [\n    {\n      \"name\": \"rowkey\",\n      \"type\": \"string\"\n    },\n    {\n      \"name\": \"info: age\",\n      \"type\": \"string\"\n    }\n  ]\n}\n
    "},{"location":"reader/hbase20xreader/#range","title":"range","text":"

    \u6307\u5b9a\u8bfb\u53d6\u7684 rowkey \u8303\u56f4

    \u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"range\": {\n    \"startRowkey\": \"aaa\",\n    \"endRowkey\": \"ccc\",\n    \"isBinaryRowkey\": false\n  }\n}\n
    "},{"location":"reader/hbase20xreader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u4e0b\u9762\u5217\u51fa\u652f\u6301\u7684\u8bfb\u53d6HBase\u6570\u636e\u7c7b\u578b:

    Addax \u5185\u90e8\u7c7b\u578b HBase \u6570\u636e\u7c7b\u578b Long int, short ,long Double float, double String string, binarystring Date date Boolean boolean

    \u8bf7\u6ce8\u610f:

    \u9664\u4e0a\u8ff0\u7f57\u5217\u5b57\u6bb5\u7c7b\u578b\u5916\uff0c\u5176\u4ed6\u7c7b\u578b\u5747\u4e0d\u652f\u6301

    "},{"location":"reader/hbase20xreader/#_4","title":"\u9650\u5236","text":"
    1. \u76ee\u524d\u4e0d\u652f\u6301\u52a8\u6001\u5217\u7684\u8bfb\u53d6\u3002\u8003\u8651\u7f51\u7edc\u4f20\u8f93\u6d41\u91cf\uff08\u652f\u6301\u52a8\u6001\u5217\uff0c\u9700\u8981\u5148\u5c06hbase\u6240\u6709\u5217\u7684\u6570\u636e\u8bfb\u53d6\u51fa\u6765\uff0c\u518d\u6309\u89c4\u5219\u8fdb\u884c\u8fc7\u6ee4\uff09\uff0c\u73b0\u652f\u6301\u7684\u4e24\u79cd\u8bfb\u53d6\u6a21\u5f0f\u4e2d\u9700\u8981\u7528\u6237\u660e\u786e\u6307\u5b9a\u8981\u8bfb\u53d6\u7684\u5217\u3002
    2. \u5173\u4e8e\u540c\u6b65\u4f5c\u4e1a\u7684\u5207\u5206\uff1a\u76ee\u524d\u7684\u5207\u5206\u65b9\u5f0f\u662f\u6839\u636e\u7528\u6237hbase\u8868\u6570\u636e\u7684region\u5206\u5e03\u8fdb\u884c\u5207\u5206\u3002\u5373\uff1a\u5728\u7528\u6237\u586b\u5199\u7684 [startrowkey\uff0cendrowkey] \u8303\u56f4\u5185\uff0c\u4e00\u4e2aregion\u4f1a\u5207\u5206\u6210\u4e00\u4e2atask\uff0c\u5355\u4e2aregion\u4e0d\u8fdb\u884c\u5207\u5206\u3002
    3. multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u4e0d\u652f\u6301\u589e\u52a0\u5e38\u91cf\u5217
    "},{"location":"reader/hbase20xsqlreader/","title":"HBase20 SQL Reader","text":"

    HBase20 SQL Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Phoenix(HBase SQL) \u8bfb\u53d6\u6570\u636e\uff0c\u5bf9\u5e94\u7248\u672c\u4e3a HBase2.X \u548c Phoenix5.X\u3002

    "},{"location":"reader/hbase20xsqlreader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4ece Phoenix \u540c\u6b65\u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a:

    {\n  \"job\": {\n    \"content\": [\n      {\n        \"reader\": {\n          \"name\": \"hbase20xsqlreader\",\n          \"parameter\": {\n            \"queryServerAddress\": \"http://127.0.0.1:8765\",\n            \"serialization\": \"PROTOBUF\",\n            \"table\": \"TEST\",\n            \"column\": [\"ID\", \"NAME\"],\n            \"splitKey\": \"ID\"\n          }\n        },\n        \"writer\": {\n          \"name\": \"streamwriter\",\n          \"parameter\": {\n            \"encoding\": \"UTF-8\",\n            \"print\": true\n          }\n        }\n      }\n    ],\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3,\n        \"bytes\": -1\n      }\n    }\n  }\n}\n
    "},{"location":"reader/hbase20xsqlreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 queryServerAddress \u662f string \u65e0 Phoenix QueryServer \u5730\u5740, \u8be5\u63d2\u4ef6\u901a\u8fc7 PQS \u8fdb\u884c\u8fde\u63a5 serialization \u5426 string PROTOBUF QueryServer \u4f7f\u7528\u7684\u5e8f\u5217\u5316\u534f\u8bae table \u662f string \u65e0 \u6240\u8981\u8bfb\u53d6\u8868\u540d schema \u5426 string \u65e0 \u8868\u6240\u5728\u7684 schema column \u5426 list `` \u586b\u5199\u9700\u8981\u4ece phoenix \u8868\u4e2d\u8bfb\u53d6\u7684\u5217\u540d\u96c6\u5408\uff0c\u7a7a\u503c\u8868\u793a\u8bfb\u53d6\u6240\u6709\u5217 splitKey \u662f string \u65e0 \u6839\u636e\u6570\u636e\u7279\u5f81\u52a8\u6001\u6307\u5b9a\u5207\u5206\u70b9\uff0c\u5bf9\u8868\u6570\u636e\u6309\u7167\u6307\u5b9a\u7684\u5217\u7684\u6700\u5927\u3001\u6700\u5c0f\u503c\u8fdb\u884c\u5207\u5206,\u4ec5\u652f\u6301\u6574\u578b\u548c\u5b57\u7b26\u4e32\u7c7b\u578b splitPoints \u5426 string \u65e0 \u6309\u7167\u8868\u7684 split \u8fdb\u884c\u5207\u5206 where \u5426 string \u65e0 \u652f\u6301\u5bf9\u8868\u67e5\u8be2\u589e\u52a0\u8fc7\u6ee4\u6761\u4ef6\uff0c\u6bcf\u4e2a\u5207\u5206\u90fd\u4f1a\u643a\u5e26\u8be5\u8fc7\u6ee4\u6761\u4ef6 querySql \u5426 string \u65e0 \u652f\u6301\u6307\u5b9a\u591a\u4e2a\u67e5\u8be2\u8bed\u53e5\uff0c\u4f46\u67e5\u8be2\u5217\u7c7b\u578b\u548c\u6570\u76ee\u5fc5\u987b\u4fdd\u6301\u4e00\u81f4"},{"location":"reader/hbase20xsqlreader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u76ee\u524d\u652f\u6301\u5927\u90e8\u5206 Phoenix \u7c7b\u578b\uff0c\u4f46\u4e5f\u5b58\u5728\u90e8\u5206\u4e2a\u522b\u7c7b\u578b\u6ca1\u6709\u652f\u6301\u7684\u60c5\u51b5\uff0c\u8bf7\u6ce8\u610f\u68c0\u67e5\u4f60\u7684\u7c7b\u578b\u3002

    Addax \u5185\u90e8\u7c7b\u578b Phoenix \u6570\u636e\u7c7b\u578b String CHAR, VARCHAR Bytes BINARY, VARBINARY Bool BOOLEAN Long INTEGER, TINYINT, SMALLINT, BIGINT Double FLOAT, DECIMAL, DOUBLE, Date DATE, TIME, TIMESTAMP"},{"location":"reader/hbase20xsqlreader/#_4","title":"\u7ea6\u675f\u9650\u5236","text":""},{"location":"reader/hdfsreader/","title":"HDFS Reader","text":"

    HDFS Reader \u63d0\u4f9b\u4e86\u8bfb\u53d6\u5206\u5e03\u5f0f\u6587\u4ef6\u7cfb\u7edf Hadoop HDFS \u6570\u636e\u5b58\u50a8\u7684\u80fd\u529b\u3002

    \u76ee\u524d HdfsReader \u652f\u6301\u7684\u6587\u4ef6\u683c\u5f0f\u5982\u4e0b\uff1a

    "},{"location":"reader/hdfsreader/#_1","title":"\u529f\u80fd\u4e0e\u9650\u5236","text":"
    1. \u652f\u6301 textfile, orcfile, parquet, rcfile, sequence file \u548c csv \u683c\u5f0f\u7684\u6587\u4ef6\uff0c\u4e14\u8981\u6c42\u6587\u4ef6\u5185\u5bb9\u5b58\u653e\u7684\u662f\u4e00\u5f20\u903b\u8f91\u610f\u4e49\u4e0a\u7684\u4e8c\u7ef4\u8868\u3002
    2. \u652f\u6301\u591a\u79cd\u7c7b\u578b\u6570\u636e\u8bfb\u53d6(\u4f7f\u7528 String \u8868\u793a)\uff0c\u652f\u6301\u5217\u88c1\u526a\uff0c\u652f\u6301\u5217\u5e38\u91cf
    3. \u652f\u6301\u9012\u5f52\u8bfb\u53d6\u3001\u652f\u6301\u6b63\u5219\u8868\u8fbe\u5f0f\uff08*\u548c ?\uff09\u3002
    4. \u652f\u6301\u5e38\u89c1\u7684\u538b\u7f29\u7b97\u6cd5\uff0c\u5305\u62ec GZIP\uff0c SNAPPY\uff0c ZLIB \u7b49\u3002
    5. \u591a\u4e2a File \u53ef\u4ee5\u652f\u6301\u5e76\u53d1\u8bfb\u53d6\u3002
    6. \u652f\u6301 sequence file \u6570\u636e\u538b\u7f29\uff0c\u76ee\u524d\u652f\u6301 lzo \u538b\u7f29\u65b9\u5f0f\u3002
    7. csv \u7c7b\u578b\u652f\u6301\u538b\u7f29\u683c\u5f0f\u6709\uff1agzip\u3001bz2\u3001zip\u3001lzo\u3001lzo_deflate\u3001snappy\u3002
    8. \u76ee\u524d\u63d2\u4ef6\u4e2d Hive \u7248\u672c\u4e3a 3.1.1\uff0cHadoop \u7248\u672c\u4e3a3.1.1, \u5728 Hadoop 2.7.x, Hadoop 3.1.x \u548c Hive 2.x, hive 3.1.x \u6d4b\u8bd5\u73af\u5883\u4e2d\u5199\u5165\u6b63\u5e38\uff1b\u5176\u5b83\u7248\u672c\u7406\u8bba\u4e0a\u90fd\u652f\u6301\uff0c\u4f46\u5728\u751f\u4ea7\u73af\u5883\u4f7f\u7528\u524d\uff0c\u8bf7\u8fdb\u4e00\u6b65\u6d4b\u8bd5\uff1b
    9. \u652f\u6301kerberos \u8ba4\u8bc1
    "},{"location":"reader/hdfsreader/#_2","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"hdfsreader\",\n        \"parameter\": {\n          \"path\": \"/user/hive/warehouse/mytable01/*\",\n          \"defaultFS\": \"hdfs://xxx:port\",\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"long\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"boolean\"\n            },\n            {\n              \"type\": \"string\",\n              \"value\": \"hello\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"double\"\n            }\n          ],\n          \"fileType\": \"orc\",\n          \"encoding\": \"UTF-8\",\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/hdfsreader/#_3","title":"\u914d\u7f6e\u9879\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e path \u662f string \u65e0 \u8981\u8bfb\u53d6\u7684\u6587\u4ef6\u8def\u5f84 defaultFS \u662f string \u65e0 HDFS NAMENODE \u8282\u70b9\u5730\u5740\uff0c\u5982\u679c\u914d\u7f6e\u4e86 HA \u6a21\u5f0f\uff0c\u5219\u4e3a defaultFS \u7684\u503c fileType \u662f string \u65e0 \u6587\u4ef6\u7684\u7c7b\u578b column \u662f list<map> \u65e0 \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868 fieldDelimiter \u5426 char , \u6307\u5b9a\u6587\u672c\u6587\u4ef6\u7684\u5b57\u6bb5\u5206\u9694\u7b26\uff0c\u4e8c\u8fdb\u5236\u6587\u4ef6\u4e0d\u9700\u8981\u6307\u5b9a\u8be5\u9879 encoding \u5426 string utf-8 \u8bfb\u53d6\u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e\uff0c \u76ee\u524d\u4ec5\u652f\u6301 utf-8 nullFormat \u5426 string \u65e0 \u53ef\u4ee5\u8868\u793a\u4e3a\u7a7a\u7684\u5b57\u7b26\uff0c\u5982\u679c\u7528\u6237\u914d\u7f6e: \"\\\\N\" \uff0c\u90a3\u4e48\u5982\u679c\u6e90\u5934\u6570\u636e\u662f \"\\N\" \uff0c\u89c6\u4f5c null \u5b57\u6bb5 haveKerberos \u5426 boolean \u65e0 \u662f\u5426\u542f\u7528 Kerberos \u8ba4\u8bc1\uff0c\u5982\u679c\u542f\u7528\uff0c\u5219\u9700\u8981\u540c\u65f6\u914d\u7f6e\u4e0b\u9762\u4e24\u9879 kerberosKeytabFilePath \u5426 string \u65e0 Kerberos \u8ba4\u8bc1\u7684\u51ed\u8bc1\u6587\u4ef6\u8def\u5f84, \u6bd4\u5982 /your/path/addax.service.keytab kerberosPrincipal \u5426 string \u65e0 Kerberos \u8ba4\u8bc1\u7684\u51ed\u8bc1\u4e3b\u4f53, \u6bd4\u5982 addax/node1@WGZHAO.COM compress \u5426 string \u65e0 \u6307\u5b9a\u8981\u8bfb\u53d6\u7684\u6587\u4ef6\u7684\u538b\u7f29\u683c\u5f0f hadoopConfig \u5426 map \u65e0 \u91cc\u53ef\u4ee5\u914d\u7f6e\u4e0e Hadoop \u76f8\u5173\u7684\u4e00\u4e9b\u9ad8\u7ea7\u53c2\u6570\uff0c\u6bd4\u5982 HA \u7684\u914d\u7f6e hdfsSitePath \u5426 string \u65e0 hdfs-site.xml \u7684\u8def\u5f84\uff0c\u8be6\u7ec6\u89e3\u91ca\u89c1\u4e0b"},{"location":"reader/hdfsreader/#path","title":"path","text":"

    \u8981\u8bfb\u53d6\u7684\u6587\u4ef6\u8def\u5f84\uff0c\u5982\u679c\u8981\u8bfb\u53d6\u591a\u4e2a\u6587\u4ef6\uff0c\u53ef\u4ee5\u4f7f\u7528\u6b63\u5219\u8868\u8fbe\u5f0f *\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84\uff1a

    1. \u5f53\u6307\u5b9a\u5355\u4e2a Hdfs \u6587\u4ef6\uff0cHdfsReader \u6682\u65f6\u53ea\u80fd\u4f7f\u7528\u5355\u7ebf\u7a0b\u8fdb\u884c\u6570\u636e\u62bd\u53d6\u3002\u4e8c\u671f\u8003\u8651\u5728\u975e\u538b\u7f29\u6587\u4ef6\u60c5\u51b5\u4e0b\u9488\u5bf9\u5355\u4e2a File \u53ef\u4ee5\u8fdb\u884c\u591a\u7ebf\u7a0b\u5e76\u53d1\u8bfb\u53d6\u3002
    2. \u5f53\u6307\u5b9a\u591a\u4e2a Hdfs \u6587\u4ef6\uff0cHdfsReader \u652f\u6301\u4f7f\u7528\u591a\u7ebf\u7a0b\u8fdb\u884c\u6570\u636e\u62bd\u53d6\u3002\u7ebf\u7a0b\u5e76\u53d1\u6570\u901a\u8fc7\u901a\u9053\u6570\u6307\u5b9a\u3002
    3. \u5f53\u6307\u5b9a\u901a\u914d\u7b26\uff0cHdfsReader \u5c1d\u8bd5\u904d\u5386\u51fa\u591a\u4e2a\u6587\u4ef6\u4fe1\u606f\u3002\u4f8b\u5982: \u6307\u5b9a /* \u4ee3\u8868\u8bfb\u53d6 / \u76ee\u5f55\u4e0b\u6240\u6709\u7684\u6587\u4ef6\uff0c\u6307\u5b9a /bazhen/* \u4ee3\u8868\u8bfb\u53d6 bazhen \u76ee\u5f55\u4e0b\u6e38\u6240\u6709\u7684\u6587\u4ef6\u3002HdfsReader \u76ee\u524d\u53ea\u652f\u6301 *\u548c ? \u4f5c\u4e3a\u6587\u4ef6\u901a\u914d\u7b26\u3002

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0cAddax \u4f1a\u5c06\u4e00\u4e2a\u4f5c\u4e1a\u4e0b\u540c\u6b65\u7684\u6240\u6709\u7684\u6587\u4ef6\u89c6\u4f5c\u540c\u4e00\u5f20\u6570\u636e\u8868\u3002\u7528\u6237\u5fc5\u987b\u81ea\u5df1\u4fdd\u8bc1\u6240\u6709\u7684 File \u80fd\u591f\u9002\u914d\u540c\u4e00\u5957 schema \u4fe1\u606f\u3002\u5e76\u4e14\u63d0\u4f9b\u7ed9 Addax \u6743\u9650\u53ef\u8bfb\u3002

    "},{"location":"reader/hdfsreader/#filetype","title":"fileType","text":"

    \u63cf\u8ff0\uff1a\u6587\u4ef6\u7684\u7c7b\u578b\uff0c\u76ee\u524d\u53ea\u652f\u6301\u7528\u6237\u914d\u7f6e\u4e3a

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0cHdfsReader \u80fd\u591f\u81ea\u52a8\u8bc6\u522b\u6587\u4ef6\u662f orcfile\u3001textfile \u6216\u8005\u8fd8\u662f\u5176\u5b83\u7c7b\u578b\u7684\u6587\u4ef6\uff0c\u4f46\u8be5\u9879\u662f\u5fc5\u586b\u9879\uff0cHdfsReader \u5219\u4f1a\u53ea\u8bfb\u53d6\u7528\u6237\u914d\u7f6e\u7684\u7c7b\u578b\u7684\u6587\u4ef6\uff0c\u5ffd\u7565\u8def\u5f84\u4e0b\u5176\u4ed6\u683c\u5f0f\u7684\u6587\u4ef6

    \u53e6\u5916\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0c\u7531\u4e8e textfile \u548c orcfile \u662f\u4e24\u79cd\u5b8c\u5168\u4e0d\u540c\u7684\u6587\u4ef6\u683c\u5f0f\uff0c\u6240\u4ee5 HdfsReader \u5bf9\u8fd9\u4e24\u79cd\u6587\u4ef6\u7684\u89e3\u6790\u65b9\u5f0f\u4e5f\u5b58\u5728\u5dee\u5f02\uff0c\u8fd9\u79cd\u5dee\u5f02\u5bfc\u81f4 hive \u652f\u6301\u7684\u590d\u6742\u590d\u5408\u7c7b\u578b(\u6bd4\u5982 map,array,struct,union)\u5728\u8f6c\u6362\u4e3a\u652f\u6301\u7684 String \u7c7b\u578b\u65f6\uff0c\u8f6c\u6362\u7684\u7ed3\u679c\u683c\u5f0f\u7565\u6709\u5dee\u5f02\uff0c\u6bd4\u5982\u4ee5 map \u7c7b\u578b\u4e3a\u4f8b\uff1a

    \u4ece\u4e0a\u9762\u7684\u8f6c\u6362\u7ed3\u679c\u53ef\u4ee5\u770b\u51fa\uff0c\u6570\u636e\u672c\u8eab\u6ca1\u6709\u53d8\u5316\uff0c\u4f46\u662f\u8868\u793a\u7684\u683c\u5f0f\u7565\u6709\u5dee\u5f02\uff0c\u6240\u4ee5\u5982\u679c\u7528\u6237\u914d\u7f6e\u7684\u6587\u4ef6\u8def\u5f84\u4e2d\u8981\u540c\u6b65\u7684\u5b57\u6bb5\u5728 Hive \u4e2d\u662f\u590d\u5408\u7c7b\u578b\u7684\u8bdd\uff0c\u5efa\u8bae\u914d\u7f6e\u7edf\u4e00\u7684\u6587\u4ef6\u683c\u5f0f\u3002

    \u5982\u679c\u9700\u8981\u7edf\u4e00\u590d\u5408\u7c7b\u578b\u89e3\u6790\u51fa\u6765\u7684\u683c\u5f0f\uff0c\u6211\u4eec\u5efa\u8bae\u7528\u6237\u5728 hive \u5ba2\u6237\u7aef\u5c06 textfile \u683c\u5f0f\u7684\u8868\u5bfc\u6210 orcfile \u683c\u5f0f\u7684\u8868

    "},{"location":"reader/hdfsreader/#column","title":"column","text":"

    \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype \u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cindex \u6307\u5b9a\u5f53\u524d\u5217\u6765\u81ea\u4e8e\u6587\u672c\u7b2c\u51e0\u5217(\u4ee5 0 \u5f00\u59cb)\uff0cvalue \u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece\u6e90\u5934\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636e value \u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u7528\u6237\u53ef\u4ee5\u5168\u90e8\u6309\u7167 String \u7c7b\u578b\u8bfb\u53d6\u6570\u636e\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"column\": [\"*\"]\n}\n

    \u7528\u6237\u53ef\u4ee5\u6307\u5b9a Column \u5b57\u6bb5\u4fe1\u606f\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    [\n  {\n    \"type\": \"long\",\n    \"index\": 0,\n    \"description\": \"\u4ece\u672c\u5730\u6587\u4ef6\u6587\u672c\u7b2c\u4e00\u5217\u83b7\u53d6int\u5b57\u6bb5\"\n  },\n  {\n    \"type\": \"string\",\n    \"value\": \"addax\",\n    \"description\": \"HdfsReader\u5185\u90e8\u751f\u6210alibaba\u7684\u5b57\u7b26\u4e32\u5b57\u6bb5\u4f5c\u4e3a\u5f53\u524d\u5b57\u6bb5\"\n  }\n]\n

    \u5bf9\u4e8e\u7528\u6237\u6307\u5b9a Column \u4fe1\u606f\uff0ctype \u5fc5\u987b\u586b\u5199\uff0cindex/value \u5fc5\u987b\u9009\u62e9\u5176\u4e00\u3002

    "},{"location":"reader/hdfsreader/#compress","title":"compress","text":"

    \u5f53 fileType\uff08\u6587\u4ef6\u7c7b\u578b\uff09\u4e3a csv \u4e0b\u7684\u6587\u4ef6\u538b\u7f29\u65b9\u5f0f\uff0c\u76ee\u524d\u4ec5\u652f\u6301 gzip\u3001bz2\u3001zip\u3001lzo\u3001lzo_deflate\u3001hadoop-snappy\u3001framing-snappy \u538b\u7f29\uff1b \u503c\u5f97\u6ce8\u610f\u7684\u662f\uff0clzo \u5b58\u5728\u4e24\u79cd\u538b\u7f29\u683c\u5f0f\uff1alzo \u548c lzo_deflate\uff0c\u7528\u6237\u5728\u914d\u7f6e\u7684\u65f6\u5019\u9700\u8981\u7559\u5fc3\uff0c\u4e0d\u8981\u914d\u9519\u4e86\uff1b

    \u53e6\u5916\uff0c\u7531\u4e8e snappy \u76ee\u524d\u6ca1\u6709\u7edf\u4e00\u7684 stream format\uff0caddax \u76ee\u524d\u53ea\u652f\u6301\u6700\u4e3b\u6d41\u7684\u4e24\u79cd\uff1ahadoop-snappy\uff08hadoop \u4e0a\u7684 snappy stream format\uff09 \u548c framing-snappy\uff08google \u5efa\u8bae\u7684 snappy stream format\uff09;

    "},{"location":"reader/hdfsreader/#hadoopconfig","title":"hadoopConfig","text":"

    hadoopConfig \u91cc\u53ef\u4ee5\u914d\u7f6e\u4e0e Hadoop \u76f8\u5173\u7684\u4e00\u4e9b\u9ad8\u7ea7\u53c2\u6570\uff0c\u6bd4\u5982 HA \u7684\u914d\u7f6e

    {\n  \"hadoopConfig\": {\n    \"dfs.nameservices\": \"cluster\",\n    \"dfs.ha.namenodes.cluster\": \"nn1,nn2\",\n    \"dfs.namenode.rpc-address.cluster.nn1\": \"node1.example.com:8020\",\n    \"dfs.namenode.rpc-address.cluster.nn2\": \"node2.example.com:8020\",\n    \"dfs.client.failover.proxy.provider.cluster\": \"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\"\n  }\n}\n

    \u8fd9\u91cc\u7684 cluster \u8868\u793a HDFS \u914d\u7f6e\u6210 HA \u65f6\u7684\u540d\u5b57\uff0c\u4e5f\u662f defaultFS \u914d\u7f6e\u9879\u4e2d\u7684\u540d\u5b57 \u5982\u679c\u5b9e\u9645\u73af\u5883\u4e2d\u7684\u540d\u5b57\u4e0d\u662f cluster \uff0c\u5219\u4e0a\u8ff0\u914d\u7f6e\u4e2d\u6240\u6709\u5199\u6709 cluster \u90fd\u9700\u8981\u66ff\u6362

    "},{"location":"reader/hdfsreader/#csvreaderconfig","title":"csvReaderConfig","text":"

    \u8bfb\u53d6 CSV \u7c7b\u578b\u6587\u4ef6\u53c2\u6570\u914d\u7f6e\uff0cMap \u7c7b\u578b\u3002\u8bfb\u53d6 CSV \u7c7b\u578b\u6587\u4ef6\u4f7f\u7528\u7684 CsvReader \u8fdb\u884c\u8bfb\u53d6\uff0c\u4f1a\u6709\u5f88\u591a\u914d\u7f6e\uff0c\u4e0d\u914d\u7f6e\u5219\u4f7f\u7528\u9ed8\u8ba4\u503c\u3002

    \u5e38\u89c1\u914d\u7f6e\uff1a

    {\n  \"csvReaderConfig\": {\n    \"safetySwitch\": false,\n    \"skipEmptyRecords\": false,\n    \"useTextQualifier\": false\n  }\n}\n

    \u6240\u6709\u914d\u7f6e\u9879\u53ca\u9ed8\u8ba4\u503c,\u914d\u7f6e\u65f6 csvReaderConfig \u7684 map \u4e2d\u8bf7 \u4e25\u683c\u6309\u7167\u4ee5\u4e0b\u5b57\u6bb5\u540d\u5b57\u8fdb\u884c\u914d\u7f6e\uff1a

    boolean caseSensitive = true;\nchar textQualifier = 34;\nboolean trimWhitespace = true;\nboolean useTextQualifier = true;//\u662f\u5426\u4f7f\u7528csv\u8f6c\u4e49\u5b57\u7b26\nchar delimiter = 44;//\u5206\u9694\u7b26\nchar recordDelimiter = 0;\nchar comment = 35;\nboolean useComments = false;\nint escapeMode = 1;\nboolean safetySwitch = true;//\u5355\u5217\u957f\u5ea6\u662f\u5426\u9650\u5236100000\u5b57\u7b26\nboolean skipEmptyRecords = true;//\u662f\u5426\u8df3\u8fc7\u7a7a\u884c\nboolean captureRawRecord = true;\n
    "},{"location":"reader/hdfsreader/#hdfssitepath","title":"hdfsSitePath","text":"

    \u8fd9\u662f 4.2.4 \u5f15\u5165\u7684\u65b0\u914d\u7f6e\u60f3\uff0c\u7528\u4e8e\u6307\u5b9a hdfs-site.xml \u6587\u4ef6\u7684\u8def\u5f84\uff0c\u6bd4\u5982\u5bf9 HDP/CDH \u800c\u8a00\uff0c\u53ef\u4ee5\u8fd9\u6837\u914d\u7f6e\uff1a

    {\n  \"hdfsSitePath\": \"/etc/hadoop/conf/hdfs-site.xml\"\n}\n

    \u5982\u679c\u914d\u7f6e\u4e86 hdfsSitePath , \u5219\u63d2\u4ef6\u4f1a\u4ece\u8be5\u6587\u4ef6\u4e2d\u83b7\u5f97\u8bbf\u95ee HDFS \u6587\u4ef6\u7cfb\u7edf\u5fc5\u8981\u7684\u914d\u7f6e\uff0c\u4ece\u800c\u5728\u5927\u90e8\u5206\u60c5\u51b5\u4e0b\u4e0d\u5728\u9700\u8981\u914d\u7f6e hadoopConfig\uff0c\u51cf\u5c11\u914d\u7f6e\u91cf\u3002

    \u5bf9\u4e8e\u628a Addax \u90e8\u7f72\u5728 Hadoop \u96c6\u7fa4\u4e0a\u7684\u573a\u666f\uff0c\u63a8\u8350\u4f7f\u7528\u8fd9\u79cd\u65b9\u5f0f\u3002

    "},{"location":"reader/hdfsreader/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b Hive \u8868 \u6570\u636e\u7c7b\u578b Long TINYINT, SMALLINT, INT, BIGINT Double FLOAT, DOUBLE String String, CHAR, VARCHAR, STRUCT, MAP, ARRAY, UNION, BINARY Boolean BOOLEAN Date Date, TIMESTAMP Bytes BINARY

    \u5176\u4e2d\uff1a

    \u7279\u522b\u63d0\u9192\uff1a

    "},{"location":"reader/hdfsreader/#faq","title":"FAQ","text":"

    Q: \u5982\u679c\u62a5 java.io.IOException: Maximum column length of 100,000 exceeded in column...\u5f02\u5e38\u4fe1\u606f\uff0c\u8bf4\u660e\u6570\u636e\u6e90 column \u5b57\u6bb5\u957f\u5ea6\u8d85\u8fc7\u4e86 100000 \u5b57\u7b26\u3002

    A: \u9700\u8981\u5728 json \u7684 reader \u91cc\u589e\u52a0\u5982\u4e0b\u914d\u7f6e

    {\n  \"csvReaderConfig\": {\n    \"safetySwitch\": false,\n    \"skipEmptyRecords\": false,\n    \"useTextQualifier\": false\n  }\n}\n

    safetySwitch = false \u8868\u793a\u5355\u5217\u957f\u5ea6\u4e0d\u9650\u5236 100000 \u5b57\u7b26

    "},{"location":"reader/hivereader/","title":"Hive Reader","text":"

    Hive Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Apache Hive \u6570\u636e\u5e93\u8bfb\u53d6\u6570\u636e\u7684\u80fd\u529b

    \u65b0\u589e\u8be5\u63d2\u4ef6\u7684\u4e3b\u8981\u76ee\u7684\u662f\u89e3\u51b3\u4f7f\u7528 RDBMS Reader \u63d2\u4ef6\u8bfb\u53d6 Hive \u6570\u636e\u5e93\u65f6\u4e0d\u80fd\u89e3\u51b3 Kerberos \u8ba4\u8bc1\u7684\u95ee\u9898\uff0c \u5982\u679c\u4f60\u7684 Hive \u6570\u636e\u5e93\u6ca1\u6709\u542f\u7528 Kerberos \u8ba4\u8bc1\uff0c\u90a3\u4e48\u76f4\u63a5\u4f7f\u7528 RDBMS Reader \u4e5f\u53ef\u4ee5\u3002 \u5982\u679c\u542f\u7528\u4e86 Kerberos \u8ba4\u8bc1\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u8be5\u63d2\u4ef6\u3002

    "},{"location":"reader/hivereader/#_1","title":"\u793a\u4f8b","text":"

    \u6211\u4eec\u5728 Hive \u7684 test \u5e93\u4e0a\u521b\u5efa\u5982\u4e0b\u8868\uff0c\u5e76\u63d2\u5165\u4e00\u6761\u8bb0\u5f55

    create table default.hive_reader\n(\n    col1 int,\n    col2 string,\n    col3 timestamp\n)\nstored as orc;\n\n\ninsert into hive_reader values(1, 'hello', current_timestamp()), (2, 'world', current_timestamp());\n

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/hive2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"hivereader\",\n        \"parameter\": {\n          \"column\": [\n            \"*\"\n          ],\n          \"username\": \"hive\",\n          \"password\": \"\",\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:hive2://localhost:10000/default;principal=hive/_HOST@EXAMPLE.COM\",\n            \"table\": [\n              \"hive_reader\"\n            ]\n          },\n          \"where\": \"logdate='20211013'\",\n          \"haveKerberos\": true,\n          \"kerberosKeytabFilePath\": \"/etc/security/keytabs/hive.headless.keytab\",\n          \"kerberosPrincipal\": \"hive@EXAMPLE.COM\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/hive2stream.json

    "},{"location":"reader/hivereader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/hive2stream.json\n
    "},{"location":"reader/hivereader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 jdbcUrl \u662f list \u65e0 \u5bf9\u7aef\u6570\u636e\u5e93\u7684 JDBC \u8fde\u63a5\u4fe1\u606f driver \u5426 string \u65e0 \u81ea\u5b9a\u4e49\u9a71\u52a8\u7c7b\u540d\uff0c\u89e3\u51b3\u517c\u5bb9\u6027\u95ee\u9898\uff0c\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0 username \u662f string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801\uff0c\u82e5\u65e0\u5bc6\u7801\uff0c\u53ef\u4e0d\u6307\u5b9a table \u662f list \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d,\u4f7f\u7528 JSON \u6570\u636e\u683c\u5f0f column \u662f list<map> \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1 rdbmreader splitPk \u5426 string \u65e0 \u4f7f\u7528 splitPk \u4ee3\u8868\u7684\u5b57\u6bb5\u8fdb\u884c\u6570\u636e\u5206\u7247\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1 rdbmreader where \u5426 string \u65e0 \u9488\u5bf9\u8868\u7684\u7b5b\u9009\u6761\u4ef6 querySql \u5426 list \u65e0 \u4f7f\u7528 SQL \u6765\u83b7\u53d6\u6570\u636e\uff0c\u5f53\u914d\u7f6e\u4e86\u8fd9\u4e00\u9879\u4e4b\u540e\uff0c table\uff0ccolumn \u914d\u7f6e\u9879\u65e0\u6548 haveKerberos \u5426 string \u65e0 \u662f\u5426\u542f\u7528 Kerberos \u8ba4\u8bc1\uff0c\u5982\u679c\u542f\u7528\uff0c\u5219\u9700\u8981\u540c\u65f6\u914d\u7f6e\u4e0b\u9762\u4e24\u9879 kerberosKeytabFilePath \u5426 string \u65e0 Kerberos \u8ba4\u8bc1\u7684\u51ed\u8bc1\u6587\u4ef6\u8def\u5f84, \u6bd4\u5982 /your/path/addax.service.keytab kerberosPrincipal \u5426 string \u65e0 Kerberos \u8ba4\u8bc1\u7684\u51ed\u8bc1\u4e3b\u4f53, \u6bd4\u5982 addax/node1@EXAMPLE.COM"},{"location":"reader/hivereader/#jdbcurl","title":"jdbcUrl","text":"

    \u8fde\u63a5 Hive \u7684 JDBC URL \u6709\u591a\u79cd\u5199\u6cd5\uff0c\u4e00\u79cd\u662f\u76f4\u63a5\u6307\u5b9a HiveServer/HiveServer2 \u670d\u52a1\u7684\u4e3b\u673a\u540d\u548c\u7aef\u53e3\u5373\u53ef\uff0c\u6bd4\u5982\uff1a jdbc:hive2://node1:10000/default

    \u5982\u679c\u4f60\u6709\u591a\u4e2a HiveServer/HiveServer2 \u670d\u52a1\uff0c\u5e76\u91c7\u53d6\u7528\u4e86\u670d\u52a1\u53d1\u73b0\uff0c\u5219\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9a zookeeper \u7684\u65b9\u5f0f\u6765\u83b7\u5f97\u6545\u969c\u8f6c\u79fb\u529f\u80fd\uff0c\u7c7b\u4f3c\u5982\u4e0b\uff1a

    jdbc:hive2://node1:2181,node2:2181,node3:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2\n

    \u5982\u679c\u4f60\u7684 Hive \u542f\u7528\u4e86 Kerberos \u8ba4\u8bc1\uff0c\u8fd8\u9700\u8981\u5728 URL \u540e\u6307\u5b9a principal \u53c2\u6570\uff0c\u4e00\u822c\u4e3a principal=hive/_HOST@EXAMPLE.COM\uff0c\u5176\u4e2d EXAMPLE.COM \u4e3a realm \u503c\u3002

    "},{"location":"reader/hivereader/#driver","title":"driver","text":"

    \u5f53\u524d Addax \u91c7\u7528\u7684 Hive JDBC \u9a71\u52a8\u4e3a 3.1.0 \u4ee5\u4e0a\u7248\u672c\uff0c\u9a71\u52a8\u7c7b\u540d\u4f7f\u7528\u7684 org.apache.hive.jdbc.HiveDriver\uff0c \u5982\u679c\u5f53\u524d\u7684 Hive JDBC \u9a71\u52a8\u4e0d\u517c\u5bb9 Hive \u6570\u636e\u5e93\uff0c \u5219\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u6b65\u9aa4\u66ff\u6362\u9a71\u52a8\u3002

    \u66ff\u6362\u63d2\u4ef6\u5185\u7f6e\u7684\u9a71\u52a8

    rm -f plugin/reader/hivereader/libs/hive-jdbc-*.jar

    \u62f7\u8d1d\u517c\u5bb9\u9a71\u52a8\u5230\u63d2\u4ef6\u76ee\u5f55

    cp hive-jdbc-<version>.jar plugin/reader/hivereader/libs/

    \u6307\u5b9a\u9a71\u52a8\u7c7b\u540d\u79f0

    \u5728\u4f60\u7684 json \u6587\u4ef6\u7c7b\uff0c\u914d\u7f6e \"driver\": \"<your jdbc class name>\"

    "},{"location":"reader/hivereader/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u76ee\u524d HiveReader \u652f\u6301\u5927\u90e8\u5206 Hive \u7c7b\u578b\uff0c\u4f46\u4e5f\u5b58\u5728\u90e8\u5206\u4e2a\u522b\u7c7b\u578b\u6ca1\u6709\u652f\u6301\u7684\u60c5\u51b5\uff0c\u8bf7\u6ce8\u610f\u68c0\u67e5\u4f60\u7684\u7c7b\u578b\u3002

    \u4e0b\u9762\u5217\u51fa HiveReader \u9488\u5bf9 Hive \u7c7b\u578b\u8f6c\u6362\u5217\u8868:

    Addax \u5185\u90e8\u7c7b\u578b Hive \u6570\u636e\u7c7b\u578b Long int, tinyint, smallint, mediumint, int, bigint Double float, double, decimal String varchar, char, string Date date, timestamp Boolean boolean Bytes binary"},{"location":"reader/httpreader/","title":"HTTP Reader","text":"

    HTTP Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u8bfb\u53d6 Restful API \u6570\u636e\u7684\u80fd\u529b

    "},{"location":"reader/httpreader/#_1","title":"\u793a\u4f8b","text":""},{"location":"reader/httpreader/#_2","title":"\u793a\u4f8b\u63a5\u53e3\u4e0e\u6570\u636e","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u6f14\u793a\u4e86\u5982\u4f55\u4ece\u4e00\u4e2a\u6307\u5b9a\u7684 API \u4e2d\u83b7\u53d6\u6570\u636e\uff0c\u5047\u5b9a\u8bbf\u95ee\u7684\u63a5\u53e3\u4e3a\uff1a

    http://127.0.0.1:9090/mock/17/LDJSC/ASSET

    \u63a5\u53e3\u63a5\u53d7 GET \u8bf7\u6c42\uff0c\u8bf7\u6c42\u7684\u53c2\u6570\u6709

    \u53c2\u6570\u540d\u79f0 \u53c2\u6570\u503c\u793a\u4f8b CURR_DATE 2021-01-17 DEPT 9400 USERNAME andi

    \u4ee5\u4e0b\u662f\u8bbf\u95ee\u7684\u6570\u636e\u6837\u4f8b\uff0c\uff08\u5b9e\u9645\u8fd4\u56de\u6570\u636e\u7565\u6709\u4e0d\u540c\uff09

    {\n  \"result\": [\n    {\n      \"CURR_DATE\": \"2019-12-09\",\n      \"DEPT\": \"9700\",\n      \"TOTAL_MANAGED_MARKET_VALUE\": 1581.03,\n      \"TOTAL_MANAGED_MARKET_VALUE_GROWTH\": 36.75,\n      \"TMMARKET_VALUE_DOD_GROWTH_RATE\": -0.009448781026677719,\n      \"TMMARKET_VALUE_GROWTH_MON\": -0.015153586011995693,\n      \"TMMARKET_VALUE_GROWTH_YEAR\": 0.0652347643813081,\n      \"TMMARKET_VALUE_SHARECOM\": 0.024853621341525287,\n      \"TMMARKET_VALUE_SHARE_GROWTH_RATE\": -0.005242133578517903,\n      \"AVERAGE_NEW_ASSETS_DAYINMON\": 1645.1193961136973,\n      \"YEAR_NEW_ASSET_SSHARECOM\": 0.16690149257388515,\n      \"YN_ASSET_SSHARECOM_GROWTH_RATE\": 0.017886267801303465,\n      \"POTENTIAL_LOST_ASSETS\": 56.76,\n      \"TOTAL_LIABILITIES\": 57.81,\n      \"TOTAL_ASSETS\": 1306.33,\n      \"TOTAL_ASSETS_DOD_GROWTH\": 4.79,\n      \"TOTAL_ASSETS_DOD_GROWTH_RATE\": -0.006797058194980485,\n      \"NEW_ASSETS_DAY\": 14.92,\n      \"NEW_ASSETS_MON\": 90.29,\n      \"NEW_ASSETS_YEAR\": 297.32,\n      \"NEW_ASSETS_DOD_GROWTH_RATE\": -0.04015576541561927,\n      \"NEW_FUNDS_DAY\": 18.16,\n      \"INFLOW_FUNDS_DAY\": 2.12,\n      \"OUTFLOW_FUNDS_DAY\": 9.73,\n      \"OVERALL_POSITION\": 0.810298404938773,\n      \"OVERALL_POSITION_DOD_GROWTH_RATE\": -0.03521615634095476,\n      \"NEW_CUST_FUNDS_MON\": 69.44,\n      \"INFLOW_FUNDS_MONTH\": 62.26,\n      \"OUTFLOW_FUNDS_MONTH\": 32.59\n    },\n    {\n      \"CURR_DATE\": \"2019-08-30\",\n      \"DEPT\": \"8700\",\n      \"TOTAL_MANAGED_MARKET_VALUE\": 1596.74,\n      \"TOTAL_MANAGED_MARKET_VALUE_GROWTH\": 41.86,\n      \"TMMARKET_VALUE_DOD_GROWTH_RATE\": 0.03470208565515685,\n      \"TMMARKET_VALUE_GROWTH_MON\": 0.07818120801111743,\n      \"TMMARKET_VALUE_GROWTH_YEAR\": -0.05440250244736409,\n      \"TMMARKET_VALUE_SHARECOM\": 0.09997733019626448,\n      \"TMMARKET_VALUE_SHARE_GROWTH_RATE\": -0.019726478499825697,\n      \"AVERAGE_NEW_ASSETS_DAYINMON\": 1007.9314679742108,\n      \"YEAR_NEW_ASSET_SSHARECOM\": 0.15123738798885086,\n      \"YN_ASSET_SSHARECOM_GROWTH_RATE\": 0.04694052069678048,\n      \"POTENTIAL_LOST_ASSETS\": 52.48,\n      \"TOTAL_LIABILITIES\": 55.28,\n      \"TOTAL_ASSETS\": 1366.72,\n      \"TOTAL_ASSETS_DOD_GROWTH\": 10.12,\n      \"TOTAL_ASSETS_DOD_GROWTH_RATE\": 0.009708491982487952,\n      \"NEW_ASSETS_DAY\": 12.42,\n      \"NEW_ASSETS_MON\": 41.14,\n      \"NEW_ASSETS_YEAR\": 279.32,\n      \"NEW_ASSETS_DOD_GROWTH_RATE\": -0.025878627161898062,\n      \"NEW_FUNDS_DAY\": 3.65,\n      \"INFLOW_FUNDS_DAY\": 14.15,\n      \"OUTFLOW_FUNDS_DAY\": 17.08,\n      \"OVERALL_POSITION\": 0.9098432997243932,\n      \"OVERALL_POSITION_DOD_GROWTH_RATE\": 0.02111922282868306,\n      \"NEW_CUST_FUNDS_MON\": 57.21,\n      \"INFLOW_FUNDS_MONTH\": 61.16,\n      \"OUTFLOW_FUNDS_MONTH\": 15.83\n    },\n    {\n      \"CURR_DATE\": \"2019-06-30\",\n      \"DEPT\": \"6501\",\n      \"TOTAL_MANAGED_MARKET_VALUE\": 1506.72,\n      \"TOTAL_MANAGED_MARKET_VALUE_GROWTH\": -13.23,\n      \"TMMARKET_VALUE_DOD_GROWTH_RATE\": -0.0024973354204176554,\n      \"TMMARKET_VALUE_GROWTH_MON\": -0.015530793150701896,\n      \"TMMARKET_VALUE_GROWTH_YEAR\": -0.08556724628979398,\n      \"TMMARKET_VALUE_SHARECOM\": 0.15000077963967678,\n      \"TMMARKET_VALUE_SHARE_GROWTH_RATE\": -0.049629446804825755,\n      \"AVERAGE_NEW_ASSETS_DAYINMON\": 1250.1040863177336,\n      \"YEAR_NEW_ASSET_SSHARECOM\": 0.19098445630488178,\n      \"YN_ASSET_SSHARECOM_GROWTH_RATE\": -0.007881179708853471,\n      \"POTENTIAL_LOST_ASSETS\": 50.53,\n      \"TOTAL_LIABILITIES\": 56.62,\n      \"TOTAL_ASSETS\": 1499.53,\n      \"TOTAL_ASSETS_DOD_GROWTH\": 29.56,\n      \"TOTAL_ASSETS_DOD_GROWTH_RATE\": -0.02599813232345556,\n      \"NEW_ASSETS_DAY\": 28.81,\n      \"NEW_ASSETS_MON\": 123.24,\n      \"NEW_ASSETS_YEAR\": 263.63,\n      \"NEW_ASSETS_DOD_GROWTH_RATE\": 0.0073986669331394875,\n      \"NEW_FUNDS_DAY\": 18.52,\n      \"INFLOW_FUNDS_DAY\": 3.26,\n      \"OUTFLOW_FUNDS_DAY\": 6.92,\n      \"OVERALL_POSITION\": 0.8713692113306709,\n      \"OVERALL_POSITION_DOD_GROWTH_RATE\": 0.02977644553289545,\n      \"NEW_CUST_FUNDS_MON\": 85.14,\n      \"INFLOW_FUNDS_MONTH\": 23.35,\n      \"OUTFLOW_FUNDS_MONTH\": 92.95\n    },\n    {\n      \"CURR_DATE\": \"2019-12-07\",\n      \"DEPT\": \"8705\",\n      \"TOTAL_MANAGED_MARKET_VALUE\": 1575.85,\n      \"TOTAL_MANAGED_MARKET_VALUE_GROWTH\": 8.94,\n      \"TMMARKET_VALUE_DOD_GROWTH_RATE\": -0.04384846980627058,\n      \"TMMARKET_VALUE_GROWTH_MON\": -0.022962456288549656,\n      \"TMMARKET_VALUE_GROWTH_YEAR\": -0.005047009316021089,\n      \"TMMARKET_VALUE_SHARECOM\": 0.07819484815809447,\n      \"TMMARKET_VALUE_SHARE_GROWTH_RATE\": -0.008534369960890256,\n      \"AVERAGE_NEW_ASSETS_DAYINMON\": 1340.0339240689955,\n      \"YEAR_NEW_ASSET_SSHARECOM\": 0.19019952857677042,\n      \"YN_ASSET_SSHARECOM_GROWTH_RATE\": 0.01272353909992914,\n      \"POTENTIAL_LOST_ASSETS\": 54.63,\n      \"TOTAL_LIABILITIES\": 53.17,\n      \"TOTAL_ASSETS\": 1315.08,\n      \"TOTAL_ASSETS_DOD_GROWTH\": 49.31,\n      \"TOTAL_ASSETS_DOD_GROWTH_RATE\": 0.0016538407028265922,\n      \"NEW_ASSETS_DAY\": 29.17,\n      \"NEW_ASSETS_MON\": 44.75,\n      \"NEW_ASSETS_YEAR\": 172.87,\n      \"NEW_ASSETS_DOD_GROWTH_RATE\": 0.045388692595736746,\n      \"NEW_FUNDS_DAY\": 18.46,\n      \"INFLOW_FUNDS_DAY\": 12.93,\n      \"OUTFLOW_FUNDS_DAY\": 10.38,\n      \"OVERALL_POSITION\": 0.8083127036694828,\n      \"OVERALL_POSITION_DOD_GROWTH_RATE\": -0.02847453515632541,\n      \"NEW_CUST_FUNDS_MON\": 49.74,\n      \"INFLOW_FUNDS_MONTH\": 81.93,\n      \"OUTFLOW_FUNDS_MONTH\": 18.17\n    }\n  ]\n}\n

    \u6211\u4eec\u9700\u8981\u628a result \u7ed3\u679c\u4e2d\u7684\u90e8\u5206 key \u503c\u6570\u636e\u83b7\u53d6

    "},{"location":"reader/httpreader/#_3","title":"\u914d\u7f6e","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u5b9e\u73b0\u4ece\u63a5\u53e3\u83b7\u53d6\u6570\u636e\u5e76\u6253\u5370\u5230\u7ec8\u7aef

    job/httpreader2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"httpreader\",\n        \"parameter\": {\n          \"connection\": [\n            {\n              \"url\": \"http://127.0.0.1:9090/mock/17/LDJSC/ASSET\",\n              \"proxy\": {\n                \"host\": \"http://127.0.0.1:3128\",\n                \"auth\": \"user:pass\"\n              }\n            }\n          ],\n          \"reqParams\": {\n            \"CURR_DATE\": \"2021-01-18\",\n            \"DEPT\": \"9700\"\n          },\n          \"resultKey\": \"result\",\n          \"method\": \"GET\",\n          \"column\": [\n            \"CURR_DATE\",\n            \"DEPT\",\n            \"TOTAL_MANAGED_MARKET_VALUE\",\n            \"TOTAL_MANAGED_MARKET_VALUE_GROWTH\"\n          ],\n          \"username\": \"user\",\n          \"password\": \"passw0rd\",\n          \"headers\": {\n            \"X-Powered-by\": \"Addax\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": \"true\"\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u5185\u5bb9\u4fdd\u5b58\u4e3a job/httpreader2stream.json \u6587\u4ef6\u3002

    "},{"location":"reader/httpreader/#_4","title":"\u6267\u884c","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8fdb\u884c\u91c7\u96c6

    bin/addax.sh job/httpreader2stream.json\n

    \u4e0a\u8ff0\u547d\u4ee4\u7684\u8f93\u51fa\u7ed3\u679c\u5927\u81f4\u5982\u4e0b\uff1a

    2021-01-20 09:07:41.864 [main] INFO  VMInfo - VMInfo# operatingSystem class => com.sun.management.internal.OperatingSystemImpl\n2021-01-20 09:07:41.877 [main] INFO  Engine - the machine info  =>\n\n    osInfo:     Mac OS X x86_64 10.15.1\n    jvmInfo:    AdoptOpenJDK 14 14.0.2+12\n    cpu num:    8\n\n    totalPhysicalMemory:    -0.00G\n    freePhysicalMemory: -0.00G\n    maxFileDescriptorCount: -1\n    currentOpenFileDescriptorCount: -1\n\n    GC Names    [G1 Young Generation, G1 Old Generation]\n\n    MEMORY_NAME                    | allocation_size                | init_size\n    CodeHeap 'profiled nmethods'   | 117.21MB                       | 2.44MB\n    G1 Old Gen                     | 2,048.00MB                     | 39.00MB\n    G1 Survivor Space              | -0.00MB                        | 0.00MB\n    CodeHeap 'non-profiled nmethods' | 117.21MB                       | 2.44MB\n    Compressed Class Space         | 1,024.00MB                     | 0.00MB\n    Metaspace                      | -0.00MB                        | 0.00MB\n    G1 Eden Space                  | -0.00MB                        | 25.00MB\n    CodeHeap 'non-nmethods'        | 5.57MB                         | 2.44MB\n\n\n2021-01-20 09:07:41.903 [main] INFO  Engine -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"reqParams\":{\n                        \"CURR_DATE\":\"2021-01-18\",\n                        \"DEPT\":\"9700\"\n                    },\n                    \"method\":\"GET\",\n                    \"column\":[\n                        \"CURR_DATE\",\n                        \"DEPT\",\n                        \"TOTAL_MANAGED_MARKET_VALUE\",\n                        \"TOTAL_MANAGED_MARKET_VALUE_GROWTH\"\n                    ],\n                    \"resultKey\":\"result\",\n                    \"connection\":[\n                        {\n                            \"url\":\"http://127.0.0.1:9090/mock/17/LDJSC/ASSET\"\n                        }\n                    ]\n                },\n                \"name\":\"httpreader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":\"true\"\n                },\n                \"name\":\"streamwriter\"\n            }\n    },\n    \"setting\":{\n        \"speed\":{\n            \"bytes\":-1,\n            \"channel\":1\n        }\n    }\n}\n\n2021-01-20 09:07:41.926 [main] INFO  PerfTrace - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-01-20 09:07:41.927 [main] INFO  JobContainer - Addax jobContainer starts job.\n2021-01-20 09:07:41.928 [main] INFO  JobContainer - Set jobId = 0\n2021-01-20 09:07:42.002 [taskGroup-0] INFO  TaskGroupContainer - taskGroup[0] taskId[0] attemptCount[1] is started\n\n2019-08-30  9700    1539.85 -14.78\n2019-10-01  9700    1531.71 47.66\n2020-12-03  9700    1574.38 7.34\n2020-11-31  9700    1528.13 41.62\n2019-03-01  9700    1554.28 -9.29\n\n2021-01-20 09:07:45.006 [job-0] INFO  JobContainer -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-01-20 09:07:41\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-01-20 09:07:44\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :               42B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :              1rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                   5\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"reader/httpreader/#_5","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e url \u662f string \u65e0 \u8981\u8bbf\u95ee\u7684 HTTP \u5730\u5740 reqParams \u5426 map \u65e0 \u63a5\u53e3\u8bf7\u6c42\u53c2\u6570 resultKey \u5426 string \u65e0 \u8981\u83b7\u53d6\u7ed3\u679c\u7684\u90a3\u4e2a key \u503c\uff0c\u5982\u679c\u662f\u83b7\u53d6\u6574\u4e2a\u8fd4\u56de\u503c\uff0c\u5219\u53ef\u4ee5\u4e0d\u7528\u586b\u5199 method \u5426 string get \u8bf7\u6c42\u6a21\u5f0f\uff0c\u4ec5\u652f\u6301 GET\uff0cPOST \u4e24\u79cd\uff0c\u4e0d\u533a\u5206\u5927\u5c0f\u5199 column \u662f list \u65e0 \u8981\u83b7\u53d6\u7684 key\uff0c\u914d\u7f6e\u4e3a \"*\" \u8868\u793a\u83b7\u53d6\u6240\u6709 key \u503c username \u5426 string \u65e0 \u63a5\u53e3\u8bf7\u6c42\u9700\u8981\u7684\u8ba4\u8bc1\u5e10\u53f7(\u5982\u6709) password \u5426 string \u65e0 \u63a5\u53e3\u8bf7\u6c42\u9700\u8981\u7684\u5bc6\u7801(\u5982\u6709) proxy \u5426 map \u65e0 \u4ee3\u7406\u5730\u5740,\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0 headers \u5426 map \u65e0 \u5b9a\u5236\u7684\u8bf7\u6c42\u5934\u4fe1\u606f isPage \u5426 boolean \u65e0 \u63a5\u53e3\u662f\u5426\u5206\u652f\u5206\u9875 pageParams \u5426 map \u65e0 \u5206\u9875\u53c2\u6570"},{"location":"reader/httpreader/#proxy","title":"proxy","text":"

    \u5982\u679c\u8bbf\u95ee\u7684\u63a5\u53e3\u9700\u8981\u901a\u8fc7\u4ee3\u7406\uff0c\u5219\u53ef\u4ee5\u914d\u7f6e proxy \u914d\u7f6e\u9879\uff0c\u8be5\u914d\u7f6e\u9879\u662f\u4e00\u4e2a json \u5b57\u5178\uff0c\u5305\u542b\u4e00\u4e2a\u5fc5\u9009\u7684 host \u5b57\u6bb5\u548c\u4e00\u4e2a\u53ef\u9009\u7684 auth \u5b57\u6bb5\u3002

    {\n  \"proxy\": {\n    \"host\": \"http://127.0.0.1:8080\",\n    \"auth\": \"user:pass\"\n  }\n}\n

    \u5982\u679c\u662f sock \u4ee3\u7406 (V4,v5)\uff0c\u5219\u53ef\u4ee5\u5199

    {\n  \"proxy\": {\n    \"host\": \"socks://127.0.0.1:8080\",\n    \"auth\": \"user:pass\"\n  }\n}\n

    host \u662f\u4ee3\u7406\u5730\u5740\uff0c\u5305\u542b\u4ee3\u7406\u7c7b\u578b\uff0c\u76ee\u524d\u4ec5\u652f\u6301 http \u4ee3\u7406\u548c socks(V4, V5 \u5747\u53ef) \u4ee3\u7406\u3002 \u5982\u679c\u4ee3\u7406\u9700\u8981\u8ba4\u8bc1\uff0c\u5219\u53ef\u4ee5\u914d\u7f6e auth , \u5b83\u7531\u7528\u6237\u540d\u548c\u5bc6\u7801\u7ec4\u6210\uff0c\u4e24\u8005\u4e4b\u95f4\u7528\u5192\u53f7(:) \u9694\u5f00\u3002

    "},{"location":"reader/httpreader/#column","title":"column","text":"

    column \u9664\u4e86\u76f4\u63a5\u6307\u5b9a key \u4e4b\u5916\uff0c\u8fd8\u5141\u8bb8\u7528 JSON Xpath \u98ce\u683c\u6765\u6307\u5b9a\u9700\u8981\u83b7\u53d6\u7684 key \u503c\uff0c\u5047\u5b9a\u4f60\u8981\u8bfb\u53d6\u7684 JSON \u6587\u4ef6\u5982\u4e0b\uff1a

    {\n  \"result\": [\n    {\n      \"CURR_DATE\": \"2019-12-09\",\n      \"DEPT\": {\n        \"ID\": \"9700\"\n      },\n      \"KK\": [\n        {\n          \"COL1\": 1\n        },\n        {\n          \"COL2\": 2\n        }\n      ]\n    },\n    {\n      \"CURR_DATE\": \"2021-11-09\",\n      \"DEPT\": {\n        \"ID\": \"6500\"\n      },\n      \"KK\": [\n        {\n          \"COL1\": 3\n        },\n        {\n          \"COL2\": 4\n        }\n      ]\n    }\n  ]\n}\n

    \u6211\u4eec\u5e0c\u671b\u628a CURR_DATE, ID, COL1, COL2 \u5f53\u4f5c\u56db\u4e2a\u5b57\u6bb5\u8bfb\u53d6\uff0c\u90a3\u4e48\u4f60\u7684 column \u53ef\u4ee5\u8fd9\u6837\u914d\u7f6e\uff1a

    {\n  \"column\": [\n    \"CURR_DATE\",\n    \"DEPT.ID\",\n    \"KK[0].COL1\",\n    \"KK[1].COL2\"\n  ]\n}\n

    \u5176\u6267\u884c\u7ed3\u679c\u5982\u4e0b\uff1a

    ...\n2021-10-30 14:01:50.273 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\n\n2019-12-09  9700    1   2\n2021-11-09  6500    3   4\n\n2021-10-30 14:01:53.283 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2021-10-30 14:01:53.284 [       job-0] INFO  JobContainer         - Addax Writer.Job [streamwriter] do post work.\n2021-10-30 14:01:53.284 [       job-0] INFO  JobContainer         - Addax Reader.Job [httpreader] do post work.\n2021-10-30 14:01:53.286 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2021-10-30 14:01:53.289 [       job-0] INFO  JobContainer         -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-10-30 14:01:50\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-10-30 14:01:53\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :               10B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :              0rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                   2\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n

    \u6ce8\u610f\uff1a \u5982\u679c\u4f60\u6307\u5b9a\u4e86\u4e0d\u5b58\u5728\u7684 Key\uff0c\u5219\u76f4\u63a5\u8fd4\u56de\u4e3a NULL \u503c\u3002

    "},{"location":"reader/httpreader/#ispage","title":"isPage","text":"

    isPage \u53c2\u6570\u7528\u6765\u6307\u5b9a\u63a5\u53e3\u662f\u5426\u5206\u9875\uff0c\u5b83\u662f\u4e00\u4e2a\u5e03\u5c14\u503c\uff0c\u5982\u679c\u4e3a true \u5219\u8868\u793a\u63a5\u53e3\u5206\u9875\uff0c\u5426\u5219\u8868\u793a\u4e0d\u5206\u9875\u3002

    \u5f53\u63a5\u53e3\u652f\u6301\u5206\u9875\u65f6\uff0c\u8be5\u76f4\u63a5\u4f1a\u81ea\u52a8\u5206\u9875\u8bfb\u53d6\uff0c\u76f4\u5230\u63a5\u53e3\u8fd4\u56de\u7684\u6700\u540e\u4e00\u6b21\u8fd4\u56de\u7684\u6570\u636e\u7684\u8bb0\u5f55\u6570\u5c0f\u4e8e\u6bcf\u9875\u7684\u8bb0\u5f55\u6570\u4e3a\u6b62\u3002

    "},{"location":"reader/httpreader/#pageparams","title":"pageParams","text":"

    pageParams \u53c2\u6570\u4ec5\u5728 isPage \u53c2\u6570\u4e3a true \u65f6\u751f\u6548\uff0c\u5b83\u662f\u4e00\u4e2a JSON \u5b57\u5178\uff0c\u5305\u542b\u4e24\u4e2a\u53ef\u9009\u5b57\u6bb5 pageIndex \u548c pageSize \u3002

    pageIndex \u7528\u6765\u8868\u793a\u7528\u4e8e\u5206\u9875\u6307\u793a\u7684\u5f53\u524d\u9875\u9762\uff0c\u4ed6\u662f\u4e00\u4e2a JSON \u5b57\u6bb5\uff0c\u5305\u542b\u4e24\u4e2a\u53ef\u9009\u5b57\u6bb5 key \u548c value \uff0c\u5176\u4e2d key \u7528\u6765\u6307\u5b9a\u8868\u793a\u9875\u7801\u7684\u53c2\u6570\u540d\uff0cvalue \u7528\u6765\u6307\u5b9a\u5f53\u524d\u9875\u7801\u7684\u503c\u3002

    pageSize \u7528\u6765\u8868\u793a\u7528\u4e8e\u5206\u9875\u6307\u793a\u7684\u6bcf\u9875\u5927\u5c0f\uff0c\u4ed6\u662f\u4e00\u4e2a JSON \u5b57\u6bb5\uff0c\u5305\u542b\u4e24\u4e2a\u53ef\u9009\u5b57\u6bb5 key \u548c value \uff0c\u5176\u4e2d key \u7528\u6765\u6307\u5b9a\u8868\u793a\u6bcf\u9875\u5927\u5c0f\u7684\u53c2\u6570\u540d\uff0cvalue \u7528\u6765\u6307\u5b9a\u6bcf\u9875\u5927\u5c0f\u7684\u503c\u3002

    \u8fd9\u4e24\u4e2a\u53c2\u6570\u7684\u9ed8\u8ba4\u503c\u5982\u4e0b\uff1a

    {\n  \"pageParams\": {\n    \"pageIndex\": {\n      \"key\": \"pageIndex\",\n      \"value\": 1\n    },\n    \"pageSize\": {\n      \"key\": \"pageSize\",\n      \"value\": 100\n    }\n  }\n}\n

    \u5982\u679c\u4f60\u7684\u63a5\u53e3\u5206\u9875\u53c2\u6570\u4e0d\u662f pageIndex \u548c pageSize \uff0c\u5219\u53ef\u4ee5\u901a\u8fc7 pageParams \u53c2\u6570\u6765\u6307\u5b9a\u3002\u6bd4\u5982

    {\n  \"isPage\": true,\n  \"pageParams\": {\n    \"pageIndex\": {\n      \"key\": \"page\",\n      \"value\": 1\n    },\n    \"pageSize\": {\n      \"key\": \"size\",\n      \"value\": 100\n    }\n  }\n}\n

    \u8fd9\u8868\u793a\u4f60\u4f20\u9012\u7ed9\u63a5\u53e3\u7684\u5206\u9875\u53c2\u6570\u4e3a page=1&size=100 \u3002

    "},{"location":"reader/httpreader/#_6","title":"\u9650\u5236\u8bf4\u660e","text":"
    1. \u8fd4\u56de\u7684\u7ed3\u679c\u5fc5\u987b\u662f JSON \u7c7b\u578b
    2. \u5f53\u524d\u6240\u6709 key \u7684\u503c\u5747\u5f53\u4f5c\u5b57\u7b26\u4e32\u7c7b\u578b
    3. \u6682\u4e0d\u652f\u6301\u63a5\u53e3 Token \u9274\u6743\u6a21\u5f0f
    4. \u6682\u4e0d\u652f\u6301\u5206\u9875\u83b7\u53d6
    5. \u4ee3\u7406\u4ec5\u652f\u6301 http \u6a21\u5f0f
    "},{"location":"reader/influxdb2reader/","title":"InfluxDB2 Reader","text":"

    InfluxDB2 Reader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece InfluxDB 2.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u8bfb\u53d6\u6570\u636e\u3002

    \u6ce8\u610f\uff0c\u5982\u679c\u4f60\u7684 InfluxDB \u662f 1.8\u53ca\u4ee5\u4e0b\u7248\u672c\uff0c\u5219\u5e94\u8be5\u4f7f\u7528 InfluxDBReader \u63d2\u4ef6

    "},{"location":"reader/influxdb2reader/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u793a\u4f8b\u7528\u6765\u6f14\u793a\u8be5\u63d2\u4ef6\u5982\u4f55\u4ece\u6307\u5b9a\u8868(\u5373\u6307\u6807)\u4e0a\u8bfb\u53d6\u6570\u636e\u5e76\u8f93\u51fa\u5230\u7ec8\u7aef

    "},{"location":"reader/influxdb2reader/#job","title":"\u521b\u5efa job \u6587\u4ef6","text":"

    \u521b\u5efa job/influx2stream.json \u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    job/influx2stream.json 
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"influxdb2reader\",\n        \"parameter\": {\n          \"column\": [\n            \"location\",\n            \"height\",\n            \"wet\"\n          ],\n          \"connection\": {\n            \"endpoint\": \"http://localhost:8086\",\n            \"bucket\": \"test\",\n            \"table\": [\n              \"temperature\"\n            ],\n            \"org\": \"com.wgzhao\"\n          },\n          \"token\": \"YOUR_SECURE_TOKEN\",\n          \"range\": [\n            \"-1h\",\n            \"-5m\"\n          ],\n          \"limit\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": \"true\"\n        }\n      }\n    },\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    }\n  }\n}\n
    "},{"location":"reader/influxdb2reader/#_2","title":"\u8fd0\u884c","text":"

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/influx2stream.json\n
    "},{"location":"reader/influxdb2reader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 InfluxDB \u8fde\u63a5\u4e32 \uff5c token \u662f string \u65e0 \u8bbf\u95ee\u6570\u636e\u5e93\u7684 token table \u5426 list \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d(\u5373\u6307\u6807) org \u662f string \u65e0 \u6307\u5b9a InfluxDB \u7684 org \u540d\u79f0 bucket \u662f string \u65e0 \u6307\u5b9a InfluxDB \u7684 bucket \u540d\u79f0 column \u5426 list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1 rdbmreader range \u662f list \u65e0 \u8bfb\u53d6\u6570\u636e\u7684\u65f6\u95f4\u8303\u56f4 limit \u5426 int \u65e0 \u9650\u5236\u83b7\u53d6\u8bb0\u5f55\u6570"},{"location":"reader/influxdb2reader/#column","title":"column","text":"

    \u5982\u679c\u4e0d\u6307\u5b9a column, \u6216\u8005\u6307\u5b9a column \u4e3a [\"*\"] \uff0c \u5219\u4f1a\u8bfb\u53d6\u6240\u6709\u6709\u6548\u7684 _field \u5b57\u6bb5\u4ee5\u53ca _time \u5b57\u6bb5 \u5426\u5219\u6309\u7167\u6307\u5b9a\u5b57\u6bb5\u8bfb\u53d6

    "},{"location":"reader/influxdb2reader/#range","title":"range","text":"

    range \u7528\u6765\u6307\u5b9a\u8bfb\u53d6\u6307\u6807\u7684\u65f6\u95f4\u8303\u56f4\uff0c\u5176\u683c\u5f0f\u5982\u4e0b:

    {\n  \"range\": [\"start_time\", \"end_time\"]\n}\n

    range \u7531\u4e24\u4e2a\u5b57\u7b26\u4e32\u7ec4\u6210\u7684\u5217\u8868\u7ec4\u6210\uff0c\u7b2c\u4e00\u4e2a\u5b57\u7b26\u4e32\u8868\u793a\u5f00\u59cb\u65f6\u95f4\uff0c\u7b2c\u4e8c\u4e2a\u8868\u793a\u7ed3\u675f\u65f6\u95f4\u3002\u5176\u65f6\u95f4\u8868\u8fbe\u65b9\u5f0f\u8981\u6c42\u7b26\u5408 Flux \u683c\u5f0f\u8981\u6c42,\u7c7b\u4f3c\u8fd9\u6837:

    {\n  \"range\": [\"-15h\", \"-2h\"]\n}\n

    \u5176\u4e2d\u7b2c\u4e8c\u4e2a\u7ed3\u675f\u65f6\u95f4\u5982\u679c\u4e0d\u60f3\u6307\u5b9a\uff0c\u53ef\u4ee5\u4e0d\u5199\uff0c\u7c7b\u4f3c\u8fd9\u6837\uff1a

    {\n  \"range\": [\"-15h\"]\n}\n
    "},{"location":"reader/influxdb2reader/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u5f53\u524d\u5b9e\u73b0\u662f\u5c06\u6240\u6709\u5b57\u6bb5\u5f53\u4f5c\u5b57\u7b26\u4e32\u5904\u7406

    "},{"location":"reader/influxdb2reader/#_5","title":"\u9650\u5236","text":"
    1. \u5f53\u524d\u63d2\u4ef6\u4ec5\u652f\u6301 2.0 \u53ca\u4ee5\u4e0a\u7248\u672c
    "},{"location":"reader/influxdbreader/","title":"InfluxDB Reader","text":"

    InfluxDBReader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece InfluxDB \u8bfb\u53d6\u6570\u636e\u3002\u5e95\u5c42\u5b9e\u73b0\u4e0a\uff0c\u662f\u901a\u8fc7\u8c03\u7528 InfluQL \u8bed\u8a00\u67e5\u8be2\u8868\uff0c\u7136\u540e\u83b7\u5f97\u8fd4\u56de\u6570\u636e\u3002

    "},{"location":"reader/influxdbreader/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u793a\u4f8b\u7528\u6765\u6f14\u793a\u8be5\u63d2\u4ef6\u5982\u4f55\u4ece\u6307\u5b9a\u8868(\u5373\u6307\u6807)\u4e0a\u8bfb\u53d6\u6570\u636e\u5e76\u8f93\u51fa\u5230\u7ec8\u7aef

    "},{"location":"reader/influxdbreader/#_2","title":"\u521b\u5efa\u9700\u8981\u7684\u5e93\u8868\u548c\u6570\u636e","text":"

    \u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u6765\u521b\u5efa\u9700\u8981\u8bfb\u53d6\u7684\u8868\u4ee5\u53ca\u6570\u636e

    # create database\ninflux --execute \"CREATE DATABASE NOAA_water_database\"\n# download sample data\ncurl https://s3.amazonaws.com/noaa.water-database/NOAA_data.txt -o NOAA_data.txt\n# import data via influx-cli\ninflux -import -path=NOAA_data.txt -precision=s -database=NOAA_water_database\n
    "},{"location":"reader/influxdbreader/#job","title":"\u521b\u5efa job \u6587\u4ef6","text":"

    \u521b\u5efa job/influxdb2stream.json \u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    job/influxdb2stream.json 
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"influxdbreader\",\n        \"parameter\": {\n          \"column\": [\n            \"*\"\n          ],\n          \"where\": \"1=1\",\n          \"connection\": {\n            \"endpoint\": \"http://localhost:8086\",\n            \"database\": \"NOAA_water_database\",\n            \"table\": \"h2o_feet\"\n          },\n          \"connTimeout\": 15,\n          \"readTimeout\": 20,\n          \"writeTimeout\": 20,\n          \"username\": \"influx\",\n          \"password\": \"influx123\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": \"true\"\n        }\n      }\n    },\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    }\n  }\n}\n
    "},{"location":"reader/influxdbreader/#_3","title":"\u8fd0\u884c","text":"

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/influxdb2stream.json\n
    "},{"location":"reader/influxdbreader/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 InfluxDB \u8fde\u63a5\u4e32 username \u662f string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801 database \u662f string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7684\u6570\u636e\u5e93 table \u662f string \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1 rdbmreader connTimeout \u5426 int 15 \u8bbe\u7f6e\u8fde\u63a5\u8d85\u65f6\u503c\uff0c\u5355\u4f4d\u4e3a\u79d2 readTimeout \u5426 int 20 \u8bbe\u7f6e\u8bfb\u53d6\u8d85\u65f6\u503c\uff0c\u5355\u4f4d\u4e3a\u79d2 writeTimeout \u5426 int 20 \u8bbe\u7f6e\u5199\u5165\u8d85\u65f6\u503c\uff0c\u5355\u4f4d\u4e3a\u79d2 where \u5426 string \u65e0 \u9488\u5bf9\u8868\u7684\u7b5b\u9009\u6761\u4ef6 querySql \u5426 string \u65e0 \u4f7f\u7528 SQL \u67e5\u8be2\u83b7\u53d6\u6570\u636e\uff0c\u5982\u914d\u7f6e\u8be5\u9879\uff0c\u5219 table\uff0ccolumn \u914d\u7f6e\u9879\u65e0\u6548"},{"location":"reader/influxdbreader/#_5","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u5f53\u524d\u5b9e\u73b0\u662f\u5c06\u6240\u6709\u5b57\u6bb5\u5f53\u4f5c\u5b57\u7b26\u4e32\u5904\u7406

    "},{"location":"reader/influxdbreader/#_6","title":"\u9650\u5236","text":"
    1. \u5f53\u524d\u63d2\u4ef6\u4ec5\u652f\u6301 1.x \u7248\u672c\uff0c2.0 \u53ca\u4ee5\u4e0a\u5e76\u4e0d\u652f\u6301
    "},{"location":"reader/jsonfilereader/","title":"JSON File Reader","text":"

    JSON File Reader \u63d0\u4f9b\u4e86\u8bfb\u53d6\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u6570\u636e\u5b58\u50a8\u7684\u80fd\u529b\u3002

    "},{"location":"reader/jsonfilereader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"job/json2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": \"true\"\n        }\n      },\n      \"reader\": {\n        \"name\": \"jsonfilereader\",\n        \"parameter\": {\n          \"path\": [\n            \"/tmp/test*.json\"\n          ],\n          \"column\": [\n            {\n              \"index\": \"$.id\",\n              \"type\": \"long\"\n            },\n            {\n              \"index\": \"$.name\",\n              \"type\": \"string\"\n            },\n            {\n              \"index\": \"$.age\",\n              \"type\": \"long\"\n            },\n            {\n              \"index\": \"$.score.math\",\n              \"type\": \"double\"\n            },\n            {\n              \"index\": \"$.score.english\",\n              \"type\": \"double\"\n            },\n            {\n              \"index\": \"$.pubdate\",\n              \"type\": \"date\"\n            },\n            {\n              \"type\": \"string\",\n              \"value\": \"constant string\"\n            }\n          ]\n        }\n      }\n    }\n  }\n}\n

    \u5176\u4e2d /tmp/test*.json \u4e3a\u540c\u4e00\u4e2a json \u6587\u4ef6\u7684\u591a\u4e2a\u590d\u5236\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    {\"name\": \"zhangshan\",\"id\": 19890604,\"age\": 12,\"score\": {\"math\": 92.5,\"english\": 97.5,\"chinese\": 95},\"pubdate\": \"2020-09-05\"}\n{\"name\": \"lisi\",\"id\": 19890605,\"age\": 12,\"score\": {\"math\": 90.5,\"english\": 77.5,\"chinese\": 90},\"pubdate\": \"2020-09-05\"}\n{\"name\": \"wangwu\",\"id\": 19890606,\"age\": 12,\"score\": {\"math\": 89,\"english\": 100,\"chinese\": 92},\"pubdate\": \"2020-09-05\"}\n
    "},{"location":"reader/jsonfilereader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f list \u65e0 \u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84,\u8be6\u7ec6\u63cf\u8ff0\u89c1\u4e0b\u6587 column \u662f list \u65e0 \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype\u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0c\u8be6\u89c1\u4e0b\u6587 fieldDelimiter \u662f string , \u63cf\u8ff0\uff1a\u8bfb\u53d6\u7684\u5b57\u6bb5\u5206\u9694\u7b26 compress \u5426 string \u65e0 \u6587\u672c\u538b\u7f29\u7c7b\u578b\uff0c\u9ed8\u8ba4\u4e0d\u586b\u5199\u610f\u5473\u7740\u6ca1\u6709\u538b\u7f29\u3002\u652f\u6301\u538b\u7f29\u7c7b\u578b\u4e3azip\u3001gzip\u3001bzip2 encoding \u5426 string utf-8 \u8bfb\u53d6\u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e singleLine \u5426 boolean true \u6bcf\u6761\u6570\u636e\u662f\u5426\u4e3a\u4e00\u884c\uff0c \u8be6\u89c1\u4e0b\u6587"},{"location":"reader/jsonfilereader/#path","title":"path","text":"

    \u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84\uff0c\u6bd4\u5982\uff1a

    {\n  \"path\": [\n    \"/var/ftp/test.json\", // \u8bfb\u53d6 /var/ftp \u76ee\u5f55\u4e0b\u7684 test.json \u6587\u4ef6\n    \"/var/tmp/*.json\", // \u8bfb\u53d6 /var/tmp \u76ee\u5f55\u4e0b\u6240\u6709 json \u6587\u4ef6\n    \"/public/ftp\", // \u8bfb\u53d6 /public/ftp \u76ee\u5f55\u4e0b\u6240\u6709\u6587\u4ef6, \u5982\u679c ftp \u662f\u6587\u4ef6\u7684\u8bdd\uff0c\u5219\u76f4\u63a5\u8bfb\u53d6\n    \"/public/a??.json\" // \u8bfb\u53d6 /public \u76ee\u5f55\u4e0b\u6240\u6709 a \u5f00\u5934\uff0c\u540e\u9762\u8ddf\u4e24\u4e2a\u5b57\u7b26\uff0c\u6700\u540e\u662f json \u7ed3\u5c3e\u7684\u6587\u4ef6\n  ]\n}\n

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0c\u5982\u679cPath\u6307\u5b9a\u7684\u8def\u5f84\u4e0b\u6ca1\u6709\u7b26\u5408\u5339\u914d\u7684\u6587\u4ef6\u62bd\u53d6\uff0cAddax\u5c06\u62a5\u9519\u3002

    "},{"location":"reader/jsonfilereader/#column","title":"column","text":"

    \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype\u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cindex\u6307\u5b9a\u5f53\u524d\u5217\u6765\u81ea\u4e8ejson\u7684\u6307\u5b9a\uff0c\u8bed\u6cd5\u4e3a Jayway JsonPath \u7684\u8bed\u6cd5\uff0cvalue\u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece\u6e90\u5934\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636evalue\u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002 \u7528\u6237\u5fc5\u987b\u6307\u5b9aColumn\u5b57\u6bb5\u4fe1\u606f

    \u5bf9\u4e8e\u7528\u6237\u6307\u5b9aColumn\u4fe1\u606f\uff0ctype\u5fc5\u987b\u586b\u5199\uff0cindex/value \u5fc5\u987b\u9009\u62e9\u5176\u4e00

    "},{"location":"reader/jsonfilereader/#singleline","title":"singleLine","text":"

    \u4f7f\u7528 JSON \u683c\u5f0f\u5b58\u50a8\u6570\u636e\uff0c\u4e1a\u754c\u6709\u4e24\u79cd\u65b9\u5f0f\uff0c\u4e00\u79cd\u662f\u6bcf\u884c\u4e00\u4e2a JSON \u5bf9\u8c61\uff0c\u4e5f\u5c31\u662f Single Line JSON(aka. JSONL or JSON Lines); \u53e6\u4e00\u79cd\u662f\u6574\u4e2a\u6587\u4ef6\u662f\u4e00\u4e2a JSON \u6570\u7ec4\uff0c\u6bcf\u4e2a\u5143\u7d20\u662f\u4e00\u4e2a JSON \u5bf9\u8c61\uff0c\u4e5f\u5c31\u662f Multiline JSON\u3002

    Addax \u9ed8\u8ba4\u652f\u6301\u6bcf\u884c\u4e00\u4e2a JSON \u5bf9\u8c61\u7684\u683c\u5f0f\uff0c\u5373 singeLine = true, \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u8981\u6ce8\u610f\u7684\u662f\uff1a

    1. \u6bcf\u884c JSON \u5bf9\u8c61\u7684\u672b\u5c3e\u4e0d\u80fd\u6709\u9017\u53f7\uff0c\u5426\u5219\u4f1a\u89e3\u6790\u5931\u8d25\u3002
    2. \u4e00\u4e2aJSON \u5bf9\u8c61\u4e0d\u80fd\u8de8\u884c\uff0c\u5426\u5219\u4f1a\u89e3\u6790\u5931\u8d25\u3002

    \u5982\u679c\u6570\u636e\u662f\u6574\u4e2a\u6587\u4ef6\u662f\u4e00\u4e2a JSON \u6570\u7ec4\uff0c\u6bcf\u4e2a\u5143\u7d20\u662f\u4e00\u4e2a JSON \u5bf9\u8c61\uff0c\u9700\u8981\u8bbe\u7f6e singeLine \u4e3a false\u3002 \u5047\u8bbe\u4e0a\u8ff0\u5217\u5b50\u4e2d\u7684\u6570\u636e\u7528\u4e0b\u9762\u7684\u683c\u5f0f\u8868\u793a\uff1a

    {\n  \"result\": [\n    {\n      \"name\": \"zhangshan\",\n      \"id\": 19890604,\n      \"age\": 12,\n      \"score\": {\n        \"math\": 92.5,\n        \"english\": 97.5,\n        \"chinese\": 95\n      },\n      \"pubdate\": \"2020-09-05\"\n    },\n    {\n      \"name\": \"lisi\",\n      \"id\": 19890605,\n      \"age\": 12,\n      \"score\": {\n        \"math\": 90.5,\n        \"english\": 77.5,\n        \"chinese\": 90\n      },\n      \"pubdate\": \"2020-09-05\"\n    },\n    {\n      \"name\": \"wangwu\",\n      \"id\": 19890606,\n      \"age\": 12,\n      \"score\": {\n        \"math\": 89,\n        \"english\": 100,\n        \"chinese\": 92\n      },\n      \"pubdate\": \"2020-09-05\"\n    }\n  ]\n}\n

    \u56e0\u4e3a\u8fd9\u79cd\u683c\u5f0f\u662f\u5408\u6cd5\u7684 JSON \u683c\u5f0f\uff0c\u56e0\u6b64\u6bcf\u4e2a JSON \u5bf9\u8c61\u53ef\u4ee5\u8de8\u884c\u3002\u76f8\u5e94\u7684\uff0c\u8fd9\u7c7b\u6570\u636e\u8bfb\u53d6\u65f6\uff0c\u5176 path \u914d\u7f6e\u5e94\u8be5\u5982\u4e0b\u586b\u5199\uff1a

    {\n  \"singleLine\": false,\n  \"column\": [\n    {\n      \"index\": \"$.result[*].id\",\n      \"type\": \"long\"\n    },\n    {\n      \"index\": \"$.result[*].name\",\n      \"type\": \"string\"\n    },\n    {\n      \"index\": \"$.result[*].age\",\n      \"type\": \"long\"\n    },\n    {\n      \"index\": \"$.result[*].score.math\",\n      \"type\": \"double\"\n    },\n    {\n      \"index\": \"$.result[*].score.english\",\n      \"type\": \"double\"\n    },\n    {\n      \"index\": \"$..result[*].pubdate\",\n      \"type\": \"date\"\n    },\n    {\n      \"type\": \"string\",\n      \"value\": \"constant string\"\n    }\n  ]\n}\n

    \u66f4\u8be6\u7ec6\u7684\u4f7f\u7528\u8bf4\u660e\u8bf7\u53c2\u8003 Jayway JsonPath \u7684\u8bed\u6cd5\u3002

    \u6ce8\u610f: \u8fd9\u79cd\u6570\u636e\u5728\u4e00\u4e2a JSON \u6570\u7ec4\u91cc\u65f6\uff0c\u7a0b\u5e8f\u53ea\u80fd\u91c7\u53d6\u5c06\u6574\u4e2a\u6587\u4ef6\u8bfb\u53d6\u5230\u5185\u5b58\u4e2d\uff0c\u7136\u540e\u89e3\u6790\u7684\u65b9\u5f0f\uff0c\u56e0\u6b64\u4e0d\u9002\u5408\u5927\u6587\u4ef6\u7684\u8bfb\u53d6\u3002 \u5bf9\u4e8e\u5927\u6587\u4ef6\u7684\u8bfb\u53d6\uff0c\u5efa\u8bae\u4f7f\u7528\u6bcf\u884c\u4e00\u4e2a JSON \u5bf9\u8c61\u7684\u683c\u5f0f\uff0c\u4e5f\u5c31\u662f Single Line JSON \u7684\u683c\u5f0f\uff0c\u8fd9\u79cd\u683c\u5f0f\u53ef\u4ee5\u91c7\u53d6\u9010\u884c\u8bfb\u53d6\u7684\u65b9\u5f0f\uff0c\u4e0d\u4f1a\u5360\u7528\u592a\u591a\u5185\u5b58\u3002

    "},{"location":"reader/jsonfilereader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b \u672c\u5730\u6587\u4ef6 \u6570\u636e\u7c7b\u578b Long Long Double Double String String Boolean Boolean Date Date"},{"location":"reader/kafkareader/","title":"Kafka Reader","text":"

    Kafka Reader \u63d2\u4ef6\u5b9e\u73b0\u4ece Kafka \u961f\u5217\u4e2d\u8bfb\u53d6 JSON \u683c\u5f0f\u6d88\u606f\u7684\u529f\u80fd\u3002 \u8be5\u63d2\u4ef6\u5728 4.0.10 \u7248\u672c\u4e2d\u5f15\u5165\u3002

    "},{"location":"reader/kafkareader/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u6f14\u793a\u4e86\u5982\u4f55\u4ece\u4ece kafka \u7684\u8bfb\u53d6\u6307\u5b9a topic \u4e2d\uff0c\u5e76\u8f93\u51fa\u5230\u7ec8\u7aef\u4e0a\u3002

    "},{"location":"reader/kafkareader/#_2","title":"\u521b\u5efa\u4efb\u52a1\u6587\u4ef6","text":"

    \u9996\u5148\u521b\u5efa\u4e00\u4e2a\u4efb\u52a1\u6587\u4ef6 kafka2stream.json , \u5185\u5bb9\u5982\u4e0b\uff1a

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1\n      }\n    },\n    \"content\": [\n      {\n        \"writer\": {\n          \"name\": \"streamwriter\",\n          \"parameter\": {\n            \"print\": true\n          }\n        },\n        \"reader\": {\n          \"name\": \"kafkareader\",\n          \"parameter\": {\n            \"brokerList\": \"wgzhao-laptop:9092\",\n            \"topic\": \"test-1\",\n            \"column\": [\n              \"col1\",\n              \"col3\",\n              \"col0\",\n              \"col9\"\n            ],\n            \"missingKeyValue\": \"\\\\N\",\n            \"properties\": {\n              \"auto.offset.reset\": \"earliest\"\n            }\n          }\n        }\n      }\n    ]\n  }\n}\n
    "},{"location":"reader/kafkareader/#_3","title":"\u8fd0\u884c","text":"

    \u6267\u884c bin/addax.sh kafka2stream.json \u547d\u4ee4\u3002

    "},{"location":"reader/kafkareader/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 brokerList \u662f string \u65e0 \u8fde\u63a5 kafka \u670d\u52a1\u7684 broker \u914d\u7f6e\uff0c\u7c7b\u4f3c localhost:9092 \uff0c\u591a\u4e2a broker\u4e4b\u95f4\u7528\u9017\u53f7(,)\u5206\u9694 topic \u662f string \u65e0 \u8981\u5199\u5165\u7684 topic column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u4ee5\u4e0b\u8be6\u8ff0 missingKeyValue \u5426 string \u65e0 \u5b57\u6bb5\u4e0d\u5b58\u5728\u65f6\u7528\u4ec0\u4e48\u503c\u586b\u5145\uff0c\u4ee5\u4e0b\u8be6\u8ff0 properties \u5426 map \u65e0 \u9700\u8981\u8bbe\u7f6e\u7684\u5176\u4ed6 kafka \u8fde\u63a5\u53c2\u6570"},{"location":"reader/kafkareader/#column","title":"column","text":"

    column \u7528\u6765\u6307\u5b9a\u8981\u8bfb\u53d6\u7684 JSON \u6d88\u606f\u4e2d\u7684 key\uff0c\u5982\u679c\u586b\u5199\u4e3a * \uff0c\u5219\u8868\u793a\u8bfb\u53d6\u6d88\u606f\u4e2d\u7684\u6240\u6709 key\u3002\u4f46\u8981\u6ce8\u610f\uff0c\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u4e00\u662f\u8f93\u51fa\u4e0d\u4f1a\u6709\u6392\u5e8f\uff0c\u4e5f\u5c31\u662f\u8bf4\u7b2c\u6bcf\u6761\u8bb0\u5f55\u7684 key \u7684 \u8f93\u51fa\u987a\u5e8f\u4e0d\u786e\u4fdd\u4e00\u81f4\u3002

    \u4e5f\u53ef\u4ee5\u6307\u5b9a key \u6765\u8fdb\u884c\u8bfb\u53d6\uff0c\u6bd4\u5982

    {\n  \"column\": [\"col1\", \"col2\", \"col3\"]\n}\n

    \u8fd9\u6837\uff0c\u63d2\u4ef6\u4f1a\u5c1d\u8bd5\u6309\u7167\u7ed9\u5b9a\u7684\u987a\u5e8f\u53bb\u8bfb\u53d6\u76f8\u5e94\u7684 key\uff0c\u5982\u679c\u4e00\u6761\u6d88\u606f\u4e2d\u8981\u8bfb\u53d6\u7684 key \u4e0d\u5b58\u5728\uff0c\u63d2\u4ef6\u4f1a\u62a5\u9519\u5e76\u9000\u51fa\u3002\u5982\u679c\u5e0c\u671b\u4e0d\u9000\u51fa\uff0c\u5219\u53ef\u4ee5\u8bbe\u7f6e missingKeyValue \u4ed6\u8868\u793a\u5f53\u8981\u8bfb\u53d6\u7684 key \u4e0d\u5b58\u5728\u65f6\uff0c\u7528\u8be5\u914d\u7f6e\u7684\u503c\u6765\u586b\u5145\u3002

    \u53e6\u5916\uff0c\u8bfb\u53d6\u7684 key \u7684\u503c\u7684\u7c7b\u578b\uff0c\u63d2\u4ef6\u4f1a\u81ea\u52a8\u53bb\u731c\u6d4b\uff0c\u5982\u679c\u7c7b\u578b\u65e0\u6cd5\u731c\u6d4b\uff0c\u5219\u4f1a\u5f53\u4f5c String \u7c7b\u578b\u3002

    "},{"location":"reader/kafkareader/#_5","title":"\u9650\u5236","text":"
    1. \u4ec5\u652f\u6301 Kafka 1.0 \u53ca\u4ee5\u4e0a\u7248\u672c\uff0c\u4f4e\u4e8e\u8be5\u7248\u672c\u7684\u65e0\u6cd5\u786e\u5b9a\u662f\u5426\u80fd\u5199\u5165
    2. \u5f53\u524d\u4e0d\u652f\u6301\u542f\u7528\u4e86 kerberos \u8ba4\u8bc1\u7684 kafka \u670d\u52a1
    "},{"location":"reader/kudureader/","title":"Kudu Reader","text":"

    Kudu Reader \u63d2\u4ef6\u5229\u7528 Kudu \u7684 java\u5ba2\u6237\u7aef KuduClient \u8fdb\u884c Kudu \u7684\u8bfb\u64cd\u4f5c\u3002

    "},{"location":"reader/kudureader/#_1","title":"\u914d\u7f6e\u793a\u4f8b","text":"

    \u6211\u4eec\u901a\u8fc7 Trino \u7684 kudu connector \u8fde\u63a5 kudu \u670d\u52a1\uff0c\u7136\u540e\u8fdb\u884c\u8868\u521b\u5efa\u4ee5\u53ca\u6570\u636e\u63d2\u5165

    "},{"location":"reader/kudureader/#_2","title":"\u5efa\u8868\u8bed\u53e5\u4ee5\u53ca\u6570\u636e\u63d2\u5165\u8bed\u53e5","text":"
    CREATE TABLE kudu.default.users (\n  user_id int WITH (primary_key = true),\n  user_name varchar with (nullable=true),\n  age int with (nullable=true),\n  salary double with (nullable=true),\n  longtitue decimal(18,6) with (nullable=true),\n  latitude decimal(18,6) with (nullable=true),\n  p decimal(21,20) with (nullable=true),\n  mtime timestamp with (nullable=true)\n) WITH (\n  partition_by_hash_columns = ARRAY['user_id'],\n  partition_by_hash_buckets = 2\n);\n\ninsert into kudu.default.users \nvalues \n(1, cast('wgzhao' as varchar), 18, cast(18888.88 as double), \n cast(123.282424 as decimal(18,6)), cast(23.123456 as decimal(18,6)),\n cast(1.12345678912345678912 as decimal(21,20)), \n timestamp '2021-01-10 14:40:41'),\n(2, cast('anglina' as varchar), 16, cast(23456.12 as double), \n cast(33.192123 as decimal(18,6)), cast(56.654321 as decimal(18,6)), \n cast(1.12345678912345678912 as decimal(21,20)), \n timestamp '2021-01-10 03:40:41');\n-- ONLY insert primary key value\n insert into kudu.default.users(user_id) values  (3);\n
    "},{"location":"reader/kudureader/#_3","title":"\u914d\u7f6e","text":"

    \u4ee5\u4e0b\u662f\u8bfb\u53d6kudu\u8868\u5e76\u8f93\u51fa\u5230\u7ec8\u7aef\u7684\u914d\u7f6e

    job/kudu2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"kudureader\",\n        \"parameter\": {\n          \"masterAddress\": \"localhost:7051,localhost:7151,localhost:7251\",\n          \"table\": \"users\",\n          \"splitPk\": \"user_id\",\n          \"lowerBound\": 1,\n          \"upperBound\": 100,\n          \"readTimeout\": 5,\n          \"scanTimeout\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u628a\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/kudu2stream.json

    "},{"location":"reader/kudureader/#_4","title":"\u6267\u884c","text":"

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91c7\u96c6

    bin/addax.sh job/kudu2stream.json\n
    "},{"location":"reader/kudureader/#_5","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 masterAddress \u662f string \u65e0 Kudu Master \u96c6\u7fa4RPC\u5730\u5740,\u591a\u4e2a\u5730\u5740\u7528\u9017\u53f7(,)\u5206\u9694 table \u662f string \u65e0 kudu \u8868\u540d splitPk \u5426 string \u65e0 \u5e76\u884c\u8bfb\u53d6\u6570\u636e\u5206\u7247\u5b57\u6bb5 lowerBound \u5426 string \u65e0 \u5e76\u884c\u8bfb\u53d6\u6570\u636e\u5206\u7247\u8303\u56f4\u4e0b\u754c upperBound \u5426 string \u65e0 \u5e76\u884c\u8bfb\u53d6\u6570\u636e\u5206\u7247\u8303\u56f4\u4e0a\u754c readTimeout \u5426 int 10 \u8bfb\u53d6\u6570\u636e\u8d85\u65f6(\u79d2) scanTimeout \u5426 int 20 \u6570\u636e\u626b\u63cf\u8bf7\u6c42\u8d85\u65f6(\u79d2) column \u5426 list \u65e0 \u6307\u5b9a\u8981\u83b7\u53d6\u7684\u5b57\u6bb5 where \u5426 list \u65e0 \u6307\u5b9a\u5176\u4ed6\u8fc7\u6ee4\u6761\u4ef6\uff0c\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0"},{"location":"reader/kudureader/#where","title":"where","text":"

    where \u7528\u6765\u5b9a\u5236\u66f4\u591a\u7684\u8fc7\u6ee4\u6761\u4ef6\uff0c\u4ed6\u662f\u4e00\u4e2a\u6570\u7ec4\u7c7b\u578b\uff0c\u6570\u7ec4\u7684\u6bcf\u4e2a\u5143\u7d20\u90fd\u662f\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u6bd4\u5982

    {\n  \"where\": [\"age > 1\", \"user_name = 'wgzhao'\"] \n}\n

    \u4e0a\u8ff0\u5b9a\u4e49\u4e86\u4e24\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u6bcf\u4e2a\u8fc7\u6ee4\u6761\u4ef6\u7531\u4e09\u90e8\u5206\u7ec4\u6210\uff0c\u683c\u5f0f\u4e3a column operator value

    \u8fd9\u91cc\u8fd8\u6709\u5176\u4ed6\u4e00\u4e9b\u9650\u5b9a\uff0c\u5728\u4f7f\u7528\u65f6\uff0c\u8981\u7279\u522b\u6ce8\u610f\uff1a

    1. \u591a\u4e2a\u8fc7\u6ee4\u6761\u4ef6\u4e4b\u95f4\u7684\u903b\u8f91\u4e0e\u5173\u7cfb(AND)\uff0c\u6682\u4e0d\u652f\u6301\u903b\u8f91\u6216(OR)\u5173\u7cfb
    "},{"location":"reader/kudureader/#_6","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b Kudu \u6570\u636e\u7c7b\u578b Long byte, short, int, long Double float, double, decimal String string Date timestamp Boolean boolean Bytes binary"},{"location":"reader/mongodbreader/","title":"MongoDB Reader","text":"

    MongoDBReader \u63d2\u4ef6\u5229\u7528 MongoDB \u7684java\u5ba2\u6237\u7aefMongoClient\u8fdb\u884cMongoDB\u7684\u8bfb\u64cd\u4f5c\u3002

    "},{"location":"reader/mongodbreader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u8be5\u793a\u4f8b\u4eceMongoDB\u4e2d\u8bfb\u4e00\u5f20\u8868\u5e76\u6253\u5370\u5230\u7ec8\u7aef

    job/mongo2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"mongodbreader\",\n        \"parameter\": {\n          \"connection\": {\n            \"address\": [\n              \"127.0.0.1:27017\"\n            ],\n            \"database\": \"tag_per_data\",\n            \"collection\": \"tag_data\",\n            \"authDb\": \"admin\"\n          },\n          \"username\": \"\",\n          \"password\": \"\",\n          \"column\": [\n            \"unique_id\",\n            \"sid\",\n            \"user_id\",\n            \"auction_id\",\n            \"content_type\",\n            \"pool_type\",\n            \"frontcat_id\",\n            \"catagoryid\",\n            \"gmt_create\",\n            \"taglist\",\n            \"property\",\n            \"scorea\",\n            \"scoreb\",\n            \"scorec\"\n          ]\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": \"true\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/mongodbreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 address \u662f list \u65e0 MongoDB \u7684\u6570\u636e\u5730\u5740\u4fe1\u606f, \u53ef\u5199\u591a\u4e2a username \u5426 string \u65e0 MongoDB \u7528\u6237\u540d password \u5426 string \u65e0 MongoDB \u5bc6\u7801 database \u662f string \u65e0 MongoDB \u6570\u636e\u5e93 collection \u662f string \u65e0 MongoDB \u7684\u96c6\u5408\u540d column \u662f list \u65e0 MongoDB \u7684\u6587\u6863\u5217\u540d\uff0c\u4e0d\u652f\u6301 [\"*\"] \u83b7\u53d6\u6240\u6709\u5217\u65b9\u5f0f query \u5426 string \u65e0 \u81ea\u5b9a\u4e49\u67e5\u8be2\u6761\u4ef6 fetchSize \u5426 int 2048 \u6279\u91cf\u83b7\u53d6\u7684\u8bb0\u5f55\u6570"},{"location":"reader/mongodbreader/#collection","title":"collection","text":"

    \u8fd9\u91cc\u7684 collection \u76ee\u524d\u53ea\u652f\u6301\u5355\u4e00 collection\uff0c\u56e0\u6b64\u8bbe\u7f6e\u7c7b\u578b\u4e3a\u5b57\u7b26\u4e32\uff0c\u800c\u4e0d\u662f\u5176\u4ed6\u63d2\u4ef6\u5e38\u89c1\u7684\u6570\u7ec4\u7c7b\u578b\uff0c\u8fd9\u4e00\u70b9\u5c24\u4e3a\u6ce8\u610f\u3002

    "},{"location":"reader/mongodbreader/#column","title":"column","text":"

    column \u7528\u6765\u6307\u5b9a\u9700\u8981\u8bfb\u53d6\u7684\u5b57\u6bb5\u540d\u79f0\uff0c\u8fd9\u91cc\u6211\u4eec\u505a\u4e86\u5b57\u6bb5\u540d\u79f0\u7684\u7ec4\u6210\u4e24\u4e2a\u5047\u5b9a\uff1a

    \u57fa\u4e8e\u4ee5\u4e0a\u5047\u5b9a\uff0c\u6211\u4eec\u53ef\u4ee5\u5728\u7b80\u5316 column \u914d\u7f6e\u7684\u540c\u65f6\uff0c\u8fd8\u53ef\u4ee5\u6307\u5b9a\u4e00\u4e9b\u5e38\u91cf\u4f5c\u4e3a\u8865\u5145\u5b57\u6bb5\uff0c\u6bd4\u5982\u4e00\u822c\u91c7\u96c6\u4e00\u5f20\u8868\uff0c\u6211\u4eec\u9700\u8981\u589e\u52a0\u91c7\u96c6\u65f6\u95f4\uff0c\u91c7\u96c6\u6e90\u7b49\u5e38\u91cf\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u914d\u7f6e

    {\n  \"column\": [\n    \"col1\",\n    \"col2\",\n    \"col3\",\n    \"'source_mongodb'\",\n    \"20211026\",\n    \"123.12\"\n  ]\n}\n

    \u4e0a\u8ff0\u914d\u7f6e\u7684\u540e\u4e09\u4e2a\u5b57\u6bb5\u5c31\u662f\u5e38\u91cf\uff0c\u5206\u522b\u5f53\u4f5c\u5b57\u7b26\u7c7b\u578b\uff0c\u6574\u578b\u548c\u6d6e\u70b9\u578b\u5904\u7406\u3002

    "},{"location":"reader/mongodbreader/#query","title":"query","text":"

    query \u662f\u53ea\u7b26\u5408 MongoDB \u67e5\u8be2\u683c\u5f0f\u7684 BSON \u5b57\u7b26\u4e32\uff0c\u6bd4\u5982\uff1a

    {\n  \"query\": \"{amount: {$gt: 140900}, oc_date: {$gt: 20190110}}\"\n}\n

    \u4e0a\u8ff0\u67e5\u8be2\u7c7b\u4f3c SQL \u4e2d\u7684 where amount > 140900 and oc_date > 20190110

    "},{"location":"reader/mongodbreader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b MongoDB \u6570\u636e\u7c7b\u578b Long int, Long Double double String string, array Date date Boolean boolean Bytes bytes"},{"location":"reader/mysqlreader/","title":"MySQL Reader","text":"

    MysqlReader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece MySQL \u8bfb\u53d6\u6570\u636e\u7684\u80fd\u529b

    "},{"location":"reader/mysqlreader/#_1","title":"\u793a\u4f8b","text":"

    \u6211\u4eec\u5728 MySQL \u7684 test \u5e93\u4e0a\u521b\u5efa\u5982\u4e0b\u8868\uff0c\u5e76\u63d2\u5165\u4e00\u6761\u8bb0\u5f55

    CREATE TABLE addax_reader\n(\n    c_bigint     bigint,\n    c_varchar    varchar(100),\n    c_timestamp  timestamp,\n    c_text       text,\n    c_decimal    decimal(8, 3),\n    c_mediumtext mediumtext,\n    c_longtext   longtext,\n    c_int        int,\n    c_time       time,\n    c_datetime   datetime,\n    c_enum       enum('one', 'two', 'three'),\n    c_float      float,\n    c_smallint   smallint,\n    c_bit        bit,\n    c_double     double,\n    c_blob       blob,\n    c_char       char(5),\n    c_varbinary  varbinary(100),\n    c_tinyint    tinyint,\n    c_json       json,\n    c_set SET ('a', 'b', 'c', 'd'),\n    c_binary     binary,\n    c_longblob   longblob,\n    c_mediumblob mediumblob\n);\nINSERT INTO addax_reader\nVALUES (2E18,\n        'a varchar data',\n        '2021-12-12 12:12:12',\n        'a long text',\n        12345.122,\n        'a medium text',\n        'a long text',\n         2 ^ 32 - 1,\n        '12:13:14',\n        '2021-12-12 12:13:14',\n        'one',\n        17.191,\n        126,\n        0,\n        1114.1114,\n        'blob',\n        'a123b',\n        'a var binary content',\n        126,\n        '{\"k1\":\"val1\",\"k2\":\"val2\"}',\n        'b',\n        binary(1),\n        x'89504E470D0A1A0A0000000D494844520000001000000010080200000090916836000000017352474200',\n        x'89504E470D0A1A0A0000000D');\n

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/mysql2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"mysqlreader\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"root\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"addax_reader\"\n            ],\n            \"jdbcUrl\": \"jdbc:mysql://127.0.0.1:3306/test\",\n            \"driver\": \"com.mysql.jdbc.Driver\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/mysql2stream.json

    "},{"location":"reader/mysqlreader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/mysql2stream.json\n
    "},{"location":"reader/mysqlreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/mysqlreader/#driver","title":"driver","text":"

    \u5f53\u524d Addax \u91c7\u7528\u7684 MySQL JDBC \u9a71\u52a8\u4e3a 8.0 \u4ee5\u4e0a\u7248\u672c\uff0c\u9a71\u52a8\u7c7b\u540d\u4f7f\u7528\u7684 com.mysql.cj.jdbc.Driver\uff0c\u800c\u4e0d\u662f com.mysql.jdbc.Driver\u3002 \u5982\u679c\u4f60\u9700\u8981\u91c7\u96c6\u7684 MySQL \u670d\u52a1\u4f4e\u4e8e 5.6\uff0c\u9700\u8981\u4f7f\u7528\u5230 Connector/J 5.1 \u9a71\u52a8\uff0c\u5219\u53ef\u4ee5\u91c7\u53d6\u4e0b\u9762\u7684\u6b65\u9aa4\uff1a

    \u66ff\u6362\u63d2\u4ef6\u5185\u7f6e\u7684\u9a71\u52a8

    rm -f plugin/reader/mysqlreader/libs/mysql-connector-java-*.jar

    \u62f7\u8d1d\u8001\u7684\u9a71\u52a8\u5230\u63d2\u4ef6\u76ee\u5f55

    cp mysql-connector-java-5.1.48.jar plugin/reader/mysqlreader/libs/

    \u6307\u5b9a\u9a71\u52a8\u7c7b\u540d\u79f0

    \u5728\u4f60\u7684 json \u6587\u4ef6\u7c7b\uff0c\u914d\u7f6e \"driver\": \"com.mysql.jdbc.Driver\"

    "},{"location":"reader/mysqlreader/#_4","title":"\u7c7b\u578b\u8f6c\u6362\u6ce8\u610f\u4e8b\u9879","text":""},{"location":"reader/oraclereader/","title":"Oracle Reader","text":"

    Oracle Reader \u63d2\u4ef6\u7528\u4e8e\u4ece Oracle \u8bfb\u53d6\u6570\u636e

    "},{"location":"reader/oraclereader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4eceOracle\u6570\u636e\u5e93\u540c\u6b65\u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a:

    job/oracle2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": 1048576,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"oraclereader\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"root\",\n          \"column\": [\n            \"id\",\n            \"name\"\n          ],\n          \"splitPk\": \"db_id\",\n          \"connection\": {\n            \"table\": [\n              \"table\"\n            ],\n            \"jdbcUrl\": \"jdbc:oracle:thin:@<HOST_NAME>:PORT:<DATABASE_NAME>\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/oraclereader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/oraclereader/#geometry","title":"\u5bf9 GEOMETRY \u7c7b\u578b\u7684\u652f\u6301","text":"

    \u4ece Addax 4.0.13 \u5f00\u59cb\uff0c\u5b9e\u9a8c\u6027\u7684\u652f\u6301 Oracle GEOMETRY \u7c7b\u578b\uff0c\u8be5\u63d2\u4ef6\u4f1a\u628a\u8be5\u7c7b\u578b\u7684\u6570\u636e\u8f6c\u4e3a JSON \u6570\u7ec4\u5b57\u7b26\u4e32\u3002

    \u5047\u5b9a\u4f60\u6709\u8fd9\u6837\u7684\u7684\u8868\u548c\u6570\u636e

    --8<-- \"assets/sql/oracle_geom.sql\n

    \u8bfb\u53d6\u8868\u8be5\u7684\u6570\u636e\u7684\u6700\u540e\u8f93\u51fa\u7ed3\u679c\u7c7b\u4f3c\u5982\u4e0b\uff1a

    --8<-- \"assets/output/oracle_geom_reader.txt\n

    \u6ce8\u610f\uff1a\u8be5\u6570\u636e\u7c7b\u578b\u76ee\u524d\u8fd8\u5904\u4e8e\u5b9e\u9a8c\u652f\u6301\u9636\u6bb5\uff0c\u4f5c\u8005\u5bf9\u6b64\u6570\u636e\u7c7b\u578b\u7684\u7406\u89e3\u5e76\u4e0d\u6df1\u523b\uff0c\u4e5f\u672a\u7ecf\u8fc7\u5168\u9762\u7684\u6d4b\u8bd5\uff0c\u8bf7\u52ff\u76f4\u63a5\u5728\u751f\u4ea7\u73af\u5883\u4f7f\u7528\u3002

    "},{"location":"reader/postgresqlreader/","title":"PostgreSQL Reader","text":"

    PostgreSQL Reader \u63d2\u4ef6\u7528\u4e8e\u4ece PostgreSQL \u8bfb\u53d6\u6570\u636e

    "},{"location":"reader/postgresqlreader/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u5efa\u8868\u8bed\u53e5\u4ee5\u53ca\u8f93\u5165\u63d2\u5165\u8bed\u53e5\u5982\u4e0b\uff1a

    create table if not exists addax_tbl\n(\n    c_bigint bigint,\n    c_bit bit(3),\n    c_bool boolean,\n    c_byte bytea,\n    c_char char(10),\n    c_varchar varchar(20),\n    c_date date,\n    c_double float8,\n    c_int integer,\n    c_json json,\n    c_number decimal(8, 3),\n    c_real real,\n    c_small smallint,\n    c_text text,\n    c_ts timestamp,\n    c_uuid uuid,\n    c_xml xml,\n    c_money money,\n    c_inet inet,\n    c_cidr cidr,\n    c_macaddr macaddr\n    );\n\ninsert into addax_tbl\nvalues (999988887777,\n        b'101',\n        TRUE,\n        '\\xDEADBEEF',\n        'hello',\n        'hello, world',\n        '2021-01-04',\n        999888.9972,\n        9876542,\n        '{\"bar\": \"baz\", \"balance\": 7.77, \"active\": false}'::json,\n        12345.123,\n        123.123,\n        126,\n        'this is a long text ',\n        '2020-01-04 12:13:14',\n        'A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11'::uuid,\n        '<foo>bar</foo>'::xml,\n        '52093.89'::money,\n        '192.168.1.1'::inet,\n        '192.168.1/24'::cidr,\n        '08002b:010203'::macaddr);\n

    \u914d\u7f6e\u4e00\u4e2a\u4ecePostgreSQL\u6570\u636e\u5e93\u540c\u6b65\u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a:

    job/postgres2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"postgresqlreader\",\n        \"parameter\": {\n          \"username\": \"pgtest\",\n          \"password\": \"pgtest\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"addax_tbl\"\n            ],\n            \"jdbcUrl\": \"jdbc:postgresql://127.0.0.1:5432/pgtest\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/postgres2stream.json

    "},{"location":"reader/postgresqlreader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/postgres2stream.json\n
    "},{"location":"reader/postgresqlreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/rdbmsreader/","title":"RDBMS Reader","text":"

    RDBMS Reader \u63d2\u4ef6\u652f\u6301\u4ece\u4f20\u7edf RDBMS \u8bfb\u53d6\u6570\u636e\u3002\u8fd9\u662f\u4e00\u4e2a\u901a\u7528\u5173\u7cfb\u6570\u636e\u5e93\u8bfb\u53d6\u63d2\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u6ce8\u518c\u6570\u636e\u5e93\u9a71\u52a8\u7b49\u65b9\u5f0f\u652f\u6301\u66f4\u591a\u5173\u7cfb\u6570\u636e\u5e93\u8bfb\u53d6\u3002

    \u540c\u65f6 RDBMS Reader \u53c8\u662f\u5176\u4ed6\u5173\u7cfb\u578b\u6570\u636e\u5e93\u8bfb\u53d6\u63d2\u4ef6\u7684\u7684\u57fa\u7840\u7c7b\u3002\u4ee5\u4e0b\u8bfb\u53d6\u63d2\u4ef6\u5747\u4f9d\u8d56\u8be5\u63d2\u4ef6

    \u6ce8\u610f\uff0c \u5982\u679c\u5df2\u7ecf\u63d0\u4f9b\u4e86\u4e13\u95e8\u7684\u6570\u636e\u5e93\u8bfb\u53d6\u63d2\u4ef6\u7684\uff0c\u63a8\u8350\u4f7f\u7528\u4e13\u7528\u63d2\u4ef6\uff0c\u5982\u679c\u4f60\u9700\u8981\u8bfb\u53d6\u7684\u6570\u636e\u5e93\u6ca1\u6709\u4e13\u95e8\u63d2\u4ef6\uff0c\u5219\u8003\u8651\u4f7f\u7528\u8be5\u901a\u7528\u63d2\u4ef6\u3002 \u5728\u4f7f\u7528\u4e4b\u524d\uff0c\u8fd8\u9700\u8981\u6267\u884c\u4ee5\u4e0b\u64cd\u4f5c\u624d\u53ef\u4ee5\u6b63\u5e38\u8fd0\u884c\uff0c\u5426\u5219\u8fd0\u884c\u4f1a\u51fa\u73b0\u5f02\u5e38\u3002

    "},{"location":"reader/rdbmsreader/#_1","title":"\u914d\u7f6e\u9a71\u52a8","text":"

    \u5047\u5b9a\u4f60\u9700\u8981\u8bfb\u53d6 IBM DB2 \u7684\u6570\u636e\uff0c\u56e0\u4e3a\u6ca1\u6709\u63d0\u4f9b\u4e13\u95e8\u7684\u8bfb\u53d6\u63d2\u4ef6\uff0c\u6240\u4ee5\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528\u8be5\u63d2\u4ef6\u6765\u5b9e\u73b0\uff0c\u5728\u4f7f\u7528\u4e4b\u524d\uff0c\u9700\u8981\u4e0b\u8f7d\u5bf9\u5e94\u7684 JDBC \u9a71\u52a8\uff0c\u5e76\u62f7\u8d1d\u5230 plugin/reader/rdbmsreader/libs \u76ee\u5f55\u3002 \u5982\u679c\u4f60\u7684\u9a71\u52a8\u7c7b\u540d\u6bd4\u8f83\u7279\u6b8a\uff0c\u5219\u9700\u8981\u5728\u4efb\u52a1\u914d\u7f6e\u6587\u4ef6\u4e2d\u627e\u5230 driver \u4e00\u9879\uff0c\u586b\u5199\u6b63\u786e\u7684 JDBC \u9a71\u52a8\u540d\uff0c\u6bd4\u5982 DB2 \u7684\u9a71\u52a8\u540d\u4e3a com.ibm.db2.jcc.DB2Driver\u3002\u5982\u679c\u4e0d\u586b\u5199\uff0c\u5219\u63d2\u4ef6\u4f1a\u81ea\u52a8\u731c\u6d4b\u9a71\u52a8\u540d\u3002

    \u4ee5\u4e0b\u5217\u51fa\u5e38\u89c1\u7684\u6570\u636e\u5e93\u4ee5\u53ca\u5bf9\u5e94\u7684\u9a71\u52a8\u540d\u79f0

    "},{"location":"reader/rdbmsreader/#_2","title":"\u914d\u7f6e\u8bf4\u660e","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u5c55\u793a\u4e86\u5982\u4f55\u4ece Presto \u6570\u636e\u5e93\u8bfb\u53d6\u6570\u636e\u5230\u7ec8\u7aef

    job/rdbms2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": 1048576,\n        \"channel\": 1\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"rdbmsreader\",\n        \"parameter\": {\n          \"username\": \"hive\",\n          \"password\": \"\",\n          \"column\": [\n            \"*\"\n          ],\n          \"driver\": \"io.prestosql.jdbc.PrestoDriver\",\n          \"connection\": {\n            \"table\": [\n              \"default.table\"\n            ],\n            \"jdbcUrl\": \"jdbc:presto://127.0.0.1:8080/hive\"\n          },\n          \"fetchSize\": 1024,\n          \"where\": \"1 = 1\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/rdbmsreader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 jdbcUrl \u662f list \u65e0 \u5bf9\u7aef\u6570\u636e\u5e93\u7684JDBC\u8fde\u63a5\u4fe1\u606f\uff0cjdbcUrl\u6309\u7167RDBMS\u5b98\u65b9\u89c4\u8303\uff0c\u5e76\u53ef\u4ee5\u586b\u5199\u8fde\u63a5\u9644\u4ef6\u63a7\u5236\u4fe1\u606f driver \u5426 string \u65e0 \u81ea\u5b9a\u4e49\u9a71\u52a8\u7c7b\u540d\uff0c\u89e3\u51b3\u517c\u5bb9\u6027\u95ee\u9898\uff0c\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0 username \u662f string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801 table \u662f list \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d,\u4f7f\u7528JSON\u6570\u636e\u683c\u5f0f\uff0c\u5f53\u914d\u7f6e\u4e3a\u591a\u5f20\u8868\u65f6\uff0c\u7528\u6237\u81ea\u5df1\u9700\u4fdd\u8bc1\u591a\u5f20\u8868\u662f\u540c\u4e00\u8868\u7ed3\u6784 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1\u540e splitPk \u5426 string \u65e0 \u4f7f\u7528splitPk\u4ee3\u8868\u7684\u5b57\u6bb5\u8fdb\u884c\u6570\u636e\u5206\u7247\uff0c\u8fd9\u6837\u53ef\u4ee5\u5927\u5927\u63d0\u4f9b\u6570\u636e\u540c\u6b65\u7684\u6548\u80fd\uff0c\u6ce8\u610f\u4e8b\u9879\u89c1\u540e autoPk \u5426 boolean false \u662f\u5426\u81ea\u52a8\u731c\u6d4b\u5206\u7247\u4e3b\u952e\uff0c3.2.6 \u7248\u672c\u5f15\u5165\uff0c\u8be6\u89c1\u540e\u9762\u63cf\u8ff0 where \u5426 string \u65e0 \u9488\u5bf9\u8868\u7684\u7b5b\u9009\u6761\u4ef6 session \u662f\u5426 list \u65e0 \u9488\u5bf9\u672c\u5730\u8fde\u63a5,\u4fee\u6539\u4f1a\u8bdd\u914d\u7f6e,\u8be6\u89c1\u4e0b\u6587 querySql \u5426 string \u65e0 \u4f7f\u7528\u81ea\u5b9a\u4e49\u7684SQL\u800c\u4e0d\u662f\u6307\u5b9a\u8868\u6765\u83b7\u53d6\u6570\u636e\uff0c\u5f53\u914d\u7f6e\u4e86\u8fd9\u4e00\u9879\u4e4b\u540e\uff0c\u5ffd\u7565 table\uff0ccolumn\u914d\u7f6e\u9879\u5ffd\u7565 fetchSize \u5426 int 1024 \u5b9a\u4e49\u4e86\u63d2\u4ef6\u548c\u6570\u636e\u5e93\u670d\u52a1\u5668\u7aef\u6bcf\u6b21\u6279\u91cf\u6570\u636e\u83b7\u53d6\u6761\u6570\uff0c\u8c03\u9ad8\u8be5\u503c\u53ef\u80fd\u5bfc\u81f4 Addax \u51fa\u73b0OOM"},{"location":"reader/rdbmsreader/#jdbcurl","title":"jdbcUrl","text":"

    jdbcUrl \u914d\u7f6e\u9664\u4e86\u914d\u7f6e\u5fc5\u8981\u7684\u4fe1\u606f\u5916\uff0c\u6211\u4eec\u8fd8\u53ef\u4ee5\u5728\u589e\u52a0\u6bcf\u79cd\u7279\u5b9a\u9a71\u52a8\u7684\u7279\u5b9a\u914d\u7f6e\u5c5e\u6027\uff0c\u8fd9\u91cc\u7279\u522b\u63d0\u5230\u6211\u4eec\u53ef\u4ee5\u5229\u7528\u914d\u7f6e\u5c5e\u6027\u5bf9\u4ee3\u7406\u7684\u652f\u6301\u4ece\u800c\u5b9e\u73b0\u901a\u8fc7\u4ee3\u7406\u8bbf\u95ee\u6570\u636e\u5e93\u7684\u529f\u80fd\u3002 \u6bd4\u5982\u5bf9\u4e8e PrestoSQL \u6570\u636e\u5e93\u7684 JDBC \u9a71\u52a8\u800c\u8a00\uff0c\u652f\u6301 socksProxy \u53c2\u6570\uff0c\u4e8e\u662f\u4e0a\u8ff0\u914d\u7f6e\u7684 jdbcUrl \u53ef\u4ee5\u4fee\u6539\u4e3a

    jdbc:presto://127.0.0.1:8080/hive?socksProxy=192.168.1.101:1081

    \u5927\u90e8\u5206\u5173\u7cfb\u578b\u6570\u636e\u5e93\u7684 JDBC \u9a71\u52a8\u652f\u6301 socksProxyHost,socksProxyPort \u53c2\u6570\u6765\u652f\u6301\u4ee3\u7406\u8bbf\u95ee\u3002\u4e5f\u6709\u4e00\u4e9b\u7279\u522b\u7684\u60c5\u51b5\u3002

    \u4ee5\u4e0b\u662f\u5404\u7c7b\u6570\u636e\u5e93 JDBC \u9a71\u52a8\u6240\u652f\u6301\u7684\u4ee3\u7406\u7c7b\u578b\u4ee5\u53ca\u914d\u7f6e\u65b9\u5f0f

    \u6570\u636e\u5e93 \u4ee3\u7406\u7c7b\u578b \u4ee3\u7406\u914d\u7f6e \u4f8b\u5b50 MySQL socks socksProxyHost,socksProxyPort socksProxyHost=192.168.1.101&socksProxyPort=1081 Presto socks socksProxy socksProxy=192.168.1.101:1081 Presto http httpProxy httpProxy=192.168.1.101:3128"},{"location":"reader/rdbmsreader/#driver","title":"driver","text":"

    \u5927\u90e8\u5206\u60c5\u51b5\u4e0b\uff0c\u4e00\u4e2a\u6570\u636e\u5e93\u7684JDBC\u9a71\u52a8\u662f\u56fa\u5b9a\u7684\uff0c\u4f46\u6709\u4e9b\u56e0\u4e3a\u7248\u672c\u7684\u4e0d\u540c\uff0c\u6240\u5efa\u8bae\u7684\u9a71\u52a8\u7c7b\u540d\u4e0d\u540c\uff0c\u6bd4\u5982 MySQL\u3002 \u65b0\u7684 MySQL JDBC \u9a71\u52a8\u7c7b\u578b\u63a8\u8350\u4f7f\u7528 com.mysql.cj.jdbc.Driver \u800c\u4e0d\u662f\u4ee5\u524d\u7684 com.mysql.jdbc.Drver \u3002\u5982\u679c\u60f3\u8981\u4f7f\u7528\u5c31\u7684\u9a71\u52a8\u540d\u79f0\uff0c\u5219\u53ef\u4ee5\u914d\u7f6e driver \u914d\u7f6e\u9879\u3002\u5426\u5219\u63d2\u4ef6\u4f1a\u81ea\u52a8\u4f9d\u636e jdbcUrl \u4e2d\u7684\u5b57\u7b26\u4e32\u6765\u731c\u6d4b\u9a71\u52a8\u540d\u79f0.

    "},{"location":"reader/rdbmsreader/#column","title":"column","text":"

    \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u4f7f\u7528JSON\u7684\u6570\u7ec4\u63cf\u8ff0\u5b57\u6bb5\u4fe1\u606f\u3002\u7528\u6237\u4f7f\u7528 * \u4ee3\u8868\u9ed8\u8ba4\u4f7f\u7528\u6240\u6709\u5217\u914d\u7f6e\uff0c\u4f8b\u5982 [\"*\"]\u3002

    \u652f\u6301\u5217\u88c1\u526a\uff0c\u5373\u5217\u53ef\u4ee5\u6311\u9009\u90e8\u5206\u5217\u8fdb\u884c\u5bfc\u51fa\u3002

    \u652f\u6301\u5217\u6362\u5e8f\uff0c\u5373\u5217\u53ef\u4ee5\u4e0d\u6309\u7167\u8868schema\u4fe1\u606f\u8fdb\u884c\u5bfc\u51fa\u3002

    \u652f\u6301\u5e38\u91cf\u914d\u7f6e\uff0c\u7528\u6237\u9700\u8981\u6309\u7167JSON\u683c\u5f0f:

    [\"id\", \"`table`\", \"1\", \"'bazhen.csy'\", \"null\", \"to_char(a + 1)\", \"2.3\" , \"true\"]

    Column\u5fc5\u987b\u663e\u793a\u586b\u5199\uff0c\u4e0d\u5141\u8bb8\u4e3a\u7a7a\uff01

    "},{"location":"reader/rdbmsreader/#splitpk","title":"splitPk","text":"

    \u5982\u679c\u6307\u5b9a splitPk\uff0c\u8868\u793a\u7528\u6237\u5e0c\u671b\u4f7f\u7528 splitPk \u4ee3\u8868\u7684\u5b57\u6bb5\u8fdb\u884c\u6570\u636e\u5206\u7247\uff0c\u56e0\u6b64\u4f1a\u542f\u52a8\u5e76\u53d1\u4efb\u52a1\u8fdb\u884c\u6570\u636e\u540c\u6b65\uff0c\u8fd9\u6837\u53ef\u4ee5\u5927\u5927\u63d0\u4f9b\u6570\u636e\u540c\u6b65\u7684\u6548\u80fd\u3002

    \u63a8\u8350 splitPk \u7528\u6237\u4f7f\u7528\u8868\u4e3b\u952e\uff0c\u56e0\u4e3a\u8868\u4e3b\u952e\u901a\u5e38\u60c5\u51b5\u4e0b\u6bd4\u8f83\u5747\u5300\uff0c\u56e0\u6b64\u5207\u5206\u51fa\u6765\u7684\u5206\u7247\u4e5f\u4e0d\u5bb9\u6613\u51fa\u73b0\u6570\u636e\u70ed\u70b9\u3002

    \u76ee\u524d splitPk \u4ec5\u652f\u6301\u6574\u5f62\u3001\u5b57\u7b26\u4e32\u578b\u6570\u636e(ASCII\u7c7b\u578b) \u5207\u5206\uff0c\u4e0d\u652f\u6301\u6d6e\u70b9\u3001\u65e5\u671f\u7b49\u5176\u4ed6\u7c7b\u578b\u3002 \u5982\u679c\u7528\u6237\u6307\u5b9a\u5176\u4ed6\u975e\u652f\u6301\u7c7b\u578b\uff0cRDBMSReader \u5c06\u62a5\u9519\uff01

    splitPk \u5982\u679c\u4e0d\u586b\u5199\uff0c\u5c06\u89c6\u4f5c\u7528\u6237\u4e0d\u5bf9\u5355\u8868\u8fdb\u884c\u5207\u5206\uff0c\u800c\u4f7f\u7528\u5355\u901a\u9053\u540c\u6b65\u5168\u91cf\u6570\u636e\u3002

    "},{"location":"reader/rdbmsreader/#autopk","title":"autoPk","text":"

    \u4ece 3.2.6 \u7248\u672c\u5f00\u59cb\uff0c\u652f\u6301\u81ea\u52a8\u83b7\u53d6\u8868\u4e3b\u952e\u6216\u552f\u4e00\u7d22\u5f15\uff0c\u5982\u679c\u8bbe\u7f6e\u4e3a true \uff0c\u5c06\u5c1d\u8bd5\u901a\u8fc7\u67e5\u8be2\u6570\u636e\u5e93\u7684\u5143\u6570\u636e\u4fe1\u606f\u83b7\u53d6\u6307\u5b9a\u8868\u7684\u4e3b\u952e\u5b57\u6bb5\u6216\u552f\u4e00\u7d22\u5f15\u5b57\u6bb5\uff0c\u5982\u679c\u83b7\u53d6\u53ef\u7528\u4e8e\u5206\u9694\u7684 \u5b57\u6bb5\u4e0d\u6b62\u4e00\u4e2a\uff0c\u5219\u9ed8\u8ba4\u53d6\u7b2c\u4e00\u4e2a\u3002

    \u8be5\u7279\u6027\u76ee\u524d\u652f\u6301\u7684\u6570\u636e\u5e93\u6709\uff1a

    "},{"location":"reader/rdbmsreader/#session","title":"session","text":"

    \u63a7\u5236\u5199\u5165\u6570\u636e\u7684\u65f6\u95f4\u683c\u5f0f\uff0c\u65f6\u533a\u7b49\u7684\u914d\u7f6e\uff0c\u76ee\u524d\u4ec5\u5bf9 MySQL, Oracle, SQLServer \u6709\u6548\u3002\u4e0b\u9762\u662f\u4e00\u4e2a\u9488\u5bf9 Oracle \u6570\u636e\u5e93\u914d\u7f6e session \u7684\u4f8b\u5b50\u3002

    {\n  \"session\": [\n    \"alter session set NLS_DATE_FORMAT='yyyy-mm-dd hh24:mi:ss'\",\n    \"alter session set NLS_TIMESTAMP_FORMAT='yyyy-mm-dd hh24:mi:ss'\",\n    \"alter session set NLS_TIMESTAMP_TZ_FORMAT='yyyy-mm-dd hh24:mi:ss'\",\n    \"alter session set TIME_ZONE='Asia/Chongqing'\"\n  ]\n}\n

    \u6ce8\u610f &quot;\u662f \" \u7684\u8f6c\u4e49\u5b57\u7b26\u4e32

    "},{"location":"reader/rdbmsreader/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b RDBMS \u6570\u636e\u7c7b\u578b Long int, tinyint, smallint, mediumint, int, bigint Double float, double, decimal String varchar, char, tinytext, text, mediumtext, longtext, year,xml Date date, datetime, timestamp, time Boolean bit, bool Bytes tinyblob, mediumblob, blob, longblob, varbinary"},{"location":"reader/redisreader/","title":"Redis Reader","text":"

    Redis Reader \u63d2\u4ef6\u7528\u4e8e\u8bfb\u53d6 Redis RDB \u6570\u636e

    "},{"location":"reader/redisreader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"redisreader\",\n        \"parameter\": {\n          \"connection\": {\n            \"uri\": [\"tcp://127.0.0.1:6379\", \"file:///data/dump.rdb\", \"http://localhost/dump.rdb\"],\n            \"auth\": \"password\"\n          },\n          \"include\": [\n            \"^user\"\n          ],\n          \"exclude\": [\n            \"^password\"\n          ],\n          \"db\": [\n            0,\n            1\n          ]\n        }\n      },\n      \"writer\": {\n        \"name\": \"rediswriter\",\n        \"parameter\": {\n          \"connection\": {\n            \"uri\": \"tcp://127.0.0.1:6379\",\n            \"auth\": \"123456\"\n          },\n          \"timeout\": 60000\n        }\n      }\n    },\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1\n      }\n    }\n  }\n}\n
    "},{"location":"reader/redisreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u9ed8\u8ba4\u503c \u63cf\u8ff0 uri \u662f \u5426 redis\u94fe\u63a5,\u652f\u6301\u591a\u4e2a\u672c\u5730rdb\u6587\u4ef6/\u7f51\u7edcrdb\u6587\u4ef6,\u5982\u679c\u662f\u96c6\u7fa4,\u586b\u5199\u6240\u6709master\u8282\u70b9\u5730\u5740 db \u5426 \u65e0 \u9700\u8981\u8bfb\u53d6\u7684db\u7d22\u5f15,\u82e5\u4e0d\u586b\u5199,\u5219\u8bfb\u53d6\u6240\u6709db include \u5426 \u65e0 \u8981\u5305\u542b\u7684 key, \u652f\u6301\u6b63\u5219\u8868\u8fbe\u5f0f exclude \u5426 \u65e0 \u8981\u6392\u9664\u7684 key,\u652f\u6301\u6b63\u5219\u8868\u8fbe\u5f0f"},{"location":"reader/redisreader/#_3","title":"\u7ea6\u675f\u9650\u5236","text":"
    1. \u4e0d\u652f\u6301\u76f4\u63a5\u8bfb\u53d6\u4efb\u4f55\u4e0d\u652f\u6301 sync \u547d\u4ee4\u7684 redis server\uff0c\u5982\u679c\u9700\u8981\u8bf7\u5907\u4efd\u7684rdb\u6587\u4ef6\u8fdb\u884c\u8bfb\u53d6\u3002
    2. \u5982\u679c\u662f\u539f\u751fredis cluster\u96c6\u7fa4\uff0c\u8bf7\u586b\u5199\u6240\u6709master\u8282\u70b9\u7684tcp\u5730\u5740\uff0credisreader \u63d2\u4ef6\u4f1a\u81ea\u52a8dump \u6240\u6709\u8282\u70b9\u7684rdb\u6587\u4ef6\u3002
    3. \u4ec5\u89e3\u6790 String \u6570\u636e\u7c7b\u578b\uff0c\u5176\u4ed6\u590d\u5408\u7c7b\u578b(Sets, List \u7b49\u4f1a\u5ffd\u7565)
    "},{"location":"reader/s3reader/","title":"S3 Reader","text":"

    S3 Reader \u63d2\u4ef6\u7528\u4e8e\u8bfb\u53d6 Amazon AWS S3 \u5b58\u50a8\u4e0a\u7684\u6570\u636e\u3002\u5728\u5b9e\u73b0\u4e0a\uff0c\u672c\u63d2\u4ef6\u57fa\u4e8e S3 \u5b98\u65b9\u7684 SDK 2.0 \u7f16\u5199\u3002

    \u540c\u65f6\u672c\u63d2\u4ef6\u4e5f\u652f\u6301\u8bfb\u53d6\u517c\u5bb9 S3 \u534f\u8bae\u7684\u5b58\u50a8\u670d\u52a1\uff0c\u6bd4\u5982 MinIO

    "},{"location":"reader/s3reader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u4ee5\u4e0b\u6837\u4f8b\u914d\u7f6e\u7528\u4e8e\u4ece S3 \u5b58\u50a8\u4e0a\u8bfb\u53d6\u4e24\u4e2a\u6587\u4ef6\uff0c\u5e76\u6253\u5370\u51fa\u6765

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"s3reader\",\n        \"parameter\": {\n          \"endpoint\": \"https://s3.amazonaws.com\",\n          \"accessId\": \"xxxxxxxxxxxx\",\n          \"accessKey\": \"xxxxxxxxxxxxxxxxxxxxxxx\",\n          \"bucket\": \"test\",\n          \"object\": [\n            \"1.csv\",\n            \"aa.csv\",\n            \"upload_*.csv\",\n            \"bb_??.csv\"\n          ],\n          \"column\": [\n            \"*\"\n          ],\n          \"region\": \"ap-northeast-1\",\n          \"fileFormat\": \"csv\",\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/s3reader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 S3 Server\u7684 EndPoint\u5730\u5740\uff0c\u4f8b\u5982 s3.xx.amazonaws.com region \u662f string \u65e0 S3 Server\u7684 Region \u5730\u5740\uff0c\u4f8b\u5982 ap-southeast-1 accessId \u662f string \u65e0 \u8bbf\u95ee ID accessKey \u662f string \u65e0 \u8bbf\u95ee Key bucket \u662f string \u65e0 \u8981\u8bfb\u53d6\u7684 bucket object \u662f list \u65e0 \u8981\u8bfb\u53d6\u7684 object\uff0c\u53ef\u4ee5\u586b\u5199\u591a\u4e2a\u4ee5\u53ca\u901a\u914d\u7b26\u65b9\u5f0f\uff0c\u8be6\u89c1\u4e0b\u9762\u8bf4\u660e column \u662f list \u65e0 \u8bfb\u53d6\u7684 object \u7684\u5217\u4fe1\u606f\uff0c\u586b\u5199\u65b9\u5f0f\u89c1RDBMS Reader \u4e2d column \u63cf\u8ff0 fieldDelimiter \u5426 string , \u8bfb\u53d6\u7684\u5b57\u6bb5\u5206\u9694\u7b26\uff0c\u4ec5\u652f\u6301\u5355\u5b57\u7b26 compress \u5426 string \u65e0 \u6587\u4ef6\u538b\u7f29\u683c\u5f0f\uff0c\u9ed8\u8ba4\u4e0d\u538b\u7f29 encoding \u5426 string utf8 \u6587\u4ef6\u7f16\u7801\u683c\u5f0f writeMode \u5426 string nonConflict"},{"location":"reader/s3reader/#object","title":"object","text":"

    \u5f53\u6307\u5b9a\u5355\u4e2a object\uff0c\u63d2\u4ef6\u6682\u65f6\u53ea\u80fd\u4f7f\u7528\u5355\u7ebf\u7a0b\u8fdb\u884c\u6570\u636e\u62bd\u53d6\u3002

    \u5f53\u6307\u5b9a\u591a\u4e2a object\uff0c\u63d2\u4ef6\u652f\u6301\u4f7f\u7528\u591a\u7ebf\u7a0b\u8fdb\u884c\u6570\u636e\u62bd\u53d6\u3002\u7ebf\u7a0b\u5e76\u53d1\u6570\u901a\u8fc7\u901a\u9053\u6570\u6307\u5b9a\u3002

    \u5f53\u6307\u5b9a\u901a\u914d\u7b26\uff0c\u63d2\u4ef6\u5c1d\u8bd5\u904d\u5386\u51fa\u591a\u4e2a object \u4fe1\u606f\u3002

    \u4f8b\u5982: \u6307\u5b9a /* \u4ee3\u8868\u8bfb\u53d6 bucket \u4e0b\u6240\u6709\u7684 object\uff0c\u6307\u5b9a /foo/* \u4ee3\u8868\u8bfb\u53d6 foo \u76ee\u5f55\u4e0b\u6240\u6709\u7684 object\u3002

    "},{"location":"reader/s3reader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b S3 \u6570\u636e\u7c7b\u578b Long int, tinyint, smallint, mediumint, int, bigint Double float, double, decimal String varchar, char, tinytext, text, mediumtext, longtext, year,xml Date date, datetime, timestamp, time Boolean bit, bool Bytes tinyblob, mediumblob, blob, longblob, varbinary"},{"location":"reader/s3reader/#_4","title":"\u9650\u5236\u8bf4\u660e","text":"
    1. \u4ec5\u652f\u6301\u8bfb\u53d6\u6587\u672c\u6587\u4ef6
    "},{"location":"reader/sqlitereader/","title":"SQLite Reader","text":"

    SQLite Reader \u63d2\u4ef6\u7528\u4e8e\u8bfb\u53d6\u6307\u5b9a\u76ee\u5f55\u4e0b\u7684 sqlite \u6587\u4ef6\uff0c \u4ed6\u7ee7\u627f\u4e8e RDBMS Reader

    "},{"location":"reader/sqlitereader/#_1","title":"\u793a\u4f8b","text":"

    \u6211\u4eec\u521b\u5efa\u793a\u4f8b\u6587\u4ef6\uff1a

    $ sqlite3  /tmp/test.sqlite3\nSQLite version 3.7.17 2013-05-20 00:56:22\nEnter \".help\" for instructions\nEnter SQL statements terminated with a \";\"\nsqlite> create table test(id int, name varchar(10), salary double);\nsqlite> insert into test values(1,'foo', 12.13),(2,'bar',202.22);\nsqlite> .q\n

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/sqlite2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"sqlitereader\",\n        \"parameter\": {\n          \"username\": \"fakeuser\",\n          \"password\": \"\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:sqlite:/tmp/test.sqlite3\",\n            \"table\": [\n              \"test\"\n            ]\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/sqlite2stream.json

    "},{"location":"reader/sqlitereader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/sqlite2stream.json\n
    "},{"location":"reader/sqlitereader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/sqlserverreader/","title":"SQLServer Reader","text":"

    SqlServerReader \u63d2\u4ef6\u7528\u4e8e\u4ece\u4ece SQLServer \u8bfb\u53d6\u6570\u636e\u3002

    "},{"location":"reader/sqlserverreader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4ece SQLServer \u6570\u636e\u5e93\u540c\u6b65\u62bd\u53d6\u6570\u636e\u5230\u672c\u5730\u7684\u4f5c\u4e1a:

    job/sqlserver2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"sqlserverreader\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"root\",\n          \"column\": [\n            \"*\"\n          ],\n          \"splitPk\": \"db_id\",\n          \"connection\": {\n            \"table\": [\n              \"table\"\n            ],\n            \"jdbcUrl\": \"jdbc:sqlserver://localhost:3433;DatabaseName=dbname\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true,\n          \"encoding\": \"UTF-8\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/sqlserverreader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/streamreader/","title":"Stream Reader","text":"

    Stream Reader \u662f\u4e00\u4e2a\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\u7684\u63d2\u4ef6\uff0c \u4ed6\u4e3b\u8981\u7528\u6765\u5feb\u901f\u751f\u6210\u671f\u671b\u7684\u6570\u636e\u5e76\u5bf9\u5199\u5165\u63d2\u4ef6\u8fdb\u884c\u6d4b\u8bd5

    \u4e00\u4e2a\u5b8c\u6574\u7684 StreamReader \u914d\u7f6e\u6587\u4ef6\u5982\u4e0b\uff1a

    {\n  \"reader\": {\n    \"name\": \"streamreader\",\n    \"parameter\": {\n      \"column\": [\n        {\n          \"value\": \"unique_id\",\n          \"type\": \"string\"\n        },\n        {\n          \"value\": \"1989-06-04 08:12:13\",\n          \"type\": \"date\",\n          \"dateFormat\": \"yyyy-MM-dd HH:mm:ss\"\n        },\n        {\n          \"value\": 1984,\n          \"type\": \"long\"\n        },\n        {\n          \"value\": 1989.64,\n          \"type\": \"double\"\n        },\n        {\n          \"value\": true,\n          \"type\": \"bool\"\n        },\n        {\n          \"value\": \"a long text\",\n          \"type\": \"bytes\"\n        }\n      ],\n      \"sliceRecordCount\": 10\n    }\n  }\n}\n

    \u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u5c06\u4f1a\u751f\u6210 10\u6761\u8bb0\u5f55\uff08\u5047\u5b9achannel\u4e3a1\uff09\uff0c\u6bcf\u6761\u8bb0\u5f55\u7684\u5185\u5bb9\u5982\u4e0b\uff1a

    unique_id,'1989-06-04 08:12:13',1984,1989.64,true,'a long text'

    \u76ee\u524d StreamReader \u652f\u6301\u7684\u8f93\u51fa\u6570\u636e\u7c7b\u578b\u5168\u90e8\u5217\u5728\u4e0a\u9762\uff0c\u5206\u522b\u662f\uff1a

    \u5176\u4e2d date \u7c7b\u578b\u8fd8\u652f\u6301 dateFormat \u914d\u7f6e\uff0c\u7528\u6765\u6307\u5b9a\u8f93\u5165\u7684\u65e5\u671f\u7684\u683c\u5f0f\uff0c\u9ed8\u8ba4\u4e3a yyyy-MM-dd HH:mm:ss\u3002\u6bd4\u5982\u4f60\u7684\u8f93\u5165\u53ef\u4ee5\u8fd9\u6837\uff1a

    {\n  \"value\": \"1989/06/04 12:13:14\",\n  \"type\": \"date\",\n  \"dateFormat\": \"yyyy/MM/dd HH:mm:ss\"\n}\n

    \u6ce8\u610f\uff0c\u65e5\u671f\u7c7b\u578b\u4e0d\u7ba1\u8f93\u5165\u662f\u4f55\u79cd\u683c\u5f0f\uff0c\u5185\u90e8\u90fd\u8f6c\u4e3a yyyy-MM-dd HH:mm:ss \u683c\u5f0f\u3002

    StreamReader \u8fd8\u652f\u6301\u968f\u673a\u8f93\u5165\u529f\u80fd\uff0c\u6bd4\u5982\u6211\u4eec\u8981\u968f\u673a\u5f97\u52300-10\u4e4b\u95f4\u7684\u4efb\u610f\u4e00\u4e2a\u6574\u6570\uff0c\u6211\u4eec\u53ef\u4ee5\u8fd9\u6837\u914d\u7f6e\u5217\uff1a

    {\n  \"random\": \"0,10\",\n  \"type\": \"long\"\n}\n

    \u83b7\u5f97\u4e00\u4e2a 0 \u81f3 100 \u4e4b\u95f4\u7684\u968f\u673a\u6d6e\u70b9\u6570\uff0c\u53ef\u4ee5\u8fd9\u6837\u914d\u7f6e\uff1a

    {\n  \"random\": \"0,100\",\n  \"type\": \"double\"\n}\n

    \u5982\u679c\u8981\u6307\u5b9a\u6d6e\u70b9\u6570\u7684\u5c0f\u6570\u4f4d\u6570\uff0c\u6bd4\u5982\u6307\u5b9a\u5c0f\u6570\u4f4d\u4e3a2\u4f4d\uff0c\u5219\u53ef\u4ee5\u8fd9\u6837\u8bbe\u5b9a

    {\n  \"random\": \"0,100,2\",\n  \"type\": \"double\"\n}\n

    \u6ce8\u610f\uff1a \u5e76\u4e0d\u80fd\u4fdd\u8bc1\u6bcf\u6b21\u751f\u6210\u7684\u5c0f\u6570\u6070\u597d\u662f2\u4f4d\uff0c\u5982\u679c\u5c0f\u6570\u4e3a\u6570\u4e3a0 \uff0c\u5219\u5c0f\u6570\u4f4d\u6570\u4f1a\u5c11\u4e8e\u6307\u5b9a\u7684\u4f4d\u6570\u3002

    \u8fd9\u91cc\u4f7f\u7528 random \u8fd9\u4e2a\u5173\u952e\u5b57\u6765\u8868\u793a\u5176\u503c\u4e3a\u968f\u673a\u503c\uff0c\u5176\u503c\u7684\u8303\u56f4\u4e3a\u5de6\u53f3\u95ed\u533a\u95f4\u3002

    \u5176\u4ed6\u7c7b\u578b\u7684\u968f\u673a\u7c7b\u578b\u914d\u7f6e\u5982\u4e0b\uff1a

    StreamReader \u8fd8\u652f\u6301\u9012\u589e\u51fd\u6570\uff0c\u6bd4\u5982\u6211\u4eec\u8981\u5f97\u5230\u4e00\u4e2a\u4ece1\u5f00\u59cb\uff0c\u6bcf\u6b21\u52a05\u7684\u7b49\u5dee\u6570\u5217\uff0c\u53ef\u4ee5\u8fd9\u6837\u914d\u7f6e\uff1a

    {\n  \"incr\": \"1,5\",\n  \"type\": \"long\"\n}\n

    \u5982\u679c\u9700\u8981\u83b7\u5f97\u4e00\u4e2a\u9012\u51cf\u7684\u6570\u5217\uff0c\u5219\u628a\u7b2c\u4e8c\u4e2a\u53c2\u6570\u7684\u6b65\u957f\uff08\u4e0a\u4f8b\u4e2d\u76845\uff09\u6539\u4e3a\u8d1f\u6570\u5373\u53ef\u3002\u6b65\u957f\u9ed8\u8ba4\u503c\u4e3a1\u3002

    \u9012\u589e\u8fd8\u652f\u6301\u65e5\u671f\u7c7b\u578b( 4.0.1 \u7248\u672c\u5f15\u5165)\uff0c\u6bd4\u5982\u4e0b\u9762\u7684\u914d\u7f6e\uff1a

    {\n  \"incr\": \"1989-06-04 09:01:02,2,d\",\n  \"type\": \"date\"\n}\n

    incr \u7531\u4e09\u90e8\u5206\u7ec4\u6210\uff0c\u5206\u522b\u662f\u5f00\u59cb\u65e5\u671f\uff0c\u6b65\u957f\u4ee5\u53ca\u6b65\u957f\u5355\u4f4d\uff0c\u4e2d\u95f4\u7528\u82f1\u6587\u9017\u53f7(,)\u5206\u9694\u3002

    \u914d\u7f6e\u9879 sliceRecordCount \u7528\u6765\u6307\u5b9a\u8981\u751f\u6210\u7684\u6570\u636e\u6761\u6570\uff0c\u5982\u679c\u6307\u5b9a\u7684 channel\uff0c\u5219\u5b9e\u9645\u751f\u6210\u7684\u8bb0\u5f55\u6570\u4e3a sliceRecordCount * channel

    "},{"location":"reader/sybasereader/","title":"Sybase Reader","text":"

    SybaseReader \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece Sybase \u8bfb\u53d6\u6570\u636e

    "},{"location":"reader/sybasereader/#_1","title":"\u793a\u4f8b","text":"

    \u6211\u4eec\u53ef\u4ee5\u7528 Docker \u5bb9\u5668\u6765\u542f\u52a8\u4e00\u4e2a Sybase \u6570\u636e\u5e93

    docker run -tid --rm  -h dksybase --name sybase  -p 5000:5000  ifnazar/sybase_15_7 bash /sybase/start\n

    \u4e0b\u9762\u7684\u914d\u7f6e\u662f\u8bfb\u53d6\u8be5\u8868\u5230\u7ec8\u7aef\u7684\u4f5c\u4e1a:

    job/sybasereader.json 
    {\n  \"job\": {\n    \"content\": [\n      {\n        \"reader\": {\n          \"name\": \"sybasereader\",\n          \"parameter\": {\n            \"column\": [\n              \"*\"\n            ],\n            \"connection\": {\n              \"jdbcUrl\": \"jdbc:sybase:Tds:127.0.0.1:5000/master\",\n              \"table\": [\n                \"dbo.ijdbc_function_escapes\"\n              ]\n            },\n            \"username\": \"sa\",\n            \"password\": \"password\"\n          }\n        },\n        \"writer\": {\n          \"name\": \"streamwriter\",\n          \"parameter\": {\n            \"print\": \"true\"\n          }\n        }\n      }\n    ],\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/sybase2stream.json

    "},{"location":"reader/sybasereader/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/sybase2stream.json\n
    "},{"location":"reader/sybasereader/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Reader \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Reader \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"reader/tdenginereader/","title":"TDengine Reader","text":"

    TDengine Reader \u63d2\u4ef6\u7528\u4e8e\u4ece\u6d9b\u601d\u516c\u53f8\u7684 TDengine \u8bfb\u53d6\u6570\u636e\u3002

    "},{"location":"reader/tdenginereader/#_1","title":"\u524d\u7f6e\u6761\u4ef6","text":"

    \u8003\u8651\u5230\u6027\u80fd\u95ee\u9898\uff0c\u8be5\u63d2\u4ef6\u4f7f\u7528\u4e86 TDengine \u7684 JDBC-JNI \u9a71\u52a8\uff0c \u8be5\u9a71\u52a8\u76f4\u63a5\u8c03\u7528\u5ba2\u6237\u7aef API\uff08libtaos.so \u6216 taos.dll\uff09\u5c06\u5199\u5165\u548c\u67e5\u8be2\u8bf7\u6c42\u53d1\u9001\u5230 taosd \u5b9e\u4f8b\u3002\u56e0\u6b64\u5728\u4f7f\u7528\u4e4b\u524d\u9700\u8981\u914d\u7f6e\u597d\u52a8\u6001\u5e93\u94fe\u63a5\u6587\u4ef6\u3002

    \u9996\u5148\u5c06 plugin/reader/tdenginereader/libs/libtaos.so.2.0.16.0 \u62f7\u8d1d\u5230 /usr/lib64 \u76ee\u5f55\uff0c\u7136\u540e\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u521b\u5efa\u8f6f\u94fe\u63a5

    ln -sf /usr/lib64/libtaos.so.2.0.16.0 /usr/lib64/libtaos.so.1\nln -sf /usr/lib64/libtaos.so.1 /usr/lib64/libtaos.so\n
    "},{"location":"reader/tdenginereader/#_2","title":"\u793a\u4f8b","text":"

    TDengine \u6570\u636e\u81ea\u5e26\u4e86\u4e00\u4e2a\u6f14\u793a\u6570\u636e\u5e93 taosdemo , \u6211\u4eec\u4ece\u6f14\u793a\u6570\u636e\u5e93\u8bfb\u53d6\u90e8\u5206\u6570\u636e\u5e76\u6253\u5370\u5230\u7ec8\u7aef

    \u4ee5\u4e0b\u662f\u914d\u7f6e\u6587\u4ef6

    job/tdengine2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0.02\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"tdenginereader\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"taosdata\",\n          \"beginDateTime\": \"2017-07-14 10:40:00\",\n          \"endDateTime\": \"2017-08-14 10:40:00\",\n          \"splitInterval\": \"1d\",\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:TAOS://127.0.0.1:6030/test\",\n            \"querySql\": [\n              \"select * from test.meters where ts <'2017-07-14 10:40:02' and  loc='beijing' limit 10\"\n            ]\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"streamwriter\",\n        \"parameter\": {\n          \"print\": true\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/tdengine2stream.json

    "},{"location":"reader/tdenginereader/#_3","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/tdengine2stream.json\n

    \u547d\u4ee4\u8f93\u51fa\u7c7b\u4f3c\u5982\u4e0b\uff1a

    2021-02-20 15:32:23.161 [main] INFO  VMInfo - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl\n2021-02-20 15:32:23.229 [main] INFO  Engine -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"password\":\"*****\",\n                    \"connection\":[\n                        {\n                            \"querySql\":[\n                                \"select * from test.meters where ts <'2017-07-14 10:40:02' and  loc='beijing' limit 100\"\n                            ],\n                            \"jdbcUrl\":[\n                                \"jdbc:TAOS://127.0.0.1:6030/test\"\n                            ]\n                        }\n                    ],\n                    \"username\":\"root\"\n                },\n                \"name\":\"tdenginereader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"print\":true\n                },\n                \"name\":\"streamwriter\"\n            }\n    },\n    \"setting\":{\n        \"errorLimit\":{\n            \"record\":0,\n            \"percentage\":0.02\n        },\n        \"speed\":{\n            \"channel\":3\n        }\n    }\n}\n\n2021-02-20 15:32:23.277 [main] INFO  PerfTrace - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-02-20 15:32:23.278 [main] INFO  JobContainer - Addax jobContainer starts job.\n2021-02-20 15:32:23.281 [main] INFO  JobContainer - Set jobId = 0\njava.library.path:/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib\n....\n2021-02-20 15:32:23.687 [0-0-0-reader] INFO  CommonRdbmsReader$Task - Begin to read record by Sql: [select * from test.meters where ts <'2017-07-14 10:40:02' and  loc='beijing' limit 100\n] jdbcUrl:[jdbc:TAOS://127.0.0.1:6030/test].\n2021-02-20 15:32:23.692 [0-0-0-reader] WARN  DBUtil - current database does not supoort TYPE_FORWARD_ONLY/CONCUR_READ_ONLY\n2021-02-20 15:32:23.740 [0-0-0-reader] INFO  CommonRdbmsReader$Task - Finished read record by Sql: [select * from test.meters where ts <'2017-07-14 10:40:02' and  loc='beijing' limit 100\n] jdbcUrl:[jdbc:TAOS://127.0.0.1:6030/test].\n\n1500000001000   5   5   0   1   beijing\n1500000001000   0   6   2   1   beijing\n1500000001000   7   0   0   1   beijing\n1500000001000   8   9   6   1   beijing\n1500000001000   9   9   1   1   beijing\n1500000001000   8   2   0   1   beijing\n1500000001000   4   5   5   3   beijing\n1500000001000   3   3   3   3   beijing\n1500000001000   5   4   8   3   beijing\n1500000001000   9   4   6   3   beijing\n\n2021-02-20 15:32:26.689 [job-0] INFO  JobContainer -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-02-20 15:32:23\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-02-20 15:32:26\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :              800B/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :             33rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                 100\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"reader/tdenginereader/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 jdbcUrl \u662f list \u65e0 \u5bf9\u7aef\u6570\u636e\u5e93\u7684JDBC\u8fde\u63a5\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u7684 TAOS \u5fc5\u987b\u5927\u5199 username \u662f string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801 table \u662f list \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d,\u4f7f\u7528JSON\u6570\u636e\u683c\u5f0f\uff0c\u5f53\u914d\u7f6e\u4e3a\u591a\u5f20\u8868\u65f6\uff0c\u7528\u6237\u81ea\u5df1\u9700\u4fdd\u8bc1\u591a\u5f20\u8868\u662f\u540c\u4e00\u8868\u7ed3\u6784 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0rdbmreader where \u5426 string \u65e0 \u9488\u5bf9\u8868\u7684\u7b5b\u9009\u6761\u4ef6 querySql \u5426 list \u65e0 \u4f7f\u7528\u81ea\u5b9a\u4e49\u7684SQL\u800c\u4e0d\u662f\u6307\u5b9a\u8868\u6765\u83b7\u53d6\u6570\u636e\uff0c\u5f53\u914d\u7f6e\u4e86\u8fd9\u4e00\u9879\u4e4b\u540e\uff0cAddax\u7cfb\u7edf\u5c31\u4f1a\u5ffd\u7565 table\uff0ccolumn\u8fd9\u4e9b\u914d\u7f6e\u9879 beginDateTime \u662f string \u65e0 \u6570\u636e\u7684\u5f00\u59cb\u65f6\u95f4\uff0cJob\u8fc1\u79fb\u4ece begineDateTime \u5230 endDateTime \u7684\u6570\u636e\uff0c\u683c\u5f0f\u4e3a yyyy-MM-dd HH:mm:ss endDateTime \u662f string \u65e0 \u6570\u636e\u7684\u7ed3\u675f\u65f6\u95f4\uff0cJob\u8fc1\u79fb\u4ece begineDateTime \u5230 endDateTime \u7684\u6570\u636e\uff0c\u683c\u5f0f\u4e3a yyyy-MM-dd HH:mm:ss splitInterval \u662f string \u65e0 \u6309\u7167 splitInterval \u6765\u5212\u5206 task, \u6bcf splitInterval \u521b\u5efa\u4e00\u4e2a task"},{"location":"reader/tdenginereader/#splitinterval","title":"splitInterval","text":"

    \u7528\u6765\u5212\u5206 task\u3002 \u4f8b\u5982\uff0c20d \u4ee3\u8868\u6309\u7167\u6bcf 20 \u5929\u7684\u6570\u636e\u5212\u5206\u4e3a 1 \u4e2a task\u3002 \u53ef\u4ee5\u914d\u7f6e\u7684\u65f6\u95f4

    "},{"location":"reader/tdenginereader/#jdbc-restful","title":"\u4f7f\u7528 JDBC-RESTful \u63a5\u53e3","text":"

    \u5982\u679c\u4e0d\u60f3\u4f9d\u8d56\u672c\u5730\u5e93\uff0c\u6216\u8005\u6ca1\u6709\u6743\u9650\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528 JDBC-RESTful \u63a5\u53e3\u6765\u5199\u5165\u8868\uff0c\u76f8\u6bd4 JDBC-JNI \u800c\u8a00\uff0c\u914d\u7f6e\u533a\u522b\u662f\uff1a

    \u6240\u4ee5\u4e0a\u8ff0\u914d\u7f6e\u4e2d\u7684 connection \u5e94\u8be5\u4fee\u6539\u4e3a\u5982\u4e0b\uff1a

    {\n  \"connection\": [\n    {\n      \"querySql\": [\n        \"select * from test.meters where ts <'2017-07-14 10:40:02' and  loc='beijing' limit 100\"\n      ],\n      \"jdbcUrl\": [\n        \"jdbc:TAOS-RS://127.0.0.1:6041/test\"\n      ],\n      \"driver\": \"com.taosdata.jdbc.rs.RestfulDriver\"\n    }\n  ]\n}\n
    "},{"location":"reader/tdenginereader/#_5","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b TDengine \u6570\u636e\u7c7b\u578b Long SMALLINT, TINYINT, INT, BIGINT, TIMESTAMP Double FLOAT, DOUBLE String BINARY, NCHAR Boolean BOOL"},{"location":"reader/tdenginereader/#_6","title":"\u5f53\u524d\u652f\u6301\u7248\u672c","text":"

    TDengine 2.0.16

    "},{"location":"reader/tdenginereader/#_7","title":"\u6ce8\u610f\u4e8b\u9879","text":""},{"location":"reader/txtfilereader/","title":"TxtFile Reader","text":"

    TxtFile Reader \u63d0\u4f9b\u4e86\u8bfb\u53d6\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u6570\u636e\u5b58\u50a8\u7684\u80fd\u529b\u3002

    "},{"location":"reader/txtfilereader/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"job/txtfile2stream.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"txtfilereader\",\n        \"parameter\": {\n          \"path\": [\n            \"/tmp/data\"\n          ],\n          \"encoding\": \"UTF-8\",\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"long\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"boolean\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"double\"\n            },\n            {\n              \"index\": 3,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 4,\n              \"type\": \"date\",\n              \"format\": \"yyyy.MM.dd\"\n            }\n          ],\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"txtfilewriter\",\n        \"parameter\": {\n          \"path\": \"/tmp/result\",\n          \"fileName\": \"txt_\",\n          \"writeMode\": \"truncate\",\n          \"format\": \"yyyy-MM-dd\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"reader/txtfilereader/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f list \u65e0 \u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84,\u8be6\u7ec6\u63cf\u8ff0\u89c1\u4e0b\u6587 column \u662f list<map>\\|* \u65e0 \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype\u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0c\u8be6\u89c1\u4e0b\u6587 fieldDelimiter \u662f string , \u63cf\u8ff0\uff1a\u8bfb\u53d6\u7684\u5b57\u6bb5\u5206\u9694\u7b26 encoding \u5426 string utf-8 \u8bfb\u53d6\u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e skipHeader \u5426 boolean false \u7c7bCSV\u683c\u5f0f\u6587\u4ef6\u53ef\u80fd\u5b58\u5728\u8868\u5934\u4e3a\u6807\u9898\u60c5\u51b5\uff0c\u9700\u8981\u8df3\u8fc7\u3002\u9ed8\u8ba4\u4e0d\u8df3\u8fc7 csvReaderConfig \u5426 string \u65e0 \u8bfb\u53d6CSV\u7c7b\u578b\u6587\u4ef6\u53c2\u6570\u914d\u7f6e\uff0cMap\u7c7b\u578b\u3002\u4e0d\u914d\u7f6e\u5219\u4f7f\u7528\u9ed8\u8ba4\u503c,\u8be6\u89c1\u4e0b\u6587"},{"location":"reader/txtfilereader/#path","title":"path","text":"

    \u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u6ce8\u610f\u8fd9\u91cc\u53ef\u4ee5\u652f\u6301\u586b\u5199\u591a\u4e2a\u8def\u5f84\u3002

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0cAddax\u4f1a\u5c06\u4e00\u4e2a\u4f5c\u4e1a\u4e0b\u540c\u6b65\u7684\u6240\u6709Text File\u89c6\u4f5c\u540c\u4e00\u5f20\u6570\u636e\u8868\u3002\u7528\u6237\u5fc5\u987b\u81ea\u5df1\u4fdd\u8bc1\u6240\u6709\u7684File\u80fd\u591f\u9002\u914d\u540c\u4e00\u5957schema\u4fe1\u606f\u3002\u8bfb\u53d6\u6587\u4ef6\u7528\u6237\u5fc5\u987b\u4fdd\u8bc1\u4e3a\u7c7bCSV\u683c\u5f0f\uff0c\u5e76\u4e14\u63d0\u4f9b\u7ed9Addax\u6743\u9650\u53ef\u8bfb\u3002

    \u7279\u522b\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0c\u5982\u679cPath\u6307\u5b9a\u7684\u8def\u5f84\u4e0b\u6ca1\u6709\u7b26\u5408\u5339\u914d\u7684\u6587\u4ef6\u62bd\u53d6\uff0cAddax\u5c06\u62a5\u9519\u3002

    \u4ece 3.2.3 \u7248\u672c\u8d77\uff0c path \u4e0b\u5141\u8bb8\u6df7\u5408\u4e0d\u540c\u538b\u7f29\u683c\u5f0f\u7684\u6587\u4ef6\uff0c\u63d2\u4ef6\u4f1a\u5c1d\u8bd5\u81ea\u52a8\u731c\u6d4b\u538b\u7f29\u683c\u5f0f\u5e76\u81ea\u52a8\u89e3\u538b\uff0c\u76ee\u524d\u652f\u6301\u7684\u538b\u7f29\u683c\u5f0f\u6709\uff1a

    "},{"location":"reader/txtfilereader/#column","title":"column","text":"

    \u8bfb\u53d6\u5b57\u6bb5\u5217\u8868\uff0ctype\u6307\u5b9a\u6e90\u6570\u636e\u7684\u7c7b\u578b\uff0cindex\u6307\u5b9a\u5f53\u524d\u5217\u6765\u81ea\u4e8e\u6587\u672c\u7b2c\u51e0\u5217(\u4ee50\u5f00\u59cb)\uff0cvalue\u6307\u5b9a\u5f53\u524d\u7c7b\u578b\u4e3a\u5e38\u91cf\uff0c\u4e0d\u4ece\u6e90\u5934\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\uff0c\u800c\u662f\u6839\u636evalue\u503c\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684\u5217\u3002

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u7528\u6237\u53ef\u4ee5\u5168\u90e8\u6309\u7167String\u7c7b\u578b\u8bfb\u53d6\u6570\u636e\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"column\": [\n    \"*\"\n  ]\n}\n

    \u7528\u6237\u53ef\u4ee5\u6307\u5b9aColumn\u5b57\u6bb5\u4fe1\u606f\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    [\n  {\n    \"type\": \"long\",\n    \"index\": 0\n  },\n  {\n    \"type\": \"string\",\n    \"value\": \"alibaba\"\n  }\n]\n

    \u5bf9\u4e8e\u7528\u6237\u6307\u5b9aColumn\u4fe1\u606f\uff0ctype\u5fc5\u987b\u586b\u5199\uff0cindex/value\u5fc5\u987b\u9009\u62e9\u5176\u4e00\u3002

    \u4ece 4.0.1 \u5f00\u59cb\uff0c\u8868\u793a\u5b57\u6bb5\u9664\u4e86\u4f7f\u7528 index \u6765\u6307\u5b9a\u5b57\u6bb5\u7684\u987a\u5e8f\u5916\uff0c\u8fd8\u652f\u6301 name \u65b9\u5f0f\uff0c\u8fd9\u9700\u8981\u6240\u8bfb\u53d6\u7684\u6587\u4ef6\u7684\u90fd\u5305\u542b\u4e86\u6587\u4ef6\u5934\uff0c\u63d2\u4ef6\u4f1a\u5c1d\u8bd5\u5c06\u6307\u5b9a\u7684 name \u53bb\u5339\u914d\u4ece\u6587\u4ef6\u8bfb\u53d6\u7684\u6587\u4ef6\u5934\uff0c \u7136\u540e\u5f97\u5230\u5bf9\u5e94\u7684 index \u503c\uff0c\u5e76\u56de\u5199\u5230 \u914d\u7f6e\u6587\u4ef6\u4e2d\u3002\u540c\u65f6\uff0cindex \u548c name \u53ef\u4ee5\u5728\u4e0d\u540c\u7684\u5217\u4e0a\u8fdb\u884c\u6df7\u5408\u4f7f\u7528\uff0c\u6bd4\u5982\u4e0b\u9762\u8fd9\u6837\uff1a

    [\n  {\n    \"type\": \"long\",\n    \"index\": 0\n  },\n  {\n    \"name\": \"region\",\n    \"type\": \"string\"\n  },\n  {\n    \"type\": \"string\",\n    \"value\": \"alibaba\"\n  }\n]\n

    \u6ce8\uff1a \u8fd9\u79cd\u65b9\u5f0f\u4ee5\u4e3a\u8fd9\u5728\u51c6\u5907\u9636\u6bb5\u5c31\u8981\u5c1d\u8bd5\u8bfb\u53d6\u6587\u4ef6\uff0c\u56e0\u4e3a\u4f1a\u6709\u4e00\u5b9a\u7684\u6027\u80fd\u635f\u5931\uff0c\u5982\u975e\u5fc5\u8981\uff0c\u4e0d\u5efa\u8bae\u914d\u7f6e name \u65b9\u5f0f\u3002

    "},{"location":"reader/txtfilereader/#csvreaderconfig","title":"csvReaderConfig","text":"

    \u8bfb\u53d6CSV\u7c7b\u578b\u6587\u4ef6\u53c2\u6570\u914d\u7f6e\uff0cMap\u7c7b\u578b\u3002\u8bfb\u53d6CSV\u7c7b\u578b\u6587\u4ef6\u4f7f\u7528\u7684CsvReader\u8fdb\u884c\u8bfb\u53d6\uff0c\u4f1a\u6709\u5f88\u591a\u914d\u7f6e\uff0c\u4e0d\u914d\u7f6e\u5219\u4f7f\u7528\u9ed8\u8ba4\u503c\u3002

    \u5e38\u89c1\u914d\u7f6e\uff1a

    {\n  \"csvReaderConfig\": {\n    \"safetySwitch\": false,\n    \"skipEmptyRecords\": false,\n    \"useTextQualifier\": false\n  }\n}\n

    \u6240\u6709\u914d\u7f6e\u9879\u53ca\u9ed8\u8ba4\u503c,\u914d\u7f6e\u65f6 csvReaderConfig \u7684map\u4e2d\u8bf7**\u4e25\u683c\u6309\u7167\u4ee5\u4e0b\u5b57\u6bb5\u540d\u5b57\u8fdb\u884c\u914d\u7f6e**\uff1a

    boolean caseSensitive = true;\nchar textQualifier = 34;\nboolean trimWhitespace = true;\nboolean useTextQualifier = true;//\u662f\u5426\u4f7f\u7528csv\u8f6c\u4e49\u5b57\u7b26\nchar delimiter = 44;//\u5206\u9694\u7b26\nchar recordDelimiter = 0;\nchar comment = 35;\nboolean useComments = false;\nint escapeMode = 1;\nboolean safetySwitch = true;//\u5355\u5217\u957f\u5ea6\u662f\u5426\u9650\u5236100000\u5b57\u7b26\nboolean skipEmptyRecords = true;//\u662f\u5426\u8df3\u8fc7\u7a7a\u884c\nboolean captureRawRecord = true;\n
    "},{"location":"reader/txtfilereader/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b \u672c\u5730\u6587\u4ef6 \u6570\u636e\u7c7b\u578b Long Long Double Double String String Boolean Boolean Date Date"},{"location":"writer/accesswriter/","title":"Access Writer","text":"

    Access Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 Access \u76ee\u7684\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/accesswriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684 Access \u8868\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    create table tbl_test(name varchar(20), file_size int, file_date date, file_open boolean, memo blob);\n

    \u8fd9\u91cc\u4f7f\u7528\u4e00\u4efd\u4ece\u5185\u5b58\u4ea7\u751f\u5230 Access \u5bfc\u5165\u7684\u6570\u636e\u3002

    job/stream2access.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"accesswriter\",\n        \"parameter\": {\n          \"username\": \"wgzhao\",\n          \"password\": \"\",\n          \"column\": [\n            \"name\",\n            \"file_size\",\n            \"file_date\",\n            \"file_open\",\n            \"memo\"\n          ],\n          \"ddl\": \"create table tbl_test(name varchar(20), file_size int, file_date date, file_open boolean, memo blob);\",\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:ucanaccess:////Users/wgzhao/Downloads/AccessThemeDemo.mdb\",\n            \"table\": [\n              \"tbl_test\"\n            ]\n          }\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2access.json

    "},{"location":"writer/accesswriter/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/stream2access.json\n
    "},{"location":"writer/accesswriter/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/cassandrawriter/","title":"Cassandra Writer","text":"

    Cassandra Writer \u63d2\u4ef6\u7528\u4e8e\u5411 Cassandra \u5199\u5165\u6570\u636e\u3002

    "},{"location":"writer/cassandrawriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4ece\u5185\u5b58\u4ea7\u751f\u5230 Cassandra \u5bfc\u5165\u7684\u4f5c\u4e1a:

    jobs/stream2cassandra.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 5,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"name\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"false\",\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": \"addr\",\n              \"type\": \"bytes\"\n            },\n            {\n              \"value\": 1.234,\n              \"type\": \"double\"\n            },\n            {\n              \"value\": 12345678,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": 2.345,\n              \"type\": \"double\"\n            },\n            {\n              \"value\": 3456789,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"4a0ef8c0-4d97-11d0-db82-ebecdb03ffa5\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"value\",\n              \"type\": \"bytes\"\n            },\n            {\n              \"value\": \"-838383838,37377373,-383883838,27272772,393993939,-38383883,83883838,-1350403181,817650816,1630642337,251398784,-622020148\",\n              \"type\": \"string\"\n            }\n          ],\n          \"sliceRecordCount\": 10000000\n        }\n      },\n      \"writer\": {\n        \"name\": \"cassandrawriter\",\n        \"parameter\": {\n          \"host\": \"localhost\",\n          \"port\": 9042,\n          \"useSSL\": false,\n          \"keyspace\": \"stresscql\",\n          \"table\": \"dst\",\n          \"batchSize\": 10,\n          \"column\": [\n            \"name\",\n            \"choice\",\n            \"date\",\n            \"address\",\n            \"dbl\",\n            \"lval\",\n            \"fval\",\n            \"ival\",\n            \"uid\",\n            \"value\",\n            \"listval\"\n          ]\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/cassandrawriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 host \u662f string \u65e0 \u8fde\u63a5\u70b9\u7684\u57df\u540d\u6216 ip\uff0c\u591a\u4e2a node \u4e4b\u95f4\u7528\u9017\u53f7\u5206\u9694 port \u662f int 9042 Cassandra \u7aef\u53e3 username \u5426 string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801 useSSL \u5426 boolean false \u662f\u5426\u4f7f\u7528 SSL \u8fde\u63a5 connectionsPerHost \u5426 int 8 \u5ba2\u6237\u7aef\u8fde\u63a5\u6c60\u914d\u7f6e\uff1a\u4e0e\u670d\u52a1\u5668\u6bcf\u4e2a\u8282\u70b9\u5efa\u591a\u5c11\u4e2a\u8fde\u63a5 maxPendingPerConnection \u5426 int 128 \u5ba2\u6237\u7aef\u8fde\u63a5\u6c60\u914d\u7f6e\uff1a\u6bcf\u4e2a\u8fde\u63a5\u6700\u5927\u8bf7\u6c42\u6570 keyspace \u662f string \u65e0 \u9700\u8981\u540c\u6b65\u7684\u8868\u6240\u5728\u7684 keyspace table \u662f string \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u96c6\u5408 consistancyLevel \u5426 string LOCAL_QUORUM \u6570\u636e\u4e00\u81f4\u6027\u7ea7\u522b, batchSize \u5426 int 1 \u4e00\u6b21\u6279\u91cf\u63d0\u4ea4(UNLOGGED BATCH)\u7684\u8bb0\u5f55\u6570\u5927\u5c0f\uff08\u6761\u6570\uff09"},{"location":"writer/cassandrawriter/#column","title":"column","text":"

    \u5185\u5bb9\u53ef\u4ee5\u662f\u5217\u7684\u540d\u79f0\u6216 writetime()\u3002\u5982\u679c\u5c06\u5217\u540d\u914d\u7f6e\u4e3a writetime()\uff0c\u4f1a\u5c06\u8fd9\u4e00\u5217\u7684\u5185\u5bb9\u4f5c\u4e3a\u65f6\u95f4\u6233

    "},{"location":"writer/cassandrawriter/#consistancylevel","title":"consistancyLevel","text":"

    \u53ef\u9009 ONE, QUORUM, LOCAL_QUORUM, EACH_QUORUM, ALL, ANY, TWO, THREE, LOCAL_ONE

    "},{"location":"writer/cassandrawriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b Cassandra \u6570\u636e\u7c7b\u578b Long int, tinyint, smallint,varint,bigint,time Double float, double, decimal String ascii,varchar, text,uuid,timeuuid,duration,list,map,set,tuple,udt,inet Date date, timestamp Boolean bool Bytes blob

    \u8bf7\u6ce8\u610f:

    \u76ee\u524d\u4e0d\u652f\u6301 counter \u7c7b\u578b\u548c custom \u7c7b\u578b\u3002

    "},{"location":"writer/cassandrawriter/#_4","title":"\u7ea6\u675f\u9650\u5236","text":""},{"location":"writer/cassandrawriter/#batchsize","title":"batchSize","text":"
    1. \u4e0d\u80fd\u8d85\u8fc7 65535
    2. batch \u4e2d\u7684\u5185\u5bb9\u5927\u5c0f\u53d7\u5230\u670d\u52a1\u5668\u7aef batch_size_fail_threshold_in_kb \u7684\u9650\u5236\u3002
    3. \u5982\u679c batch \u4e2d\u7684\u5185\u5bb9\u8d85\u8fc7\u4e86 batch_size_warn_threshold_in_kb \u7684\u9650\u5236\uff0c\u4f1a\u6253\u51fa warn \u65e5\u5fd7\uff0c\u4f46\u5e76\u4e0d\u5f71\u54cd\u5199\u5165\uff0c\u5ffd\u7565\u5373\u53ef\u3002
    4. \u5982\u679c\u6279\u91cf\u63d0\u4ea4\u5931\u8d25\uff0c\u4f1a\u628a\u8fd9\u4e2a\u6279\u91cf\u7684\u6240\u6709\u5185\u5bb9\u91cd\u65b0\u9010\u6761\u5199\u5165\u4e00\u904d\u3002
    "},{"location":"writer/clickhousewriter/","title":"ClickHouse Writer","text":"

    ClickHouse Writer \u63d2\u4ef6\u7528\u4e8e\u5411 ClickHouse \u5199\u5165\u6570\u636e\u3002

    "},{"location":"writer/clickhousewriter/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u793a\u4f8b\u6211\u4eec\u6f14\u793a\u4ece clickhouse \u4e2d\u8bfb\u53d6\u4e00\u5f20\u8868\u7684\u5185\u5bb9\uff0c\u5e76\u5199\u5165\u5230\u76f8\u540c\u8868\u7ed3\u6784\u7684\u53e6\u5916\u4e00\u5f20\u8868\u4e2d\uff0c\u7528\u6765\u6d4b\u8bd5\u63d2\u4ef6\u6240\u652f\u6301\u7684\u6570\u636e\u7ed3\u6784

    "},{"location":"writer/clickhousewriter/#_2","title":"\u8868\u7ed3\u6784\u4ee5\u6570\u636e","text":"

    \u5047\u5b9a\u8981\u8bfb\u53d6\u7684\u8868\u7ed3\u6784\u53ca\u6570\u636e\u5982\u4e0b\uff1a

    CREATE TABLE ck_addax (\n    c_int8 Int8,\n    c_int16 Int16,\n    c_int32 Int32,\n    c_int64 Int64,\n    c_uint8 UInt8,\n    c_uint16 UInt16,\n    c_uint32 UInt32,\n    c_uint64 UInt64,\n    c_float32 Float32,\n    c_float64 Float64,\n    c_decimal Decimal(38,10),\n    c_string String,\n    c_fixstr FixedString(36),\n    c_uuid UUID,\n    c_date Date,\n    c_datetime DateTime('Asia/Chongqing'),\n    c_datetime64 DateTime64(3, 'Asia/Chongqing'),\n    c_enum Enum('hello' = 1, 'world'=2)\n) ENGINE = MergeTree() ORDER BY (c_int8, c_int16) SETTINGS index_granularity = 8192;\n\ninsert into ck_addax values(\n    127,\n    -32768,\n    2147483647,\n    -9223372036854775808,\n    255,\n    65535,\n    4294967295,\n    18446744073709551615,\n    0.999999999999,\n    0.99999999999999999,\n    1234567891234567891234567891.1234567891,\n    'Hello String',\n    '2c:16:db:a3:3a:4f',\n    '5F042A36-5B0C-4F71-ADFD-4DF4FCA1B863',\n    '2021-01-01',\n    '2021-01-01 11:22:33',\n    '2021-01-01 10:33:23.123',\n    'hello'\n);\n

    \u8981\u5199\u5165\u7684\u8868\u91c7\u53d6\u548c\u8bfb\u53d6\u8868\u7ed3\u6784\u76f8\u540c\uff0c\u5176\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    create table ck_addax_writer as ck_addax;\n
    "},{"location":"writer/clickhousewriter/#_3","title":"\u914d\u7f6e","text":"

    \u4ee5\u4e0b\u4e3a\u914d\u7f6e\u6587\u4ef6

    job/clickhouse2clickhouse.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"writer\": {\n        \"name\": \"clickhousewriter\",\n        \"parameter\": {\n          \"username\": \"default\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"ck_addax_writer\"\n            ],\n            \"jdbcUrl\": \"jdbc:clickhouse://127.0.0.1:8123/default\"\n          },\n          \"preSql\": [\n            \"alter table @table delete where 1=1\"\n          ]\n        }\n      },\n      \"reader\": {\n        \"name\": \"clickhousereader\",\n        \"parameter\": {\n          \"username\": \"default\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:clickhouse://127.0.0.1:8123/\",\n            \"table\": [\n              \"ck_addax\"\n            ]\n          }\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/clickhouse2clickhouse.json

    "},{"location":"writer/clickhousewriter/#_4","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/clickhouse2clickhouse.json\n
    "},{"location":"writer/clickhousewriter/#_5","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/databendwriter/","title":"DatabendWriter","text":"

    Databend \u63d2\u4ef6\u7528\u4e8e\u5411 Databend \u6570\u636e\u5e93\u4ee5 JDBC \u65b9\u5f0f\u5199\u5165\u6570\u636e\u3002

    Databend \u662f\u4e00\u4e2a\u517c\u5bb9 MySQL \u534f\u8bae\u7684\u6570\u636e\u5e93\u540e\u7aef\uff0c\u56e0\u6b64 Databend \u5199\u5165\u53ef\u4ee5\u4f7f\u7528 MySQLWriter \u8fdb\u884c\u8bbf\u95ee\u3002

    "},{"location":"writer/databendwriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684\u8868\u7684\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    CREATE\nDATABASE example_db;\nCREATE TABLE `example_db`.`table1`\n(\n    `siteid`   INT DEFAULT CAST(10 AS INT),\n    `citycode` INT,\n    `username` VARCHAR,\n    `pv`       BIGINT\n);\n

    \u4e0b\u9762\u914d\u7f6e\u4e00\u4e2a\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\uff0c\u7136\u540e\u5199\u5165\u5230 databend \u8868\u7684\u914d\u7f6e\u6587\u4ef6

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2\n      }\n    },\n    \"content\": {\n      \"writer\": {\n        \"name\": \"databendwriter\",\n        \"parameter\": {\n          \"preSql\": [\n            \"truncate table @table\"\n          ],\n          \"postSql\": [],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:databend://localhost:8000/addax\",\n            \"table\": [\n              \"table1\"\n            ]\n          },\n          \"username\": \"u1\",\n          \"password\": \"123\",\n          \"column\": [\n            \"*\"\n          ]\n        }\n      },\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"1,500\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"1,127\",\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"this is a text\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"5,200\",\n              \"type\": \"long\"\n            }\n          ],\n          \"sliceRecordCount\": 100\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2databend.json

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4

    bin/addax.sh job/stream2Databend.json\n
    "},{"location":"writer/databendwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879, \u5e76\u589e\u52a0\u4e86\u5982\u4e0b\u914d\u7f6e\u9879\uff1a

    \u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 writeMode \u5426 string insert \u5199\u5165\u6a21\u5f0f\uff0c\u652f\u6301 insert \u548c replace \u4e24\u79cd\u6a21\u5f0f onConflictColumn \u5426 string \u65e0 \u51b2\u7a81\u5217\uff0c\u5f53 writeMode \u4e3a replace \u65f6\uff0c\u5fc5\u987b\u6307\u5b9a\u51b2\u7a81\u5217\uff0c\u5426\u5219\u4f1a\u5bfc\u81f4\u5199\u5165\u5931\u8d25\u3002"},{"location":"writer/databendwriter/#writemode","title":"writeMode","text":"

    \u7528\u6765\u652f\u6301 Databend \u7684 replace into \u8bed\u6cd5\uff0c\u5f53\u8be5\u53c2\u6570\u8bbe\u5b9a\u4e3a replace \u65f6\uff0c\u5fc5\u987b\u540c\u65f6\u6307\u5b9a onConflictColumn \u53c2\u6570\uff0c\u7528\u6765\u5224\u65ad\u6570\u636e\u662f\u63d2\u5165\u8fd8\u662f\u66f4\u65b0\u7684\u4f9d\u636e\u3002

    \u4e24\u4e2a\u53c2\u6570\u7684\u793a\u4f8b\u5982\u4e0b\uff1a

    {\n  \"writeMode\": \"replace\",\n  \"onConflictColumn\": [\n    \"id\"\n  ]\n}\n
    "},{"location":"writer/dbfwriter/","title":"DBF Writer","text":"

    Dbf Writer \u63d0\u4f9b\u4e86\u5411\u672c\u5730\u6587\u4ef6\u5199\u5165\u7c7bdbf\u683c\u5f0f\u7684\u4e00\u4e2a\u6216\u8005\u591a\u4e2a\u8868\u6587\u4ef6\u3002

    "},{"location":"writer/dbfwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 00:00:00\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"\u4e2d\u6587\u6d4b\u8bd5\",\n              \"type\": \"string\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"dbfwriter\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"name\": \"col1\",\n              \"type\": \"char\",\n              \"length\": 100\n            },\n            {\n              \"name\": \"col2\",\n              \"type\": \"numeric\",\n              \"length\": 18,\n              \"scale\": 0\n            },\n            {\n              \"name\": \"col3\",\n              \"type\": \"date\"\n            },\n            {\n              \"name\": \"col4\",\n              \"type\": \"logical\"\n            },\n            {\n              \"name\": \"col5\",\n              \"type\": \"char\",\n              \"length\": 100\n            }\n          ],\n          \"fileName\": \"test.dbf\",\n          \"path\": \"/tmp/out\",\n          \"writeMode\": \"truncate\",\n          \"encoding\": \"GBK\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/dbfwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f string \u65e0 \u6587\u4ef6\u76ee\u5f55\uff0c\u6ce8\u610f\u8fd9\u91cc\u662f\u6587\u4ef6\u5939\uff0c\u4e0d\u662f\u6587\u4ef6 column \u662f list<map> \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u96c6\u5408\uff0c\u8be6\u89c1\u793a\u4f8b\u914d\u7f6e fileName \u662f string \u65e0 \u5199\u5165\u7684\u6587\u4ef6\u540d writeMode \u662f string \u65e0 \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff0c\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0 encoding \u5426 string UTF-8 \u6587\u4ef6\u7f16\u7801\uff0c\u6bd4\u5982 GBK, UTF-8 nullFormat \u5426 string \\N \u5b9a\u4e49\u54ea\u4e2a\u5b57\u7b26\u4e32\u53ef\u4ee5\u8868\u793a\u4e3anull, dateFormat \u5426 string \u65e0 \u65e5\u671f\u7c7b\u578b\u7684\u6570\u636e\u5e8f\u5217\u5316\u5230\u6587\u4ef6\u4e2d\u65f6\u7684\u683c\u5f0f\uff0c\u4f8b\u5982 \"yyyy-MM-dd\""},{"location":"writer/dbfwriter/#writemode","title":"writeMode","text":"

    \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff1a

    "},{"location":"writer/dbfwriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u5f53\u524d\u8be5\u63d2\u4ef6\u652f\u6301\u5199\u5165\u7684\u7c7b\u578b\u4ee5\u53ca\u5bf9\u5e94\u5173\u7cfb\u5982\u4e0b\uff1a

    XBase Type XBase Symbol Java Type used in JavaDBF Character C java.lang.String Numeric N java.math.BigDecimal Floating Point F java.math.BigDecimal Logical L java.lang.Boolean Date D java.util.Date

    \u5176\u4e2d\uff1a

    "},{"location":"writer/doriswriter/","title":"Doris Writer","text":"

    DorisWriter \u63d2\u4ef6\u7528\u4e8e\u5411 Doris \u6570\u636e\u5e93\u4ee5\u6d41\u5f0f\u65b9\u5f0f\u5199\u5165\u6570\u636e\u3002 \u5176\u5b9e\u73b0\u4e0a\u662f\u901a\u8fc7\u8bbf\u95ee Doris http \u8fde\u63a5(8030)\uff0c\u7136\u540e\u901a\u8fc7 stream load \u52a0\u8f7d\u6570\u636e\u5230\u6570\u636e\u4e2d\uff0c\u76f8\u6bd4 insert into \u65b9\u5f0f\u6548\u7387\u8981\u9ad8\u4e0d\u5c11\uff0c\u4e5f\u662f\u5b98\u65b9\u63a8\u8350\u7684\u751f\u4ea7\u73af\u5883\u4e0b\u7684\u6570\u636e\u52a0\u8f7d\u65b9\u5f0f\u3002

    Doris \u662f\u4e00\u4e2a\u517c\u5bb9 MySQL \u534f\u8bae\u7684\u6570\u636e\u5e93\u540e\u7aef\uff0c\u56e0\u6b64 Doris \u8bfb\u53d6\u53ef\u4ee5\u4f7f\u7528 MySQL Reader \u8fdb\u884c\u8bbf\u95ee\u3002

    "},{"location":"writer/doriswriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684\u8868\u7684\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    CREATE\nDATABASE example_db;\nCREATE TABLE example_db.table1\n(\n    siteid   INT         DEFAULT '10',\n    citycode SMALLINT,\n    username VARCHAR(32) DEFAULT '',\n    pv       BIGINT SUM DEFAULT '0'\n) AGGREGATE KEY(siteid, citycode, username)\nDISTRIBUTED BY HASH(siteid) BUCKETS 10\nPROPERTIES(\"replication_num\" = \"1\");\n

    \u4e0b\u9762\u914d\u7f6e\u4e00\u4e2a\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\uff0c\u7136\u540e\u5199\u5165\u5230 doris \u8868\u7684\u914d\u7f6e\u6587\u4ef6

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2\n      }\n    },\n    \"content\": {\n      \"writer\": {\n        \"name\": \"doriswriter\",\n        \"parameter\": {\n          \"loadUrl\": [\n            \"127.0.0.1:8030\"\n          ],\n          \"username\": \"test\",\n          \"password\": \"123456\",\n          \"batchSize\": 1024,\n          \"column\": [\n            \"siteid\",\n            \"citycode\",\n            \"username\",\n            \"pv\"\n          ],\n          \"connection\": {\n            \"table\": \"table1\",\n            \"database\": \"example_db\",\n            \"jdbcUrl\": \"jdbc:mysql://localhost:9030/example_db\"\n          },\n          \"loadProps\": {\n            \"format\": \"json\",\n            \"strip_outer_array\": true\n          }\n        }\n      },\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"1,500\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"1,127\",\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"this is a text\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"5,200\",\n              \"type\": \"long\"\n            }\n          ],\n          \"sliceRecordCount\": 100\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2doris.json

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4

    bin/addax.sh job/stream2doris.json\n

    \u8f93\u51fa\u7c7b\u4f3c\u5982\u4e0b\uff1a

    2021-02-23 15:22:57.851 [main] INFO  VMInfo - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl\n2021-02-23 15:22:57.871 [main] INFO  Engine -\n{\n\"content\":{\n\"reader\":{\n    \"parameter\":{\n            \"column\":[\n                    {\n                            \"random\":\"1,500\",\n                            \"type\":\"long\"\n                    },\n                    {\n                            \"random\":\"1,127\",\n                            \"type\":\"long\"\n                    },\n                    {\n                            \"type\":\"string\",\n                            \"value\":\"username\"\n                    }\n            ],\n            \"sliceRecordCount\":100\n    },\n    \"name\":\"streamreader\"\n},\n\"writer\":{\n    \"parameter\":{\n            \"password\":\"*****\",\n            \"batchSize\":1024,\n            \"connection\":[\n                    {\n                            \"database\":\"example_db\",\n                            \"endpoint\":\"http://127.0.0.1:8030/\",\n                            \"table\":\"table1\"\n                    }\n            ],\n            \"username\":\"test\"\n    },\n    \"name\":\"doriswriter\"\n }\n},\n\"setting\":{\n\"speed\":{\n\"channel\":2\n}\n}\n}\n\n2021-02-23 15:22:57.886 [main] INFO  PerfTrace - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-02-23 15:22:57.886 [main] INFO  JobContainer - Addax jobContainer starts job.\n2021-02-23 15:22:57.920 [job-0] INFO  JobContainer - Scheduler starts [1] taskGroups.\n2021-02-23 15:22:57.928 [taskGroup-0] INFO  TaskGroupContainer - taskGroupId=[0] start [2] channels for [2] tasks.\n2021-02-23 15:22:57.935 [taskGroup-0] INFO  Channel - Channel set byte_speed_limit to -1, No bps activated.\n2021-02-23 15:22:57.936 [taskGroup-0] INFO  Channel - Channel set record_speed_limit to -1, No tps activated.\n2021-02-23 15:22:57.970 [0-0-1-writer] INFO  DorisWriterTask - connect DorisDB with http://127.0.0.1:8030//api/example_db/table1/_stream_load\n2021-02-23 15:22:57.970 [0-0-0-writer] INFO  DorisWriterTask - connect DorisDB with http://127.0.0.1:8030//api/example_db/table1/_stream_load\n\n2021-02-23 15:23:00.941 [job-0] INFO  JobContainer - PerfTrace not enable!\n2021-02-23 15:23:00.946 [job-0] INFO  JobContainer -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-02-23 15:22:57\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-02-23 15:23:00\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :            1.56KB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :             66rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                 200\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"writer/doriswriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 loadUrl \u662f string \u65e0 Stream Load \u7684\u8fde\u63a5\u76ee\u6807 \uff5c username \u662f string \u65e0 \u8bbf\u95eeDoris\u6570\u636e\u5e93\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u8bbf\u95eeDoris\u6570\u636e\u5e93\u7684\u5bc6\u7801 flushInterval \u5426 int 3000 \u6570\u636e\u5199\u5165\u5230\u76ee\u6807\u8868\u7684\u95f4\u9694\u65f6\u95f4\uff0c\u5355\u4f4d\u4e3a\u6beb\u79d2\uff0c\u5373\u6bcf\u9694\u591a\u5c11\u6beb\u79d2\u5199\u5165\u4e00\u6b21\u6570\u636e flushQueueLength \u5426 int 1 \u4e0a\u4f20\u6570\u636e\u7684\u961f\u5217\u957f\u5ea6 table \u662f string \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1 RBDMS Writer batchSize \u5426 int 2048 \u6bcf\u6279\u6b21\u5bfc\u5165\u6570\u636e\u7684\u6700\u5927\u884c\u6570 loadProps \u5426 map csv streamLoad \u7684\u8bf7\u6c42\u53c2\u6570\uff0c\u8be6\u60c5\u53c2\u7167StreamLoad\u4ecb\u7ecd\u9875\u9762 preSql \u5426 list \u5199\u5165\u6570\u636e\u5230\u76ee\u6807\u8868\u524d\u8981\u6267\u884c\u7684 SQL \u8bed\u53e5 postSql \u5426 list \u6570\u636e\u5199\u5b8c\u540e\u8981\u6267\u884c\u7684 SQL \u8bed\u53e5"},{"location":"writer/doriswriter/#loadurl","title":"loadUrl","text":"

    \u4f5c\u4e3a Stream Load \u7684\u8fde\u63a5\u76ee\u6807\u3002\u683c\u5f0f\u4e3a \"ip:port\"\u3002\u5176\u4e2d IP \u662f FE \u8282\u70b9 IP\uff0cport \u662f FE \u8282\u70b9\u7684 http_port\u3002\u53ef\u4ee5\u586b\u5199\u591a\u4e2a\uff0c\u5f53\u586b\u5199\u591a\u4e2a\u65f6\uff0c\u63d2\u4ef6\u4f1a\u6bcf\u4e2a\u6279\u6b21\u968f\u673a\u9009\u62e9\u4e00\u4e2a\u6709\u6548 FE \u8282\u70b9\u8fdb\u884c\u8fde\u63a5\u3002

    "},{"location":"writer/doriswriter/#column","title":"column","text":"

    \u5141\u8bb8\u914d\u7f6e\u4e3a [\"*\"] \uff0c \u5982\u679c\u662f \"*\" , \u5219\u5c1d\u8bd5\u4ece Doris \u6570\u636e\u5e93\u4e2d\u76f4\u63a5\u8bfb\u53d6\u8868\u5b57\u6bb5\uff0c\u7136\u540e\u8fdb\u884c\u62fc\u88c5\u3002

    "},{"location":"writer/doriswriter/#loadprops","title":"loadProps","text":"

    StreamLoad \u7684\u8bf7\u6c42\u53c2\u6570\uff0c\u8be6\u60c5\u53c2\u7167StreamLoad\u4ecb\u7ecd\u9875\u9762\u3002Stream load - Apache Doris

    \u8fd9\u91cc\u5305\u62ec\u5bfc\u5165\u7684\u6570\u636e\u683c\u5f0f\uff1aformat\u7b49\uff0c\u5bfc\u5165\u6570\u636e\u683c\u5f0f\u9ed8\u8ba4\u6211\u4eec\u4f7f\u7528csv\uff0c\u652f\u6301JSON\uff0c\u5177\u4f53\u53ef\u4ee5\u53c2\u7167\u4e0b\u9762\u7c7b\u578b\u8f6c\u6362\u90e8\u5206\uff0c\u4e5f\u53ef\u4ee5\u53c2\u7167\u4e0a\u9762Stream load \u5b98\u65b9\u4fe1\u606f

    "},{"location":"writer/doriswriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u9ed8\u8ba4\u4f20\u5165\u7684\u6570\u636e\u5747\u4f1a\u88ab\u8f6c\u4e3a\u5b57\u7b26\u4e32\uff0c\u5e76\u4ee5\\t\u4f5c\u4e3a\u5217\u5206\u9694\u7b26\uff0c\\n\u4f5c\u4e3a\u884c\u5206\u9694\u7b26\uff0c\u7ec4\u6210csv\u6587\u4ef6\u8fdb\u884cStreamLoad\u5bfc\u5165\u64cd\u4f5c\u3002

    \u9ed8\u8ba4\u662fcsv\u683c\u5f0f\u5bfc\u5165\uff0c\u5982\u9700\u66f4\u6539\u5217\u5206\u9694\u7b26\uff0c \u5219\u6b63\u786e\u914d\u7f6e loadProps \u5373\u53ef

    {\n  \"loadProps\": {\n    \"column_separator\": \"\\\\x01\",\n    \"line_delimiter\": \"\\\\x02\"\n  }\n}\n

    \u5982\u9700\u66f4\u6539\u5bfc\u5165\u683c\u5f0f\u4e3ajson\uff0c \u5219\u6b63\u786e\u914d\u7f6e loadProps \u5373\u53ef\uff1a

    {\n  \"loadProps\": {\n    \"format\": \"json\",\n    \"strip_outer_array\": true\n  }\n}\n
    "},{"location":"writer/elasticsearchwriter/","title":"ElasticSearch Writer","text":"

    ElasticSearch Writer \u63d2\u4ef6\u7528\u4e8e\u5411 ElasticSearch \u5199\u5165\u6570\u636e\u3002 \u5176\u5b9e\u73b0\u662f\u901a\u8fc7 elasticsearch \u7684 rest api \u63a5\u53e3\uff0c \u6279\u91cf\u628a\u636e\u5199\u5165 elasticsearch

    "},{"location":"writer/elasticsearchwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"job/stream2es.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"10,1000\",\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1.1.1.1\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"double\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"hello world\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"long text\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"41.12,-71.34\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"2017-05-25 11:22:33\",\n              \"type\": \"string\"\n            }\n          ],\n          \"sliceRecordCount\": 100\n        }\n      },\n      \"writer\": {\n        \"name\": \"elasticsearchwriter\",\n        \"parameter\": {\n          \"endpoint\": \"http://localhost:9200\",\n          \"index\": \"test-1\",\n          \"type\": \"default\",\n          \"cleanup\": true,\n          \"settings\": {\n            \"index\": {\n              \"number_of_shards\": 1,\n              \"number_of_replicas\": 0\n            }\n          },\n          \"discovery\": false,\n          \"batchSize\": 1000,\n          \"splitter\": \",\",\n          \"column\": [\n            {\n              \"name\": \"pk\",\n              \"type\": \"id\"\n            },\n            {\n              \"name\": \"col_ip\",\n              \"type\": \"ip\"\n            },\n            {\n              \"name\": \"col_double\",\n              \"type\": \"double\"\n            },\n            {\n              \"name\": \"col_long\",\n              \"type\": \"long\"\n            },\n            {\n              \"name\": \"col_integer\",\n              \"type\": \"integer\"\n            },\n            {\n              \"name\": \"col_keyword\",\n              \"type\": \"keyword\"\n            },\n            {\n              \"name\": \"col_text\",\n              \"type\": \"text\",\n              \"analyzer\": \"ik_max_word\"\n            },\n            {\n              \"name\": \"col_geo_point\",\n              \"type\": \"geo_point\"\n            },\n            {\n              \"name\": \"col_date\",\n              \"type\": \"date\",\n              \"format\": \"yyyy-MM-dd HH:mm:ss\"\n            },\n            {\n              \"name\": \"col_nested1\",\n              \"type\": \"nested\"\n            },\n            {\n              \"name\": \"col_nested2\",\n              \"type\": \"nested\"\n            },\n            {\n              \"name\": \"col_object1\",\n              \"type\": \"object\"\n            },\n            {\n              \"name\": \"col_object2\",\n              \"type\": \"object\"\n            },\n            {\n              \"name\": \"col_integer_array\",\n              \"type\": \"integer\",\n              \"array\": true\n            },\n            {\n              \"name\": \"col_geo_shape\",\n              \"type\": \"geo_shape\",\n              \"tree\": \"quadtree\",\n              \"precision\": \"10m\"\n            }\n          ]\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/elasticsearchwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 ElasticSearch \u7684\u8fde\u63a5\u5730\u5740,\u5982\u679c\u662f\u96c6\u7fa4\uff0c\u5219\u591a\u4e2a\u5730\u5740\u7528\u9017\u53f7(,)\u5206\u5272 accessId \u5426 string \u7a7a http auth \u4e2d\u7684 user, \u9ed8\u8ba4\u4e3a\u7a7a accessKey \u5426 string \u7a7a http auth \u4e2d\u7684 password index \u662f string \u65e0 index \u540d type \u5426 string default index \u7c7b\u578b cleanup \u5426 boolean false \u662f\u5426\u5220\u9664\u539f\u8868 batchSize \u5426 int 1000 \u6bcf\u6b21\u6279\u91cf\u6570\u636e\u7684\u6761\u6570 trySize \u5426 int 30 \u5931\u8d25\u540e\u91cd\u8bd5\u7684\u6b21\u6570 timeout \u5426 int 600000 \u5ba2\u6237\u7aef\u8d85\u65f6\u65f6\u95f4\uff0c\u5355\u4f4d\u4e3a\u6beb\u79d2(ms) discovery \u5426 boolean false \u542f\u7528\u8282\u70b9\u53d1\u73b0\u5c06(\u8f6e\u8be2)\u5e76\u5b9a\u671f\u66f4\u65b0\u5ba2\u6237\u673a\u4e2d\u7684\u670d\u52a1\u5668\u5217\u8868 compression \u5426 boolean true \u5426\u662f\u5f00\u542f http \u8bf7\u6c42\u538b\u7f29 multiThread \u5426 boolean true \u662f\u5426\u5f00\u542f\u591a\u7ebf\u7a0b http \u8bf7\u6c42 ignoreWriteError \u5426 boolean false \u5199\u5165\u9519\u8bef\u65f6\uff0c\u662f\u5426\u91cd\u8bd5\uff0c\u5982\u679c\u662f true \u5219\u8868\u793a\u4e00\u76f4\u91cd\u8bd5\uff0c\u5426\u5219\u5ffd\u7565\u8be5\u6761\u6570\u636e ignoreParseError \u5426 boolean true \u89e3\u6790\u6570\u636e\u683c\u5f0f\u9519\u8bef\u65f6\uff0c\u662f\u5426\u7ee7\u7eed\u5199\u5165 alias \u5426 string \u65e0 \u6570\u636e\u5bfc\u5165\u5b8c\u6210\u540e\u5199\u5165\u522b\u540d aliasMode \u5426 string append \u6570\u636e\u5bfc\u5165\u5b8c\u6210\u540e\u589e\u52a0\u522b\u540d\u7684\u6a21\u5f0f\uff0cappend(\u589e\u52a0\u6a21\u5f0f), exclusive(\u53ea\u7559\u8fd9\u4e00\u4e2a) settings \u5426 map \u65e0 \u521b\u5efa index \u65f6\u5019\u7684 settings, \u4e0e elasticsearch \u5b98\u65b9\u76f8\u540c splitter \u5426 string , \u5982\u679c\u63d2\u5165\u6570\u636e\u662f array\uff0c\u5c31\u4f7f\u7528\u6307\u5b9a\u5206\u9694\u7b26 column \u662f list<map> \u65e0 \u5b57\u6bb5\u7c7b\u578b\uff0c\u6587\u6863\u4e2d\u7ed9\u51fa\u7684\u6837\u4f8b\u4e2d\u5305\u542b\u4e86\u5168\u90e8\u652f\u6301\u7684\u5b57\u6bb5\u7c7b\u578b dynamic \u5426 boolean false \u4e0d\u4f7f\u7528 addax \u7684 mappings\uff0c\u4f7f\u7528 es \u81ea\u5df1\u7684\u81ea\u52a8 mappings"},{"location":"writer/elasticsearchwriter/#_3","title":"\u7ea6\u675f\u9650\u5236","text":""},{"location":"writer/excelwriter/","title":"Excel Writer","text":"

    Excel Writer \u5b9e\u73b0\u4e86\u5c06\u6570\u636e\u5199\u5165\u5230 Excel \u6587\u4ef6\u7684\u529f\u80fd

    "},{"location":"writer/excelwriter/#_1","title":"\u914d\u7f6e\u793a\u4f8b","text":"

    \u6211\u4eec\u5047\u5b9a\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\uff0c\u5e76\u5199\u5165\u5230 Excel \u6587\u4ef6\u4e2d

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"DataX\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 11:22:33\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"excelwriter\",\n        \"parameter\": {\n          \"path\": \"/tmp/out\",\n          \"fileName\": \"test\",\n          \"header\": [\n            \"str\",\n            \"\u957f\u5ea6\",\n            \"\u65e5\u671f\",\n            \"\u662f\u5426\u4e3a\u771f\",\n            \"\u5b57\u8282\u7c7b\u578b\"\n          ]\n        }\n      }\n    }\n  }\n}\n

    \u8bb2\u4e0a\u8ff0\u5185\u5bb9\u4fdd\u5b58\u4e3a job/stream2excel.json

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\uff1a

    bin/addax.sh job/stream2excel.sh\n

    \u5e94\u8be5\u5f97\u5230\u7c7b\u4f3c\u5982\u4e0b\u7684\u8f93\u51fa

    \u70b9\u51fb\u5c55\u5f00 
      ___      _     _\n / _ \\    | |   | |\n/ /_\\ \\ __| | __| | __ ___  __\n|  _  |/ _` |/ _` |/ _` \\ \\/ /\n| | | | (_| | (_| | (_| |>  <\n\\_| |_/\\__,_|\\__,_|\\__,_/_/\\_\\\n\n:: Addax version ::    (v4.0.3-SNAPSHOT)\n\n2021-09-10 22:16:38.247 [        main] INFO  VMInfo               - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl\n2021-09-10 22:16:38.269 [        main] INFO  Engine               -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"column\":[\n                        {\n                            \"type\":\"string\",\n                            \"value\":\"DataX\"\n                        },\n                        {\n                            \"type\":\"long\",\n                            \"value\":19890604\n                        },\n                        {\n                            \"type\":\"date\",\n                            \"value\":\"1989-06-04 11:22:33\"\n                        },\n                        {\n                            \"type\":\"bool\",\n                            \"value\":true\n                        },\n                        {\n                            \"type\":\"bytes\",\n                            \"value\":\"test\"\n                        }\n                    ],\n                    \"sliceRecordCount\":1000\n                },\n                \"name\":\"streamreader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"path\":\"/tmp/out\",\n                    \"fileName\":\"test\",\n                    \"header\":[\n                        \"str\",\n                        \"\u957f\u5ea6\",\n                        \"\u65e5\u671f\",\n                        \"\u662f\u5426\u4e3a\u771f\",\n                        \"\u5b57\u8282\u7c7b\u578b\"\n                    ],\n                    \"writeMode\":\"truncate\"\n                },\n                \"name\":\"excelwriter\"\n            }\n        },\n    \"setting\":{\n        \"speed\":{\n            \"byte\":-1,\n            \"channel\":1\n        }\n    }\n}\n\n2021-09-10 22:16:38.287 [        main] INFO  PerfTrace            - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-09-10 22:16:38.287 [        main] INFO  JobContainer         - Addax jobContainer starts job.\n2021-09-10 22:16:38.289 [        main] INFO  JobContainer         - Set jobId = 0\n2021-09-10 22:16:38.303 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do prepare work .\n2021-09-10 22:16:38.304 [       job-0] INFO  JobContainer         - Addax Writer.Job [excelwriter] do prepare work .\n2021-09-10 22:16:38.304 [       job-0] INFO  JobContainer         - Job set Channel-Number to 1 channels.\n2021-09-10 22:16:38.304 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] splits to [1] tasks.\n2021-09-10 22:16:38.305 [       job-0] INFO  JobContainer         - Addax Writer.Job [excelwriter] splits to [1] tasks.\n2021-09-10 22:16:38.325 [       job-0] INFO  JobContainer         - Scheduler starts [1] taskGroups.\n2021-09-10 22:16:38.332 [ taskGroup-0] INFO  TaskGroupContainer   - taskGroupId=[0] start [1] channels for [1] tasks.\n2021-09-10 22:16:38.335 [ taskGroup-0] INFO  Channel              - Channel set byte_speed_limit to -1, No bps activated.\n2021-09-10 22:16:38.336 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\n2021-09-10 22:16:41.345 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2021-09-10 22:16:41.346 [       job-0] INFO  JobContainer         - Addax Writer.Job [excelwriter] do post work.\n2021-09-10 22:16:41.346 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do post work.\n2021-09-10 22:16:41.348 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2021-09-10 22:16:41.349 [       job-0] INFO  StandAloneJobContainerCommunicator - Total 1000 records, 26000 bytes | Speed 8.46KB/s, 333 records/s | Error 0 records, 0 bytes |  All Task WaitWriterTime 0.528s |  All Task WaitReaderTime 0.000s | Percentage 100.00%\n2021-09-10 22:16:41.350 [       job-0] INFO  JobContainer         -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-09-10 22:16:38\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-09-10 22:16:41\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :            8.46KB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :            333rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                1000\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"writer/excelwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f string \u65e0 \u6307\u5b9a\u6587\u4ef6\u4fdd\u5b58\u7684\u76ee\u5f55, \u6307\u5b9a\u7684\u76ee\u5f55\u5982\u679c\u4e0d\u5b58\u5728\uff0c\u5219\u5c1d\u8bd5\u521b\u5efa fileName \u662f string \u65e0 \u8981\u751f\u6210\u7684excel \u6587\u4ef6\u540d\uff0c\u8be6\u8ff0\u5982\u4e0b header \u5426 list \u65e0 Excel \u8868\u5934"},{"location":"writer/excelwriter/#filename","title":"fileName","text":"

    \u5982\u679c\u914d\u7f6e\u7684 fileName \u6ca1\u6709\u540e\u7f00\uff0c\u5219\u81ea\u52a8\u52a0\u4e0a .xlsx\uff1b \u5982\u679c\u540e\u7f00\u4e3a .xls\uff0c\u5219\u62a5\u9519\uff0c\u56e0\u4e3a\u5f53\u524d\u4ec5\u751f\u6210 Excel 97 \u4ee5\u540e\u7684\u6587\u4ef6\u683c\u5f0f\uff0c\u5373 .xlsx \u540e\u7f00\u7684\u6587\u4ef6

    "},{"location":"writer/excelwriter/#header","title":"header","text":"

    \u5982\u679c\u4e0d\u6307\u5b9a header \uff0c\u5219\u751f\u6210\u7684 Excel \u6587\u4ef6\u6ca1\u6709\u8868\u5934\uff0c\u53ea\u6709\u6570\u636e\u3002 \u6ce8\u610f\uff0c\u63d2\u4ef6\u4e0d\u5173\u5fc3 header \u7684\u6570\u91cf\u662f\u5426\u5339\u914d\u6570\u636e\u4e2d\u7684\u5217\u6570\uff0c\u4e5f\u5c31\u662f\u8bf4\u8868\u5934\u7684\u5217\u6570\u5e76\u4e0d\u8981\u6c42\u548c\u63a5\u4e0b\u6765\u7684\u6570\u636e\u7684\u5217\u6570\u76f8\u7b49\u3002

    "},{"location":"writer/excelwriter/#_3","title":"\u9650\u5236","text":"
    1. \u5f53\u524d\u4ec5\u751f\u6210\u4e00\u4e2a Excel \u6587\u4ef6\uff0c\u4e14\u6ca1\u6709\u8003\u8651\u884c\u6570\u548c\u5217\u6570\u662f\u5426\u8d85\u8fc7\u4e86 Excel \u7684\u9650\u5b9a
    2. \u5982\u679c\u6307\u5b9a\u7684\u76ee\u5f55\u4e0b\u6709\u540c\u540d\u6587\u4ef6\uff0c\u5f53\u524d\u4f1a\u88ab\u8986\u76d6\uff0c\u540e\u7eed\u4f1a\u7edf\u4e00\u5904\u7406\u76ee\u6807\u76ee\u5f55\u7684\u95ee\u9898
    3. \u5f53\u524d\u65e5\u671f\u683c\u5f0f\u7684\u6570\u636e\uff0c\u8bbe\u7f6e\u5355\u5143\u683c\u6837\u5f0f\u4e3a yyyy-MM-dd HH:mm:ss\uff0c\u4e14\u4e0d\u80fd\u5b9a\u5236
    4. \u4e0d\u652f\u6301\u4e8c\u8fdb\u5236\u7c7b\u578b\u7684\u6570\u636e\u5199\u5165
    "},{"location":"writer/ftpwriter/","title":"Ftp Writer","text":"

    Ftp Writer \u63d0\u4f9b\u4e86\u5411\u8fdc\u7a0b FTP/SFTP \u670d\u52a1\u5199\u5165\u6587\u4ef6\u7684\u80fd\u529b\uff0c\u5f53\u524d\u4ec5\u652f\u6301\u5199\u5165\u6587\u672c\u6587\u4ef6\u3002

    "},{"location":"writer/ftpwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"job/stream2ftp.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {},\n      \"writer\": {\n        \"name\": \"ftpwriter\",\n        \"parameter\": {\n          \"protocol\": \"sftp\",\n          \"host\": \"***\",\n          \"port\": 22,\n          \"username\": \"xxx\",\n          \"password\": \"xxx\",\n          \"timeout\": \"60000\",\n          \"connectPattern\": \"PASV\",\n          \"path\": \"/tmp/data/\",\n          \"fileName\": \"test\",\n          \"writeMode\": \"truncate|append|nonConflict\",\n          \"fieldDelimiter\": \",\",\n          \"encoding\": \"UTF-8\",\n          \"nullFormat\": \"null\",\n          \"dateFormat\": \"yyyy-MM-dd\",\n          \"fileFormat\": \"csv\",\n          \"useKey\": false,\n          \"keyPath\": \"\",\n          \"keyPass\": \"\",\n          \"header\": []\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/ftpwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 protocol \u662f string ftp \u670d\u52a1\u5668\u534f\u8bae\uff0c\u76ee\u524d\u652f\u6301\u4f20\u8f93\u534f\u8bae\u6709 ftp \u548c sftp host \u662f string \u65e0 \u670d\u52a1\u5668\u5730\u5740 port \u5426 int 22/21 ftp \u9ed8\u8ba4\u4e3a 21\uff0csftp \u9ed8\u8ba4\u4e3a 22 timeout \u5426 int 60000 \u8fde\u63a5ftp\u670d\u52a1\u5668\u8fde\u63a5\u8d85\u65f6\u65f6\u95f4\uff0c\u5355\u4f4d\u6beb\u79d2(ms) connectPattern \u5426 string PASV \u8fde\u63a5\u6a21\u5f0f\uff0c\u4ec5\u652f\u6301 PORT, PASV \u6a21\u5f0f\u3002ftp\u534f\u8bae\u65f6\u4f7f\u7528 \uff5c username \u662f string \u65e0 \u7528\u6237\u540d password \u662f string \u65e0 \u8bbf\u95ee\u5bc6\u7801 useKey \u5426 boolean false \u662f\u5426\u4f7f\u7528\u79c1\u94a5\u767b\u5f55\uff0c\u4ec5\u9488\u5bf9 sftp \u767b\u5f55\u6709\u6548 keyPath \u5426 string ~/.ssh/id_rsa \u79c1\u94a5\u5730\u5740 keyPass \u5426 string \u65e0 \u79c1\u94a5\u5bc6\u7801\uff0c\u82e5\u6ca1\u6709\u8bbe\u7f6e\u79c1\u94a5\u5bc6\u7801\uff0c\u5219\u65e0\u9700\u914d\u7f6e\u8be5\u9879 path \u662f string \u65e0 \u8fdc\u7a0bFTP\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0cFtpWriter\u4f1a\u5199\u5165Path\u76ee\u5f55\u4e0b\u5c5e\u591a\u4e2a\u6587\u4ef6 fileName \u662f string \u65e0 \u5199\u5165\u7684\u6587\u4ef6\u540d\uff0c\u8be5\u6587\u4ef6\u540d\u4f1a\u6dfb\u52a0\u968f\u673a\u7684\u540e\u7f00\u4f5c\u4e3a\u6bcf\u4e2a\u7ebf\u7a0b\u5199\u5165\u5b9e\u9645\u6587\u4ef6\u540d writeMode \u662f string \u65e0 \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff0c\u8be6\u89c1\u4e0b\u6587 fieldDelimiter \u662f string , \u63cf\u8ff0\uff1a\u8bfb\u53d6\u7684\u5b57\u6bb5\u5206\u9694\u7b26 compress \u5426 string \u65e0 \u6587\u672c\u538b\u7f29\u7c7b\u578b\uff0c\u6682\u4e0d\u652f\u6301 encoding \u5426 string utf-8 \u8bfb\u53d6\u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e dateFormat \u5426 string \u65e0 \u65e5\u671f\u7c7b\u578b\u7684\u6570\u636e\u5e8f\u5217\u5316\u5230\u6587\u4ef6\u4e2d\u65f6\u7684\u683c\u5f0f\uff0c\u4f8b\u5982 \"yyyy-MM-dd\" fileFormat \u5426 string text \u6587\u4ef6\u5199\u51fa\u7684\u683c\u5f0f\uff0c\u5305\u62ec csv, text \u4e24\u79cd\uff0c header \u5426 list \u65e0 text\u5199\u51fa\u65f6\u7684\u8868\u5934\uff0c\u793a\u4f8b ['id', 'name', 'age'] nullFormat \u5426 string \\N \u5b9a\u4e49\u54ea\u4e9b\u5b57\u7b26\u4e32\u53ef\u4ee5\u8868\u793a\u4e3anull maxTraversalLevel \u5426 int 100 \u5141\u8bb8\u904d\u5386\u6587\u4ef6\u5939\u7684\u6700\u5927\u5c42\u6570 csvReaderConfig \u5426 map \u65e0 \u8bfb\u53d6CSV\u7c7b\u578b\u6587\u4ef6\u53c2\u6570\u914d\u7f6e\uff0c\u8be6\u89c1\u4e0b\u6587"},{"location":"writer/ftpwriter/#writemode","title":"writeMode","text":"

    \u63cf\u8ff0\uff1aFtpWriter\u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff1a

    1. truncate\uff0c\u5199\u5165\u524d\u6e05\u7406\u76ee\u5f55\u4e0b\u4e00fileName\u524d\u7f00\u7684\u6240\u6709\u6587\u4ef6\u3002
    2. append\uff0c\u5199\u5165\u524d\u4e0d\u505a\u4efb\u4f55\u5904\u7406\uff0cAddax FtpWriter\u76f4\u63a5\u4f7f\u7528filename\u5199\u5165\uff0c\u5e76\u4fdd\u8bc1\u6587\u4ef6\u540d\u4e0d\u51b2\u7a81\u3002
    3. nonConflict\uff0c\u5982\u679c\u76ee\u5f55\u4e0b\u6709fileName\u524d\u7f00\u7684\u6587\u4ef6\uff0c\u76f4\u63a5\u62a5\u9519\u3002
    "},{"location":"writer/ftpwriter/#_3","title":"\u8ba4\u8bc1","text":"

    \u4ece 4.0.2 \u7248\u672c\u5f00\u59cb\uff0c \u652f\u6301\u79c1\u94a5\u8ba4\u8bc1\u65b9\u5f0f\u767b\u5f55 SFTP \u670d\u52a1\u5668\uff0c\u5982\u679c\u5bc6\u7801\u548c\u79c1\u6709\u90fd\u586b\u5199\u4e86\uff0c\u5219\u4e24\u8005\u8ba4\u8bc1\u65b9\u5f0f\u90fd\u4f1a\u5c1d\u8bd5\u3002 \u6ce8\u610f\uff0c\u5982\u679c\u586b\u5199\u4e86 keyPath, keyPass \u9879\uff0c\u4f46 useKey \u8bbe\u7f6e\u4e3a false \uff0c\u63d2\u4ef6\u4f9d\u7136\u4e0d\u4f1a\u5c1d\u8bd5\u7528\u79c1\u94a5\u8fdb\u884c\u767b\u5f55\u3002

    "},{"location":"writer/ftpwriter/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    FTP\u6587\u4ef6\u672c\u8eab\u4e0d\u63d0\u4f9b\u6570\u636e\u7c7b\u578b\uff0c\u8be5\u7c7b\u578b\u662f Addax FtpWriter \u5b9a\u4e49\uff1a

    Addax \u5185\u90e8\u7c7b\u578b FTP\u6587\u4ef6 \u6570\u636e\u7c7b\u578b Long Long -> \u5b57\u7b26\u4e32\u5e8f\u5217\u5316\u8868\u793a Double Double -> \u5b57\u7b26\u4e32\u5e8f\u5217\u5316\u8868\u793a String String -> \u5b57\u7b26\u4e32\u5e8f\u5217\u5316\u8868\u793a Boolean Boolean -> \u5b57\u7b26\u4e32\u5e8f\u5217\u5316\u8868\u793a Date Date -> \u5b57\u7b26\u4e32\u5e8f\u5217\u5316\u8868\u793a"},{"location":"writer/greenplumwriter/","title":"Greenplum Writer","text":"

    Greenplum Writer \u63d2\u4ef6\u4f7f\u7528 COPY FROM \u8bed\u6cd5 \u5c06\u6570\u636e\u5199\u5165 Greenplum \u6570\u636e\u5e93\u3002

    "},{"location":"writer/greenplumwriter/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u6f14\u793a\u4ece greenplum \u6307\u5b9a\u7684\u8868\u8bfb\u53d6\u6570\u636e\uff0c\u5e76\u63d2\u5165\u5230\u5177\u6709\u76f8\u540c\u8868\u7ed3\u6784\u7684\u53e6\u5916\u4e00\u5f20\u8868\u4e2d\uff0c\u7528\u6765\u6d4b\u8bd5\u8be5\u63d2\u4ef6\u6240\u652f\u6301\u7684\u6570\u636e\u7c7b\u578b\u3002

    create table if not exists addax_tbl\n(\n    c_bigint\n    bigint,\n    c_bit\n    bit(3),\n    c_bool boolean,\n    c_byte bytea,\n    c_char char(10),\n    c_varchar varchar(20),\n    c_date date,\n    c_double float8,\n    c_int integer,\n    c_json json,\n    c_number decimal(8, 3),\n    c_real real,\n    c_small smallint,\n    c_text text,\n    c_ts timestamp,\n    c_uuid uuid,\n    c_xml xml,\n    c_money money,\n    c_inet inet,\n    c_cidr cidr,\n    c_macaddr macaddr\n    );\ninsert into addax_tbl\nvalues (999988887777,\n        B'101',\n        TRUE,\n        '\\xDEADBEEF',\n        'hello',\n        'hello, world',\n        '2021-01-04',\n        999888.9972,\n        9876542,\n        '{\"bar\": \"baz\", \"balance\": 7.77, \"active\": false}'::json,\n        12345.123,\n        123.123,\n        126,\n        'this is a long text ',\n        '2020-01-04 12:13:14',\n        'A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11'::uuid,\n        '<foo>bar</foo>'::xml,\n        '52093.89'::money,\n        '192.168.1.1'::inet,\n        '192.168.1/24'::cidr,\n        '08002b:010203'::macaddr);\n

    \u521b\u5efa\u9700\u8981\u63d2\u5165\u7684\u8868\u7684\u8bed\u53e5\u5982\u4e0b:

    create table gp_test like addax_tbl;\n
    "},{"location":"writer/greenplumwriter/#_2","title":"\u4efb\u52a1\u914d\u7f6e","text":"

    \u4ee5\u4e0b\u662f\u914d\u7f6e\u6587\u4ef6

    job/pg2gp.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"postgresqlreader\",\n        \"parameter\": {\n          \"username\": \"wgzhao\",\n          \"password\": \"wgzhao\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"addax_tbl\"\n            ],\n            \"jdbcUrl\": \"jdbc:postgresql://localhost:5432/wgzhao\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"greenplumwriter\",\n        \"parameter\": {\n          \"username\": \"wgzhao\",\n          \"password\": \"wgzhao\",\n          \"column\": [\n            \"*\"\n          ],\n          \"preSql\": [\n            \"truncate table @table\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:postgresql://localhost:5432/wgzhao\",\n            \"table\": [\n              \"gp_test\"\n            ]\n          }\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/pg2gp.json

    "},{"location":"writer/greenplumwriter/#_3","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/pg2gp.json\n
    "},{"location":"writer/greenplumwriter/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/hanawriter/","title":"HANA Writer","text":"

    HANA Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 SAP HANA \u76ee\u7684\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/hanawriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684 HANA \u8868\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    create table system.addax_tbl\n(\ncol1 varchar(200) ,\ncol2 int(4),\ncol3 date,\ncol4 boolean,\ncol5 clob\n);\n

    \u8fd9\u91cc\u4f7f\u7528\u4e00\u4efd\u4ece\u5185\u5b58\u4ea7\u751f\u5230 HANA \u5bfc\u5165\u7684\u6570\u636e\u3002

    job/hanawriter.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"hanawriter\",\n        \"parameter\": {\n          \"connection\": [\n            {\n              \"jdbcUrl\": \"jdbc:sap://wgzhao-pc:39017/system\",\n              \"table\": [\n                \"addax_tbl\"\n              ]\n            }\n          ],\n          \"username\": \"system\",\n          \"password\": \"HXEHana1\",\n          \"column\": [\n            \"*\"\n          ]\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/hana2stream.json

    "},{"location":"writer/hanawriter/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/hana2stream.json\n
    "},{"location":"writer/hanawriter/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/hbase11xsqlwriter/","title":"HBase11x SQL Writer","text":"

    HBase11x SQL Writer \u63d2\u4ef6\u5229\u7528 Phoniex\uff0c \u7528\u4e8e\u5411 HBase 1.x \u7248\u672c\u7684\u6570\u636e\u5e93\u5199\u5165\u6570\u636e\u3002

    \u5982\u679c\u4f60\u5e0c\u671b\u901a\u8fc7\u8c03\u7528\u539f\u751f\u63a5\u53e3\u5199\u5165\u6570\u636e\uff0c\u5219\u9700\u8981\u4f7f\u7528HBase11xWriter \u63d2\u4ef6

    \u5982\u679c HBase \u662f 2.X \u7248\u672c\uff0c\u5219\u9700\u8981\u4f7f\u7528 HBase20xsqlwriter \u63d2\u4ef6

    "},{"location":"writer/hbase11xsqlwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":""},{"location":"writer/hbase11xsqlwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 hbaseConfig \u662f map \u65e0 hbase \u96c6\u7fa4\u5730\u5740\uff0c\u8be6\u89c1\u793a\u4f8b\u914d\u7f6e table \u662f string \u65e0 \u8981\u5bfc\u5165\u7684\u8868\u540d\uff0c\u5927\u5c0f\u5199\u654f\u611f\uff0c\u901a\u5e38 phoenix \u8868\u90fd\u662f \u5927\u5199 \u8868\u540d column \u662f list \u65e0 \u5217\u540d\uff0c\u5927\u5c0f\u5199\u654f\u611f\uff0c\u901a\u5e38 phoenix \u7684\u5217\u540d\u90fd\u662f \u5927\u5199 batchSize \u5426 int 256 \u4e00\u6b21\u5199\u5165\u7684\u6700\u5927\u8bb0\u5f55\u6570 nullMode \u5426 string skip \u8bfb\u53d6\u5230\u7684\u5217\u503c\u4e3a null \u65f6\uff0c\u5982\u4f55\u5904\u7406\u3002 haveKerberos \u5426 bolean false \u662f\u5426\u542f\u7528 Kerberos \u8ba4\u8bc1, true \u8868\u793a\u542f\u7528, false \u8868\u793a\u4e0d\u542f\u7528 kerberosPrincipal \u5426 string \u65e0 kerberos \u51ed\u8bc1\u4fe1\u606f\uff0c\u4ec5\u5f53 havekerberos \u542f\u7528\u540e\u6709\u6548 kerberosKeytabFilePath \u5426 string \u65e0 kerberos \u51ed\u8bc1\u6587\u4ef6\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u4ec5\u5f53 havekerberos \u542f\u7528\u540e\u6709\u6548"},{"location":"writer/hbase11xsqlwriter/#nullmode","title":"nullMode","text":"

    \u652f\u6301 skip, empty,\u524d\u8005\u8868\u793a\u8df3\u8fc7\u8be5\u5217,\u540e\u8005\u8868\u793a\u63d2\u5165\u7a7a\u503c,\u6570\u503c\u7c7b\u578b\u4e3a 0,\u5b57\u7b26\u7c7b\u578b\u4e3a null

    \u6ce8\u610f\uff1a\u542f\u7528 kerberos \u8ba4\u8bc1\u540e\uff0c\u7a0b\u5e8f\u9700\u8981\u77e5\u9053hbase-site.xml \u6240\u5728\u7684\u8def\u5f84\uff0c\u4e00\u79cd\u529e\u6cd5\u662f\u8fd0\u884c\u6267\u884c\u5728\u73af\u5883\u53d8\u91cf CLASSPATH \u4e2d\u589e\u52a0\u8be5\u6587\u4ef6\u7684\u6240\u5728\u8def\u5f84\u3002

    \u53e6\u5916\u4e00\u4e2a\u89e3\u51b3\u529e\u6cd5\u662f\u5c06 hbase-site.xml \u6587\u4ef6\u62f7\u8d1d\u5230\u63d2\u4ef6\u7684 libs \u76ee\u5f55\u91cc\u3002

    "},{"location":"writer/hbase11xwriter/","title":"HBase11X Writer","text":"

    Hbase11X Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u4ece\u5411 HBase \u5199\u6570\u636e\u7684\u80fd\u529b\u3002\u5728\u5e95\u5c42\u5b9e\u73b0\u4e0a\uff0cHBase11X Writer \u901a\u8fc7 HBase \u7684 Java \u5ba2\u6237\u7aef\u8fde\u63a5\u8fdc\u7a0b HBase \u670d\u52a1\uff0c\u5e76\u901a\u8fc7 put \u65b9\u5f0f\u5199\u5165Hbase\u3002

    \u5982\u679c HBase \u662f 2.X \u7248\u672c\uff0c\u5219\u9700\u8981\u4f7f\u7528 HBase20xsqlwriter \u63d2\u4ef6

    "},{"location":"writer/hbase11xwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u914d\u7f6e\u4e00\u4e2a\u4ece\u672c\u5730\u5199\u5165hbase1.1.x\u7684\u4f5c\u4e1a\uff1a

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 5,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"txtfilereader\",\n        \"parameter\": {\n          \"path\": \"/tmp/normal.txt\",\n          \"charset\": \"UTF-8\",\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"String\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 3,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 4,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 5,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 6,\n              \"type\": \"string\"\n            }\n          ],\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"hbase11xwriter\",\n        \"parameter\": {\n          \"hbaseConfig\": {\n            \"hbase.zookeeper.quorum\": \"***\"\n          },\n          \"table\": \"writer\",\n          \"mode\": \"normal\",\n          \"rowkeyColumn\": [\n            {\n              \"index\": 0,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": -1,\n              \"type\": \"string\",\n              \"value\": \"_\"\n            }\n          ],\n          \"column\": [\n            {\n              \"index\": 1,\n              \"name\": \"cf1:q1\",\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 2,\n              \"name\": \"cf1:q2\",\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 3,\n              \"name\": \"cf1:q3\",\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 4,\n              \"name\": \"cf2:q1\",\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 5,\n              \"name\": \"cf2:q2\",\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 6,\n              \"name\": \"cf2:q3\",\n              \"type\": \"string\"\n            }\n          ],\n          \"versionColumn\": {\n            \"index\": -1,\n            \"value\": \"123456789\"\n          },\n          \"encoding\": \"utf-8\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/hbase11xwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 hbaseConfig \u662f map \u65e0 \u8fde\u63a5 HBase \u96c6\u7fa4\u9700\u8981\u7684\u914d\u7f6e\u4fe1\u606f,\u8be6\u89c1\u793a\u4f8b\u914d\u7f6e mode \u662f string \u65e0 \u5199\u5165 HBase \u7684\u6a21\u5f0f\uff0c\u76ee\u524d\u4ec5\u652f\u6301 normal \u6a21\u5f0f table \u662f string \u65e0 HBase \u8868\u540d\uff08\u5927\u5c0f\u5199\u654f\u611f\uff09 encoding \u5426 string UTF-8 \u7f16\u7801\u65b9\u5f0f\uff0cUTF-8 \u6216\u662f GBK\uff0c\u7528\u4e8e\u5bf9\u4e8c\u8fdb\u5236\u5b58\u50a8\u7684 HBase byte[] \u8f6c\u4e3a String \u65f6\u7684\u7f16\u7801 column \u662f list<map> \u65e0 \u8981\u5199\u5165\u7684\u5b57\u6bb5\uff0cnormal \u6a21\u5f0f\u4e0e multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b\u9879, \u8be6\u7ec6\u8bf4\u660e\u89c1\u4e0b\u6587 rowkeyColumn \u662f list<map> \u65e0 \u8981\u5199\u5165\u7684 rowkey \u5217, \u8be6\u7ec6\u8bf4\u660e\u89c1\u4e0b\u6587 versionColumn \u5426 string \u65e0 \u6307\u5b9a\u5199\u5165\u7684\u65f6\u95f4\u6233,\u8be6\u89c1\u4e0b\u6587 nullMode \u5426 string skip \u8bfb\u53d6\u7684null\u503c\u65f6\uff0c\u5982\u4f55\u5904\u7406, walFlag \u5426 boolean false \u662f\u5426\u5199 WAL, true \u8868\u793a\u5199\u5165, false \u8868\u793a\u4e0d\u5199 writeBufferSize \u5426 int 8388608 \u8bbe\u7f6e\u5199 buffer \u5927\u5c0f\uff0c\u5355\u4f4d\u5b57\u8282, \u9ed8\u8ba48M maxVersion \u662f string \u65e0 \u6307\u5b9a\u5728\u591a\u7248\u672c\u6a21\u5f0f\u4e0b\u8bfb\u53d6\u7684\u7248\u672c\u6570\uff0c-1 \u8868\u793a\u8bfb\u53d6\u6240\u6709\u7248\u672c, multiVersionFixedColumn \u6a21\u5f0f\u4e0b\u5fc5\u586b range \u5426 string \u65e0 \u6307\u5b9a\u8bfb\u53d6\u7684 rowkey \u8303\u56f4, \u8be6\u89c1\u4e0b\u6587 scanCacheSize \u5426 string 256 \u6bcf\u6b21\u4ece\u670d\u52a1\u5668\u7aef\u8bfb\u53d6\u7684\u884c\u6570 scanBatchSize \u5426 string 100 \u6bcf\u6b21\u4ece\u670d\u52a1\u5668\u7aef\u8bfb\u53d6\u7684\u5217\u6570"},{"location":"writer/hbase11xwriter/#column","title":"column","text":"

    \u8981\u5199\u5165\u7684hbase\u5b57\u6bb5\u3002

    \u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"column\": [\n    {\n      \"index\": 1,\n      \"name\": \"cf1:q1\",\n      \"type\": \"string\"\n    },\n    {\n      \"index\": 2,\n      \"name\": \"cf1:q2\",\n      \"type\": \"string\"\n    }\n  ]\n}\n
    "},{"location":"writer/hbase11xwriter/#rowkey","title":"rowkey","text":"

    \u8981\u5199\u5165\u7684 rowkey \u5217\u3002

    \u63d2\u4ef6\u4f1a\u5c06 rowkeyColumn \u4e2d\u6240\u6709\u5217\u6309\u7167\u914d\u7f6e\u987a\u5e8f\u8fdb\u884c\u62fc\u63a5\u4f5c\u4e3a\u5199\u5165 hbase \u7684 rowkey\uff0c\u4e0d\u80fd\u5168\u4e3a\u5e38\u91cf\u3002\u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"rowkeyColumn\": [\n    {\n      \"index\": 0,\n      \"type\": \"string\"\n    },\n    {\n      \"index\": -1,\n      \"type\": \"string\",\n      \"value\": \"_\"\n    }\n  ]\n}\n
    "},{"location":"writer/hbase11xwriter/#versioncolumn","title":"versionColumn","text":"

    \u6307\u5b9a\u5199\u5165\u7684\u65f6\u95f4\u6233\u3002\u652f\u6301\uff1a\u5f53\u524d\u65f6\u95f4\u3001\u6307\u5b9a\u65f6\u95f4\u5217\uff0c\u6307\u5b9a\u65f6\u95f4\uff0c\u4e09\u8005\u9009\u4e00\u3002\u82e5\u4e0d\u914d\u7f6e\u8868\u793a\u7528\u5f53\u524d\u65f6\u95f4\u3002

    index\uff1a\u6307\u5b9a\u5bf9\u5e94 reader \u7aef column \u7684\u7d22\u5f15\uff0c\u4ece 0 \u5f00\u59cb\uff0c\u9700\u4fdd\u8bc1\u80fd\u8f6c\u6362\u4e3a long,\u82e5\u662f Date \u7c7b\u578b\uff0c \u4f1a\u5c1d\u8bd5\u7528 yyyy-MM-dd HH:mm:ss \u548c yyyy-MM-dd HH:mm:ss SSS \u53bb\u89e3\u6790\uff1b \u82e5\u4e3a\u6307\u5b9a\u65f6\u95f4 index \u4e3a \uff0d1\uff0c\u5219 value \u4e3a\u6307\u5b9a\u65f6\u95f4\u7684\u503c\u3002\u914d\u7f6e\u683c\u5f0f\u5982\u4e0b\uff1a

    {\n  \"versionColumn\": {\n    \"index\": 1\n  }\n}\n

    \u6216\u8005

    {\n  \"versionColumn\": {\n    \"index\": -1,\n    \"value\": 123456789\n  }\n}\n
    "},{"location":"writer/hbase11xwriter/#nullmode","title":"nullMode","text":"

    skip \u8868\u793a\u4e0d\u5411hbase\u5199\u8fd9\u5217\uff1bempty\uff1a\u5199\u5165 HConstants.EMPTY_BYTE_ARRAY\uff0c\u5373new byte [0]

    "},{"location":"writer/hbase11xwriter/#_3","title":"\u652f\u6301\u7684\u5217\u7c7b\u578b","text":"

    \u8bf7\u6ce8\u610f: \u9664\u4e0a\u8ff0\u7f57\u5217\u5b57\u6bb5\u7c7b\u578b\u5916\uff0c\u5176\u4ed6\u7c7b\u578b\u5747\u4e0d\u652f\u6301

    "},{"location":"writer/hbase20xsqlwriter/","title":"HBase20x SQL Writer","text":"

    HBase20x SQL Writer \u63d2\u4ef6\u5229\u7528 Phoenix \u5411 HBase 2.x \u5199\u5165\u6570\u636e\u3002

    \u5982\u679c HBase \u662f 1.X \u7248\u672c\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528 HBase11xsqlWriter \u6216HBase11xWriter \u63d2\u4ef6

    "},{"location":"writer/hbase20xsqlwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"txtfilereader\",\n        \"parameter\": {\n          \"path\": \"/tmp/normal.txt\",\n          \"charset\": \"UTF-8\",\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"String\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 3,\n              \"type\": \"string\"\n            }\n          ],\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"hbase20xsqlwriter\",\n        \"parameter\": {\n          \"batchSize\": \"100\",\n          \"column\": [\n            \"UID\",\n            \"TS\",\n            \"EVENTID\",\n            \"CONTENT\"\n          ],\n          \"queryServerAddress\": \"http://127.0.0.1:8765\",\n          \"nullMode\": \"skip\",\n          \"table\": \"TEST_TBL\"\n        }\n      }\n    },\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 5,\n        \"bytes\": -1\n      }\n    }\n  }\n}\n
    "},{"location":"writer/hbase20xsqlwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 jdbcUrl \u662f string \u65e0 Phoenix \u8fde\u63a5\u5730\u5740 table \u662f string \u65e0 \u6240\u8981\u8bfb\u53d6\u8868\u540d schema \u5426 string \u65e0 \u8868\u6240\u5728\u7684 schema batchSize \u5426 int 256 \u4e00\u6b21\u6279\u91cf\u5199\u5165\u7684\u6700\u5927\u884c\u6570 column \u5426 list \u65e0 \u5217\u540d\uff0c\u5927\u5c0f\u5199\u654f\u611f\uff0c\u901a\u5e38phoenix\u7684\u5217\u540d\u90fd\u662f**\u5927\u5199** nullMode \u5426 string skip \u8bfb\u53d6\u7684 null \u503c\u65f6\uff0c\u5982\u4f55\u5904\u7406, \u8be6\u8ff0\u89c1\u4e0b haveKerberos \u5426 boolean false \u662f\u5426\u542f\u7528Kerberos\u8ba4\u8bc1, true \u8868\u793a\u542f\u7528, false \u8868\u793a\u4e0d\u542f\u7528 kerberosPrincipal \u5426 string \u65e0 kerberos \u51ed\u8bc1\u4fe1\u606f\uff0c\u4ec5\u5f53 havekerberos \u542f\u7528\u540e\u6709\u6548 kerberosKeytabFilePath \u5426 string \u65e0 kerberos \u51ed\u8bc1\u6587\u4ef6\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u4ec5\u5f53 havekerberos \u542f\u7528\u540e\u6709\u6548"},{"location":"writer/hbase20xsqlwriter/#jdbcurl","title":"jdbcUrl","text":"

    queryServerAddress \u662f\u6ee1\u8db3 Phoenix \u94fe\u63a5\u7684\u5730\u5740\uff0c\u5177\u4f53\u683c\u5f0f\u548c\u8981\u6c42\u53ef\u4ee5\u53c2\u8003\u5b98\u65b9\u6587\u6863 \uff0c\u5176 jdbc \u8fde\u63a5\u4e32\u683c\u5f0f\u5982\u4e0b\uff1a

    jdbc:phoenix [ :<zookeeper quorum> [ :<port number> [ :<root node> [ :<principal> [ :<keytab file> ] ] ] ] ] \n

    \u5982\u679c\u4f60\u5e0c\u671b\u901a\u8fc7\u8fde\u63a5 Phoenix Query Server (a.k.a PQS) \uff0c\u5219 JDBC \u8fde\u63a5\u4e32\u5982\u4e0b\uff1a

    jdbc:phoenix:thin:url=<scheme>://<server-hostname>:<port>[;option=value...]\n

    \u66f4\u8be6\u7ec6\u7684\u63cf\u8ff0\uff0c\u53ef\u4ee5\u53c2\u8003\u5b98\u65b9\u6587\u6863

    "},{"location":"writer/hbase20xsqlwriter/#nullmode","title":"nullMode","text":"

    skip \u8868\u793a\u4e0d\u5411hbase\u5199\u8fd9\u5217\uff1bempty\uff1a\u5199\u5165 HConstants.EMPTY_BYTE_ARRAY\uff0c\u5373new byte [0]

    \u6ce8\u610f\uff1a\u542f\u7528kerberos\u8ba4\u8bc1\u540e\uff0c\u7a0b\u5e8f\u9700\u8981\u77e5\u9053hbase-site.xml \u6240\u5728\u7684\u8def\u5f84\uff0c\u4e00\u79cd\u529e\u6cd5\u662f\u8fd0\u884c\u6267\u884c\u5728\u73af\u5883\u53d8\u91cf CLASSPATH \u4e2d\u589e\u52a0\u8be5\u6587\u4ef6\u7684\u6240\u5728\u8def\u5f84\u3002

    \u53e6\u5916\u4e00\u4e2a\u89e3\u51b3\u529e\u6cd5\u662f\u5c06 hbase-site.xml \u6587\u4ef6\u62f7\u8d1d\u5230\u63d2\u4ef6\u7684 libs \u76ee\u5f55\u91cc\u3002

    "},{"location":"writer/hdfswriter/","title":"HDFS Writer","text":"

    HDFS Writer \u63d0\u4f9b\u5411 HDFS \u6587\u4ef6\u7cfb\u7edf\u6307\u5b9a\u8def\u5f84\u4e2d\u5199\u5165 TextFile \uff0c ORCFile , Parquet \u7b49\u683c\u5f0f\u6587\u4ef6\u7684\u80fd\u529b\uff0c \u6587\u4ef6\u5185\u5bb9\u53ef\u4e0e hive \u4e2d\u8868\u5173\u8054\u3002

    "},{"location":"writer/hdfswriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 00:00:00\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        },\n        \"writer\": {\n          \"name\": \"hdfswriter\",\n          \"parameter\": {\n            \"defaultFS\": \"hdfs://xxx:port\",\n            \"fileType\": \"orc\",\n            \"path\": \"/user/hive/warehouse/writerorc.db/orcfull\",\n            \"fileName\": \"xxxx\",\n            \"column\": [\n              {\n                \"name\": \"col1\",\n                \"type\": \"string\"\n              },\n              {\n                \"name\": \"col2\",\n                \"type\": \"int\"\n              },\n              {\n                \"name\": \"col3\",\n                \"type\": \"string\"\n              },\n              {\n                \"name\": \"col4\",\n                \"type\": \"boolean\"\n              },\n              {\n                \"name\": \"col5\",\n                \"type\": \"string\"\n              }\n            ],\n            \"writeMode\": \"overwrite\",\n            \"fieldDelimiter\": \"\\u0001\",\n            \"compress\": \"SNAPPY\"\n          }\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/hdfswriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e path \u662f string \u65e0 \u8981\u8bfb\u53d6\u7684\u6587\u4ef6\u8def\u5f84 defaultFS \u662f string \u65e0 \u8be6\u8ff0\u89c1\u4e0b fileType \u662f string \u65e0 \u6587\u4ef6\u7684\u7c7b\u578b\uff0c\u8be6\u8ff0\u89c1\u4e0b fileName \u662f string \u65e0 \u8981\u5199\u5165\u7684\u6587\u4ef6\u540d\uff0c\u7528\u4e8e\u5f53\u4f5c\u524d\u7f00 column \u662f list<map> \u65e0 \u5199\u5165\u7684\u5b57\u6bb5\u5217\u8868 writeMode \u662f string \u65e0 \u5199\u5165\u6a21\u5f0f\uff0c\u8be6\u8ff0\u89c1\u4e0b skipTrash \u5426 boolean false \u662f\u5426\u8df3\u8fc7\u5783\u573e\u56de\u6536\u7ad9\uff0c\u548c writeMode \u914d\u7f6e\u76f8\u5173\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0 fieldDelimiter \u5426 string , \u6587\u672c\u6587\u4ef6\u7684\u5b57\u6bb5\u5206\u9694\u7b26\uff0c\u4e8c\u8fdb\u5236\u6587\u4ef6\u4e0d\u9700\u8981\u6307\u5b9a\u8be5\u9879 encoding \u5426 string utf-8 \u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e\uff0c \u76ee\u524d\u4ec5\u652f\u6301 utf-8 nullFormat \u5426 string \u65e0 \u5b9a\u4e49\u8868\u793a\u4e3a\u7a7a\u7684\u5b57\u7b26\uff0c\u4f8b\u5982\u5982\u679c\u7528\u6237\u914d\u7f6e: \"\\\\N\" \uff0c\u90a3\u4e48\u5982\u679c\u6e90\u5934\u6570\u636e\u662f \"\\N\" \uff0c\u89c6\u4f5c null \u5b57\u6bb5 haveKerberos \u5426 boolean false \u662f\u5426\u542f\u7528 Kerberos \u8ba4\u8bc1\uff0c\u5982\u679c\u542f\u7528\uff0c\u5219\u9700\u8981\u540c\u65f6\u914d\u7f6e\u4ee5\u4e0b\u4e24\u9879 kerberosKeytabFilePath \u5426 string \u65e0 \u7528\u4e8e Kerberos \u8ba4\u8bc1\u7684\u51ed\u8bc1\u6587\u4ef6\u8def\u5f84, \u6bd4\u5982 /your/path/addax.service.keytab kerberosPrincipal \u5426 string \u65e0 \u7528\u4e8e Kerberos \u8ba4\u8bc1\u7684\u51ed\u8bc1\u4e3b\u4f53, \u6bd4\u5982 addax/node1@WGZHAO.COM compress \u5426 string \u65e0 \u6587\u4ef6\u7684\u538b\u7f29\u683c\u5f0f\uff0c\u8be6\u89c1\u4e0b\u6587 hadoopConfig \u5426 map \u65e0 \u91cc\u53ef\u4ee5\u914d\u7f6e\u4e0e Hadoop \u76f8\u5173\u7684\u4e00\u4e9b\u9ad8\u7ea7\u53c2\u6570\uff0c\u6bd4\u5982HA\u7684\u914d\u7f6e preShell \u5426 list \u65e0 \u5199\u5165\u6570\u636e\u524d\u6267\u884c\u7684shell\u547d\u4ee4\uff0c\u6bd4\u5982 hive -e \"truncate table test.hello\" postShell \u5426 list \u65e0 \u5199\u5165\u6570\u636e\u540e\u6267\u884c\u7684shell\u547d\u4ee4\uff0c\u6bd4\u5982 hive -e \"select count(1) from test.hello\" ignoreError \u5426 boolean false \u662f\u5426\u5ffd\u7565preShell, postShell \u547d\u4ee4\u7684\u9519\u8bef hdfsSitePath \u5426 string \u65e0 hdfs-site.xml \u7684\u8def\u5f84\uff0c\u8be6\u7ec6\u89e3\u91ca\u89c1\u4e0b"},{"location":"writer/hdfswriter/#path","title":"path","text":"

    \u5b58\u50a8\u5230 Hadoop hdfs\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0cHdfsWriter \u4f1a\u6839\u636e\u5e76\u53d1\u914d\u7f6e\u5728 Path \u76ee\u5f55\u4e0b\u5199\u5165\u591a\u4e2a\u6587\u4ef6\u3002\u4e3a\u4e0ehive\u8868\u5173\u8054\uff0c\u8bf7\u586b\u5199hive\u8868\u5728hdfs\u4e0a\u7684\u5b58\u50a8\u8def\u5f84\u3002 \u4f8b\uff1aHive\u4e0a\u8bbe\u7f6e\u7684\u6570\u636e\u4ed3\u5e93\u7684\u5b58\u50a8\u8def\u5f84\u4e3a\uff1a/user/hive/warehouse/ \uff0c\u5df2\u5efa\u7acb\u6570\u636e\u5e93\uff1atest\uff0c\u8868\uff1ahello\uff1b \u5219\u5bf9\u5e94\u7684\u5b58\u50a8\u8def\u5f84\u4e3a\uff1a/user/hive/warehouse/test.db/hello (\u5982\u679c\u5efa\u8868\u65f6\u6307\u5b9a\u4e86location \u5c5e\u6027\uff0c\u5219\u4f9d\u636e\u8be5\u5c5e\u6027\u7684\u8def\u5f84)

    "},{"location":"writer/hdfswriter/#defaultfs","title":"defaultFS","text":"

    Hadoop hdfs \u6587\u4ef6\u7cfb\u7edf namenode \u8282\u70b9\u5730\u5740\u3002\u683c\u5f0f\uff1ahdfs://ip:port \uff1b\u4f8b\u5982\uff1ahdfs://127.0.0.1:9000 , \u5982\u679c\u542f\u7528\u4e86HA\uff0c\u5219\u4e3a servicename \u6a21\u5f0f\uff0c\u6bd4\u5982 hdfs://sandbox

    "},{"location":"writer/hdfswriter/#filetype","title":"fileType","text":"

    \u63cf\u8ff0\uff1a\u6587\u4ef6\u7684\u7c7b\u578b\uff0c\u76ee\u524d\u53ea\u652f\u6301\u7528\u6237\u914d\u7f6e\u4e3a

    "},{"location":"writer/hdfswriter/#column","title":"column","text":"

    \u5199\u5165\u6570\u636e\u7684\u5b57\u6bb5\uff0c\u4e0d\u652f\u6301\u5bf9\u90e8\u5206\u5217\u5199\u5165\u3002\u4e3a\u4e0ehive\u4e2d\u8868\u5173\u8054\uff0c\u9700\u8981\u6307\u5b9a\u8868\u4e2d\u6240\u6709\u5b57\u6bb5\u540d\u548c\u5b57\u6bb5\u7c7b\u578b\uff0c \u5176\u4e2d\uff1aname \u6307\u5b9a\u5b57\u6bb5\u540d\uff0ctype \u6307\u5b9a\u5b57\u6bb5\u7c7b\u578b\u3002

    \u7528\u6237\u53ef\u4ee5\u6307\u5b9a column \u5b57\u6bb5\u4fe1\u606f\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"column\": [\n    {\n      \"name\": \"userName\",\n      \"type\": \"string\"\n    },\n    {\n      \"name\": \"age\",\n      \"type\": \"long\"\n    },\n    {\n      \"name\": \"salary\",\n      \"type\": \"decimal(8,2)\"\n    }\n  ]\n}\n

    \u5bf9\u4e8e\u6570\u636e\u7c7b\u578b\u662f decimal \u7c7b\u578b\u7684\uff0c\u9700\u8981\u6ce8\u610f\uff1a

    1. \u5982\u679c\u6ca1\u6709\u6307\u5b9a\u7cbe\u5ea6\u548c\u5c0f\u6570\u4f4d\uff0c\u5219\u4f7f\u7528\u9ed8\u8ba4\u7684 decimal(38,10) \u8868\u793a
    2. \u5982\u679c\u4ec5\u6307\u5b9a\u4e86\u7cbe\u5ea6\u4f46\u672a\u6307\u5b9a\u5c0f\u6570\u4f4d\uff0c\u5219\u5c0f\u6570\u4f4d\u75280\u8868\u793a\uff0c\u5373 decimal(p,0)
    3. \u5982\u679c\u90fd\u6307\u5b9a\uff0c\u5219\u4f7f\u7528\u6307\u5b9a\u7684\u89c4\u683c\uff0c\u5373 decimal(p,s)
    "},{"location":"writer/hdfswriter/#writemode","title":"writeMode","text":"

    \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff1a

    "},{"location":"writer/hdfswriter/#skiptrash","title":"skipTrash","text":"

    \u5f53 writeMode \u4e3a overwrite \u6a21\u5f0f\u65f6\uff0c\u5f53\u524d\u8981\u5220\u9664\u7684\u6587\u4ef6\u6216\u6587\u4ef6\u5939\u662f\u5426\u8fdb\u5165\u56de\u6536\u7ad9\uff0c\u9ed8\u8ba4\u4e3a\u8fdb\u56de\u6536\u7ad9\uff0c\u4ec5\u5f53\u914d\u7f6e\u4e3a true \u65f6\u4e3a\u76f4\u63a5\u5220\u9664\u3002

    \u8be5\u529f\u80fd\u7684\u5b9e\u73b0\u65b9\u5f0f\u4e3a\u83b7\u53d6 Hadoop HDFS \u7684 fs.trash.interval \u53c2\u6570\uff0c\u5982\u679c\u8be5\u53c2\u6570\u6ca1\u6709\u8bbe\u7f6e\uff0c\u6216\u8bbe\u7f6e\u4e3a0\u65f6\uff0c\u4f1a\u5728\u5220\u9664\u65f6\uff0c\u8bbe\u7f6e\u8be5\u53c2\u6570\u4e3a 10080 \uff0c\u8868\u793a 7 \u5929\u3002

    \u8fd9\u6837\uff0c\u8fdb\u5165\u56de\u6536\u7ad9\u7684\u6587\u4ef6\u4f1a\u4fdd\u75597\u5929\u3002

    \u4fee\u6539\u5220\u9664\u7684\u9ed8\u8ba4\u884c\u4e3a\u662f\u4e3a\u4e86\u7ed9\u56e0\u4e3a\u9519\u8bef\u7684\u91c7\u96c6\u800c\u5bfc\u81f4\u5220\u9664\u7684\u6570\u636e\u6709\u633d\u56de\u7684\u673a\u4f1a\u3002

    "},{"location":"writer/hdfswriter/#compress","title":"compress","text":"

    \u5f53 fileType\uff08\u6587\u4ef6\u7c7b\u578b\uff09\u4e3a csv \u4e0b\u7684\u6587\u4ef6\u538b\u7f29\u65b9\u5f0f\uff0c\u76ee\u524d\u4ec5\u652f\u6301 gzip\u3001bz2\u3001zip\u3001lzo\u3001lzo_deflate\u3001hadoop-snappy\u3001framing-snappy \u538b\u7f29\uff1b \u503c\u5f97\u6ce8\u610f\u7684\u662f\uff0clzo \u5b58\u5728\u4e24\u79cd\u538b\u7f29\u683c\u5f0f\uff1alzo \u548c lzo_deflate\uff0c\u7528\u6237\u5728\u914d\u7f6e\u7684\u65f6\u5019\u9700\u8981\u7559\u5fc3\uff0c\u4e0d\u8981\u914d\u9519\u4e86\uff1b

    \u53e6\u5916\uff0c\u7531\u4e8e snappy \u76ee\u524d\u6ca1\u6709\u7edf\u4e00\u7684 stream format\uff0caddax \u76ee\u524d\u53ea\u652f\u6301\u6700\u4e3b\u6d41\u7684\u4e24\u79cd\uff1ahadoop-snappy\uff08hadoop \u4e0a\u7684 snappy stream format\uff09 \u548c framing-snappy\uff08google \u5efa\u8bae\u7684 snappy stream format\uff09;

    "},{"location":"writer/hdfswriter/#hadoopconfig","title":"hadoopConfig","text":"

    hadoopConfig \u91cc\u53ef\u4ee5\u914d\u7f6e\u4e0e Hadoop \u76f8\u5173\u7684\u4e00\u4e9b\u9ad8\u7ea7\u53c2\u6570\uff0c\u6bd4\u5982HA\u7684\u914d\u7f6e

    {\n  \"hadoopConfig\": {\n    \"dfs.nameservices\": \"cluster\",\n    \"dfs.ha.namenodes.cluster\": \"nn1,nn2\",\n    \"dfs.namenode.rpc-address.cluster.nn1\": \"node1.example.com:8020\",\n    \"dfs.namenode.rpc-address.cluster.nn2\": \"node2.example.com:8020\",\n    \"dfs.client.failover.proxy.provider.cluster\": \"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\"\n  }\n}\n

    \u8fd9\u91cc\u7684 cluster \u8868\u793a HDFS \u914d\u7f6e\u6210HA\u65f6\u7684\u540d\u5b57\uff0c\u4e5f\u662f defaultFS \u914d\u7f6e\u9879\u4e2d\u7684\u540d\u5b57 \u5982\u679c\u5b9e\u9645\u73af\u5883\u4e2d\u7684\u540d\u5b57\u4e0d\u662f cluster \uff0c\u5219\u4e0a\u8ff0\u914d\u7f6e\u4e2d\u6240\u6709\u5199\u6709 cluster \u90fd\u9700\u8981\u66ff\u6362

    "},{"location":"writer/hdfswriter/#preshell-postshell","title":"preShell \u4e0e postShell","text":"

    \u5f15\u5165 preShell \u4e0e postShell \u7684\u76ee\u7684\u662f\u4e3a\u4e86\u5728\u5199\u5165\u6570\u636e\u524d\u540e\u6267\u884c\u4e00\u4e9b\u989d\u5916\u7684\u64cd\u4f5c\uff0c\u6bd4\u5982\u5728\u5199\u5165\u6570\u636e\u524d\u6e05\u7a7a\u8868\uff0c\u5199\u5165\u6570\u636e\u540e\u67e5\u8be2\u8868\u7684\u884c\u6570\u7b49\u3002\u4e00\u4e2a\u5178\u578b\u7684\u751f\u4ea7\u73af\u5883\u573a\u666f\u65f6\uff0c\u91c7\u96c6\u7684\u6570\u636e\u6309\u65e5\u5206\u533a\u4fdd\u5b58\u5728 HDFS \u4e0a\uff0c \u91c7\u96c6\u4e4b\u524d\u9700\u8981\u521b\u5efa\u5206\u533a\uff0c\u8fd9\u6837\u5c31\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e preShell \u6765\u5b9e\u73b0\uff0c\u6bd4\u5982 hive -e \"alter table test.hello add partition(dt='${logdate}')\"

    "},{"location":"writer/hdfswriter/#ignoreerror","title":"ignoreError","text":"

    \u8be5\u914d\u7f6e\u9879\u7528\u4e8e\u63a7\u5236\u662f\u5426\u5ffd\u7565 preShell \u548c postShell \u547d\u4ee4\u7684\u9519\u8bef\uff0c\u5982\u679c\u914d\u7f6e\u4e3a true\uff0c\u5219\u5728\u6267\u884c preShell \u548c postShell \u547d\u4ee4\u65f6\uff0c\u5982\u679c\u547d\u4ee4\u6267\u884c\u5931\u8d25\uff0c\u4e0d\u4f1a\u5bfc\u81f4\u4efb\u52a1\u5931\u8d25\uff0c\u800c\u662f\u4f1a\u6253\u5370\u9519\u8bef\u65e5\u5fd7\uff0c\u7ee7\u7eed\u6267\u884c\u4efb\u52a1\u3002 \u5426\u5219\uff0c\u5982\u679c\u914d\u7f6e\u4e3a false\uff0c\u5219\u5728\u6267\u884c preShell \u548c postShell \u547d\u4ee4\u65f6\uff0c\u5982\u679c\u547d\u4ee4\u6267\u884c\u5931\u8d25\uff0c\u4f1a\u5bfc\u81f4\u4efb\u52a1\u5931\u8d25\u3002

    "},{"location":"writer/hdfswriter/#hdfssitepath","title":"hdfsSitePath","text":"

    \u8fd9\u662f 4.2.4 \u5f15\u5165\u7684\u65b0\u914d\u7f6e\u60f3\uff0c\u7528\u4e8e\u6307\u5b9a hdfs-site.xml \u6587\u4ef6\u7684\u8def\u5f84\uff0c\u6bd4\u5982\u5bf9 HDP/CDH \u800c\u8a00\uff0c\u53ef\u4ee5\u8fd9\u6837\u914d\u7f6e\uff1a

    {\n  \"hdfsSitePath\": \"/etc/hadoop/conf/hdfs-site.xml\"\n}\n

    \u5982\u679c\u914d\u7f6e\u4e86 hdfsSitePath , \u5219\u63d2\u4ef6\u4f1a\u4ece\u8be5\u6587\u4ef6\u4e2d\u83b7\u5f97\u8bbf\u95ee HDFS \u6587\u4ef6\u7cfb\u7edf\u5fc5\u8981\u7684\u914d\u7f6e\uff0c\u4ece\u800c\u5728\u5927\u90e8\u5206\u60c5\u51b5\u4e0b\u4e0d\u5728\u9700\u8981\u914d\u7f6e hadoopConfig\uff0c\u51cf\u5c11\u914d\u7f6e\u91cf\u3002

    \u5bf9\u4e8e\u628a Addax \u90e8\u7f72\u5728 Hadoop \u96c6\u7fa4\u4e0a\u7684\u573a\u666f\uff0c\u63a8\u8350\u4f7f\u7528\u8fd9\u79cd\u65b9\u5f0f\u3002

    "},{"location":"writer/hdfswriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b HIVE \u6570\u636e\u7c7b\u578b Long TINYINT,SMALLINT,INT,INTEGER,BIGINT Double FLOAT,DOUBLE,DECIMAL String STRING,VARCHAR,CHAR Boolean BOOLEAN Date DATE,TIMESTAMP Bytes BINARY"},{"location":"writer/hdfswriter/#_4","title":"\u529f\u80fd\u4e0e\u9650\u5236","text":"
    1. \u76ee\u524d\u4e0d\u652f\u6301\uff1abinary\u3001arrays\u3001maps\u3001structs\u3001union \u7c7b\u578b
    "},{"location":"writer/influxdb2writer/","title":"InfluxDB2 Writer","text":"

    InfluxDB2 Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5c06\u6570\u636e\u5199\u5165 InfluxDB 2.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u6570\u636e\u5e93\u7684\u529f\u80fd\u3002

    \u6ce8\u610f\uff0c\u5982\u679c\u4f60\u7684 InfluxDB \u662f 1.8 \u53ca\u4ee5\u4e0b\u7248\u672c\uff0c\u5219\u5e94\u8be5\u4f7f\u7528 InfluxDBWriter \u63d2\u4ef6

    "},{"location":"writer/influxdb2writer/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u793a\u4f8b\u7528\u6765\u6f14\u793a\u8be5\u63d2\u4ef6\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\u5e76\u5199\u5165\u5230\u6307\u5b9a\u8868

    "},{"location":"writer/influxdb2writer/#job","title":"\u521b\u5efa job \u6587\u4ef6","text":"

    \u521b\u5efa job/stream2influx2.json \u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    job/stream2influx2.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"incr\": \"2021-10-17 22:40:00,1,s\",\n              \"type\": \"date\"\n            },\n            {\n              \"random\": \"1,1000\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"1,10\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"1000,50000\",\n              \"type\": \"double\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"influxdb2writer\",\n        \"parameter\": {\n          \"connection\": {\n            \"endpoint\": \"http://localhost:8086\",\n            \"org\": \"com.wgzhao\",\n            \"bucket\": \"test\",\n            \"table\": \"addax_tbl\"\n          },\n          \"token\": \"YOUR_SECURE_TOKEN\",\n          \"tag\": [\n            {\n              \"location\": \"east\"\n            },\n            {\n              \"lat\": \"23.123445\"\n            }\n          ],\n          \"interval\": \"ms\",\n          \"column\": [\n            \"c_long\",\n            \"c_string\",\n            \"c_double\"\n          ],\n          \"batchSize\": 1024\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/influxdb2writer/#_2","title":"\u8fd0\u884c","text":"

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/stream2influx2.json\n
    "},{"location":"writer/influxdb2writer/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 InfluxDB \u8fde\u63a5\u4e32 table \u662f string \u65e0 \u8981\u5199\u5165\u7684\u8868\uff08\u6307\u6807\uff09 org \u662f string \u65e0 \u6307\u5b9a InfluxDB \u7684 org \u540d\u79f0 bucket \u662f string \u65e0 \u6307\u5b9a InfluxDB \u7684 bucket \u540d\u79f0 token \u662f string \u65e0 \u8bbf\u95ee\u6570\u636e\u5e93\u7684 token column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408 tag \u5426 list<map> \u65e0 \u8981\u6307\u5b9a\u7684 tag interval \u5426 string ms \u6307\u5b9a\u65f6\u95f4\u95f4\u9694\uff0c\u53ef\u4ee5\u6307\u5b9a s,ms,us, ns batchSize \u5426 int 1024 \u6279\u91cf\u5199\u5165\u7684\u5927\u5c0f"},{"location":"writer/influxdb2writer/#column","title":"column","text":"

    InfluxDB \u4f5c\u4e3a\u65f6\u5e8f\u6570\u636e\u5e93\uff0c\u9700\u8981\u6bcf\u6761\u8bb0\u5f55\u90fd\u6709\u65f6\u95f4\u6233\u5b57\u6bb5\uff0c\u56e0\u6b64\u4f1a\u628a\u6bcf\u6761\u7684\u8bb0\u5f55\u7684\u7b2c\u4e00\u4e2a\u5b57\u6bb5\u5f53\u4f5c\u65f6\u95f4\u6233\u6765\u5904\u7406\u3002 column \u53ea\u9700\u8981\u6307\u5b9a\u9664\u4e86\u7b2c\u4e00\u4e2a\u5b57\u6bb5\u5916\u7684\u5176\u4ed6\u5b57\u6bb5\u3002 \u6bd4\u5982\u793a\u4f8b\u4e2d\uff0cstreamreader \u8bbe\u7f6e\u4e864\u4e2a\u5b57\u6bb5\uff0c\u4f46\u5728 influxdb2writer \u4e2d\u7684 column \u53ea\u6307\u5b9a\u4e86\u4e09\u4e2a\u5b57\u6bb5\uff0c\u5c31\u662f\u56e0\u4e3a\u7b2c\u4e00\u4e2a\u5b57\u6bb5\u5df2\u7ecf\u9ed8\u8ba4\u4f5c\u4e3a\u65f6\u95f4\u6233\u4e86\u3002

    "},{"location":"writer/influxdb2writer/#tag","title":"tag","text":"

    \u7528\u4e8e\u6307\u5b9a\u6307\u6807\uff08\u8fd9\u91cc\u5f53\u4f5c\u8868\uff09\u7684 \u6807\u7b7e\uff0c\u6bcf\u4e2a tag \u4f7f\u7528 map \u65b9\u5f0f\u6307\u5b9a\uff0c\u6bd4\u5982\u793a\u4f8b\u4e2d\uff1a

    {\n  \"tag\": [\n    {\n      \"location\": \"east\"\n    },\n    {\n      \"lat\": 23.123445\n    }\n  ]\n}\n

    map\u4e2d\u7684 key \u8868\u793a\u6807\u7b7e\u7684\u540d\u79f0\uff0cvalue \u8868\u793a\u6807\u7b7e\u503c

    "},{"location":"writer/influxdb2writer/#interval","title":"interval","text":"

    \u8bbe\u7f6e\u65f6\u95f4\u6233\u7684\u95f4\u9694\u9891\u7387\uff0c\u8be5\u5b57\u6bb5\u7684\u5b9a\u4e49\u6765\u6e90\u4e8e influxdb-client-java \u4e2d\u7684 WritePrecision.java \u5176\u5b57\u7b26\u4e32\u8868\u8fbe\u7684\u542b\u4e49\u5206\u522b\u4e3a\uff1a

    "},{"location":"writer/influxdb2writer/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u5f53\u524d\u652f\u6301 InfluxDB 2.0 \u7684\u57fa\u672c\u7c7b\u578b

    "},{"location":"writer/influxdbwriter/","title":"InfluxDB Writer","text":"

    InfluxDB Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5c06\u6570\u636e\u5199\u5165 InfluxDB \u8bfb\u53d6\u6570\u636e\u7684\u529f\u80fd\u3002 \u5e95\u5c42\u5b9e\u73b0\u4e0a\uff0c\u662f\u901a\u8fc7\u8c03\u7528 InfluQL \u8bed\u8a00\u63a5\u53e3\uff0c\u6784\u5efa\u63d2\u5165\u8bed\u53e5\uff0c\u7136\u540e\u8fdb\u884c\u6570\u636e\u63d2\u5165\u3002

    "},{"location":"writer/influxdbwriter/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u793a\u4f8b\u7528\u6765\u6f14\u793a\u8be5\u63d2\u4ef6\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\u5e76\u5199\u5165\u5230\u6307\u5b9a\u8868

    "},{"location":"writer/influxdbwriter/#_2","title":"\u521b\u5efa\u9700\u8981\u7684\u5e93","text":"

    \u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u6765\u521b\u5efa\u9700\u8981\u5199\u5165\u7684\u5e93

    # create database\ninflux --execute \"CREATE DATABASE addax\"\n
    "},{"location":"writer/influxdbwriter/#job","title":"\u521b\u5efa job \u6587\u4ef6","text":"

    \u521b\u5efa job/stream2influxdb.json \u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    job/stream2influxdb.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"2001-01-01 00:00:00, 2016-07-07 23:59:59\",\n              \"type\": \"date\"\n            },\n            {\n              \"random\": \"1,1000\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"1,10\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"1000,50000\",\n              \"type\": \"double\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"influxdbwriter\",\n        \"parameter\": {\n          \"connection\": {\n            \"endpoint\": \"http://localhost:8086\",\n            \"database\": \"addax\",\n            \"table\": \"addax_tbl\"\n          },\n          \"connTimeout\": 15,\n          \"readTimeout\": 20,\n          \"writeTimeout\": 20,\n          \"username\": \"influx\",\n          \"password\": \"influx123\",\n          \"column\": [\n            {\n              \"name\": \"time\",\n              \"type\": \"timestamp\"\n            },\n            {\n              \"name\": \"user_id\",\n              \"type\": \"int\"\n            },\n            {\n              \"name\": \"user_name\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"salary\",\n              \"type\": \"double\"\n            }\n          ],\n          \"preSql\": [\n            \"delete from addax_tbl\"\n          ],\n          \"batchSize\": 1024,\n          \"retentionPolicy\": {\n            \"name\": \"one_day_only\",\n            \"duration\": \"1d\",\n            \"replication\": 1\n          }\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/influxdbwriter/#_3","title":"\u8fd0\u884c","text":"

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/stream2influxdb.json\n
    "},{"location":"writer/influxdbwriter/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 InfluxDB \u8fde\u63a5\u4e32 username \u662f string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801 database \u662f string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7684\u6570\u636e\u5e93 table \u662f string \u65e0 \u8981\u5199\u5165\u7684\u8868\uff08\u6307\u6807\uff09 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408 connTimeout \u5426 int 15 \u8bbe\u7f6e\u8fde\u63a5\u8d85\u65f6\u503c\uff0c\u5355\u4f4d\u4e3a\u79d2 readTimeout \u5426 int 20 \u8bbe\u7f6e\u8bfb\u53d6\u8d85\u65f6\u503c\uff0c\u5355\u4f4d\u4e3a\u79d2 writeTimeout \u5426 int 20 \u8bbe\u7f6e\u5199\u5165\u8d85\u65f6\u503c\uff0c\u5355\u4f4d\u4e3a\u79d2 preSql \u5426 list \u65e0 \u63d2\u5165\u6570\u636e\u524d\u6267\u884c\u7684SQL\u8bed\u53e5 postSql \u5426 list \u65e0 \u6570\u636e\u63d2\u5165\u5b8c\u6bd5\u540e\u9700\u8981\u6267\u884c\u7684\u8bed\u53e5 retentionPolicy \u5426 map \u65e0 \u8bbe\u7f6e\u6570\u636e\u5e93\u7684 Retention Policy \u7b56\u7565"},{"location":"writer/influxdbwriter/#column","title":"column","text":"

    InfluxDB \u4f5c\u4e3a\u65f6\u5e8f\u6570\u636e\u5e93\uff0c\u9700\u8981\u6bcf\u6761\u8bb0\u5f55\u90fd\u6709\u65f6\u95f4\u6233\u5b57\u6bb5\uff0c\u56e0\u6b64\u8fd9\u91cc\u4f1a\u628a column \u914d\u7f6e\u7684\u7b2c\u4e00\u4e2a\u5b57\u6bb5\u9ed8\u8ba4\u5f53\u4f5c\u65f6\u95f4\u6233

    "},{"location":"writer/influxdbwriter/#retentionpolicy","title":"retentionPolicy","text":"

    \u8bbe\u5b9a\u6570\u636e\u5e93\u7684 Retention Policy \u7b56\u7565\uff0c\u4f9d\u636e\u7ed9\u5b9a\u7684\u914d\u7f6e\uff0c\u5728\u6307\u5b9a\u6570\u636e\u5e93\u4e0a\u521b\u5efa\u4e00\u6761 Retention Policy \u4fe1\u606f\u3002 \u6709\u5173 Retention Policy \u66f4\u8be6\u7ec6\u7684\u4fe1\u606f\uff0c\u53ef\u4ee5\u53c2\u8003 [\u5b98\u65b9\u6587\u6863][2]

    "},{"location":"writer/influxdbwriter/#_5","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u5f53\u524d\u652f\u6301 InfluxDB \u7684\u57fa\u672c\u7c7b\u578b

    "},{"location":"writer/influxdbwriter/#_6","title":"\u9650\u5236","text":"
    1. \u5f53\u524d\u63d2\u4ef6\u4ec5\u652f\u6301 1.x \u7248\u672c\uff0c2.0 \u53ca\u4ee5\u4e0a\u5e76\u4e0d\u652f\u6301
    "},{"location":"writer/kafkawriter/","title":"Kafka Writer","text":"

    Kafka Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5c06\u6570\u636e\u4ee5 json \u683c\u5f0f\u5199\u5165 Kafka \u7684\u529f\u80fd\u3002

    "},{"location":"writer/kafkawriter/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u6f14\u793a\u4e86\u5982\u4f55\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\u5e76\u5199\u5165\u5230 kafka \u7684\u6307\u5b9a topic \u4e2d\u3002

    "},{"location":"writer/kafkawriter/#_2","title":"\u521b\u5efa\u4efb\u52a1\u6587\u4ef6","text":"

    \u9996\u5148\u521b\u5efa\u4e00\u4e2a\u4efb\u52a1\u6587\u4ef6 stream2kafka.json , \u5185\u5bb9\u5982\u4e0b\uff1a

    {\n  \"job\": {\n    \"setting\": {\n        \"speed\": {\n            \"channel\": 1\n        }\n    },\n    \"content\": [\n      {\n        \"reader\": {\n            \"name\": \"streamreader\",\n            \"parameter\": {\n              \"column\": [\n                    {\"random\": \"10,1000\", \"type\": \"long\"},\n                    {\"value\": \"1.1.1.1\", \"type\": \"string\"},\n                    {\"value\": 19890604.0, \"type\": \"double\"},\n                    {\"value\": 19890604, \"type\": \"long\"},\n                    {\"value\": 19890604, \"type\": \"long\"},\n                    {\"value\": \"hello world\", \"type\": \"string\"},\n                    {\"value\": \"long text\", \"type\": \"string\"},\n                    {\"value\": \"41.12,-71.34\", \"type\": \"string\"},\n                    {\"value\": \"2017-05-25 11:22:33\", \"type\": \"string\"}\n                    ],\n            \"sliceRecordCount\": 100\n            }\n        },\n        \"writer\": {\n          \"name\": \"kafkawriter\",\n          \"parameter\": {\n            \"brokerList\": \"localhost:9092\",\n            \"topic\": \"test-1\",\n            \"partitions\": 0,\n            \"batchSize\": 1000,\n            \"column\": [\"col1\", \"col2\",\"col3\",\"col4\",\"col5\", \"col6\", \"col7\", \"col8\", \"col9\"]\n          }\n        }\n      }\n    ]\n  }\n}\n
    "},{"location":"writer/kafkawriter/#_3","title":"\u8fd0\u884c","text":"

    \u6267\u884c bin/addax.sh stream2kafka.json \u547d\u4ee4\uff0c\u83b7\u5f97\u7c7b\u4f3c\u4e0b\u9762\u7684\u8f93\u51fa\uff1a

    2022-02-26 21:59:22.975 [        main] INFO  VMInfo               - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl\n2022-02-26 21:59:22.985 [        main] INFO  Engine               - \n{\n    \"content\":{\n        \"reader\":{\n            \"parameter\":{\n                \"column\":[\n                    {\n                        \"random\":\"10,1000\",\n                        \"type\":\"long\"\n                    },\n                    {\n                        \"type\":\"string\",\n                        \"value\":\"1.1.1.1\"\n                    },\n                    {\n                        \"type\":\"double\",\n                        \"value\":19890604.0\n                    },\n                    {\n                        \"type\":\"long\",\n                        \"value\":19890604\n                    },\n                    {\n                        \"type\":\"long\",\n                        \"value\":19890604\n                    },\n                    {\n                        \"type\":\"string\",\n                        \"value\":\"hello world\"\n                    },\n                    {\n                        \"type\":\"string\",\n                        \"value\":\"long text\"\n                    },\n                    {\n                        \"type\":\"string\",\n                        \"value\":\"41.12,-71.34\"\n                    },\n                    {\n                        \"type\":\"string\",\n                        \"value\":\"2017-05-25 11:22:33\"\n                    }\n                ],\n                \"sliceRecordCount\":100\n            },\n            \"name\":\"streamreader\"\n        },\n        \"writer\":{\n            \"parameter\":{\n                \"partitions\":0,\n                \"column\":[\n                    \"col1\",\n                    \"col2\",\n                    \"col3\",\n                    \"col4\",\n                    \"col5\",\n                    \"col6\",\n                    \"col7\",\n                    \"col8\",\n                    \"col9\"\n                ],\n                \"topic\":\"test-1\",\n                \"batchSize\":1000,\n                \"brokerList\":\"localhost:9092\"\n            },\n            \"name\":\"kafkawriter\"\n        }\n    },\n    \"setting\":{\n        \"speed\":{\n            \"channel\":1\n        }\n    }\n}\n\n2022-02-26 21:59:23.002 [        main] INFO  PerfTrace            - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2022-02-26 21:59:23.003 [        main] INFO  JobContainer         - Addax jobContainer starts job.\n2022-02-26 21:59:23.004 [        main] INFO  JobContainer         - Set jobId = 0\n2022-02-26 21:59:23.017 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do prepare work .\n2022-02-26 21:59:23.017 [       job-0] INFO  JobContainer         - Addax Writer.Job [kafkawriter] do prepare work .\n2022-02-26 21:59:23.017 [       job-0] INFO  JobContainer         - Job set Channel-Number to 1 channel(s).\n2022-02-26 21:59:23.018 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] splits to [1] tasks.\n2022-02-26 21:59:23.019 [       job-0] INFO  JobContainer         - Addax Writer.Job [kafkawriter] splits to [1] tasks.\n2022-02-26 21:59:23.039 [       job-0] INFO  JobContainer         - Scheduler starts [1] taskGroups.\n2022-02-26 21:59:23.047 [ taskGroup-0] INFO  TaskGroupContainer   - taskGroupId=[0] start [1] channels for [1] tasks.\n2022-02-26 21:59:23.050 [ taskGroup-0] INFO  Channel              - Channel set byte_speed_limit to -1, No bps activated.\n2022-02-26 21:59:23.050 [ taskGroup-0] INFO  Channel              - Channel set record_speed_limit to -1, No tps activated.\n2022-02-26 21:59:23.082 [0-0-0-writer] INFO  ProducerConfig       - ProducerConfig values: \n    acks = 1\n    batch.size = 1000\n    bootstrap.servers = [localhost:9092]\n    buffer.memory = 33554432\n    client.id = addax-kafka-writer\n    compression.type = none\n    connections.max.idle.ms = 540000\n    enable.idempotence = false\n    interceptor.classes = []\n    key.serializer = class org.apache.kafka.common.serialization.StringSerializer\n    linger.ms = 0\n    max.block.ms = 60000\n    max.in.flight.requests.per.connection = 5\n    max.request.size = 1048576\n    metadata.max.age.ms = 300000\n    metric.reporters = []\n    metrics.num.samples = 2\n    metrics.recording.level = INFO\n    metrics.sample.window.ms = 30000\n    partitioner.class = class org.apache.kafka.clients.producer.internals.DefaultPartitioner\n    receive.buffer.bytes = 32768\n    reconnect.backoff.max.ms = 1000\n    reconnect.backoff.ms = 50\n    request.timeout.ms = 30000\n    retries = 0\n    retry.backoff.ms = 100\n    sasl.client.callback.handler.class = null\n    sasl.jaas.config = null\n    sasl.kerberos.kinit.cmd = /usr/bin/kinit\n    sasl.kerberos.min.time.before.relogin = 60000\n    sasl.kerberos.service.name = null\n    sasl.kerberos.ticket.renew.jitter = 0.05\n    sasl.kerberos.ticket.renew.window.factor = 0.8\n    sasl.login.callback.handler.class = null\n    sasl.login.class = null\n    sasl.login.refresh.buffer.seconds = 300\n    sasl.login.refresh.min.period.seconds = 60\n    sasl.login.refresh.window.factor = 0.8\n    sasl.login.refresh.window.jitter = 0.05\n    sasl.mechanism = GSSAPI\n    security.protocol = PLAINTEXT\n    send.buffer.bytes = 131072\n    ssl.cipher.suites = null\n    ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]\n    ssl.endpoint.identification.algorithm = https\n    ssl.key.password = null\n    ssl.keymanager.algorithm = SunX509\n    ssl.keystore.location = null\n    ssl.keystore.password = null\n    ssl.keystore.type = JKS\n    ssl.protocol = TLS\n    ssl.provider = null\n    ssl.secure.random.implementation = null\n    ssl.trustmanager.algorithm = PKIX\n    ssl.truststore.location = null\n    ssl.truststore.password = null\n    ssl.truststore.type = JKS\n    transaction.timeout.ms = 60000\n    transactional.id = null\n    value.serializer = class org.apache.kafka.common.serialization.StringSerializer\n\n2022-02-26 21:59:23.412 [0-0-0-writer] INFO  AppInfoParser        - Kafka version : 2.0.0\n2022-02-26 21:59:23.413 [0-0-0-writer] INFO  AppInfoParser        - Kafka commitId : 3402a8361b734732\n2022-02-26 21:59:23.534 [kafka-producer-network-thread | addax-kafka-writer] INFO  Metadata             - Cluster ID: xPAQZFNDTp6y63nZO4LACA\n2022-02-26 21:59:26.061 [       job-0] INFO  AbstractScheduler    - Scheduler accomplished all tasks.\n2022-02-26 21:59:26.062 [       job-0] INFO  JobContainer         - Addax Writer.Job [kafkawriter] do post work.\n2022-02-26 21:59:26.062 [       job-0] INFO  JobContainer         - Addax Reader.Job [streamreader] do post work.\n2022-02-26 21:59:26.063 [       job-0] INFO  JobContainer         - PerfTrace not enable!\n2022-02-26 21:59:26.064 [       job-0] INFO  StandAloneJobContainerCommunicator - Total 100 records, 9200 bytes | Speed 2.99KB/s, 33 records/s | Error 0 records, 0 bytes |  All Task WaitWriterTime 0.000s |  All Task WaitReaderTime 0.000s | Percentage 100.00%\n2022-02-26 21:59:26.065 [       job-0] INFO  JobContainer         - \n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2022-02-26 21:59:23\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2022-02-26 21:59:26\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :            2.99KB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :             33rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                 100\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n

    \u6211\u4eec\u4f7f\u7528 kafka \u81ea\u5e26\u7684 kafka-console-consumer.sh \u5c1d\u8bd5\u8bfb\u53d6\u6570\u636e\uff0c\u8f93\u51fa\u5982\u4e0b\uff1a

    $ bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test-1 --from-beginning\n\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":916}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":572}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":88}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":33}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":697}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":381}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":304}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":103}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":967}\n{\"col8\":\"41.12,-71.34\",\"col9\":\"2017-05-25 11:22:33\",\"col6\":\"hello world\",\"col7\":\"long text\",\"col4\":19890604,\"col5\":19890604,\"col2\":\"1.1.1.1\",\"col3\":1.9890604E7,\"col1\":147}\n
    "},{"location":"writer/kafkawriter/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 brokerList \u662f string \u65e0 \u8fde\u63a5 kafka \u670d\u52a1\u7684 broker \u914d\u7f6e\uff0c\u591a\u4e2a broker\u4e4b\u95f4\u7528\u9017\u53f7(,)\u5206\u9694 topic \u662f string \u65e0 \u8981\u5199\u5165\u7684 topic batchSize \u5426 int 1204 \u8bbe\u7f6e Kafka \u7684 batch.size \u53c2\u6570 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u4e0d\u5141\u8bb8\u4e3a * properties \u5426 map \u65e0 \u9700\u8981\u8bbe\u7f6e\u7684\u5176\u4ed6 kafka \u8fde\u63a5\u53c2\u6570"},{"location":"writer/kafkawriter/#_5","title":"\u9650\u5236","text":"
    1. \u4ec5\u652f\u6301 Kafka 1.0 \u53ca\u4ee5\u4e0a\u7248\u672c\uff0c\u4f4e\u4e8e\u8be5\u7248\u672c\u7684\u65e0\u6cd5\u786e\u5b9a\u662f\u5426\u80fd\u5199\u5165
    2. \u5f53\u524d\u4e0d\u652f\u6301\u542f\u7528\u4e86 kerberos \u8ba4\u8bc1\u7684 kafka \u670d\u52a1
    "},{"location":"writer/kuduwriter/","title":"Kudu Writer","text":"

    Kudu Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5c06\u6570\u636e\u5199\u5165\u5230 kudu \u7684\u80fd\u529b\uff0c\u5f53\u524d\u662f\u901a\u8fc7\u8c03\u7528\u539f\u751fRPC\u63a5\u53e3\u6765\u5b9e\u73b0\u7684\u3002 \u540e\u671f\u5e0c\u671b\u901a\u8fc7 impala \u63a5\u53e3\u5b9e\u73b0\uff0c\u4ece\u800c\u589e\u52a0\u66f4\u591a\u7684\u529f\u80fd\u3002

    "},{"location":"writer/kuduwriter/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u793a\u4f8b\u6f14\u793a\u4e86\u5982\u4f55\u4ece\u5185\u5b58\u8bfb\u53d6\u6837\u4f8b\u6570\u636e\u5e76\u5199\u5165\u5230 kudu \u8868\u4e2d\u7684\u3002

    "},{"location":"writer/kuduwriter/#_2","title":"\u8868\u7ed3\u6784","text":"

    \u6211\u4eec\u7528 trino \u5de5\u5177\u8fde\u63a5\u5230 kudu \u670d\u52a1\uff0c\u7136\u540e\u901a\u8fc7\u4e0b\u9762\u7684 SQL \u8bed\u53e5\u521b\u5efa\u8868

    CREATE TABLE kudu.default.users (\n  user_id int WITH (primary_key = true),\n  user_name varchar,\n  salary double\n) WITH (\n  partition_by_hash_columns = ARRAY['user_id'],\n  partition_by_hash_buckets = 2\n);\n
    "},{"location":"writer/kuduwriter/#job","title":"job \u914d\u7f6e\u6587\u4ef6","text":"

    \u521b\u5efa job/stream2kudu.json \u6587\u4ef6\uff0c\u5185\u5bb9\u5982\u4e0b\uff1a

    job/stream2kudu.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"1,1000\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"1,10\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"1000,50000\",\n              \"type\": \"double\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"kuduwriter\",\n        \"parameter\": {\n          \"masterAddress\": \"127.0.0.1:7051,127.0.0.1:7151,127.0.0.1:7251\",\n          \"timeout\": 60,\n          \"table\": \"users\",\n          \"writeMode\": \"upsert\",\n          \"column\": [ \"user_id\", \"user_name\", \"salary\"],\n          \"batchSize\": 1024,\n          \"bufferSize\": 2048,\n          \"skipFail\": false,\n          \"encoding\": \"UTF-8\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/kuduwriter/#_3","title":"\u8fd0\u884c","text":"

    \u6267\u884c\u4e0b\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/stream2kudu.json\n
    "},{"location":"writer/kuduwriter/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 masterAddress \u662f string \u65e0 Kudu Master\u96c6\u7fa4RPC\u5730\u5740,\u591a\u4e2a\u5730\u5740\u7528\u9017\u53f7(,)\u5206\u9694 table \u662f string \u65e0 kudu \u8868\u540d writeMode \u5426 string upsert \u8868\u6570\u636e\u5199\u5165\u6a21\u5f0f\uff0c\u652f\u6301 upsert, insert \u4e24\u8005 timeout \u5426 int 100 \u5199\u5165\u6570\u636e\u8d85\u65f6\u65f6\u95f4(\u79d2), 0 \u8868\u793a\u4e0d\u53d7\u9650\u5236 column \u662f list \u65e0 \u8981\u5199\u5165\u7684\u8868\u5b57\u6bb5\uff0c\u914d\u7f6e\u65b9\u5f0f\u89c1\u4e0a\u793a\u4f8b skipFail \u5426 boolean false \u662f\u5426\u8df3\u8fc7\u63d2\u5165\u5931\u8d25\u7684\u8bb0\u5f55\uff0c\u5982\u679c\u8bbe\u7f6e\u4e3atrue\uff0c\u5219\u63d2\u4ef6\u4e0d\u4f1a\u628a\u63d2\u5165\u5931\u8d25\u7684\u5f53\u4f5c\u5f02\u5e38"},{"location":"writer/kuduwriter/#column","title":"column","text":"

    column \u53ef\u4ee5\u76f4\u63a5\u6307\u5b9a\u8981\u5199\u5165\u7684\u5217\uff0c\u5982\u540c\u4e0a\u8ff0\u4f8b\u5b50\uff0c\u4e5f\u53ef\u4ee5\u8bbe\u7f6e [\"*\"] \u6765\u8868\u793a\u5199\u5165\u6240\u6709\u5217\u3002

    "},{"location":"writer/mongodbwriter/","title":"MongoDB Writer","text":"

    MongoDB Writer \u63d2\u4ef6\u7528\u4e8e\u5411 MongoDB \u5199\u5165\u6570\u636e\u3002

    "},{"location":"writer/mongodbwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u8be5\u793a\u4f8b\u5c06\u6d41\u5f0f\u6570\u636e\u5199\u5165\u5230 MongoDB \u8868\u4e2d

    job/stream2mongo.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"unique_id\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"sid\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"user_id\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"auction_id\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"content_type\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"pool_type\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"a1 a2 a3\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"c1 c2 c3\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"2020-09-06\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"tag1 tag2 tag3\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": \"property\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 1984,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": 1900,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": 75,\n              \"type\": \"long\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"mongodbwriter\",\n        \"parameter\": {\n          \"connection\": {\n            \"address\": [\n              \"127.0.0.1:27017\"\n            ],\n            \"database\": \"my_database\",\n            \"collection\": \"addax_writer\",\n            \"authDb\": \"my_database\"\n          },\n          \"username\": \"my_user\",\n          \"password\": \"password123\",\n          \"column\": [\n            {\n              \"name\": \"unique_id\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"sid\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"user_id\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"auction_id\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"content_type\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"pool_type\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"frontcat_id\",\n              \"type\": \"Array\",\n              \"splitter\": \" \"\n            },\n            {\n              \"name\": \"categoryid\",\n              \"type\": \"Array\",\n              \"splitter\": \" \"\n            },\n            {\n              \"name\": \"gmt_create\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"taglist\",\n              \"type\": \"Array\",\n              \"splitter\": \" \"\n            },\n            {\n              \"name\": \"property\",\n              \"type\": \"string\"\n            },\n            {\n              \"name\": \"scorea\",\n              \"type\": \"int\"\n            },\n            {\n              \"name\": \"scoreb\",\n              \"type\": \"int\"\n            },\n            {\n              \"name\": \"scorec\",\n              \"type\": \"int\"\n            }\n          ],\n          \"writeMode\": \"insert\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/mongodbwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 address \u662f list \u65e0 MongoDB \u7684\u6570\u636e\u5730\u5740\u4fe1\u606f username \u5426 string \u65e0 MongoDB \u7684\u7528\u6237\u540d password \u5426 string \u65e0 MongoDB \u7684\u5bc6\u7801 collection \u662f string \u65e0 MongoDB \u7684\u96c6\u5408\u540d column \u662f list<map> \u65e0 MongoDB \u7684\u6587\u6863\u5217\u540d splitter \u5426 string \u65e0 \u7279\u6b8a\u5206\u9694\u7b26\uff0c\u8be6\u89c1\u4e0b\u6587 writeMode \u5426 string insert \u6307\u5b9a\u4e86\u4f20\u8f93\u6570\u636e\u65f6\u66f4\u65b0\u7684\u4fe1\u606f,\u652f\u6301 insert\uff0c update \u4e24\u79cd batchSize \u5426 int 2048 \u6307\u5b9a\u6279\u6b21\u8f93\u5165\u7684\u6570\u91cf isUpsert \u5426 boolean \u65e0 \u5f53\u8bbe\u7f6e\u4e3a true \u65f6\uff0c\u8868\u793a\u9488\u5bf9\u76f8\u540c\u7684 upsertKey \u505a\u66f4\u65b0\u64cd\u4f5c upsertKey \u5426 string \u65e0 upsertKey \u6307\u5b9a\u4e86\u6ca1\u884c\u8bb0\u5f55\u7684\u4e1a\u52a1\u4e3b\u952e\u3002\u7528\u6765\u505a\u66f4\u65b0\u65f6\u4f7f\u7528"},{"location":"writer/mongodbwriter/#column","title":"column","text":"

    column \u6307\u5b9a mongo collection \u7684\u5b57\u6bb5\u4ee5\u53ca\u7c7b\u578b\uff0c\u5982\u679c\u662f\u6570\u7ec4\u7c7b\u578b\uff0c\u8fd8\u9700\u8981\u6307\u5b9a\u63a5\u6536\u5230\u7684\u6570\u636e\u6309\u7167\u4ec0\u4e48\u5206\u5272\uff0c\u4e00\u4e2a column \u5b57\u6bb5\u81f3\u5c11\u9700\u8981\u6307\u5b9a name \u4ee5\u53ca type\uff0c\u6bd4\u5982

    {\n  \"column\": [\n    {\n      \"name\": \"user_id\",\n      \"type\": \"string\"\n    }\n  ]\n}\n

    \u5982\u679c\u662f\u6570\u7ec4\u7c7b\u578b\uff0c\u5219\u9700\u8981\u914d\u7f6e splitter \u6765\u544a\u77e5\u5206\u9694\u7b26\uff0c\u7c7b\u4f3c\u5982\u4e0b\uff1a

    {\n  \"column\": {\n    \"name\": \"taglist\",\n    \"type\": \"Array\",\n    \"splitter\": \" \"\n  }\n}\n
    "},{"location":"writer/mongodbwriter/#splitter","title":"splitter","text":"

    \u5f53\u4e14\u4ec5\u5f53\u8981\u5904\u7406\u7684\u5b57\u7b26\u4e32\u8981\u7528\u5206\u9694\u7b26\u5206\u9694\u4e3a\u5b57\u7b26\u6570\u7ec4\u65f6\uff0c\u624d\u4f7f\u7528\u8fd9\u4e2a\u53c2\u6570\uff0c\u901a\u8fc7\u8fd9\u4e2a\u53c2\u6570\u6307\u5b9a\u7684\u5206\u9694\u7b26\uff0c\u5c06\u5b57\u7b26\u4e32\u5206\u9694\u5b58\u50a8\u5230 MongoDB \u7684\u6570\u7ec4\u4e2d

    "},{"location":"writer/mongodbwriter/#writemode","title":"writeMode","text":"

    \u4e0d\u914d\u7f6e\u7684\u60c5\u51b5\u4e0b\uff0c\u9ed8\u8ba4\u91c7\u53d6\u76f4\u63a5\u63d2\u5165\u8bb0\u5f55\u7684\u65b9\u5f0f\uff0c\u5982\u679c\u5e0c\u671b\u5b9e\u73b0\u63d2\u5165\u66f4\u65b0\uff08\u5373\u8bb0\u5f55\u5b58\u5728\u5219\u66f4\u65b0\u5426\u5219\u63d2\u5165\uff09\uff0c\u53ef\u4ee5\u6307\u5b9a\u4e3a update \u6a21\u5f0f\uff0c\u8be5\u6a21\u5f0f\u4e0b\uff0c\u5fc5\u987b\u540c\u65f6\u66f4\u65b0\u7684\u5b57\u6bb5\u662f\u54ea\u4e2a\uff0c\u6bd4\u5982\uff1a

    {\n  \"writeMode\": \"update(unique_id)\"\n}\n

    \u4e0a\u8ff0\u914d\u7f6e\u8868\u793a\u4f9d\u636e\u5b57\u6bb5 unique_id \u6765\u51b3\u5b9a\u5f53\u524d\u8bb0\u5f55\u662f\u63d2\u5165\u8fd8\u662f\u66f4\u65b0\uff0c\u5f53\u524d\u6682\u4e0d\u652f\u6301\u6307\u5b9a\u591a\u4e2a\u5b57\u6bb5

    "},{"location":"writer/mongodbwriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b MongoDB \u6570\u636e\u7c7b\u578b Long int, Long Double double String string, array Date date Boolean boolean Bytes bytes"},{"location":"writer/mysqlwriter/","title":"MySQL Writer","text":"

    MySQL Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 MySQL \u76ee\u7684\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/mysqlwriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684 MySQL \u8868\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    create table test.addax_tbl\n(\n  col1 varchar(20) ,\n  col2 int(4),\n  col3 datetime,\n  col4 boolean,\n  col5 binary\n) default charset utf8;\n

    \u8fd9\u91cc\u4f7f\u7528\u4e00\u4efd\u4ece\u5185\u5b58\u4ea7\u751f\u5230 MySQL \u5bfc\u5165\u7684\u6570\u636e\u3002

    job/stream2mysql.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"mysqlwriter\",\n        \"parameter\": {\n          \"writeMode\": \"insert\",\n          \"username\": \"root\",\n          \"password\": \"\",\n          \"column\": [\n            \"*\"\n          ],\n          \"session\": [\n            \"set session sql_mode='ANSI'\"\n          ],\n          \"preSql\": [\n            \"delete from @table\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:mysql://127.0.0.1:3306/test?useSSL=false\",\n            \"table\": [\n              \"addax_tbl\"\n            ],\n            \"driver\": \"com.mysql.jdbc.Driver\"\n          }\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2mysql.json

    "},{"location":"writer/mysqlwriter/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/stream2mysql.json\n
    "},{"location":"writer/mysqlwriter/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\uff0c\u5e76\u4e14\u589e\u52a0\u4e86\u4e00\u4e9b MySQL \u7279\u6709\u7684\u914d\u7f6e\u9879\u3002

    \u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 writeMode \u662f string insert \u6570\u636e\u5199\u5165\u8868\u7684\u65b9\u5f0f\uff0c\u8be6\u89c1\u4e0b\u6587 batchSize \u5426 int 1024 \u5b9a\u4e49\u4e86\u63d2\u4ef6\u548c\u6570\u636e\u5e93\u670d\u52a1\u5668\u7aef\u6bcf\u6b21\u6279\u91cf\u6570\u636e\u83b7\u53d6\u6761\u6570"},{"location":"writer/mysqlwriter/#driver","title":"driver","text":"

    \u5f53\u524d\u91c7\u7528\u7684 MySQL JDBC \u9a71\u52a8\u4e3a 8.0 \u4ee5\u4e0a\u7248\u672c\uff0c\u9a71\u52a8\u7c7b\u540d\u4f7f\u7528\u7684 com.mysql.cj.jdbc.Driver\uff0c\u800c\u4e0d\u662f com.mysql.jdbc.Driver\u3002 \u5982\u679c\u4f60\u9700\u8981\u91c7\u96c6\u7684 MySQL \u670d\u52a1\u4f4e\u4e8e 5.6\uff0c\u9700\u8981\u4f7f\u7528\u5230 Connector/J 5.1 \u9a71\u52a8\uff0c\u5219\u53ef\u4ee5\u91c7\u53d6\u4e0b\u9762\u7684\u6b65\u9aa4\uff1a

    1. \u66ff\u6362\u63d2\u4ef6\u5185\u7f6e\u7684\u9a71\u52a8 rm -f plugin/writer/mysqlwriter/libs/mysql-connector-java-*.jar

    2. \u62f7\u8d1d\u8001\u7684\u9a71\u52a8\u5230\u63d2\u4ef6\u76ee\u5f55 cp mysql-connector-java-5.1.48.jar plugin/writer/mysqlwriter/libs/

    3. \u6307\u5b9a\u9a71\u52a8\u7c7b\u540d\u79f0 \u5728\u4f60\u7684 json \u6587\u4ef6\u7c7b\uff0c\u914d\u7f6e \"driver\": \"com.mysql.jdbc.Driver\"

    "},{"location":"writer/mysqlwriter/#writemode","title":"writeMode","text":""},{"location":"writer/oraclewriter/","title":"Oracle Writer","text":"

    Oracle Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 Oracle \u76ee\u7684\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/oraclewriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u8fd9\u91cc\u4f7f\u7528\u4e00\u4efd\u4ece\u5185\u5b58\u4ea7\u751f\u5230 Oracle \u5bfc\u5165\u7684\u6570\u636e\u3002

    job/stream2oracle.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"oraclewriter\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"root\",\n          \"column\": [\n            \"id\",\n            \"name\"\n          ],\n          \"preSql\": [\n            \"delete from test\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:oracle:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]\",\n            \"table\": [\n              \"test\"\n            ]\n          }\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/oraclewriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\uff0c\u5e76\u5728\u6b64\u57fa\u7840\u4e0a\u589e\u52a0\u4e86\u4e00\u4e9b OracleWriter \u7279\u6709\u7684\u914d\u7f6e\u9879\u3002

    \u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u9ed8\u8ba4\u503c \u63cf\u8ff0 writeMode \u5426 insert \u5199\u5165\u65b9\u5f0f\uff0c\u652f\u6301 insert\uff0c update\uff0c\u8be6\u89c1\u4e0b\u6587"},{"location":"writer/oraclewriter/#writemode","title":"writeMode","text":"

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c \u91c7\u53d6 insert into \u8bed\u6cd5\u5199\u5165 Oracle \u8868\uff0c\u5982\u679c\u4f60\u5e0c\u671b\u91c7\u53d6\u4e3b\u952e\u5b58\u5728\u65f6\u66f4\u65b0\uff0c\u4e0d\u5b58\u5728\u5219\u5199\u5165\u7684\u65b9\u5f0f\uff0c\u4e5f\u5c31\u662f Oracle \u7684 merge into \u8bed\u6cd5\uff0c \u53ef\u4ee5\u4f7f\u7528 update \u6a21\u5f0f\u3002\u5047\u5b9a\u8868\u7684\u4e3b\u952e\u4e3a id ,\u5219 writeMode \u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff1a

    \"writeMode\": \"update(id)\"\n

    \u5982\u679c\u662f\u8054\u5408\u552f\u4e00\u7d22\u5f15\uff0c\u5219\u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff1a

    \"writeMode\": \"update(col1, col2)\"\n
    "},{"location":"writer/paimonwriter/","title":"Paimon Writer","text":"

    Paimon Writer \u63d0\u4f9b\u5411 \u5df2\u6709\u7684paimon\u8868\u5199\u5165\u6570\u636e\u7684\u80fd\u529b\u3002

    "},{"location":"writer/paimonwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 3\n      },\n      \"errorLimit\": {\n        \"record\": 0,\n        \"percentage\": 0\n      }\n    },\n    \"content\": [\n      {\n        \"reader\": {\n          \"name\": \"rdbmsreader\",\n          \"parameter\": {\n            \"username\": \"root\",\n            \"password\": \"root\",\n            \"column\": [\n              \"*\"\n            ],\n            \"connection\": [\n              {\n                \"querySql\": [\n                  \"select 1+0 id ,'test1' as name\"\n                ],\n                \"jdbcUrl\": [\"jdbc:mysql://localhost:3306/ruoyi_vue_camunda?allowPublicKeyRetrieval=true\",]\n              }\n            ],\n            \"fetchSize\": 1024\n          }\n        },\n        \"writer\": {\n          \"name\": \"paimonwriter\",\n          \"parameter\": {\n            \"dbName\": \"test\",\n            \"tableName\": \"test2\",\n            \"writeMode\": \"truncate\",\n            \"paimonConfig\": {\n              \"warehouse\": \"file:///g:/paimon\",\n              \"metastore\": \"filesystem\"\n            }\n          }\n        }\n      }\n    ]\n  }\n}\n
    "},{"location":"writer/paimonwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e dbName \u662f string \u65e0 \u8981\u5199\u5165\u7684paimon\u6570\u636e\u5e93\u540d tableName \u662f string \u65e0 \u8981\u5199\u5165\u7684paimon\u8868\u540d writeMode \u662f string \u65e0 \u5199\u5165\u6a21\u5f0f\uff0c\u8be6\u8ff0\u89c1\u4e0b paimonConfig \u662f json {} \u91cc\u53ef\u4ee5\u914d\u7f6e\u4e0e Paimon catalog\u548cHadoop \u76f8\u5173\u7684\u4e00\u4e9b\u9ad8\u7ea7\u53c2\u6570\uff0c\u6bd4\u5982HA\u7684\u914d\u7f6e"},{"location":"writer/paimonwriter/#writemode","title":"writeMode","text":"

    \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff1a

    "},{"location":"writer/paimonwriter/#paimonconfig","title":"paimonConfig","text":"

    paimonConfig \u91cc\u53ef\u4ee5\u914d\u7f6e\u4e0e Paimon catalog\u548cHadoop \u76f8\u5173\u7684\u4e00\u4e9b\u9ad8\u7ea7\u53c2\u6570\uff0c\u6bd4\u5982HA\u7684\u914d\u7f6e

    \u672c\u5730\u76ee\u5f55\u521b\u5efapaimon\u8868

    pom.xml

    <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\"\n         xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n         xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <modelVersion>4.0.0</modelVersion>\n\n    <groupId>com.test</groupId>\n    <artifactId>paimon-java-api-test</artifactId>\n    <version>1.0-SNAPSHOT</version>\n\n    <properties>\n        <maven.compiler.source>8</maven.compiler.source>\n        <maven.compiler.target>8</maven.compiler.target>\n        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>\n        <hadoop.version>3.2.4</hadoop.version>\n        <woodstox.version>7.0.0</woodstox.version>\n    </properties>\n<dependencies>\n    <dependency>\n        <groupId>org.apache.paimon</groupId>\n        <artifactId>paimon-bundle</artifactId>\n        <version>1.0.0</version>\n    </dependency>\n\n\n    <dependency>\n        <groupId>org.apache.hadoop</groupId>\n        <artifactId>hadoop-common</artifactId>\n        <version>${hadoop.version}</version>\n        <exclusions>\n            <exclusion>\n                <groupId>com.fasterxml.jackson.core</groupId>\n                <artifactId>jackson-databind</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.codehaus.jackson</groupId>\n                <artifactId>jackson-core-asl</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.codehaus.jackson</groupId>\n                <artifactId>jackson-mapper-asl</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>com.fasterxml.woodstox</groupId>\n                <artifactId>woodstox-core</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>commons-codec</groupId>\n                <artifactId>commons-codec</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>commons-net</groupId>\n                <artifactId>commons-net</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>io.netty</groupId>\n                <artifactId>netty</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>log4j</groupId>\n                <artifactId>log4j</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>net.minidev</groupId>\n                <artifactId>json-smart</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.codehaus.jettison</groupId>\n                <artifactId>jettison</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.eclipse.jetty</groupId>\n                <artifactId>jetty-server</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.xerial.snappy</groupId>\n                <artifactId>snappy-java</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.apache.zookeeper</groupId>\n                <artifactId>zookeeper</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.eclipse.jetty</groupId>\n                <artifactId>jetty-util</artifactId>\n            </exclusion>\n        </exclusions>\n    </dependency>\n\n    <dependency>\n        <groupId>org.apache.hadoop</groupId>\n        <artifactId>hadoop-aws</artifactId>\n        <version>${hadoop.version}</version>\n        <exclusions>\n            <exclusion>\n                <groupId>com.fasterxml.jackson.core</groupId>\n                <artifactId>jackson-databind</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.codehaus.jackson</groupId>\n                <artifactId>jackson-core-asl</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.codehaus.jackson</groupId>\n                <artifactId>jackson-mapper-asl</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>com.fasterxml.woodstox</groupId>\n                <artifactId>woodstox-core</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>commons-codec</groupId>\n                <artifactId>commons-codec</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>commons-net</groupId>\n                <artifactId>commons-net</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>io.netty</groupId>\n                <artifactId>netty</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>log4j</groupId>\n                <artifactId>log4j</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>net.minidev</groupId>\n                <artifactId>json-smart</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.codehaus.jettison</groupId>\n                <artifactId>jettison</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.eclipse.jetty</groupId>\n                <artifactId>jetty-server</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.xerial.snappy</groupId>\n                <artifactId>snappy-java</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.apache.zookeeper</groupId>\n                <artifactId>zookeeper</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.eclipse.jetty</groupId>\n                <artifactId>jetty-util</artifactId>\n            </exclusion>\n        </exclusions>\n    </dependency>\n\n    <dependency>\n        <groupId>org.apache.hadoop</groupId>\n        <artifactId>hadoop-mapreduce-client-core</artifactId>\n        <version>${hadoop.version}</version>\n        <exclusions>\n            <exclusion>\n                <groupId>com.fasterxml.jackson.core</groupId>\n                <artifactId>jackson-databind</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>commons-codec</groupId>\n                <artifactId>commons-codec</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>io.netty</groupId>\n                <artifactId>netty</artifactId>\n            </exclusion>\n            <exclusion>\n                <groupId>org.eclipse.jetty</groupId>\n                <artifactId>jetty-util</artifactId>\n            </exclusion>\n        </exclusions>\n    </dependency>\n\n\n    <dependency>\n        <groupId>com.fasterxml.woodstox</groupId>\n        <artifactId>woodstox-core</artifactId>\n        <version>${woodstox.version}</version>\n    </dependency>\n</dependencies>\n</project>\n
    import org.apache.paimon.catalog.Catalog;\nimport org.apache.paimon.catalog.CatalogContext;\nimport org.apache.paimon.catalog.CatalogFactory;\nimport org.apache.paimon.catalog.Identifier;\nimport org.apache.paimon.fs.Path;\nimport org.apache.paimon.schema.Schema;\nimport org.apache.paimon.types.DataTypes;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\npublic class CreatePaimonTable {\n\n    public static Catalog createFilesystemCatalog() {\n        CatalogContext context = CatalogContext.create(new Path(\"file:///g:/paimon\"));\n        return CatalogFactory.createCatalog(context);\n    }\n    /* \u5982\u679c\u662fminio\u5219\u4f8b\u5b50\u5982\u4e0b\n\n     public static Catalog createFilesystemCatalog() {\n        Options options = new Options();\n        options.set(\"warehouse\", \"s3a://pvc-91d1e2cd-4d25-45c9-8613-6c4f7bf0a4cc/paimon\");\n        Configuration hadoopConf = new Configuration();\n        hadoopConf.set(\"fs.s3a.endpoint\", \"http://localhost:9000\");\n        hadoopConf.set(\"fs.s3a.access.key\", \"gy0dX5lALP176g6c9fYf\");\n        hadoopConf.set(\"fs.s3a.secret.key\", \"ReuUrCzzu5wKWAegtswoHIWV389BYl9AB1ZQbiKr\");\n        hadoopConf.set(\"fs.s3a.connection.ssl.enabled\", \"false\");\n        hadoopConf.set(\"fs.s3a.path.style.access\", \"true\");\n        hadoopConf.set(\"fs.s3a.impl\",\"org.apache.hadoop.fs.s3a.S3AFileSystem\");\n        CatalogContext context = CatalogContext.create(options,hadoopConf);\n\n\n        return CatalogFactory.createCatalog(context);\n    }\n     * \n     * \n     * */\n\n    public static void main(String[] args) {\n        Schema.Builder schemaBuilder = Schema.newBuilder();\n        schemaBuilder.primaryKey(\"id\");\n        schemaBuilder.column(\"id\", DataTypes.INT());\n        schemaBuilder.column(\"name\", DataTypes.STRING());\n        Map<String, String> options = new HashMap<>();\n        options.put(\"bucket\", \"1\");//\u7531\u4e8epaimon java api \u9650\u5236\u9700\u8981bucket>0\n        options.put(\"bucket-key\", \"id\");\n        options.put(\"file.format\", \"orc\");\n        options.put(\"file.compression\", \"lz4\");\n        options.put(\"lookup.cache-spill-compression\", \"lz4\");\n        options.put(\"spill-compression\", \"LZ4\");\n        options.put(\"orc.compress\", \"lz4\");\n        options.put(\"manifest.format\", \"orc\");\n\n        schemaBuilder.options(options);\n        Schema schema = schemaBuilder.build();\n\n        Identifier identifier = Identifier.create(\"test\", \"test2\");\n        try {\n            Catalog catalog = CreatePaimonTable.createFilesystemCatalog();\n            catalog.createDatabase(\"test\",true);\n            catalog.createTable(identifier, schema, true);\n        } catch (Catalog.TableAlreadyExistException e) {\n            e.printStackTrace();\n        } catch (Catalog.DatabaseNotExistException e) {\n            e.printStackTrace();\n        } catch (Catalog.DatabaseAlreadyExistException e) {\n            throw new RuntimeException(e);\n        }\n\n\n    }\n}\n

    Spark \u6216\u8005 flink \u73af\u5883\u521b\u5efa\u8868

    CREATE TABLE if not exists test.test2(id int ,name string)  tblproperties (\n    'primary-key' = 'id',\n    'bucket' = '1',\n    'bucket-key' = 'id'\n    'file.format'='orc',\n    'file.compression'='lz4',\n    'lookup.cache-spill-compression'='lz4',\n    'spill-compression'='LZ4',\n    'orc.compress'='lz4',\n    'manifest.format'='orc'\n)\n

    \u672c\u5730\u6587\u4ef6\u4f8b\u5b50

    {\n                    \"name\": \"paimonwriter\",\n                    \"parameter\": {\n                        \"dbName\": \"test\",\n                        \"tableName\": \"test2\",\n                        \"writeMode\": \"truncate\",\n                        \"paimonConfig\": {\n                           \"warehouse\": \"file:///g:/paimon\",\n                           \"metastore\": \"filesystem\"\n                         }\n                    }\n}\n

    s3 \u6216\u8005 minio catalog\u4f8b\u5b50 

    {\n    \"job\": {\n        \"setting\": {\n            \"speed\": {\n                \"channel\": 3\n            },\n            \"errorLimit\": {\n                \"record\": 0,\n                \"percentage\": 0\n            }\n        },\n        \"content\": [\n            {\n                \"reader\": {\n                    \"name\": \"rdbmsreader\",\n                    \"parameter\": {\n                        \"username\": \"root\",\n                        \"password\": \"root\",\n                        \"column\": [\n                            \"*\"\n                        ],\n                        \"connection\": [\n                            {\n                                \"querySql\": [\n                                    \"select 1+0 id ,'test1' as name\"\n                                ],\n                                \"jdbcUrl\": [\n                                    \"jdbc:mysql://localhost:3306/ruoyi_vue_camunda?allowPublicKeyRetrieval=true\"\n                                ]\n                            }\n                        ],\n                        \"fetchSize\": 1024\n                    }\n                },\n                \"writer\": {\n                    \"name\": \"paimonwriter\",\n                    \"parameter\": {\n                        \"dbName\": \"test\",\n                        \"tableName\": \"test2\",\n                        \"writeMode\": \"truncate\",\n                        \"paimonConfig\": {\n                            \"warehouse\": \"s3a://pvc-91d1e2cd-4d25-45c9-8613-6c4f7bf0a4cc/paimon\",\n                            \"metastore\": \"filesystem\",\n                            \"fs.s3a.endpoint\": \"http://localhost:9000\",\n                            \"fs.s3a.access.key\": \"gy0dX5lALP176g6c9fYf\",\n                            \"fs.s3a.secret.key\": \"ReuUrCzzu5wKWAegtswoHIWV389BYl9AB1ZQbiKr\",\n                            \"fs.s3a.connection.ssl.enabled\": \"false\",\n                            \"fs.s3a.path.style.access\": \"true\",\n                            \"fs.s3a.impl\": \"org.apache.hadoop.fs.s3a.S3AFileSystem\"\n                        }\n                    }\n                }\n            }\n        ]\n    }\n}\n

    hdfs catalog\u4f8b\u5b50

    {\n  \"paimonConfig\": {\n    \"warehouse\": \"hdfs://nameservice1/user/hive/paimon\",\n    \"metastore\": \"filesystem\",\n    \"fs.defaultFS\":\"hdfs://nameservice1\",\n    \"hadoop.security.authentication\" : \"kerberos\",\n    \"hadoop.kerberos.principal\" : \"hive/_HOST@XXXX.COM\",\n    \"hadoop.kerberos.keytab\" : \"/tmp/hive@XXXX.COM.keytab\",\n    \"ha.zookeeper.quorum\" : \"test-pr-nn1:2181,test-pr-nn2:2181,test-pr-nn3:2181\",\n    \"dfs.nameservices\" : \"nameservice1\",\n    \"dfs.namenode.rpc-address.nameservice1.namenode371\" : \"test-pr-nn2:8020\",\n    \"dfs.namenode.rpc-address.nameservice1.namenode265\": \"test-pr-nn1:8020\",\n    \"dfs.namenode.keytab.file\" : \"/tmp/hdfs@XXXX.COM.keytab\",\n    \"dfs.namenode.keytab.enabled\" : \"true\",\n    \"dfs.namenode.kerberos.principal\" : \"hdfs/_HOST@XXXX.COM\",\n    \"dfs.namenode.kerberos.internal.spnego.principal\" : \"HTTP/_HOST@XXXX.COM\",\n    \"dfs.ha.namenodes.nameservice1\" : \"namenode265,namenode371\",\n    \"dfs.datanode.keytab.file\" : \"/tmp/hdfs@XXXX.COM.keytab\",\n    \"dfs.datanode.keytab.enabled\" : \"true\",\n    \"dfs.datanode.kerberos.principal\" : \"hdfs/_HOST@XXXX.COM\",\n    \"dfs.client.use.datanode.hostname\" : \"false\",\n    \"dfs.client.failover.proxy.provider.nameservice1\" : \"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\",\n    \"dfs.balancer.keytab.file\" : \"/tmp/hdfs@XXXX.COM.keytab\",\n    \"dfs.balancer.keytab.enabled\" : \"true\",\n    \"dfs.balancer.kerberos.principal\" : \"hdfs/_HOST@XXXX.COM\"\n  }\n}\n
    "},{"location":"writer/paimonwriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b Paimon \u6570\u636e\u7c7b\u578b Integer TINYINT,SMALLINT,INT,INTEGER Long BIGINT Double FLOAT,DOUBLE,DECIMAL String STRING,VARCHAR,CHAR Boolean BOOLEAN Date DATE,TIMESTAMP Bytes BINARY"},{"location":"writer/postgresqlwriter/","title":"Postgresql Writer","text":"

    Postgresql Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 PostgreSQL \u6570\u636e\u5e93\u5e93\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/postgresqlwriter/#_1","title":"\u793a\u4f8b","text":"

    \u4ee5\u4e0b\u914d\u7f6e\u6f14\u793a\u4ece postgresql \u6307\u5b9a\u7684\u8868\u8bfb\u53d6\u6570\u636e\uff0c\u5e76\u63d2\u5165\u5230\u5177\u6709\u76f8\u540c\u8868\u7ed3\u6784\u7684\u53e6\u5916\u4e00\u5f20\u8868\u4e2d\uff0c\u7528\u6765\u6d4b\u8bd5\u8be5\u63d2\u4ef6\u6240\u652f\u6301\u7684\u6570\u636e\u7c7b\u578b\u3002

    "},{"location":"writer/postgresqlwriter/#_2","title":"\u8868\u7ed3\u6784\u4fe1\u606f","text":"

    \u5047\u5b9a\u5efa\u8868\u8bed\u53e5\u4ee5\u53ca\u8f93\u5165\u63d2\u5165\u8bed\u53e5\u5982\u4e0b\uff1a

    create table if not exists addax_tbl\n(\n    c_bigint bigint,\n    c_bit bit(3),\n    c_bool boolean,\n    c_byte bytea,\n    c_char char(10),\n    c_varchar varchar(20),\n    c_date date,\n    c_double float8,\n    c_int integer,\n    c_json json,\n    c_number decimal(8, 3),\n    c_real real,\n    c_small smallint,\n    c_text text,\n    c_ts timestamp,\n    c_uuid uuid,\n    c_xml xml,\n    c_money money,\n    c_inet inet,\n    c_cidr cidr,\n    c_macaddr macaddr\n    );\n\ninsert into addax_tbl\nvalues (999988887777,\n        b'101',\n        TRUE,\n        '\\xDEADBEEF',\n        'hello',\n        'hello, world',\n        '2021-01-04',\n        999888.9972,\n        9876542,\n        '{\"bar\": \"baz\", \"balance\": 7.77, \"active\": false}'::json,\n        12345.123,\n        123.123,\n        126,\n        'this is a long text ',\n        '2020-01-04 12:13:14',\n        'A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11'::uuid,\n        '<foo>bar</foo>'::xml,\n        '52093.89'::money,\n        '192.168.1.1'::inet,\n        '192.168.1/24'::cidr,\n        '08002b:010203'::macaddr);\n

    \u521b\u5efa\u9700\u8981\u63d2\u5165\u7684\u8868\u7684\u8bed\u53e5\u5982\u4e0b:

    create table addax_tbl1 as select * from  addax_tbl where 1=2;\n
    "},{"location":"writer/postgresqlwriter/#_3","title":"\u4efb\u52a1\u914d\u7f6e","text":"

    \u4ee5\u4e0b\u662f\u914d\u7f6e\u6587\u4ef6

    job/pg2pg.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"postgresqlreader\",\n        \"parameter\": {\n          \"username\": \"pgtest\",\n          \"password\": \"pgtest\",\n          \"column\": [\n            \"*\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"addax_tbl\"\n            ],\n            \"jdbcUrl\": \"jdbc:postgresql://localhost:5432/pgtest\"\n          }\n        }\n      },\n      \"writer\": {\n        \"name\": \"postgresqlwriter\",\n        \"parameter\": {\n          \"column\": [\n            \"*\"\n          ],\n          \"preSql\": [\n            \"truncate table @table\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:postgresql://127.0.0.1:5432/pgtest\",\n            \"table\": [\n              \"addax_tbl1\"\n            ]\n          },\n          \"username\": \"pgtest\",\n          \"password\": \"pgtest\",\n          \"writeMode\": \"insert\"\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/pg2pg.json

    "},{"location":"writer/postgresqlwriter/#_4","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/pg2pg.json\n
    "},{"location":"writer/postgresqlwriter/#_5","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/postgresqlwriter/#writemode","title":"writeMode","text":"

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c \u91c7\u53d6 insert into \u8bed\u6cd5\u5199\u5165 postgresql \u8868\uff0c\u5982\u679c\u4f60\u5e0c\u671b\u91c7\u53d6\u4e3b\u952e\u5b58\u5728\u65f6\u66f4\u65b0\uff0c\u4e0d\u5b58\u5728\u5219\u5199\u5165\u7684\u65b9\u5f0f\uff0c \u53ef\u4ee5\u4f7f\u7528 update \u6a21\u5f0f\u3002\u5047\u5b9a\u8868\u7684\u4e3b\u952e\u4e3a id ,\u5219 writeMode \u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff1a

    \"writeMode\": \"update(id)\"\n

    \u5982\u679c\u662f\u8054\u5408\u552f\u4e00\u7d22\u5f15\uff0c\u5219\u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff1a

    \"writeMode\": \"update(col1, col2)\"\n

    \u6ce8\uff1a update \u6a21\u5f0f\u5728 3.1.6 \u7248\u672c\u9996\u6b21\u589e\u52a0\uff0c\u4e4b\u524d\u7248\u672c\u5e76\u4e0d\u652f\u6301\u3002

    "},{"location":"writer/postgresqlwriter/#_6","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u76ee\u524d PostgresqlWriter \u652f\u6301\u5927\u90e8\u5206 PostgreSQL \u7c7b\u578b\uff0c\u4f46\u4e5f\u5b58\u5728\u90e8\u5206\u6ca1\u6709\u652f\u6301\u7684\u60c5\u51b5\uff0c\u8bf7\u6ce8\u610f\u68c0\u67e5\u4f60\u7684\u7c7b\u578b\u3002

    \u4e0b\u9762\u5217\u51fa PostgresqlWriter \u9488\u5bf9 PostgreSQL \u7c7b\u578b\u8f6c\u6362\u5217\u8868:

    Addax \u5185\u90e8\u7c7b\u578b PostgreSQL \u6570\u636e\u7c7b\u578b Long bigint, bigserial, integer, smallint, serial Double double precision, money, numeric, real String varchar, char, text, bit, inet,cidr,macaddr,uuid,xml,json Date date, time, timestamp Boolean bool Bytes bytea"},{"location":"writer/postgresqlwriter/#_7","title":"\u5df2\u77e5\u9650\u5236","text":"

    \u9664\u4ee5\u4e0a\u5217\u51fa\u7684\u6570\u636e\u7c7b\u578b\u5916\uff0c\u5176\u4ed6\u6570\u636e\u7c7b\u578b\u7406\u8bba\u4e0a\u5747\u4e3a\u8f6c\u4e3a\u5b57\u7b26\u4e32\u7c7b\u578b\uff0c\u4f46\u4e0d\u786e\u4fdd\u51c6\u786e\u6027

    "},{"location":"writer/rdbmswriter/","title":"RDBMS Writer","text":"

    RDBMS Writer \u63d2\u4ef6\u652f\u6301\u4ece\u4f20\u7edf RDBMS \u8bfb\u53d6\u6570\u636e\u3002\u8fd9\u662f\u4e00\u4e2a\u901a\u7528\u5173\u7cfb\u6570\u636e\u5e93\u8bfb\u53d6\u63d2\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u6ce8\u518c\u6570\u636e\u5e93\u9a71\u52a8\u7b49\u65b9\u5f0f\u652f\u6301\u66f4\u591a\u5173\u7cfb\u6570\u636e\u5e93\u8bfb\u53d6\u3002

    \u540c\u65f6 RDBMS Writer \u53c8\u662f\u5176\u4ed6\u5173\u7cfb\u578b\u6570\u636e\u5e93\u8bfb\u53d6\u63d2\u4ef6\u7684\u7684\u57fa\u7840\u7c7b\u3002\u4ee5\u4e0b\u8bfb\u53d6\u63d2\u4ef6\u5747\u4f9d\u8d56\u8be5\u63d2\u4ef6

    \u6ce8\u610f, \u5982\u679c\u5df2\u7ecf\u63d0\u4f9b\u4e86\u4e13\u95e8\u7684\u6570\u636e\u5e93\u5199\u5165\u63d2\u4ef6\u7684\uff0c\u63a8\u8350\u4f7f\u7528\u4e13\u7528\u63d2\u4ef6\uff0c\u5982\u679c\u4f60\u9700\u8981\u5199\u5165\u7684\u6570\u636e\u5e93\u6ca1\u6709\u4e13\u95e8\u63d2\u4ef6\uff0c\u5219\u8003\u8651\u4f7f\u7528\u8be5\u901a\u7528\u63d2\u4ef6\u3002 \u5728\u4f7f\u7528\u4e4b\u524d\uff0c\u8fd8\u9700\u8981\u6267\u884c\u4ee5\u4e0b\u64cd\u4f5c\u624d\u53ef\u4ee5\u6b63\u5e38\u8fd0\u884c\uff0c\u5426\u5219\u8fd0\u884c\u4f1a\u51fa\u73b0\u5f02\u5e38\u3002

    "},{"location":"writer/rdbmswriter/#_1","title":"\u914d\u7f6e\u9a71\u52a8","text":"

    \u5047\u5b9a\u4f60\u9700\u8981\u5199\u5165 IBM DB2 \u7684\u6570\u636e\uff0c\u56e0\u4e3a\u6ca1\u6709\u63d0\u4f9b\u4e13\u95e8\u7684\u8bfb\u53d6\u63d2\u4ef6\uff0c\u6240\u4ee5\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528\u8be5\u63d2\u4ef6\u6765\u5b9e\u73b0\uff0c\u5728\u4f7f\u7528\u4e4b\u524d\uff0c\u9700\u8981\u6267\u884c\u4e0b\u9762\u4e24\u4e2a\u64cd\u4f5c\uff1a

    1. \u4e0b\u8f7d\u5bf9\u5e94\u7684 JDBC \u9a71\u52a8\uff0c\u5e76\u62f7\u8d1d\u5230 plugin/writer/rdbmswriter/libs \u76ee\u5f55
    2. \u4fee\u6539\u4efb\u52a1\u914d\u7f6e\u6587\u4ef6\uff0c\u627e\u5230 driver \u4e00\u9879\uff0c\u586b\u5199\u6b63\u786e\u7684 JDBC \u9a71\u52a8\u540d\uff0c\u6bd4\u5982 DB2 \u7684\u9a71\u52a8\u540d\u4e3a com.ibm.db2.jcc.DB2Driver

    \u4ee5\u4e0b\u5217\u51fa\u5e38\u89c1\u7684\u6570\u636e\u5e93\u4ee5\u53ca\u5bf9\u5e94\u7684\u9a71\u52a8\u540d\u79f0

    "},{"location":"writer/rdbmswriter/#_2","title":"\u914d\u7f6e\u8bf4\u660e","text":"

    \u914d\u7f6e\u4e00\u4e2a\u5199\u5165RDBMS\u7684\u4f5c\u4e1a\u3002

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"rdbmswriter\",\n        \"parameter\": {\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:dm://ip:port/database\",\n            \"table\": [\n              \"table\"\n            ]\n          },\n          \"username\": \"username\",\n          \"password\": \"password\",\n          \"driver\": \"dm.jdbc.driver.DmDriver\",\n          \"column\": [\n            \"*\"\n          ],\n          \"preSql\": [\n            \"delete from XXX;\"\n          ]\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/rdbmswriter/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 jdbcUrl \u662f string \u65e0 \u5bf9\u7aef\u6570\u636e\u5e93\u7684JDBC\u8fde\u63a5\u4fe1\u606f\uff0cjdbcUrl\u6309\u7167RDBMS\u5b98\u65b9\u89c4\u8303\uff0c\u5e76\u53ef\u4ee5\u586b\u5199\u8fde\u63a5\u9644\u4ef6\u63a7\u5236\u4fe1\u606f \uff5c driver \u662f string \u65e0 \u81ea\u5b9a\u4e49\u9a71\u52a8\u7c7b\u540d\uff0c\u89e3\u51b3\u517c\u5bb9\u6027\u95ee\u9898\uff0c\u8be6\u89c1\u4e0b\u9762\u63cf\u8ff0 username \u662f string \u65e0 \u6570\u636e\u6e90\u7684\u7528\u6237\u540d password \u5426 string \u65e0 \u6570\u636e\u6e90\u6307\u5b9a\u7528\u6237\u540d\u7684\u5bc6\u7801 table \u662f list \u65e0 \u6240\u9009\u53d6\u7684\u9700\u8981\u540c\u6b65\u7684\u8868\u540d,\u4f7f\u7528JSON\u6570\u636e\u683c\u5f0f\uff0c\u5f53\u914d\u7f6e\u4e3a\u591a\u5f20\u8868\u65f6\uff0c\u7528\u6237\u81ea\u5df1\u9700\u4fdd\u8bc1\u591a\u5f20\u8868\u662f\u540c\u4e00\u8868\u7ed3\u6784 column \u662f list \u65e0 \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u8be6\u7ec6\u63cf\u8ff0\u89c1\u540e preSql \u5426 list \u65e0 \u6267\u884c\u6570\u636e\u540c\u6b65\u4efb\u52a1\u4e4b\u524d\u7387\u5148\u6267\u884c\u7684sql\u8bed\u53e5\uff0c\u76ee\u524d\u53ea\u5141\u8bb8\u6267\u884c\u4e00\u6761SQL\u8bed\u53e5\uff0c\u4f8b\u5982\u6e05\u9664\u65e7\u6570\u636e,\u6d89\u53ca\u5230\u7684\u8868\u53ef\u7528 @table\u8868\u793a postSql \u5426 list \u65e0 \u6267\u884c\u6570\u636e\u540c\u6b65\u4efb\u52a1\u4e4b\u540e\u6267\u884c\u7684sql\u8bed\u53e5\uff0c\u76ee\u524d\u53ea\u5141\u8bb8\u6267\u884c\u4e00\u6761SQL\u8bed\u53e5\uff0c\u4f8b\u5982\u52a0\u4e0a\u67d0\u4e00\u4e2a\u65f6\u95f4\u6233 batchSize \u5426 int 1024 \u5b9a\u4e49\u4e86\u63d2\u4ef6\u548c\u6570\u636e\u5e93\u670d\u52a1\u5668\u7aef\u6bcf\u6b21\u6279\u91cf\u6570\u636e\u83b7\u53d6\u6761\u6570\uff0c\u8c03\u9ad8\u8be5\u503c\u53ef\u80fd\u5bfc\u81f4 Addax \u51fa\u73b0OOM\u6216\u8005\u76ee\u6807\u6570\u636e\u5e93\u4e8b\u52a1\u63d0\u4ea4\u5931\u8d25\u5bfc\u81f4\u6302\u8d77 session \u662f\u5426 list \u65e0 \u9488\u5bf9\u672c\u5730\u8fde\u63a5,\u4fee\u6539\u4f1a\u8bdd\u914d\u7f6e,\u8be6\u89c1\u4e0b\u6587"},{"location":"writer/rdbmswriter/#column","title":"column","text":"

    \u6240\u914d\u7f6e\u7684\u8868\u4e2d\u9700\u8981\u540c\u6b65\u7684\u5217\u540d\u96c6\u5408\uff0c\u4f7f\u7528JSON\u7684\u6570\u7ec4\u63cf\u8ff0\u5b57\u6bb5\u4fe1\u606f\u3002\u7528\u6237\u4f7f\u7528 * \u4ee3\u8868\u9ed8\u8ba4\u4f7f\u7528\u6240\u6709\u5217\u914d\u7f6e\uff0c\u4f8b\u5982 [\"*\"]\u3002

    \u652f\u6301\u5217\u88c1\u526a\uff0c\u5373\u5217\u53ef\u4ee5\u6311\u9009\u90e8\u5206\u5217\u8fdb\u884c\u5bfc\u51fa\u3002

    \u652f\u6301\u5217\u6362\u5e8f\uff0c\u5373\u5217\u53ef\u4ee5\u4e0d\u6309\u7167\u8868schema\u4fe1\u606f\u8fdb\u884c\u5bfc\u51fa\u3002

    \u652f\u6301\u5e38\u91cf\u914d\u7f6e\uff0c\u7528\u6237\u9700\u8981\u6309\u7167JSON\u683c\u5f0f:

    [\"id\", \"`table`\", \"1\", \"'bazhen.csy'\", \"null\", \"to_char(a + 1)\", \"2.3\" , \"true\"]

    Column\u5fc5\u987b\u663e\u793a\u586b\u5199\uff0c\u4e0d\u5141\u8bb8\u4e3a\u7a7a\uff01

    "},{"location":"writer/rdbmswriter/#jdbcurl","title":"jdbcUrl","text":"

    jdbcUrl \u914d\u7f6e\u9664\u4e86\u914d\u7f6e\u5fc5\u8981\u7684\u4fe1\u606f\u5916\uff0c\u6211\u4eec\u8fd8\u53ef\u4ee5\u5728\u589e\u52a0\u6bcf\u79cd\u7279\u5b9a\u9a71\u52a8\u7684\u7279\u5b9a\u914d\u7f6e\u5c5e\u6027\uff0c\u8fd9\u91cc\u7279\u522b\u63d0\u5230\u6211\u4eec\u53ef\u4ee5\u5229\u7528\u914d\u7f6e\u5c5e\u6027\u5bf9\u4ee3\u7406\u7684\u652f\u6301\u4ece\u800c\u5b9e\u73b0\u901a\u8fc7\u4ee3\u7406\u8bbf\u95ee\u6570\u636e\u5e93\u7684\u529f\u80fd\u3002 \u6bd4\u5982\u5bf9\u4e8e PrestoSQL \u6570\u636e\u5e93\u7684 JDBC \u9a71\u52a8\u800c\u8a00\uff0c\u652f\u6301 socksProxy \u53c2\u6570\uff0c\u6bd4\u5982\u4e00\u4e2a\u53ef\u80fd\u7684 jdbcUrl \u4e3a

    jdbc:presto://127.0.0.1:8080/hive?socksProxy=192.168.1.101:1081

    \u5927\u90e8\u5206\u5173\u7cfb\u578b\u6570\u636e\u5e93\u7684 JDBC \u9a71\u52a8\u652f\u6301 socksProxyHost,socksProxyPort \u53c2\u6570\u6765\u652f\u6301\u4ee3\u7406\u8bbf\u95ee\u3002\u4e5f\u6709\u4e00\u4e9b\u7279\u522b\u7684\u60c5\u51b5\u3002

    \u4ee5\u4e0b\u662f\u5404\u7c7b\u6570\u636e\u5e93 JDBC \u9a71\u52a8\u6240\u652f\u6301\u7684\u4ee3\u7406\u7c7b\u578b\u4ee5\u53ca\u914d\u7f6e\u65b9\u5f0f

    \u6570\u636e\u5e93 \u4ee3\u7406\u7c7b\u578b \u4ee3\u7406\u914d\u7f6e \u4f8b\u5b50 MySQL socks socksProxyHost,socksProxyPort socksProxyHost=192.168.1.101&socksProxyPort=1081 Presto socks socksProxy socksProxy=192.168.1.101:1081 Presto http httpProxy httpProxy=192.168.1.101:3128"},{"location":"writer/rdbmswriter/#driver","title":"driver","text":"

    \u5927\u90e8\u5206\u60c5\u51b5\u4e0b\uff0c\u4e00\u4e2a\u6570\u636e\u5e93\u7684JDBC\u9a71\u52a8\u662f\u56fa\u5b9a\u7684\uff0c\u4f46\u6709\u4e9b\u56e0\u4e3a\u7248\u672c\u7684\u4e0d\u540c\uff0c\u6240\u5efa\u8bae\u7684\u9a71\u52a8\u7c7b\u540d\u4e0d\u540c\uff0c\u6bd4\u5982 MySQL\u3002 \u65b0\u7684 MySQL JDBC \u9a71\u52a8\u7c7b\u578b\u63a8\u8350\u4f7f\u7528 com.mysql.cj.jdbc.Driver \u800c\u4e0d\u662f\u4ee5\u524d\u7684 com.mysql.jdbc.Drver\u3002 \u5982\u679c\u60f3\u8981\u4f7f\u7528\u5c31\u7684\u9a71\u52a8\u540d\u79f0\uff0c\u5219\u53ef\u4ee5\u914d\u7f6e driver \u914d\u7f6e\u9879\u3002

    "},{"location":"writer/rdbmswriter/#session","title":"session","text":"

    \u63cf\u8ff0\uff1a\u8bbe\u7f6e\u6570\u636e\u5e93\u8fde\u63a5\u65f6\u7684session\u4fe1\u606f\uff0c\u6bd4\u5982\u9488\u5bf9 Oracle \u6570\u636e\u5e93\uff0c\u53ef\u4ee5\u8bbe\u7f6e\u5982\u4e0b\uff1a

    {\n  \"session\": [\n    \"alter session set nls_date_format = 'dd.mm.yyyy hh24:mi:ss';\",\n    \"alter session set NLS_LANG = 'AMERICAN';\"\n  ]\n}\n
    "},{"location":"writer/rediswriter/","title":"Redis Writer","text":"

    Redis Writer \u63d0\u4f9b\u4e86\u8fd8\u539f Redis dump \u547d\u4ee4\u7684\u80fd\u529b\uff0c\u5e76\u5199\u5165\u5230\u76ee\u6807 Redis\u3002\u652f\u6301 redis cluster \u96c6\u7fa4\u3001proxy\u3001\u4ee5\u53ca\u5355\u673a

    "},{"location":"writer/rediswriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"content\": {\n      \"reader\": {\n        \"name\": \"redisreader\",\n        \"parameter\": {\n          \"connection\": [\n            {\n              \"uri\": \"tcp://127.0.0.1:7003\"\n            }\n          ]\n        }\n      },\n      \"writer\": {\n        \"name\": \"rediswriter\",\n        \"parameter\": {\n          \"connection\": {\n            \"uri\": \"tcp://127.0.0.1:6379\",\n            \"auth\": \"123456\"\n          },\n          \"redisCluster\": false,\n          \"flushDB\": false\n        }\n      }\n    },\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    }\n  }\n}\n
    "},{"location":"writer/rediswriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 uri \u662f string \u5426 redis\u94fe\u63a5 redisCluster \u5426 boolean false \u662f\u5426\u4e3aredis cluster\u96c6\u7fa4,\u5982\u679c\u662f proxy \u6216\u5355\u673a\u5ffd\u7565\u8be5\u9879 flushDB \u5426 boolean false \u8fc1\u79fb\u524d\u662f\u5426\u6e05\u7a7a\u76ee\u6807 Redis batchSize \u5426 string 1000 \u6bcf\u6b21\u6279\u91cf\u5904\u7406\u6570\u91cf\u3002\u5982\u679ckey\u8fc7\u5927/\u5c0f,\u53ef\u4ee5\u76f8\u5e94\u7684\u8c03\u6574 timeout \u5426 string 60000 \u6bcf\u6b21\u6267\u884c\u6700\u5927\u8d85\u65f6\u65f6\u95f4, \u5355\u4f4d\u6beb\u79d2(ms)"},{"location":"writer/s3writer/","title":"S3 Writer","text":"

    S3 Writer \u63d2\u4ef6\u7528\u4e8e\u5c06\u6570\u636e\u5199\u5165 Amazon AWS S3 \u5b58\u50a8\uff0c\u4ee5\u53ca\u517c\u5bb9 S3 \u534f\u8bae\u7684\u5b58\u50a8\uff0c\u6bd4\u5982 MinIO\u3002

    \u5728\u5b9e\u73b0\u4e0a\uff0c\u672c\u63d2\u4ef6\u57fa\u4e8e S3 \u5b98\u65b9\u7684 SDK 2.0 \u7f16\u5199\u3002

    "},{"location":"writer/s3writer/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u4e0b\u9762\u7684\u914d\u7f6e\u7528\u4e8e\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\uff0c\u5e76\u5199\u5165\u5230\u6307\u5b9a\u7684 S3 bucket \u4e0a\u3002

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"byte\": -1,\n        \"channel\": 1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19890604,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1989-06-04 11:22:33\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 10\n        }\n      },\n      \"writer\": {\n        \"name\": \"s3writer\",\n        \"parameter\": {\n          \"endpoint\": \"https://s3.amazonaws.com\",\n          \"accessId\": \"xxxxxxxxxxxx\",\n          \"accessKey\": \"xxxxxxxxxxxxxxxxxxxxxxx\",\n          \"bucket\": \"test\",\n          \"object\": \"upload.csv\",\n          \"region\": \"ap-northeast-1\",\n          \"encoding\": \"\",\n          \"fieldDelimiter\": \",\",\n          \"writeMode\": \"truncate\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/s3writer/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 endpoint \u662f string \u65e0 S3 Server\u7684 EndPoint\u5730\u5740\uff0c\u4f8b\u5982 s3.xx.amazonaws.com region \u662f string \u65e0 S3 Server\u7684 Region \u5730\u5740\uff0c\u4f8b\u5982 ap-southeast-1 accessId \u662f string \u65e0 \u8bbf\u95ee ID accessKey \u662f string \u65e0 \u8bbf\u95ee Key bucket \u662f string \u65e0 \u8981\u5199\u5165\u7684 bucket object \u662f string \u65e0 \u8981\u5199\u5165\u7684 object\uff0c\u6ce8\u610f\u4e8b\u9879\u89c1\u4e0b fieldDelimiter \u5426 char ',' \u5b57\u6bb5\u7684\u5206\u9694\u7b26 nullFormat \u5426 char \\N \u5f53\u503c\u4e3a\u7a7a\u65f6\uff0c\u7528\u4ec0\u4e48\u5b57\u7b26\u8868\u793a header \u5426 list \u65e0 \u5199\u5165\u6587\u4ef6\u5934\u4fe1\u606f\uff0c\u6bd4\u5982 [\"id\",\"title\",\"url\"] maxFileSize \u5426 int 100000 \u5355\u4e2a object \u7684\u5927\u5c0f\uff0c\u5355\u4f4d\u4e3a MB encoding \u5426 string utf-8 \u6587\u4ef6\u7f16\u7801\u683c\u5f0f writeMode \u5426 string append \u5199\u5165\u6a21\u5f0f\uff0c\u8be6\u89c1 hdfswriter \u4e2d\u76f8\u5173\u63cf\u8ff0"},{"location":"writer/s3writer/#object","title":"object","text":"

    \u4e0a\u8ff0\u914d\u7f6e\u4e2d\u7684 object \u914d\u7f6e\u7684\u867d\u7136\u662f upload.csv \u6587\u4ef6\uff0c\u5b9e\u9645\u4e0a\u5728 S3 \u5199\u5165\u7684\u6587\u4ef6\u540d\u4f1a\u5728\u6307\u5b9a\u7684\u6587\u4ef6\u540d\u540e\u9762\u52a0\u4e0a uuid \u540e\u7f00\uff0c \u7c7b\u4f3c upload.csv_c0d2ca7df0444933a6f18ea76718b569\u3002 \u8fd9\u662f\u7528\u4e8e\u5728\u591a\u901a\u9053\u5199\u5165\u7684\u60c5\u51b5\u4e0b\uff0c\u786e\u4fdd\u6587\u4ef6\u540d\u4e0d\u4f1a\u91cd\u540d\u3002

    "},{"location":"writer/s3writer/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b S3 \u6570\u636e\u7c7b\u578b Long int, tinyint, smallint, mediumint, int, bigint Double float, double, decimal String varchar, char, tinytext, text, mediumtext, longtext, year,xml Date date, datetime, timestamp, time Boolean bit, bool Bytes tinyblob, mediumblob, blob, longblob, varbinary"},{"location":"writer/s3writer/#_4","title":"\u9650\u5236\u8bf4\u660e","text":"
    1. \u4ec5\u652f\u6301\u5199\u5165\u6587\u672c\u6587\u4ef6\u683c\u5f0f
    2. \u6682\u4e0d\u652f\u6301\u538b\u7f29
    "},{"location":"writer/sqlitewriter/","title":"SQLite Writer","text":"

    SQLite Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 SQLite \u6570\u636e\u5e93\u7684\u529f\u80fd\u3002

    "},{"location":"writer/sqlitewriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684\u8868\u5982\u4e0b\uff1a

    create table addax_tbl\n(\n    col1 varchar(20) ,\n    col2 int(4),\n    col3 datetime,\n    col4 boolean,\n    col5 binary\n);\n

    \u8fd9\u91cc\u4f7f\u7528\u4e00\u4efd\u4ece\u5185\u5b58\u4ea7\u751f\u5230 SQLite \u7684\u6570\u636e\u3002

    job/stream2sqlite.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"sqlitewriter\",\n        \"parameter\": {\n          \"writeMode\": \"insert\",\n          \"column\": [\n            \"*\"\n          ],\n          \"preSql\": [\n            \"delete from @table\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:sqlite://tmp/writer.sqlite3\",\n            \"table\": [\n              \"addax_tbl\"\n            ]\n          }\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2sqlite.json

    "},{"location":"writer/sqlitewriter/#_2","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/stream2sqlite.json\n
    "},{"location":"writer/sqlitewriter/#_3","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002\u56e0\u4e3a SQLite \u8fde\u63a5\u65e0\u9700\u8d26\u53f7\u5bc6\u7801\uff0c\u56e0\u6b64\u5176\u4ed6\u6570\u636e\u5e93\u5199\u5165\u63d2\u4ef6\u9700\u8981\u914d\u7f6e\u7684 username, password \u5728\u8fd9\u91cc\u4e0d\u9700\u8981\u3002

    "},{"location":"writer/sqlitewriter/#writemode","title":"writeMode","text":""},{"location":"writer/sqlitewriter/#_4","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b SQLite \u6570\u636e\u7c7b\u578b Long integer Double real String varchar Date datetime Boolean bool Bytes blob, binary"},{"location":"writer/sqlserverwriter/","title":"SQLServer Writer","text":"

    SQLServer Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 SQL Server \u5e93\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/sqlserverwriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u8fd9\u91cc\u4f7f\u7528\u4e00\u4efd\u4ece\u5185\u5b58\u4ea7\u751f\u5230 SQL Server \u5bfc\u5165\u7684\u6570\u636e\u3002

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {},\n      \"writer\": {\n        \"name\": \"sqlserverwriter\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"root\",\n          \"column\": [\n            \"db_id\",\n            \"db_type\",\n            \"db_ip\",\n            \"db_port\",\n            \"db_role\",\n            \"db_name\",\n            \"db_username\",\n            \"db_password\",\n            \"db_modify_time\",\n            \"db_modify_user\",\n            \"db_description\",\n            \"db_tddl_info\"\n          ],\n          \"connection\": {\n            \"table\": [\n              \"db_info_for_writer\"\n            ],\n            \"jdbcUrl\": \"jdbc:sqlserver://[HOST_NAME]:PORT;DatabaseName=[DATABASE_NAME]\"\n          },\n          \"preSql\": [\n            \"delete from @table where db_id = -1;\"\n          ],\n          \"postSql\": [\n            \"update @table set db_modify_time = now() where db_id = 1;\"\n          ]\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/sqlserverwriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/sqlserverwriter/#writemode","title":"writeMode","text":"

    \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c \u91c7\u53d6 insert into \u8bed\u6cd5\u5199\u5165 SQL Server \u8868\uff0c\u5982\u679c\u4f60\u5e0c\u671b\u91c7\u53d6\u4e3b\u952e\u5b58\u5728\u65f6\u66f4\u65b0\uff0c\u4e0d\u5b58\u5728\u5219\u5199\u5165\u7684\u65b9\u5f0f\uff0c\u4e5f\u5c31\u662f SQL Server \u7684 MERGE INTO \u8bed\u6cd5, \u53ef\u4ee5\u4f7f\u7528 update \u6a21\u5f0f\u3002 \u5047\u5b9a\u8868\u7684\u4e3b\u952e\u4e3a id ,\u5219 writeMode \u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff1a

    {\n  \"writeMode\": \"update(id)\"\n}\n

    \u5982\u679c\u662f\u8054\u5408\u552f\u4e00\u7d22\u5f15\uff0c\u5219\u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff1a

    {\n  \"writeMode\": \"update(col1, col2)\"\n}\n
    "},{"location":"writer/starrockswriter/","title":"StarRocks Writer","text":"

    StarRocks Writer \u63d2\u4ef6\u7528\u4e8e\u5411 Starrocks \u6570\u636e\u5e93\u4ee5\u6d41\u5f0f\u65b9\u5f0f\u5199\u5165\u6570\u636e\u3002 \u5176\u5b9e\u73b0\u4e0a\u662f\u901a\u8fc7\u8bbf\u95ee Doris http \u8fde\u63a5(8030) \uff0c\u7136\u540e\u901a\u8fc7 stream load \u52a0\u8f7d\u6570\u636e\u5230\u6570\u636e\u4e2d\uff0c\u76f8\u6bd4 insert into \u65b9\u5f0f\u6548\u7387\u8981\u9ad8\u4e0d\u5c11\uff0c\u4e5f\u662f\u5b98\u65b9\u63a8\u8350\u7684\u751f\u4ea7\u73af\u5883\u4e0b\u7684\u6570\u636e\u52a0\u8f7d\u65b9\u5f0f\u3002

    StarRocks \u662f\u4e00\u4e2a\u517c\u5bb9 MySQL \u534f\u8bae\u7684\u6570\u636e\u5e93\u540e\u7aef\uff0c\u56e0\u6b64 Doris \u8bfb\u53d6\u53ef\u4ee5\u4f7f\u7528 MySQLReader \u8fdb\u884c\u8bbf\u95ee\u3002

    "},{"location":"writer/starrockswriter/#_1","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684\u8868\u7684\u5efa\u8868\u8bed\u53e5\u5982\u4e0b\uff1a

    CREATE DATABASE example_db;\nCREATE TABLE example_db.table1\n(\n    siteid   INT         DEFAULT '10',\n    citycode SMALLINT,\n    username VARCHAR(32) DEFAULT '',\n    pv       BIGINT SUM DEFAULT '0'\n) AGGREGATE KEY(siteid, citycode, username)\nDISTRIBUTED BY HASH(siteid) BUCKETS 10\nPROPERTIES(\"replication_num\" = \"1\");\n

    \u4e0b\u9762\u914d\u7f6e\u4e00\u4e2a\u4ece\u5185\u5b58\u8bfb\u53d6\u6570\u636e\uff0c\u7136\u540e\u5199\u5165\u5230 doris \u8868\u7684\u914d\u7f6e\u6587\u4ef6

    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2\n      }\n    },\n    \"content\": {\n      \"writer\": {\n        \"name\": \"starrockswriter\",\n        \"parameter\": {\n          \"username\": \"test\",\n          \"password\": \"123456\",\n          \"database\": \"example_db\",\n          \"table\": \"table1\",\n          \"jdbcUrl\": \"jdbc:mysql://172.28.17.100:9030/\",\n          \"loadUrl\": [\"172.28.17.100:8030\", \"172.28.17.100:8030\"],\n          \"loadProps\": {\n               \"column_separator\": \"\\\\x01\",\n               \"row_delimiter\": \"\\\\x02\"\n           },\n          \"column\": [\"siteid\", \"citycode\", \"username\", \"pv\"]\n        }\n      },\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"1,500\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"1,127\",\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"this is a text\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"5,200\",\n              \"type\": \"long\"\n            }\n          ],\n          \"sliceRecordCount\": 100\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2starrocks.json

    \u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4

    bin/addax.sh job/stream2starrocks.json\n
    "},{"location":"writer/starrockswriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\uff0c\u5e76\u589e\u52a0\u4e86\u4e0b\u9762\u7684\u914d\u7f6e\u9879\u3002

    \u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 loadUrl \u662f string \u65e0 StarRocks FE \u7684\u5730\u5740\u7528\u4e8eStreamLoad1\uff0c\u53ef\u4ee5\u4e3a\u591a\u4e2afe\u5730\u5740\uff0cfe_ip:fe_http_port maxBatchRows \u5426 int 500000 \u5355\u6b21StreamLoad\u5bfc\u5165\u7684\u6700\u5927\u884c\u6570 maxBatchSize \u5426 int 104857600 \u5355\u6b21StreamLoad\u5bfc\u5165\u7684\u6700\u5927\u5b57\u8282\u6570 flushInterval \u5426 int 300000 \u4e0a\u4e00\u6b21StreamLoad\u7ed3\u675f\u81f3\u4e0b\u4e00\u6b21\u5f00\u59cb\u7684\u65f6\u95f4\u95f4\u9694(\u5355\u4f4d\uff1ams) loadProps \u5426 map streamLoad \u7684\u8bf7\u6c42\u53c2\u6570\uff0c\u8be6\u60c5\u53c2\u7167StreamLoad\u4ecb\u7ecd\u9875\u9762"},{"location":"writer/starrockswriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u9ed8\u8ba4\u4f20\u5165\u7684\u6570\u636e\u5747\u4f1a\u88ab\u8f6c\u4e3a\u5b57\u7b26\u4e32\uff0c\u5e76\u4ee5\\t\u4f5c\u4e3a\u5217\u5206\u9694\u7b26\uff0c\\n\u4f5c\u4e3a\u884c\u5206\u9694\u7b26\uff0c\u7ec4\u6210csv\u6587\u4ef6\u8fdb\u884cStreamLoad\u5bfc\u5165\u64cd\u4f5c\u3002 \u5982\u9700\u66f4\u6539\u5217\u5206\u9694\u7b26\uff0c \u5219\u6b63\u786e\u914d\u7f6e loadProps \u5373\u53ef\uff1a 

    \"loadProps\": {\n    \"column_separator\": \"\\\\x01\",\n    \"row_delimiter\": \"\\\\x02\"\n}\n

    \u5982\u9700\u66f4\u6539\u5bfc\u5165\u683c\u5f0f\u4e3ajson\uff0c \u5219\u6b63\u786e\u914d\u7f6e loadProps \u5373\u53ef\uff1a 

    \"loadProps\": {\n    \"format\": \"json\",\n    \"strip_outer_array\": true\n}\n

    "},{"location":"writer/streamwriter/","title":"Stream Writer","text":"

    Stream Writer \u662f\u4e00\u4e2a\u5c06\u6570\u636e\u5199\u5165\u5185\u5b58\u7684\u63d2\u4ef6\uff0c\u4e00\u822c\u7528\u6765\u5c06\u83b7\u53d6\u5230\u7684\u6570\u636e\u5199\u5230\u7ec8\u7aef\uff0c\u7528\u6765\u8c03\u8bd5\u8bfb\u53d6\u63d2\u4ef6\u7684\u6570\u636e\u5904\u7406\u60c5\u51b5\u3002

    \u4e00\u4e2a\u5178\u578b\u7684 Stream Writer \u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"name\": \"streamwriter\",\n  \"parameter\": {\n    \"encoding\": \"UTF-8\",\n    \"print\": true,\n    \"nullFormat\": \"NULL\"\n  }\n}\n

    \u4e0a\u8ff0\u914d\u7f6e\u4f1a\u5c06\u83b7\u53d6\u7684\u6570\u636e\u76f4\u63a5\u6253\u5370\u5230\u7ec8\u7aef\u3002 \u5176\u4e2d nullFormat \u7528\u6765\u6307\u5b9a\u5f53\u503c\u4e3a\u7a7a\u65f6\uff0c\u5982\u4f55\u5728\u7ec8\u7aef\u8868\u793a\uff0c\u9ed8\u8ba4\u662f\u5b57\u7b26\u4e32 NULL, \u5982\u679c\u4e0d\u60f3\u6253\u5370\u7a7a\u503c\uff0c\u53ef\u4ee5\u5c06\u5176\u8bbe\u7f6e\u4e3a \"\"\u3002

    \u8be5\u63d2\u4ef6\u4e5f\u652f\u6301\u5c06\u6570\u636e\u5199\u5165\u5230\u6587\u4ef6\uff0c\u914d\u7f6e\u5982\u4e0b\uff1a

    {\n  \"name\": \"streamwriter\",\n  \"parameter\": {\n    \"encoding\": \"UTF-8\",\n    \"path\": \"/tmp/out\",\n    \"fileName\": \"out.txt\",\n    \"fieldDelimiter\": \",\",\n    \"recordNumBeforeSleep\": \"100\",\n    \"sleepTime\": \"5\"\n  }\n}\n

    \u4e0a\u8ff0\u914d\u7f6e\u4e2d:

    \u4e0a\u8ff0\u914d\u7f6e\u7684\u542b\u4e49\u662f\u5c06\u6570\u636e\u5199\u5165\u5230 /tmp/out/out.txt \u6587\u4ef6\uff0c \u6bcf\u83b7\u53d6100\u6761\u8bb0\u5f55\u540e\uff0c\u4f11\u77205\u79d2\u3002

    "},{"location":"writer/sybasewriter/","title":"Sybase Writer","text":"

    Sybase Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5199\u5165\u6570\u636e\u5230 Sybase \u5e93\u8868\u7684\u529f\u80fd\u3002

    "},{"location":"writer/sybasewriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"

    \u6211\u4eec\u53ef\u4ee5\u7528 Docker \u5bb9\u5668\u6765\u542f\u52a8\u4e00\u4e2a Sybase \u6570\u636e\u5e93

    docker run -tid --rm  -h dksybase --name sybase  -p 5000:5000  ifnazar/sybase_15_7 bash /sybase/start\n

    \u7136\u540e\u521b\u5efa\u4e00\u5f20\u5982\u4e0b\u8868

    create table addax_writer \n(\n    id int,\n    name varchar(255),\n    salary float(2),\n    created_at datetime,\n    updated_at datetime\n);\n

    \u518d\u4f7f\u7528\u4e0b\u9762\u7684\u4efb\u52a1\u914d\u7f6e\u6587\u4ef6

    {\n  \"job\": {\n    \"content\": {\n      \"writer\": {\n        \"name\": \"sybasewriter\",\n        \"parameter\": {\n          \"column\": [\n            \"id\",\n            \"name\",\n            \"salary\",\n            \"created_at\",\n            \"updated_at\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:sybase:Tds:127.0.0.1:5000/master\",\n            \"table\": [\n              \"dbo.addax_writer\"\n            ]\n          },\n          \"username\": \"sa\",\n          \"password\": \"password\"\n        }\n      },\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"100,1000\",\n              \"type\": \"long\"\n            },\n            {\n              \"random\": \"10,100\",\n              \"type\": \"string\"\n            },\n            {\n              \"random\": \"10,1000\",\n              \"type\": \"double\"\n            },\n            {\n              \"incr\": \"2022-01-01 13:00:00,2,d\",\n              \"type\": \"date\"\n            },\n            {\n              \"incr\": \"2023-01-01 13:00:00,2,d\",\n              \"type\": \"date\"\n            }\n          ],\n          \"sliceRecordCount\": 100\n        }\n      }\n    },\n    \"setting\": {\n      \"speed\": {\n        \"bytes\": -1,\n        \"channel\": 1\n      }\n    }\n  }\n}\n
    "},{"location":"writer/sybasewriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/tdenginewriter/","title":"TDengine Writer","text":"

    TDengine Writer \u63d2\u4ef6\u5b9e\u73b0\u4e86\u5c06\u6570\u636e\u5199\u5165 TDengine \u6570\u636e\u5e93\u7cfb\u7edf\u3002\u5728\u5e95\u5c42\u5b9e\u73b0\u4e0a\uff0cTDengine Writer \u901a\u8fc7JDBC JNI \u9a71\u52a8\u8fde\u63a5\u8fdc\u7a0b TDengine \u6570\u636e\u5e93\uff0c \u5e76\u6267\u884c\u76f8\u5e94\u7684sql\u8bed\u53e5\u5c06\u6570\u636e\u6279\u91cf\u5199\u5165 TDengine \u5e93\u4e2d\u3002

    "},{"location":"writer/tdenginewriter/#_1","title":"\u524d\u7f6e\u6761\u4ef6","text":"

    \u8003\u8651\u5230\u6027\u80fd\u95ee\u9898\uff0c\u8be5\u63d2\u4ef6\u4f7f\u7528\u4e86 TDengine \u7684 JDBC-JNI \u9a71\u52a8\uff0c \u8be5\u9a71\u52a8\u76f4\u63a5\u8c03\u7528\u5ba2\u6237\u7aef API\uff08libtaos.so \u6216 taos.dll\uff09\u5c06\u5199\u5165\u548c\u67e5\u8be2\u8bf7\u6c42\u53d1\u9001\u5230 taosd \u5b9e\u4f8b\u3002 \u56e0\u6b64\u5728\u4f7f\u7528\u4e4b\u524d\u9700\u8981\u914d\u7f6e\u597d\u52a8\u6001\u5e93\u94fe\u63a5\u6587\u4ef6\u3002

    \u9996\u5148\u5c06 plugin/writer/tdenginewriter/libs/libtaos.so.2.0.16.0 \u62f7\u8d1d\u5230 /usr/lib64 \u76ee\u5f55\uff0c\u7136\u540e\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u521b\u5efa\u8f6f\u94fe\u63a5

    ln -sf /usr/lib64/libtaos.so.2.0.16.0 /usr/lib64/libtaos.so.1\nln -sf /usr/lib64/libtaos.so.1 /usr/lib64/libtaos.so\n
    "},{"location":"writer/tdenginewriter/#_2","title":"\u793a\u4f8b","text":"

    \u5047\u5b9a\u8981\u5199\u5165\u7684\u8868\u5982\u4e0b\uff1a

    create table test.addax_test (\n    ts timestamp,\n    name nchar(100),\n    file_size int,\n    file_date timestamp,\n    flag_open bool,\n    memo nchar(100)\n);\n

    \u4ee5\u4e0b\u662f\u914d\u7f6e\u6587\u4ef6

    job/stream2tdengine.json 
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"streamreader\",\n        \"parameter\": {\n          \"column\": [\n            {\n              \"random\": \"2017-08-01 00:01:02,2020-01-01 12:13:14\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": \"Addax\",\n              \"type\": \"string\"\n            },\n            {\n              \"value\": 19880808,\n              \"type\": \"long\"\n            },\n            {\n              \"value\": \"1988-08-08 08:08:08\",\n              \"type\": \"date\"\n            },\n            {\n              \"value\": true,\n              \"type\": \"bool\"\n            },\n            {\n              \"value\": \"test\",\n              \"type\": \"bytes\"\n            }\n          ],\n          \"sliceRecordCount\": 1000\n        }\n      },\n      \"writer\": {\n        \"name\": \"tdenginewriter\",\n        \"parameter\": {\n          \"username\": \"root\",\n          \"password\": \"taosdata\",\n          \"column\": [\n            \"ts\",\n            \"name\",\n            \"file_size\",\n            \"file_date\",\n            \"flag_open\",\n            \"memo\"\n          ],\n          \"connection\": {\n            \"jdbcUrl\": \"jdbc:TAOS://127.0.0.1:6030/test\",\n            \"table\": [\n              \"addax_test\"\n            ]\n          }\n        }\n      }\n    }\n  }\n}\n

    \u5c06\u4e0a\u8ff0\u914d\u7f6e\u6587\u4ef6\u4fdd\u5b58\u4e3a job/stream2tdengine.json

    "},{"location":"writer/tdenginewriter/#_3","title":"\u6267\u884c\u91c7\u96c6\u547d\u4ee4","text":"

    \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u91c7\u96c6

    bin/addax.sh job/tdengine2stream.json\n

    \u547d\u4ee4\u8f93\u51fa\u7c7b\u4f3c\u5982\u4e0b\uff1a

    2021-02-20 15:52:07.691 [main] INFO  VMInfo - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl\n2021-02-20 15:52:07.748 [main] INFO  Engine -\n{\n    \"content\":\n        {\n            \"reader\":{\n                \"parameter\":{\n                    \"column\":[\n                        {\n                            \"random\":\"2017-08-01 00:01:02,2020-01-01 12:13:14\",\n                            \"type\":\"date\"\n                        },\n                        {\n                            \"type\":\"string\",\n                            \"value\":\"Addax\"\n                        },\n                        {\n                            \"type\":\"long\",\n                            \"value\":19880808\n                        },\n                        {\n                            \"type\":\"date\",\n                            \"value\":\"1988-08-08 08:08:08\"\n                        },\n                        {\n                            \"type\":\"bool\",\n                            \"value\":true\n                        },\n                        {\n                            \"type\":\"bytes\",\n                            \"value\":\"test\"\n                        }\n                    ],\n                    \"sliceRecordCount\":1000\n                },\n                \"name\":\"streamreader\"\n            },\n            \"writer\":{\n                \"parameter\":{\n                    \"password\":\"*****\",\n                    \"column\":[\n                        \"ts\",\n                        \"name\",\n                        \"file_size\",\n                        \"file_date\",\n                        \"flag_open\",\n                        \"memo\"\n                    ],\n                    \"connection\":[\n                        {\n                            \"jdbcUrl\":\"jdbc:TAOS://127.0.0.1:6030/test\",\n                            \"table\":[\n                                \"addax_test\"\n                            ]\n                        }\n                    ],\n                    \"username\":\"root\",\n                    \"preSql\":[]\n                },\n                \"name\":\"tdenginewriter\"\n            }\n    },\n    \"setting\":{\n        \"speed\":{\n            \"bytes\":-1,\n            \"channel\":1\n        }\n    }\n}\n\n2021-02-20 15:52:07.786 [main] INFO  PerfTrace - PerfTrace traceId=job_-1, isEnable=false, priority=0\n2021-02-20 15:52:07.787 [main] INFO  JobContainer - Addax jobContainer starts job.\n2021-02-20 15:52:07.789 [main] INFO  JobContainer - Set jobId = 0\njava.library.path:/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib\n2021-02-20 15:52:08.048 [job-0] INFO  OriginalConfPretreatmentUtil - table:[addax_test] all columns:[ts,name,file_size,file_date,flag_open,memo].\n2021-02-20 15:52:08.056 [job-0] INFO  OriginalConfPretreatmentUtil - Write data [\nINSERT INTO %s (ts,name,file_size,file_date,flag_open,memo) VALUES(?,?,?,?,?,?)\n], which jdbcUrl like:[jdbc:TAOS://127.0.0.1:6030/test]\n\n2021-02-20 15:52:11.158 [job-0] INFO  JobContainer -\n\u4efb\u52a1\u542f\u52a8\u65f6\u523b                    : 2021-02-20 15:52:07\n\u4efb\u52a1\u7ed3\u675f\u65f6\u523b                    : 2021-02-20 15:52:11\n\u4efb\u52a1\u603b\u8ba1\u8017\u65f6                    :                  3s\n\u4efb\u52a1\u5e73\u5747\u6d41\u91cf                    :           11.07KB/s\n\u8bb0\u5f55\u5199\u5165\u901f\u5ea6                    :            333rec/s\n\u8bfb\u51fa\u8bb0\u5f55\u603b\u6570                    :                1000\n\u8bfb\u5199\u5931\u8d25\u603b\u6570                    :                   0\n
    "},{"location":"writer/tdenginewriter/#_4","title":"\u53c2\u6570\u8bf4\u660e","text":"

    \u8be5\u63d2\u4ef6\u57fa\u4e8e RDBMS Writer \u5b9e\u73b0\uff0c\u56e0\u6b64\u53ef\u4ee5\u53c2\u8003 RDBMS Writer \u7684\u6240\u6709\u914d\u7f6e\u9879\uff0c\u5e76\u4e14\u589e\u52a0\u4e86\u4e00\u4e9b TDengine \u7279\u6709\u7684\u914d\u7f6e\u9879\u3002

    "},{"location":"writer/tdenginewriter/#jdbc-restful","title":"\u4f7f\u7528 JDBC-RESTful \u63a5\u53e3","text":"

    \u5982\u679c\u4e0d\u60f3\u4f9d\u8d56\u672c\u5730\u5e93\uff0c\u6216\u8005\u6ca1\u6709\u6743\u9650\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528 JDBC-RESTful \u63a5\u53e3\u6765\u5199\u5165\u8868\uff0c\u76f8\u6bd4 JDBC-JNI \u800c\u8a00\uff0c\u914d\u7f6e\u533a\u522b\u662f\uff1a

    \u6240\u4ee5\u4e0a\u8ff0\u914d\u7f6e\u4e2d\u7684 connection \u5e94\u8be5\u4fee\u6539\u4e3a\u5982\u4e0b\uff1a

    {\n  \"connection\": [\n    {\n      \"jdbcUrl\": \"jdbc:TAOS-RS://127.0.0.1:6041/test\",\n      \"table\": [\n        \"addax_test\"\n      ],\n      \"driver\": \"com.taosdata.jdbc.rs.RestfulDriver\"\n    }\n  ]\n}\n
    "},{"location":"writer/tdenginewriter/#_5","title":"\u7c7b\u578b\u8f6c\u6362","text":"

    \u76ee\u524d TDenginereader \u652f\u6301 TDengine \u6240\u6709\u7c7b\u578b\uff0c\u5177\u4f53\u5982\u4e0b

    Addax \u5185\u90e8\u7c7b\u578b TDengine \u6570\u636e\u7c7b\u578b Long SMALLINT, TINYINT, INT, BIGINT, TIMESTAMP Double FLOAT, DOUBLE String BINARY, NCHAR Boolean BOOL"},{"location":"writer/tdenginewriter/#_6","title":"\u5f53\u524d\u652f\u6301\u7248\u672c","text":"

    TDengine 2.0.16

    "},{"location":"writer/tdenginewriter/#_7","title":"\u6ce8\u610f\u4e8b\u9879","text":""},{"location":"writer/txtfilewriter/","title":"TxtFile Writer","text":"

    TxtFile Writer \u63d0\u4f9b\u4e86\u5411\u672c\u5730\u6587\u4ef6\u5199\u5165\u7c7b CSV \u683c\u5f0f\u7684\u4e00\u4e2a\u6216\u8005\u591a\u4e2a\u8868\u6587\u4ef6\u3002

    "},{"location":"writer/txtfilewriter/#_1","title":"\u914d\u7f6e\u6837\u4f8b","text":"
    {\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 2,\n        \"bytes\": -1\n      }\n    },\n    \"content\": {\n      \"reader\": {\n        \"name\": \"txtfilereader\",\n        \"parameter\": {\n          \"path\": [\n            \"/tmp/data\"\n          ],\n          \"encoding\": \"UTF-8\",\n          \"column\": [\n            {\n              \"index\": 0,\n              \"type\": \"long\"\n            },\n            {\n              \"index\": 1,\n              \"type\": \"boolean\"\n            },\n            {\n              \"index\": 2,\n              \"type\": \"double\"\n            },\n            {\n              \"index\": 3,\n              \"type\": \"string\"\n            },\n            {\n              \"index\": 4,\n              \"type\": \"date\",\n              \"format\": \"yyyy.MM.dd\"\n            }\n          ],\n          \"fieldDelimiter\": \",\"\n        }\n      },\n      \"writer\": {\n        \"name\": \"txtfilewriter\",\n        \"parameter\": {\n          \"path\": \"/tmp/result\",\n          \"fileName\": \"luohw\",\n          \"writeMode\": \"truncate\",\n          \"dateFormat\": \"yyyy-MM-dd\"\n        }\n      }\n    }\n  }\n}\n
    "},{"location":"writer/txtfilewriter/#_2","title":"\u53c2\u6570\u8bf4\u660e","text":"\u914d\u7f6e\u9879 \u662f\u5426\u5fc5\u987b \u6570\u636e\u7c7b\u578b \u9ed8\u8ba4\u503c \u63cf\u8ff0 path \u662f string \u65e0 \u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684\u8def\u5f84\u4fe1\u606f\uff0c\u5199\u5165 Path \u76ee\u5f55\u4e0b\u5c5e\u591a\u4e2a\u6587\u4ef6 fileName \u662f string \u65e0 \u5199\u5165\u7684\u6587\u4ef6\u540d\uff0c\u8be5\u6587\u4ef6\u540d\u4f1a\u6dfb\u52a0\u968f\u673a\u7684\u540e\u7f00\u4f5c\u4e3a\u6bcf\u4e2a\u7ebf\u7a0b\u5199\u5165\u5b9e\u9645\u6587\u4ef6\u540d writeMode \u662f string \u65e0 \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff0c\u8be6\u89c1\u4e0b\u6587 fieldDelimiter \u662f string , \u63cf\u8ff0\uff1a\u8bfb\u53d6\u7684\u5b57\u6bb5\u5206\u9694\u7b26 compress \u5426 string \u65e0 \u6587\u672c\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u538b\u7f29\u7c7b\u578b\u4e3a zip\u3001lzo\u3001lzop\u3001tgz\u3001bzip2 encoding \u5426 string utf-8 \u8bfb\u53d6\u6587\u4ef6\u7684\u7f16\u7801\u914d\u7f6e nullFormat \u5426 string \\N \u5b9a\u4e49\u54ea\u4e9b\u5b57\u7b26\u4e32\u53ef\u4ee5\u8868\u793a\u4e3a null dateFormat \u5426 string \u65e0 \u65e5\u671f\u7c7b\u578b\u7684\u6570\u636e\u5e8f\u5217\u5316\u5230\u6587\u4ef6\u4e2d\u65f6\u7684\u683c\u5f0f\uff0c\u4f8b\u5982 \"yyyy-MM-dd\" fileFormat \u5426 string text \u6587\u4ef6\u5199\u51fa\u7684\u683c\u5f0f\uff0c\u8be6\u89c1\u4e0b\u6587 table \u662f string \u65e0 sql \u6a21\u5f0f\u65f6\u9700\u8981\u6307\u5b9a\u8868\u540d\uff0c column \u5426 list \u65e0 sql \u6a21\u5f0f\u65f6\u53ef\u9009\u6307\u5b9a\u5217\u540d\uff0c extendedInsert \u5426 boolean true sql \u6a21\u5f0f\u65f6\u662f\u5426\u4f7f\u7528\u6279\u91cf\u63d2\u5165\u8bed\u6cd5\uff0c\u8be6\u89c1\u4e0b\u6587 batchSize \u5426 int 2048 sql \u6a21\u5f0f\u65f6\u6279\u91cf\u63d2\u5165\u8bed\u6cd5\u7684\u6279\u6b21\u5927\u5c0f\uff0c\u8be6\u89c1\u4e0b\u6587 header \u5426 list \u65e0 text \u5199\u51fa\u65f6\u7684\u8868\u5934\uff0c\u793a\u4f8b ['id', 'name', 'age']"},{"location":"writer/txtfilewriter/#writemode","title":"writeMode","text":"

    \u5199\u5165\u524d\u6570\u636e\u6e05\u7406\u5904\u7406\u6a21\u5f0f\uff1a

    "},{"location":"writer/txtfilewriter/#fileformat","title":"fileFormat","text":"

    \u6587\u4ef6\u5199\u51fa\u7684\u683c\u5f0f\uff0c\u5305\u62ec csv \u548c text \u548c 4.1.3 \u7248\u672c\u5f15\u5165\u7684 sql \u4e09\u79cd\uff0ccsv \u662f\u4e25\u683c\u7684 csv \u683c\u5f0f\uff0c\u5982\u679c\u5f85\u5199\u6570\u636e\u5305\u62ec\u5217\u5206\u9694\u7b26\uff0c\u5219\u4f1a\u6309\u7167 csv \u7684\u8f6c\u4e49\u8bed\u6cd5\u8f6c\u4e49\uff0c\u8f6c\u4e49\u7b26\u53f7\u4e3a\u53cc\u5f15\u53f7 \"\uff1b text \u683c\u5f0f\u662f\u7528\u5217\u5206\u9694\u7b26\u7b80\u5355\u5206\u5272\u5f85\u5199\u6570\u636e\uff0c\u5bf9\u4e8e\u5f85\u5199\u6570\u636e\u5305\u62ec\u5217\u5206\u9694\u7b26\u60c5\u51b5\u4e0b\u4e0d\u505a\u8f6c\u4e49\u3002 sql \u683c\u5f0f\u8868\u793a\u5c06\u6570\u636e\u4ee5 SQL \u8bed\u53e5 (INSERT INTO ... VALUES) \u7684\u65b9\u5f0f\u5199\u5165\u5230\u6587\u4ef6

    "},{"location":"writer/txtfilewriter/#table","title":"table","text":"

    \u4ec5\u5728 sql \u6587\u4ef6\u683c\u5f0f\u4e0b\u9700\u8981\uff0c\u7528\u6765\u6307\u5b9a\u5199\u5165\u7684\u8868\u540d

    "},{"location":"writer/txtfilewriter/#column","title":"column","text":"

    \u5728 sql \u6587\u4ef6\u683c\u5f0f\u4e0b\uff0c\u53ef\u4ee5\u6307\u5b9a\u5199\u5165\u7684\u5217\u540d\uff0c\u5982\u679c\u6307\u5b9a\uff0c\u5219 sql \u8bed\u53e5\u7c7b\u4f3c INSERT INTO table (col1, col2, col3) VALUES (val1, val2, val3)\uff0c\u6a21\u5f0f \u5426\u5219\u4e3a INSERT INTO table VALUES (val1, val2, val3)\u3002\u6a21\u5f0f

    "},{"location":"writer/txtfilewriter/#extendedinsert","title":"extendedInsert","text":"

    \u662f\u5426\u542f\u7528\u6279\u91cf\u63d2\u5165\u8bed\u6cd5\uff0c\u5982\u679c\u542f\u7528\uff0c\u5219\u4f1a\u5c06 batchSize \u4e2a\u6570\u636e\u4e00\u6b21\u6027\u5199\u5165\u5230\u6587\u4ef6\u4e2d\uff0c\u5426\u5219\u6bcf\u4e2a\u6570\u636e\u4e00\u884c\u3002\u8be5\u53c2\u6570\u501f\u9274\u4e86 mysqldump \u5de5\u5177\u7684 extended-insert \u53c2\u6570\u8bed\u6cd5

    "},{"location":"writer/txtfilewriter/#batchsize","title":"batchSize","text":"

    \u6279\u91cf\u63d2\u5165\u8bed\u6cd5\u7684\u6279\u6b21\u5927\u5c0f\uff0c\u5982\u679c extendedInsert \u4e3a true\uff0c\u5219\u6bcf batchSize \u4e2a\u6570\u636e\u4e00\u6b21\u6027\u5199\u5165\u5230\u6587\u4ef6\u4e2d\uff0c\u5426\u5219\u6bcf\u4e2a\u6570\u636e\u4e00\u884c\u3002

    "},{"location":"writer/txtfilewriter/#_3","title":"\u7c7b\u578b\u8f6c\u6362","text":"Addax \u5185\u90e8\u7c7b\u578b \u672c\u5730\u6587\u4ef6 \u6570\u636e\u7c7b\u578b Long Long Double Double string string Boolean Boolean Date Date"}]} \ No newline at end of file diff --git a/develop/sitemap.xml b/develop/sitemap.xml index 04dc60a05..9e9208c12 100644 --- a/develop/sitemap.xml +++ b/develop/sitemap.xml @@ -2,310 +2,310 @@ https://wgzhao.github.io/Addax/develop/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/debug/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/encrypt_password/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/plugin_development/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/quickstart/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/setupJob/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/statsreport/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/transformer/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/accessreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/cassandrareader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/clickhousereader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/databendreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/datareader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/dbfreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/elasticsearchreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/excelreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/ftpreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/hanareader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/hbase11xreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/hbase11xsqlreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/hbase20xreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/hbase20xsqlreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/hdfsreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/hivereader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/httpreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/influxdb2reader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/influxdbreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/jsonfilereader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/kafkareader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/kudureader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/mongodbreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/mysqlreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/oraclereader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/postgresqlreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/rdbmsreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/redisreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/s3reader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/sqlitereader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/sqlserverreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/streamreader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/sybasereader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/tdenginereader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/reader/txtfilereader/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/accesswriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/cassandrawriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/clickhousewriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/databendwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/dbfwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/doriswriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/elasticsearchwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/excelwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/ftpwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/greenplumwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/hanawriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/hbase11xsqlwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/hbase11xwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/hbase20xsqlwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/hdfswriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/influxdb2writer/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/influxdbwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/kafkawriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/kuduwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/mongodbwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/mysqlwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/oraclewriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/paimonwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/postgresqlwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/rdbmswriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/rediswriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/s3writer/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/sqlitewriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/sqlserverwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/starrockswriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/streamwriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/sybasewriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/tdenginewriter/ - 2025-01-24 + 2025-02-06 https://wgzhao.github.io/Addax/develop/writer/txtfilewriter/ - 2025-01-24 + 2025-02-06 \ No newline at end of file diff --git a/develop/sitemap.xml.gz b/develop/sitemap.xml.gz index 7a0e24e0b..fb80362cf 100644 Binary files a/develop/sitemap.xml.gz and b/develop/sitemap.xml.gz differ diff --git a/develop/writer/hdfswriter/index.html b/develop/writer/hdfswriter/index.html index 1fe1d24d2..06ac35701 100644 --- a/develop/writer/hdfswriter/index.html +++ b/develop/writer/hdfswriter/index.html @@ -1744,6 +1744,15 @@ + + +
  • + + + hdfsSitePath + + +
    • @@ -2423,6 +2432,15 @@ + + +
  • + + + hdfsSitePath + + +
    • @@ -2679,6 +2697,13 @@

    参数说明

    + + + + + + +
    false 是否忽略preShell, postShell 命令的错误
    hdfsSitePath否string无hdfs-site.xml 的路径，详细解释见下

    path

    @@ -2758,6 +2783,14 @@

    preShell 与 postShellignoreError

    该配置项用于控制是否忽略 preShell 和 postShell 命令的错误，如果配置为 true，则在执行 preShell 和 postShell 命令时，如果命令执行失败，不会导致任务失败，而是会打印错误日志，继续执行任务。 否则，如果配置为 false，则在执行 preShell 和 postShell 命令时，如果命令执行失败，会导致任务失败。

    +

    hdfsSitePath

    +

    这是 4.2.4 引入的新配置想，用于指定 hdfs-site.xml 文件的路径，比如对 HDP/CDH 而言，可以这样配置：

    +
    {
+  "hdfsSitePath": "/etc/hadoop/conf/hdfs-site.xml"
+}
+
    +

    如果配置了 hdfsSitePath , 则插件会从该文件中获得访问 HDFS 文件系统必要的配置，从而在大部分情况下不在需要配置 hadoopConfig，减少配置量。

    +

    对于把 Addax 部署在 Hadoop 集群上的场景，推荐使用这种方式。

    类型转换

    diff --git a/versions.json b/versions.json index a728daf0c..dc39780c1 100644 --- a/versions.json +++ b/versions.json @@ -3,9 +3,9 @@ "version": "develop", "title": "develop", "aliases": [ - "last", "lastest", - "latest" + "latest", + "last" ] }, {