update

Author: netnr

README.md

@@ -2,20 +2,276 @@
 
 中华人民共和国行政区划：省级、地级、县级、乡级和村级
 
+> Gitee <https://gitee.com/netnr/zoning>
+
+> GitHub <https://github.com/netnr/zoning>
+
 ----------
-# 数据来源
+# 来源
 <http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/>
 
 <http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017>
 
 ----------
-# 使用方法
+# 使用
 - 打开页面 <http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/index.html>
 - 打开浏览器控制台（推荐谷歌，请不要用IE系列，谢谢）
 - 拷贝脚本`zoning.js`的内容粘贴到控制台运行
 
 ----------
-# 注意事项
-初次抓取可能会出错，请求太频繁，浏览器可能会卡顿，也会出现网络错误等问题
+# 注意
+首次抓取会出现大量失败请求，再次抓取会从浏览器缓存获取，非常快
+
+----------
+# 代码
+```
+var zoning = {
+    //版本号
+    version: "1.0.0",
+    //载入js脚本
+    getScript: function (src, success) {
+        var ele = document.createElement("SCRIPT");
+        ele.src = src;
+        ele.type = "text/javascript";
+        document.getElementsByTagName("HEAD")[0].appendChild(ele);
+        //加载完成回调
+        if (success != undefined) {
+            ele.onload = ele.onreadystatechange = function () {
+                if (!this.readyState || this.readyState == "loaded" || this.readyState == "complete") { success(); }
+            }
+        }
+    },
+    //参数配置
+    config: {
+        //jszip CDN
+        urljszip: "https://lib.baomitu.com/jszip/3.1.4/jszip.min.js",
+        //fileSaver CDN
+        urlfilesaver: "https://lib.baomitu.com/FileSaver.js/2014-11-29/FileSaver.min.js",
+        //抓取首页
+        urlprefix: "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/",
+        //起始深度
+        deep: 1,
+        //最大深度
+        //5 村 约46800
+        //4 街道 约3380
+        deepmax: 5,
+        //抓取过程信息
+        item: {
+            //父级编码
+            id: "00",
+            //请求相对地址
+            href: "index"
+        }
+    },
+    //fetch 抓取
+    grab: function (urlprefix, deep, item) {
+        if (item.href == null) {
+            return false;
+        }
+        var url = urlprefix;
+        switch (deep) {
+            case 4:
+                url += item.id.substr(0, 2) + "/";
+                break;
+            case 5:
+                url += item.id.substr(0, 2) + "/" + item.id.substr(2, 2) + "/";
+                break;
+        }
+        url += item.href + ".html";
+
+        zoning.taskcount += 1;
+
+        //fetch 抓取 gb2312
+        fetch(url).then(res => res.blob()).then(blob => {
+            var reader = new FileReader();
+            reader.onload = function () {
+                var list = zoning.matcharray(reader.result, item, deep);
+                zoning.taskcount -= 1;
+                if (list.length > 0 && deep < zoning.config.deepmax) {
+                    for (var i = 0; i < list.length; i++) {
+                        var li = list[i];
+                        deep += 1;
+                        zoning.grab(urlprefix, deep, li);
+                        deep -= 1;
+                    }
+                }
+            }
+            reader.readAsText(blob, 'GBK');
+        }).catch(function (e) {
+            var obj = {};
+            obj.item = item;
+            obj.url = url;
+            obj.error = e;
+            zoning.catchdata.push(obj);
+            zoning.taskcount -= 1;
+        });
+    },
+    //任务量
+    taskcount: 0,
+    //抓取数量
+    matchcount: 0,
+    //抓取异常记录
+    catchdata: [],
+    //抓取结果数据
+    matchdata: {},
+    //匹配抓取内容
+    matcharray: function (data, item, deep) {
+        var arr = [];
+        if (deep != 5) {
+            //匹配 市辖区 无链接 项
+            data.replace(/<td>[0-9]{12}<\/td><td>.*?<\/td>/g, function (x) {
+                var mat = x.split('</td><td>');
+                var obj = {};
+                obj.href = null;
+                obj.id = mat[0].split('>')[1];
+                obj.text = mat[1].split('<')[0];
+                arr.push(obj);
+            });
+        }
+        data = data.replace(/'/g, '"').replace(/<br\/>/g, "");
+        //匹配所有的A标签
+        var reg = /<a[^>]*href=['"]([^"]*)['"][^>]*>(.*?)<\/a>/g;
+        var matchs = data.match(reg);
+        var filename = "00";
+        switch (deep) {
+            //首页
+            case 1:
+                if (!matchs) {
+                    return [];
+                }
+                for (var i = 0; i < matchs.length; i++) {
+                    var mat = matchs[i];
+                    var obj = {};
+                    obj.id = mat.split('"')[1].split('.')[0];
+                    obj.href = obj.id;
+                    obj.text = mat.split('>')[1].split('<')[0];
+                    arr.push(obj);
+                }
+                break;
+            case 2:
+            case 3:
+            case 4:
+                if (!matchs) {
+                    return [];
+                }
+                for (var i = 0; i < matchs.length; i++) {
+                    var mat = matchs[i];
+                    var obj = {};
+                    obj.href = mat.split('"')[1].split('.')[0];
+                    obj.id = mat.split('>')[1].split('<')[0];
+                    mat = matchs[++i];
+                    obj.text = mat.split('>')[1].split('<')[0];
+                    arr.push(obj);
+                }
+                break;
+            case 5:
+                //匹配 村委会 无连接
+                data.replace(/<td>[0-9]{12}<\/td><td>[0-9]{3}<\/td><td>.*?<\/td>/g, function (x) {
+                    var mat = x.split('</td><td>');
+                    var obj = {};
+                    obj.href = null;
+                    obj.id = mat[0].split('>')[1];
+                    obj.text = mat[2].split('<')[0];
+                    arr.push(obj);
+                });
+                break;
+        }
+        //根据深度 得到文件名（编码）
+        switch (deep) {
+            case 2:
+                filename = item.id;
+                break;
+            case 3:
+                filename = item.id.substr(0, 4);
+                break;
+            case 4:
+                filename = item.id.substr(0, 6);
+                break;
+            case 5:
+                filename = item.id.substr(0, 9);
+                break;
+        }
+        zoning.matchdata[filename] = arr;
+        //记录请求结果数量
+        zoning.matchcount += 1;
+        return arr;
+    },
+    //外部调用生成下载
+    zip: function () {
+        zoning.ziping(zoning.matchdata, zoning.catchdata);
+    },
+    //内部调用生成下载
+    ziping: function (matchdata, catchdata) {
+        zoning.getScript(zoning.config.urljszip, function () {
+            zoning.getScript(zoning.config.urlfilesaver, function () {
+                var zip = new JSZip();
+                var data = {};
+                for (var i in matchdata) {
+                    var di = matchdata[i];
+                    for (var j = 0; j < di.length; j++) {
+                        delete di[j].href;
+                        switch (i.length) {
+                            case 2:
+                                di[j].id = di[j].id.substr(0, 4);
+                                break;
+                            case 4:
+                                di[j].id = di[j].id.substr(0, 6);
+                                break;
+                            case 6:
+                                di[j].id = di[j].id.substr(0, 9);
+                                break;
+                        }
+                    }
+                    data[i] = di;
+                    zip.file(i + ".json", JSON.stringify(di));
+                }
+                zip.file("all.json", JSON.stringify(data));
+                if (catchdata.length) {
+                    zip.file('catch.json', JSON.stringify(catchdata));
+                }
+                zip.generateAsync({ type: "blob" }).then(function (content) {
+                    saveAs(content, "zoning.zip");
+                });
+            });
+        });
+    },
+    //开始运行
+    run: function () {
+        zoning.startTime = new Date().valueOf();
+        zoning.taskid = setInterval(function () {
+            console.log("count:" + zoning.matchcount, "taskcount:" + zoning.taskcount);
+            if (zoning.taskcount == 0) {
+                clearInterval(zoning.taskid);
+                zoning.zip();
+            }
+        }, 1000 * 4);
+        console.log('fetching ... please see the network tab');
+        zoning.grab(zoning.config.urlprefix, zoning.config.deep, zoning.config.item);
+    }
+};
+
+//开始运行 可手动调用
+zoning.run();
+
+//下载zip，抓取完成后
+//zoning.zip();
+
+/*
+ * 注意：
+ * 
+ * 首次抓取会出现大量失败请求，再次抓取会从浏览器缓存获取，非常快。
+ * 
+ * 文件：
+ * 00.json 根数据
+ * 12.json 二级数据
+ * 1234.json 三级数据
+ * 123456.json 四级数据
+ * 123456789.json 五级数据
+ * 
+ * 其他：
+ * all.json 所有数据
+ * catch.json 抓取异常记录（有异常时，经测试有5个页面请求失败）
+ */
+```
 
-由于浏览器有缓存机制，第二次开始从缓存获取页面内容，很快能完成
+> [联系打赏](https://ss.netnr.com/contact)

dist/zoning/11.json

@@ -1 +1 @@
-[{"id":"110100000000","text":"市辖区"}]
+[{"id":"1101","text":"市辖区"}]

dist/zoning/1101.json

@@ -1 +1 @@
-[{"id":"110101000000","text":"东城区"},{"id":"110102000000","text":"西城区"},{"id":"110105000000","text":"朝阳区"},{"id":"110106000000","text":"丰台区"},{"id":"110107000000","text":"石景山区"},{"id":"110108000000","text":"海淀区"},{"id":"110109000000","text":"门头沟区"},{"id":"110111000000","text":"房山区"},{"id":"110112000000","text":"通州区"},{"id":"110113000000","text":"顺义区"},{"id":"110114000000","text":"昌平区"},{"id":"110115000000","text":"大兴区"},{"id":"110116000000","text":"怀柔区"},{"id":"110117000000","text":"平谷区"},{"id":"110118000000","text":"密云区"},{"id":"110119000000","text":"延庆区"}]
+[{"id":"110101","text":"东城区"},{"id":"110102","text":"西城区"},{"id":"110105","text":"朝阳区"},{"id":"110106","text":"丰台区"},{"id":"110107","text":"石景山区"},{"id":"110108","text":"海淀区"},{"id":"110109","text":"门头沟区"},{"id":"110111","text":"房山区"},{"id":"110112","text":"通州区"},{"id":"110113","text":"顺义区"},{"id":"110114","text":"昌平区"},{"id":"110115","text":"大兴区"},{"id":"110116","text":"怀柔区"},{"id":"110117","text":"平谷区"},{"id":"110118","text":"密云区"},{"id":"110119","text":"延庆区"}]

dist/zoning/110101.json

@@ -1 +1 @@
-[{"id":"110101001000","text":"东华门街道办事处"},{"id":"110101002000","text":"景山街道办事处"},{"id":"110101003000","text":"交道口街道办事处"},{"id":"110101004000","text":"安定门街道办事处"},{"id":"110101005000","text":"北新桥街道办事处"},{"id":"110101006000","text":"东四街道办事处"},{"id":"110101007000","text":"朝阳门街道办事处"},{"id":"110101008000","text":"建国门街道办事处"},{"id":"110101009000","text":"东直门街道办事处"},{"id":"110101010000","text":"和平里街道办事处"},{"id":"110101011000","text":"前门街道办事处"},{"id":"110101012000","text":"崇文门外街道办事处"},{"id":"110101013000","text":"东花市街道办事处"},{"id":"110101014000","text":"龙潭街道办事处"},{"id":"110101015000","text":"体育馆路街道办事处"},{"id":"110101016000","text":"天坛街道办事处"},{"id":"110101017000","text":"永定门外街道办事处"}]
+[{"id":"110101001","text":"东华门街道办事处"},{"id":"110101002","text":"景山街道办事处"},{"id":"110101003","text":"交道口街道办事处"},{"id":"110101004","text":"安定门街道办事处"},{"id":"110101005","text":"北新桥街道办事处"},{"id":"110101006","text":"东四街道办事处"},{"id":"110101007","text":"朝阳门街道办事处"},{"id":"110101008","text":"建国门街道办事处"},{"id":"110101009","text":"东直门街道办事处"},{"id":"110101010","text":"和平里街道办事处"},{"id":"110101011","text":"前门街道办事处"},{"id":"110101012","text":"崇文门外街道办事处"},{"id":"110101013","text":"东花市街道办事处"},{"id":"110101014","text":"龙潭街道办事处"},{"id":"110101015","text":"体育馆路街道办事处"},{"id":"110101016","text":"天坛街道办事处"},{"id":"110101017","text":"永定门外街道办事处"}]

dist/zoning/110101001.json

@@ -0,0 +1 @@
+[{"id":"110101001001","text":"多福巷社区居委会"},{"id":"110101001002","text":"银闸社区居委会"},{"id":"110101001005","text":"东厂社区居委会"},{"id":"110101001006","text":"智德社区居委会"},{"id":"110101001007","text":"南池子社区居委会"},{"id":"110101001008","text":"黄图岗社区居委会"},{"id":"110101001009","text":"灯市口社区居委会"},{"id":"110101001010","text":"正义路社区居委会"},{"id":"110101001011","text":"甘雨社区居委会"},{"id":"110101001013","text":"台基厂社区居委会"},{"id":"110101001014","text":"韶九社区居委会"},{"id":"110101001015","text":"王府井社区居委会"}]

dist/zoning/110101002.json

@@ -0,0 +1 @@
+[{"id":"110101002001","text":"隆福寺社区居委会"},{"id":"110101002002","text":"吉祥社区居委会"},{"id":"110101002003","text":"黄化门社区居委会"},{"id":"110101002004","text":"钟鼓社区居委会"},{"id":"110101002005","text":"魏家社区居委会"},{"id":"110101002006","text":"汪芝麻社区居委会"},{"id":"110101002008","text":"景山东街社区居委会"},{"id":"110101002009","text":"皇城根北街社区居委会"}]

dist/zoning/110101003.json

@@ -0,0 +1 @@
+[{"id":"110101003001","text":"交东社区居委会"},{"id":"110101003002","text":"福祥社区居委会"},{"id":"110101003003","text":"大兴社区居委会"},{"id":"110101003005","text":"府学社区居委会"},{"id":"110101003007","text":"鼓楼苑社区居委会"},{"id":"110101003008","text":"菊儿社区居委会"},{"id":"110101003009","text":"南锣鼓巷社区居委会"}]

dist/zoning/110101004.json

@@ -0,0 +1 @@
+[{"id":"110101004001","text":"交北头条社区居委会"},{"id":"110101004002","text":"北锣鼓巷社区居委会"},{"id":"110101004003","text":"国子监社区居委会"},{"id":"110101004004","text":"钟楼湾社区居委会"},{"id":"110101004005","text":"宝钞南社区居委会"},{"id":"110101004006","text":"五道营社区居委会"},{"id":"110101004009","text":"分司厅社区居委会"},{"id":"110101004011","text":"国旺社区居委会"},{"id":"110101004012","text":"花园社区居委会"}]

dist/zoning/110101005.json

@@ -0,0 +1 @@
+[{"id":"110101005001","text":"海运仓社区居委会"},{"id":"110101005002","text":"北新仓社区居委会"},{"id":"110101005004","text":"门楼社区居委会"},{"id":"110101005005","text":"十三条社区居委会"},{"id":"110101005006","text":"民安社区居委会"},{"id":"110101005008","text":"九道湾社区居委会"},{"id":"110101005009","text":"北官厅社区居委会"},{"id":"110101005010","text":"青龙社区居委会"},{"id":"110101005011","text":"小菊社区居委会"},{"id":"110101005012","text":"藏经馆社区居委会"},{"id":"110101005014","text":"草园社区居委会"},{"id":"110101005016","text":"前永康社区居委会"}]

dist/zoning/110101006.json

@@ -0,0 +1 @@
+[{"id":"110101006001","text":"南门仓社区居委会"},{"id":"110101006002","text":"七条社区居委会"},{"id":"110101006003","text":"二条社区居委会"},{"id":"110101006006","text":"六条社区居委会"},{"id":"110101006007","text":"豆瓣社区居委会"},{"id":"110101006008","text":"八条社区居委会"},{"id":"110101006009","text":"总院社区居委会"}]