爬取高德地图地址信息（周边设施经纬度）

2024-03-29 18:19:10

需求：给定进百万小区名称及对应城市信息，通过高德获取其周边设施信息（交通、医疗、教育、生活设施）

高德JS API

爬取灵感来自高德开发支持的JS API，当然你也可以直接调用高德提供的api服务获取地址信息，但是api服务是有次数限制。而JS API的调用并没有次数限制，经我的验证一个爬取节点控制在500毫秒的抓取频率是没有限制的。
https://lbs.amap.com/api/javascript-api/example/poi-search/keywords-search

高德地址信息获取原理

通过JS引入高德的AMap对象，调用相关方法获取对应数据。JS调用高德地图显示周边信息也是这个逻辑。

准备工作

注册高德地图账号->控制台创建应用->申请Key
需要申请Key才可以在自己js代码中使用高德js

数据抓取

数据服务API：
读取小区数据放入list中每来一次请求从list中pop一条数据返回，这样前端可以多个节点多个任务同时抓取。

数据获取：
要想使用高德需引入如下2个js，其中的key即为高德官网申请的key

<script src="http://cache.amap.com/lbs/static/jquery-1.9.1.js"></script>
<script src="http://webapi.amap.com/maps?v=1.3&key=************&plugin=AMap.ArrivalRange,AMap.Scale,AMap.Geocoder,AMap.Transfer,AMap.Autocomplete" id="amap_key"></script>

get请求获取小区数据，创建高德对象

$.ajax({
        type:"get", 
        url:"http://127.0.0.1:5075/api/community",
        dataType:"json",
        contentType:"application/json",
        success:function(data){ 
            params = data;
            console.log('Getapi data',data)
            MAP = new AMap.Map("container", {
                resizeEnable: true
            });
            getAddress(data)
        }
     });

调用高德JS搜素小区信息：
1.在AMap.service方法中构造构造地点查询类
2.调用地点查询类的search方法返回结果result
3.使用返回结果中的小区经纬度调用searchNearBy去获取周边数据

AMap.service(["AMap.PlaceSearch"], function() {
            var placeSearch = new AMap.PlaceSearch({ //构造地点查询类
                pageSize: 3,
                pageIndex: 1,
                city: city, //城市
                map: MAP,
                type:"商务住宅|门牌信息",
                citylimit:true,
                extensions:"all",
                // panel: "panel"
            });
            //关键字查询
     
            placeSearch.search(keyword, function(status, result) {
                console.log('getAddress info',result.info,result);
                
                searchNearBy(city,cpoint,keyword);
            });
        });

前端打印的搜索结果，获取的信息非常详细包括省市县及其编码，以及经纬度信息等：

完整代码

将代码保存至index.html，在同级目录下python（python -m SimpleHTTPServer 8000）本地启动http服务。
浏览器请求：http://localhost:8000?key=***************************

<!doctype html>
<html>

<head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="initial-scale=1.0, user-scalable=no, width=device-width">
    <title>高德房源地址获取</title>
    <script src="http://cache.amap.com/lbs/static/jquery-1.9.1.js"></script>
    <!-- <script src="http://webapi.amap.com/maps?v=1.3&key=01aa9e655798548a06c7697db75a9903&plugin=AMap.ArrivalRange,AMap.Scale,AMap.Geocoder,AMap.Transfer,AMap.Autocomplete" id="amap_key"></script> -->
    <script type="text/javascript">
        // 根据请求url获取小区信息 及构造高德js
        function GetRequest() {
           var url = decodeURIComponent(location.search); //获取url中"?"符后的字串  
           var theRequest = new Object();  
           if (url.indexOf("?") != -1) {  
              var str = url.substr(1);  
              strs = str.split("&");  
              for(var i = 0; i < strs.length; i ++) {  
                 theRequest[strs[i].split("=")[0]]=unescape(strs[i].split("=")[1]);  
              }  
           }  
           return theRequest;  
        }

        var params = GetRequest();
        var key = params.key;
       // 动态构造引入高德js
        var mapSrc = 'http://webapi.amap.com/maps?v=1.3&key='+key+'&plugin=AMap.ArrivalRange,AMap.Scale,AMap.Geocoder,AMap.Transfer,AMap.Autocomplete';
        console.log(mapSrc)
        document.write('<script src='+mapSrc+'></sc' + 'ript>')
    </script>

</head>

<body id='body'>
<div id="container"></div>

<script type="text/javascript">
    function send_mongodb(save_data){
        save_url =  decodeURIComponent(location.href)+'&'+params.id; 
        console.log('amap_data done:',save_data,save_url)
        params = {
          "_template": "amap_gps_nearby",
          "_title": save_data.community+"周边信息",
          "_timestamp": Date.parse(new Date()),
          "_hostname": '',
          "_url": save_url,
          "_data": {
            "data":save_data
            },
        "_timestampstr": new Date()
        }
        $.ajax({
            type:"POST", 
            url:"http://127.0.0.1:5125/upload",
            dataType:"json",
            contentType:"application/json",
            data:JSON.stringify(params), 
            success:function(data){ 
                console.log('POST mongodb:',data)
            }
         });
        setTimeout(function(){
            flag_over = true;
            // location.reload();
        }, 400);
        
    }

    function getAddress(params) {
        // alert('')
        var city = params.city;
        var district = params.district;
        var keyword = params.xiaoqu;
        var xiaoqu_code = params.xiaoqu_code;
        var xiaoqu = keyword;
        console.log('---------------------',city,keyword);

        AMap.service(["AMap.PlaceSearch"], function() {
            var placeSearch = new AMap.PlaceSearch({ //构造地点查询类
                pageSize: 3,
                pageIndex: 1,
                city: city, //城市
                map: MAP,
                type:"商务住宅|门牌信息",
                citylimit:true,
                extensions:"all",
                // panel: "panel"
            });
            //关键字查询
     
            placeSearch.search(keyword, function(status, result) {
                if(status==0){
                    console.log('请求失败')
                    return
                }
                console.log('getAddress info',result.info,result);
                if(keyword!='' && result.poiList &&result.poiList.pois.length>0){
                    var detail = [];
                    var detail = result.poiList.pois[0];
                    for(i=0; i<result.poiList.pois.length; i++){
                        // 对返回的数据通过区县进行二次筛选
                        var item = result.poiList.pois[i];
                        if(item.adname && district && (item.adname.indexOf(district)!=-1 || district.indexOf(item.adname)!=-1) &&
                            (xiaoqu.indexOf(item.name)!=-1 || item.name.indexOf(xiaoqu)!=-1)){
                            detail = item;
                            break;
                        }
                    }
                    var cpoint = [detail["location"]["lng"],detail["location"]["lat"]];
                    save_data = {"city":city,"district":district,"community":xiaoqu, "community_code":xiaoqu_code,"detail":detail}
                    searchNearBy(city,cpoint,keyword);
                }else{
                    flag_over = true;
                    console.log(keyword+' not find in amap');
                    // location.reload();
                }
            });
        });
    }

    function searchNearBy(city,cpoint,keyword){
        console.log('searchNearBy:',city,cpoint,keyword)
        var placeSearchNearBy = new AMap.PlaceSearch({ //构造地点查询类
                pageSize: 5,
                type: '',
                pageIndex: 1,
                city: city, //城市
                citylimit:true,
                map: MAP,
            });
        var nearby = {};
        // len = 30;
        var types = [
            {'subway':'地铁站'},{'bus':'公交车站'},{'parking_lot':'停车场'},{'other_traffic':'飞机场|火车站'},
            {'kids_school':'幼儿园'},{'primary_school':'小学'},{'middle_school':'初中'},{'high_school':'高等院校'},{'college':'大学'},{'train_school':'培训机构'},{'library':'图书馆'},{'science_museum':'科技馆'},{'AAA_hospital':'三级甲等医院'},{'special_hospital':'专科医院'},{'hospital':'综合医院'},{'clinic':'诊所'},{'pharmacy':'医药保健销售店'},{'shopping':'商场'},{'store':'便民商店|便利店'},{'supermarket':'超级市场'},{'comprehensive_market':'综合市场'},{'bank':'银行'},{'ATM':'ATM'},{'food':'中餐厅|外国餐厅|快餐厅'},{'drink':'咖啡厅|茶艺馆|冷饮店|甜品店'},{'park':'公园'},{'movies':'电影院'},{'sports':'运动场馆'},{'entertainment':'娱乐场所'},{'gym':'健身中心'}
            ]

        Search(placeSearchNearBy, types)
        function Search(placeSearchNearBy, types){
            if(types.length==0){
                flagSuccess = true;
                save_data['nearby'] = nearby;
                send_mongodb(save_data);
                return
            }
            var item = types.pop();
            for(var name in item){
                type = item[name]
            }
            placeSearchNearBy.setType(type);
            placeSearchNearBy.searchNearBy('', cpoint, 5000, function(status, result) {
                if(result.poiList && result.poiList.pois.length>0){
                    nearby[name] = result.poiList.pois;
                }
                else{
                    nearby[name] = [];
                }
                setTimeout(function(){
                    Search(placeSearchNearBy, types)
                }, 400);
                
            });
        }
    }

    save_data = {};

    setInterval(function(){
        location.reload();
    }, 1000*60*10);

    $.ajax({
            type:"get", 
            url:"http://127.0.0.1:5075/api/community",
            dataType:"json",
            contentType:"application/json",
            success:function(data){ 
                params = data;
                console.log('Getapi data',data)

                setTimeout(function(){
                    MAP = new AMap.Map("container", {
                        resizeEnable: true
                    });
                    getAddress(data)

                }, 400*10);
            }
         });

</script>
</body>
</html>

码农公寓

高德JS API

高德地址信息获取原理

准备工作

数据抓取

完整代码

相关文章