python爬虫，矢量数据地铁线路获取

import requests
import shapefile# -*- coding: utf-8 -*-
import json
import urllib
import math
import pandas as pd
import numpy as np#坐标转换&＃xff0c;转WGS84
x_pi &＃61; 3.14159265358979324 * 3000.0 / 180.0
pi &＃61; 3.1415926535897932384626 # π
a &＃61; 6378245.0 # 长半轴
ee &＃61; 0.00669342162296594323 # 偏心率平方
def gcj02_to_wgs84(lng, lat):"""坐标转化script来源于github,感谢作者的贡献&＃xff01;GCJ02(火星坐标系)转GPS84:param lng:火星坐标系的经度:param lat:火星坐标系纬度:return:"""#if out_of_china(lng, lat):#return [lng, lat]dlat &＃61; _transformlat(lng - 105.0, lat - 35.0)dlng &＃61; _transformlng(lng - 105.0, lat - 35.0)radlat &＃61; lat / 180.0 * pimagic &＃61; math.sin(radlat)magic &＃61; 1 - ee * magic * magicsqrtmagic &＃61; math.sqrt(magic)dlat &＃61; (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)dlng &＃61; (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * pi)mglat &＃61; lat &＃43; dlatmglng &＃61; lng &＃43; dlngreturn [lng * 2 - mglng, lat * 2 - mglat]def _transformlat(lng, lat):ret &＃61; -100.0 &＃43; 2.0 * lng &＃43; 3.0 * lat &＃43; 0.2 * lat * lat &＃43; \0.1 * lng * lat &＃43; 0.2 * math.sqrt(math.fabs(lng))ret &＃43;&＃61; (20.0 * math.sin(6.0 * lng * pi) &＃43; 20.0 *math.sin(2.0 * lng * pi)) * 2.0 / 3.0ret &＃43;&＃61; (20.0 * math.sin(lat * pi) &＃43; 40.0 *math.sin(lat / 3.0 * pi)) * 2.0 / 3.0ret &＃43;&＃61; (160.0 * math.sin(lat / 12.0 * pi) &＃43; 320 *math.sin(lat * pi / 30.0)) * 2.0 / 3.0return retdef _transformlng(lng, lat):ret &＃61; 300.0 &＃43; lng &＃43; 2.0 * lat &＃43; 0.1 * lng * lng &＃43; \0.1 * lng * lat &＃43; 0.1 * math.sqrt(math.fabs(lng))ret &＃43;&＃61; (20.0 * math.sin(6.0 * lng * pi) &＃43; 20.0 *math.sin(2.0 * lng * pi)) * 2.0 / 3.0ret &＃43;&＃61; (20.0 * math.sin(lng * pi) &＃43; 40.0 *math.sin(lng / 3.0 * pi)) * 2.0 / 3.0ret &＃43;&＃61; (150.0 * math.sin(lng / 12.0 * pi) &＃43; 300.0 *math.sin(lng / 30.0 * pi)) * 2.0 / 3.0return ret
#获取原始数据
def get_single_line(mykey,city_name,line_name):&＃39;&＃39;&＃39;下载单条地铁线路:param city_c: 城市名称:param line_c: 线路名称&＃xff0c;如11路、地铁一号线:return: 一条公交线路数据,追加到已存的数据后面&＃39;&＃39;&＃39;url &＃61; "https://restapi.amap.com/v3/bus/linename?s&＃61;rsv3&extensions&＃61;all&key&＃61;{}&output&＃61;json&" \"city&＃61;{}&offset&＃61;1&keywords&＃61;{}&platform&＃61;JS".format(mykey,city_name, line_name)#url &＃61; "https://restapi.amap.com/v3/bus/linename?s&＃61;rsv3&extensions&＃61;all&key&＃61;{}&output&＃61;json&" \#"city&＃61;{}&offset&＃61;1&keywords&＃61;{}&platform&＃61;JS".format(mykey,city_name, line_name)try:print("开始获取{}数据".format(line_name))response &＃61; requests.get(url).json()dt &＃61; {}if response["buslines"]:print("服务器成功返回数据")if len(response["buslines"]) &＃61;&＃61; 0:print("返回的内容没有数据")else:# 提取地铁线路名称dt["line_name"] &＃61; response["buslines"][0]["name"]#获取线路沿线的经纬度&＃xff0c;以便生成线路dt["polyline"] &＃61; response["buslines"][0]["polyline"]#获取地铁线路的最早、最晚运营时间dt["start_time"] &＃61; response["buslines"][0]["start_time"]dt["end_time"] &＃61; response["buslines"][0]["end_time"]#获取地铁线路的首末站站点名称dt["start_stop"] &＃61; response["buslines"][0]["start_stop"]dt["end_stop"] &＃61; response["buslines"][0]["end_stop"]#获取地铁线路的首末站站点名称dt["distance"] &＃61; response["buslines"][0]["distance"]#获取地铁的起始价和全程价dt["basic_price"] &＃61; response["buslines"][0]["basic_price"]dt["total_price"] &＃61; response["buslines"][0]["total_price"]#获取站点的名称和经纬度st_name &＃61; []st_coords &＃61; []for i in response["buslines"][0]["busstops"]:station_name &＃61; i["name"]station_corrds &＃61; i["location"]st_name.append(station_name)st_coords.append(station_corrds)dt["station_name"] &＃61; st_namedt["station_coords"] &＃61; st_coords#保存数据print("{}数据爬取成功".format(line_name))data &＃61; pd.DataFrame([dt])data.to_csv(r"./{}_subway_data.csv".format(city_name), mode&＃61;"a", header&＃61;False, encoding&＃61;"utf_8_sig")else:print("内容返回失败")#记录失败数据with open(r"./error.txt",mode&＃61;"a") as f:f.write("{}数据爬取失败".format(line_name))except:#记录失败数据print("{}数据爬取失败".format(line_name))with open(r"./error.txt", mode&＃61;"a") as f:f.write("{}数据爬取失败".format(line_name))#得到站点数据
def get_station():data &＃61; pd.read_csv(r"./{}_subway_data.csv".format(city_name),encoding&＃61;"gbk",engine&＃61;"python",names&＃61;["id","line_name","polyline","start_time","end_time","start_stop","end_stop","distance","basic_price","total_price","station_name","station_coords"])print(data)df &＃61; data[["line_name",&＃39;station_coords&＃39;, &＃39;station_name&＃39;]]#将字符串处理并分割成列表df["station_coords"] &＃61; df["station_coords"].apply(lambda x: x.replace("[", "").replace("]", "").replace("\&＃39;", "").split(", "))df["station_name"] &＃61; df["station_name"].apply(lambda x: x.replace("[", "").replace("]", "").replace("\&＃39;", "").split(", "))#将单元格的列表数值进行竖铺sta_name &＃61; np.hstack(df["station_name"])sta_coords &＃61; np.hstack(df["station_coords"])li_name &＃61; np.hstack(df["line_name"].repeat(list(map(len,df["station_name"]))))#将线路、站点名称、站点经纬度进行对应匹配station_data &＃61; pd.DataFrame(np.column_stack((li_name,sta_name,sta_coords)),columns&＃61;["line_name","station_name","station_coords"])#删除相同的站点station_data &＃61; station_data.drop_duplicates(["station_name"])#分别提取经度、纬度station_data["lng"] &＃61; station_data["station_coords"].apply(lambda x: x.split(",")[0])station_data["lat"] &＃61; station_data["station_coords"].apply(lambda x: x.split(",")[1])#重新设置索引&＃xff0c;使得索引从0 开始&＃xff0c;1为间隔station_data &＃61; station_data.reset_index(drop&＃61;True)#转化坐标station_data["lng84"] &＃61; 0.00station_data["lat84"] &＃61; 0.00for i in range(len(station_data)):station_data["lng84"][i] &＃61; gcj02_to_wgs84(float(station_data["lng"][i]),float(station_data["lat"][i]))[0]station_data["lat84"][i] &＃61; gcj02_to_wgs84(float(station_data["lng"][i]),float(station_data["lat"][i]))[1]#station_data.to_csv(r"./{}_subway_station_data.csv".format(city_name),encoding&＃61;"gbk")#转化为shp文件w &＃61; shapefile.Writer(r"./{}/station.shp".format(city_name))w.field("line_name","C")w.field("station_name","C")w.field("lng","C")w.field("lat","C")w.field("lng84","C")w.field("lat84","C")for i in range(len(station_data)):w.point(station_data["lng84"][i],station_data["lat84"][i])w.record(station_data["line_name"][i],station_data["station_name"][i],station_data["lng"][i],station_data["lat"][i],station_data["lng84"][i],station_data["lat84"][i],encode &＃61; "gbk")w.close()#获取线路数据
def get_line():data &＃61; pd.read_csv(r"./{}_subway_data.csv".format(city_name), encoding&＃61;"gbk", engine&＃61;"python",names&＃61;["id", "line_name", "polyline", "start_time", "end_time", "start_stop", "end_stop","distance", "basic_price", "total_price", "station_name", "station_coords"])data["polyline"] &＃61; data["polyline"].apply(lambda x: x.split(";"))#转化坐标for i in range(len(data["polyline"])):list &＃61; []all_point &＃61; data["polyline"][i]for j in range(len(all_point)):lng &＃61; all_point[j].split(",")[0]lat &＃61; all_point[j].split(",")[1]list.append(gcj02_to_wgs84(float(lng),float(lat)))data["polyline"][i] &＃61; list# 转化为shp文件w &＃61; shapefile.Writer(r"./{}/line.shp".format(city_name))w.field("line_name", "C")w.field("start_time", "C")w.field("end_time","C")w.field("start_stop","C")w.field("end_stop","C")w.field("distance", "C")w.field("basic_price", "C")w.field("total_price", "C")for i in range(len(data)):w.line([data["polyline"][i]])w.record(data["line_name"][i],data["start_time"][i],data["end_time"][i],data["start_stop"][i],data["end_stop"][i],data["distance"][i],data["basic_price"][i],data["total_price"][i], encode&＃61;"gbk")w.close()if __name__ &＃61;&＃61; "__main__":key &＃61; "***" #这里请输入key值city_name &＃61; "武汉市" #这里请输入所需要的城市名称all_line_name &＃61; [&＃39;1号线&＃39;] #这里请输入所需要的地铁线路名称for line_name in all_line_name:get_single_line(key,city_name,line_name)#get_station()#get_line()